<commit_before>package modelhelper

import (
	"koding/db/models"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type Bongo struct {
	ConstructorName string `json:"constructorName"`
	InstanceId      string `json:"instanceId"`
}

type MachineContainer struct {
	Bongo Bongo           `json:"bongo_"`
	Data  *models.Machine `json:"data"`
	*models.Machine
}

var (
	MachineColl            = "jMachines"
	MachineConstructorName = "JMachine"
)

func GetMachines(userId bson.ObjectId) ([]*MachineContainer, error) {
	machines := []*models.Machine{}

	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{"users.id": userId}).All(&machines)
	}

	err := Mongo.Run(MachineColl, query)
	if err != nil {
		return nil, err
	}

	containers := []*MachineContainer{}

	for _, machine := range machines {
		bongo := Bongo{
			ConstructorName: MachineConstructorName,
			InstanceId:      machine.ObjectId.Hex(),
		}
		container := &MachineContainer{bongo, machine, machine}

		containers = append(containers, container)
	}

	return containers, nil
}

var (
	MachineStateRunning = "Running"
)

func GetRunningVms() ([]*models.Machine, error) {
	query := bson.M{"status.state": MachineStateRunning}
	return findMachine(query)
}

func GetMachinesByUsername(username string) ([]*models.Machine, error) {
	user, err := GetUser(username)
	if err != nil {
		return nil, err
	}

	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": user.ObjectId, "owner": true},
	}}

	return findMachine(query)
}

func GetOwnMachines(userId bson.ObjectId) ([]*MachineContainer, error) {
	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": userId, "owner": true},
	}}

	return findMachineContainers(query)
}

func GetSharedMachines(userId bson.ObjectId) ([]*MachineContainer, error) {
	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": userId, "owner": false, "permanent": true},
	}}

	return findMachineContainers(query)
}

func GetCollabMachines(userId bson.ObjectId) ([]*MachineContainer, error) {
	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": userId, "owner": false, "permanent": false},
	}}

	return findMachineContainers(query)
}

func findMachineContainers(query bson.M) ([]*MachineContainer, error) {
	machines, err := findMachine(query)
	if err != nil {
		return nil, err
	}

	containers := []*MachineContainer{}

	for _, machine := range machines {
		bongo := Bongo{
			ConstructorName: MachineConstructorName,
			InstanceId:      "1", // TODO: what should go here?
		}

		container := &MachineContainer{bongo, machine, machine}
		containers = append(containers, container)
	}

	return containers, nil
}

func findMachine(query bson.M) ([]*models.Machine, error) {
	machines := []*models.Machine{}

	queryFn := func(c *mgo.Collection) error {
		iter := c.Find(query).Iter()

		var machine models.Machine
		for iter.Next(&machine) {
			var newMachine models.Machine
			newMachine = machine

			machines = append(machines, &newMachine)
		}

		return iter.Close()
	}

	if err := Mongo.Run(MachineColl, queryFn); err != nil {
		return nil, err
	}

	return machines, nil
}

func UpdateMachineAlwaysOn(machineId bson.ObjectId, alwaysOn bool) error {
	query := func(c *mgo.Collection) error {
		return c.Update(
			bson.M{"_id": machineId},
			bson.M{"$set": bson.M{"meta.alwaysOn": alwaysOn}},
		)
	}

	return Mongo.Run(MachineColl, query)
}

func CreateMachine(m *models.Machine) error {
	query := func(c *mgo.Collection) error {
		return c.Insert(m)
	}

	return Mongo.Run(MachineColl, query)
}
<commit_msg>go-webserver: check for non-truthy value for machine#users.permanent<commit_after>package modelhelper

import (
	"koding/db/models"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type Bongo struct {
	ConstructorName string `json:"constructorName"`
	InstanceId      string `json:"instanceId"`
}

type MachineContainer struct {
	Bongo Bongo           `json:"bongo_"`
	Data  *models.Machine `json:"data"`
	*models.Machine
}

var (
	MachineColl            = "jMachines"
	MachineConstructorName = "JMachine"
)

func GetMachines(userId bson.ObjectId) ([]*MachineContainer, error) {
	machines := []*models.Machine{}

	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{"users.id": userId}).All(&machines)
	}

	err := Mongo.Run(MachineColl, query)
	if err != nil {
		return nil, err
	}

	containers := []*MachineContainer{}

	for _, machine := range machines {
		bongo := Bongo{
			ConstructorName: MachineConstructorName,
			InstanceId:      machine.ObjectId.Hex(),
		}
		container := &MachineContainer{bongo, machine, machine}

		containers = append(containers, container)
	}

	return containers, nil
}

var (
	MachineStateRunning = "Running"
)

func GetRunningVms() ([]*models.Machine, error) {
	query := bson.M{"status.state": MachineStateRunning}
	return findMachine(query)
}

func GetMachinesByUsername(username string) ([]*models.Machine, error) {
	user, err := GetUser(username)
	if err != nil {
		return nil, err
	}

	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": user.ObjectId, "owner": true},
	}}

	return findMachine(query)
}

func GetOwnMachines(userId bson.ObjectId) ([]*MachineContainer, error) {
	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": userId, "owner": true},
	}}

	return findMachineContainers(query)
}

func GetSharedMachines(userId bson.ObjectId) ([]*MachineContainer, error) {
	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": userId, "owner": false, "permanent": true},
	}}

	return findMachineContainers(query)
}

func GetCollabMachines(userId bson.ObjectId) ([]*MachineContainer, error) {
	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": userId, "owner": false,
			"permanent": bson.M{"$ne": true}},
	}}

	return findMachineContainers(query)
}

func findMachineContainers(query bson.M) ([]*MachineContainer, error) {
	machines, err := findMachine(query)
	if err != nil {
		return nil, err
	}

	containers := []*MachineContainer{}

	for _, machine := range machines {
		bongo := Bongo{
			ConstructorName: MachineConstructorName,
			InstanceId:      "1", // TODO: what should go here?
		}

		container := &MachineContainer{bongo, machine, machine}
		containers = append(containers, container)
	}

	return containers, nil
}

func findMachine(query bson.M) ([]*models.Machine, error) {
	machines := []*models.Machine{}

	queryFn := func(c *mgo.Collection) error {
		iter := c.Find(query).Iter()

		var machine models.Machine
		for iter.Next(&machine) {
			var newMachine models.Machine
			newMachine = machine

			machines = append(machines, &newMachine)
		}

		return iter.Close()
	}

	if err := Mongo.Run(MachineColl, queryFn); err != nil {
		return nil, err
	}

	return machines, nil
}

func UpdateMachineAlwaysOn(machineId bson.ObjectId, alwaysOn bool) error {
	query := func(c *mgo.Collection) error {
		return c.Update(
			bson.M{"_id": machineId},
			bson.M{"$set": bson.M{"meta.alwaysOn": alwaysOn}},
		)
	}

	return Mongo.Run(MachineColl, query)
}

func CreateMachine(m *models.Machine) error {
	query := func(c *mgo.Collection) error {
		return c.Insert(m)
	}

	return Mongo.Run(MachineColl, query)
}
<|endoftext|>


<commit_before>package kontrol

import (
	"errors"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"net"
	"net/url"

	"github.com/koding/kite"
	"github.com/koding/kite/protocol"
)

type GetKodingKitesResult struct {
	Kites []*KodingKiteWithToken `json:"kites"`
}

// KodingKiteWithToken implements a KiteWithToken with a handful of additional
// fields pertaining
type KodingKiteWithToken struct {
	// These fields match protocol.KiteWithToken
	Kite  protocol.Kite `json:"kite"`
	URL   string        `json:"url"`
	KeyID string        `json:"keyId,omitempty"`
	Token string        `json:"token"`

	// The machine label, as seen from the Koding UI
	MachineLabel string `json:"machineLabel"`

	// The team names, if any.
	Teams []string `json:"teams"`
}

// HandleGetKodingKites implements GetKites with additional Koding specific response
// data.
func HandleGetKodingKites(handleGetKites kite.HandlerFunc) kite.HandlerFunc {
	return func(req *kite.Request) (interface{}, error) {
		// getKitesResponse is an interface, not the actual GetKitesResult value.
		getKitesResponse, err := handleGetKites(req)
		if err != nil {
			req.LocalKite.Log.Error("GetKitesHandler returned an error: %s", err)
			return nil, err
		}

		getKitesResult, ok := getKitesResponse.(*protocol.GetKitesResult)
		if !ok {
			return nil, errors.New("GetKites returned unexpected protocol result")
		}

		// Get all machines for the given user.
		machines, err := modelhelper.GetMachinesByUsername(req.Username)
		if err != nil {
			return nil, err
		}

		// A map of GroupId a slice of all KodingKiteWithToken that contain that group.
		kkwtsByGroupId := map[string][]*KodingKiteWithToken{}

		// Create our slice of unique group ids that we will use to query all the
		// groups that we need.
		uniqueGroups := []string{}

		// Because we are using the Kite's IP as a means to identify the machine,
		// we are sorting the machines by IP so that we can easily look them up.
		machinesByIp := map[string]*models.Machine{}
		for _, m := range machines {
			if m.IpAddress != "" {
				machinesByIp[m.IpAddress] = m
			}
		}

		// Create our KodingKitesResult which will be returned after being populated.
		kodingKitesResult := &GetKodingKitesResult{
			Kites: make([]*KodingKiteWithToken, len(getKitesResult.Kites)),
		}

		// Populate the koding kites
		for i, kwt := range getKitesResult.Kites {
			kkwt := &KodingKiteWithToken{
				Kite:  kwt.Kite,
				URL:   kwt.URL,
				KeyID: kwt.KeyID,
				Token: kwt.Token,
			}
			kodingKitesResult.Kites[i] = kkwt

			host, err := hostFromUrl(kwt.URL)
			if err != nil {
				req.LocalKite.Log.Warning(
					"Kite with badly formed URL, unable to extract host. Koding data will not be provided for this Kite. [username:%s, kiteId:%s, host:%s]",
					kwt.Kite.Username,
					kwt.Kite.ID,
					kwt.URL,
				)
				continue
			}

			// With a valid host, get the koding specific information such as Label
			// or Team names and apply it to each KodingKiteWithToken
			if machine, ok := machinesByIp[host]; ok {
				kkwt.MachineLabel = machine.Label

				for _, g := range machine.Groups {
					id := g.Id.Hex()
					// Get all of the kites (not including this one) which have
					// a reference to this kite, so we can append this kite to them.
					kkwts, ok := kkwtsByGroupId[id]

					// If the id is *not* in the kkwts map yet, it is unique.
					// Store the group id in the unique slice, so we can query for the groups.
					if !ok {
						uniqueGroups = append(uniqueGroups, id)
					}

					// kkwts could be a nil map, but that's okay as append will resolve
					// that issue.
					//
					// Note that if a Machine somehow has two entries for the same group,
					// this will be appended twice, but should not cause a problem.
					kkwtsByGroupId[id] = append(kkwts, kkwt)
				}
			}
		}

		// Now that we have all of our group ids that we want to query, get our Groups
		// from that.
		groups, err := modelhelper.GetGroupsByIds(uniqueGroups...)
		if err != nil {
			req.LocalKite.Log.Error(
				"Error returned when querying Team Names. [groupIds:%s, err:%s",
				uniqueGroups, err.Error(),
			)
		}

		// Now that we have the groups, go through our KodingKiteWithToken slice and
		// assign the team names.
		for _, group := range groups {
			id := group.Id.Hex()

			kkwts, ok := kkwtsByGroupId[id]

			// This shouldn't happen, so log a warning to aid in identifying a problem,
			// and continue.
			if !ok {
				req.LocalKite.Log.Warning(
					"Queried GroupId not present in KodingKitesWithToken by GroupId map. This is unexpected, and may represent a problem. [id:%s, ids:%s]",
					id, uniqueGroups,
				)
				continue
			}

			// Append this groups team title to the KodingKiteWithToken Teams slice.
			for _, kkwt := range kkwts {
				kkwt.Teams = append(kkwt.Teams, group.Title)
			}
		}

		return kodingKitesResult, nil
	}
}

func hostFromUrl(s string) (string, error) {
	u, err := url.Parse(s)
	if err != nil {
		return "", err
	}

	host, _, err := net.SplitHostPort(u.Host)
	if err != nil {
		return "", err
	}

	return host, nil
}
<commit_msg>styleguide: Fixed GoLint complaints<commit_after>package kontrol

import (
	"errors"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"net"
	"net/url"

	"github.com/koding/kite"
	"github.com/koding/kite/protocol"
)

// GetKodingKitesResult mirrors the Kite GetKitesResult protocol.
type GetKodingKitesResult struct {
	Kites []*KodingKiteWithToken `json:"kites"`
}

// KodingKiteWithToken mirrors the Kite library KiteWithToken with a handful of
// additional fields pertaining to Koding itself. Such as Machine Label, and
// Teams information.
type KodingKiteWithToken struct {
	// These fields match protocol.KiteWithToken
	Kite  protocol.Kite `json:"kite"`
	URL   string        `json:"url"`
	KeyID string        `json:"keyId,omitempty"`
	Token string        `json:"token"`

	// The machine label, as seen from the Koding UI
	MachineLabel string `json:"machineLabel"`

	// The team names, if any.
	Teams []string `json:"teams"`
}

// HandleGetKodingKites implements GetKites with additional Koding specific response
// data.
func HandleGetKodingKites(handleGetKites kite.HandlerFunc) kite.HandlerFunc {
	return func(req *kite.Request) (interface{}, error) {
		// getKitesResponse is an interface, not the actual GetKitesResult value.
		getKitesResponse, err := handleGetKites(req)
		if err != nil {
			req.LocalKite.Log.Error("GetKitesHandler returned an error: %s", err)
			return nil, err
		}

		getKitesResult, ok := getKitesResponse.(*protocol.GetKitesResult)
		if !ok {
			return nil, errors.New("GetKites returned unexpected protocol result")
		}

		// Get all machines for the given user.
		machines, err := modelhelper.GetMachinesByUsername(req.Username)
		if err != nil {
			return nil, err
		}

		// A map of GroupId a slice of all KodingKiteWithToken that contain that group.
		kkwtsByGroupID := map[string][]*KodingKiteWithToken{}

		// Create our slice of unique group ids that we will use to query all the
		// groups that we need.
		uniqueGroups := []string{}

		// Because we are using the Kite's IP as a means to identify the machine,
		// we are sorting the machines by IP so that we can easily look them up.
		machinesByIP := map[string]*models.Machine{}
		for _, m := range machines {
			if m.IpAddress != "" {
				machinesByIP[m.IpAddress] = m
			}
		}

		// Create our KodingKitesResult which will be returned after being populated.
		kodingKitesResult := &GetKodingKitesResult{
			Kites: make([]*KodingKiteWithToken, len(getKitesResult.Kites)),
		}

		// Populate the koding kites
		for i, kwt := range getKitesResult.Kites {
			kkwt := &KodingKiteWithToken{
				Kite:  kwt.Kite,
				URL:   kwt.URL,
				KeyID: kwt.KeyID,
				Token: kwt.Token,
			}
			kodingKitesResult.Kites[i] = kkwt

			host, err := hostFromURL(kwt.URL)
			if err != nil {
				req.LocalKite.Log.Warning(
					"Kite with badly formed URL, unable to extract host. Koding data will not be provided for this Kite. [username:%s, kiteId:%s, host:%s]",
					kwt.Kite.Username,
					kwt.Kite.ID,
					kwt.URL,
				)
				continue
			}

			// With a valid host, get the koding specific information such as Label
			// or Team names and apply it to each KodingKiteWithToken
			if machine, ok := machinesByIP[host]; ok {
				kkwt.MachineLabel = machine.Label

				for _, g := range machine.Groups {
					id := g.Id.Hex()
					// Get all of the kites (not including this one) which have
					// a reference to this kite, so we can append this kite to them.
					kkwts, ok := kkwtsByGroupID[id]

					// If the id is *not* in the kkwts map yet, it is unique.
					// Store the group id in the unique slice, so we can query for the groups.
					if !ok {
						uniqueGroups = append(uniqueGroups, id)
					}

					// kkwts could be a nil map, but that's okay as append will resolve
					// that issue.
					//
					// Note that if a Machine somehow has two entries for the same group,
					// this will be appended twice, but should not cause a problem.
					kkwtsByGroupID[id] = append(kkwts, kkwt)
				}
			}
		}

		// Now that we have all of our group ids that we want to query, get our Groups
		// from that.
		groups, err := modelhelper.GetGroupsByIds(uniqueGroups...)
		if err != nil {
			req.LocalKite.Log.Error(
				"Error returned when querying Team Names. [groupIds:%s, err:%s",
				uniqueGroups, err.Error(),
			)
		}

		// Now that we have the groups, go through our KodingKiteWithToken slice and
		// assign the team names.
		for _, group := range groups {
			id := group.Id.Hex()

			kkwts, ok := kkwtsByGroupID[id]

			// This shouldn't happen, so log a warning to aid in identifying a problem,
			// and continue.
			if !ok {
				req.LocalKite.Log.Warning(
					"Queried GroupId not present in KodingKitesWithToken by GroupId map. This is unexpected, and may represent a problem. [id:%s, ids:%s]",
					id, uniqueGroups,
				)
				continue
			}

			// Append this groups team title to the KodingKiteWithToken Teams slice.
			for _, kkwt := range kkwts {
				kkwt.Teams = append(kkwt.Teams, group.Title)
			}
		}

		return kodingKitesResult, nil
	}
}

// hostFromURL parses the given string, extracting the host (ip/domain) from the
// given string. Ignoring other data such as Port or Url Parameters.
func hostFromURL(s string) (string, error) {
	u, err := url.Parse(s)
	if err != nil {
		return "", err
	}

	host, _, err := net.SplitHostPort(u.Host)
	if err != nil {
		return "", err
	}

	return host, nil
}
<|endoftext|>


<commit_before>////////////////////////////////////////////////////////////////////////////////

package j2tf

////////////////////////////////////////////////////////////////////////////////

import (
	"errors"
	"flag"
	"fmt"
	"image/jpeg"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/recogni/sorten/gcloud"
	"github.com/recogni/sorten/logger"

	"github.com/recogni/tfutils"
	"google.golang.org/api/iterator"
)

////////////////////////////////////////////////////////////////////////////////

var (
	CLI = struct {
		// Command line args common to all commands
		inputDir       string
		outputDir      string
		recordPrefix   string
		filesPerRecord int
		imageClassId   int

		// Private stuff
		numWorkers int
	}{}
)

////////////////////////////////////////////////////////////////////////////////

type jpegToTfRecordNameFactory struct {
	sync.RWMutex
	recordCount int
}

func (jf *jpegToTfRecordNameFactory) GetNextTfRecordName(prefix string) string {
	jf.Lock()
	defer func() {
		jf.recordCount += 1
		jf.Unlock()
	}()

	return fmt.Sprintf("%s%05d.tfrecord", prefix, jf.recordCount)
}

// Global instance that is go-routine safe to access.  This is used
// by the various workers to fetch the appropriate name for the next
// record that they might want to write.  It is expected that a worker
// will only ask for a name once he finds at-least one record to write.
var j2tfFactory = &jpegToTfRecordNameFactory{recordCount: 0}

////////////////////////////////////////////////////////////////////////////////

// jpegToTfRecordStatus keeps track of a single workers status for a
// given running job.
type jpegToTfRecordStatus struct {
	writer     *tfutils.RecordWriter
	writerFile string   // path to the writer's file, if tfdst is local, this is tfdst
	workerId   int      // id of the worker that made this file
	tfdst      string   // output record path name
	count      int      // number of records that we have seen so far
	files      []string // files that we have converted so far
}

func newJpegToTfRecordStatus(wId int, dst string) *jpegToTfRecordStatus {
	writerOutFile := dst
	if gcloud.IsBucketPath(dst) {
		// Create a temporary file name for this worker to write to - if it already
		// exists, nuke the file so the writer starts again.
		writerOutFile = path.Join(os.TempDir(), fmt.Sprintf("worker_%d.tfrecord", wId))
	} else {
		// Local file - verify that its parent directory exists
		dir := path.Dir(writerOutFile)
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			if err := os.MkdirAll(dir, 0755); err != nil {
				// TODO: This should be done from a mutex :)
				// TODO: Ignore this error for now
				// return err
			}
		}
	}

	w, err := tfutils.NewWriter(writerOutFile, nil)
	if err != nil {
		panic(err)
	}

	return &jpegToTfRecordStatus{
		writer:     w,
		writerFile: writerOutFile,
		workerId:   wId,
		tfdst:      dst,
		count:      0,
		files:      []string{},
	}
}

func (js *jpegToTfRecordStatus) Marshal() []byte {
	ret := fmt.Sprintf(`# Generated meta-data file
Worker ID:     %d
Record File:   %s
Num Files:     %d
File List:
`, js.workerId, js.tfdst, js.count)
	for _, f := range js.files {
		ret += f + "\n"
	}
	return []byte(ret)
}

func (js *jpegToTfRecordStatus) AddFile(file string, commonFeatures map[string]interface{}) error {
	sourceFile := file
	if gcloud.IsBucketPath(file) {
		sourceFile = path.Join(os.TempDir(), fmt.Sprintf("worker_%d.jpeg", js.workerId))
		bp, err := gcloud.NewBucketPath(file)
		if err != nil {
			return err
		}

		bm, err := gcloud.GetBucketManager(bp.Bucket)
		if err := bm.BucketDownloadFile(sourceFile, bp); err != nil {
			return err
		}
	}

	// Load the local jpeg image and examine its bounds.
	r, err := os.Open(sourceFile)
	if err != nil {
		return err
	}

	cfg, err := jpeg.DecodeConfig(r)
	if err != nil {
		return err
	}
	r.Close()

	bs, err := ioutil.ReadFile(sourceFile)
	if err != nil {
		return err
	}

	imageFeatures := map[string]interface{}{
		"image/colorspace":  "RGB",
		"image/width":       cfg.Width,
		"image/height":      cfg.Height,
		"image/channels":    3,
		"image/format":      "JPEG",
		"image/filename":    file,
		"image/encoded":     bs,
		"image/class/label": CLI.imageClassId,
	}
	for k, v := range commonFeatures {
		imageFeatures[k] = v
	}

	fs, err := tfutils.GetFeaturesFromMap(imageFeatures)
	if err != nil {
		return err
	}

	bs, err = tfutils.GetTFRecordStringForFeatures(fs)
	if err != nil {
		return err
	}

	js.writer.WriteRecord(bs)
	js.count += 1
	js.files = append(js.files, file)

	return nil
}

func (js *jpegToTfRecordStatus) Flush() error {
	if js == nil || js.count == 0 {
		return nil
	}

	// If the destination file is a bucket path, we upload the intermediate
	// file to the expected destination.
	ext := path.Ext(js.tfdst)
	if gcloud.IsBucketPath(js.tfdst) {
		bp, err := gcloud.NewBucketPath(js.tfdst)
		if err != nil {
			return err
		}

		bm, err := gcloud.GetBucketManager(bp.Bucket)
		if err != nil {
			return err
		}

		err = bm.BucketUploadFile(bp, js.writerFile)
		if err != nil {
			fmt.Printf("Warning, unable to upload file %s\n", js.tfdst)
			return err
		}

		metaFile := strings.Replace(js.tfdst, ext, ".metadata", -1)
		bp, err = gcloud.NewBucketPath(metaFile)
		if err != nil {
			return err
		}
		return bm.BucketUpload(bp, js.Marshal())
	} else {
		metaFile := strings.Replace(js.tfdst, ext, ".metadata", -1)
		ioutil.WriteFile(metaFile, js.Marshal(), 0600)
	}

	return nil
}

////////////////////////////////////////////////////////////////////////////////

// jpegToTfRecordWorker is ...
func jpegToTfRecordWorker(workerId int, commonMap map[string]interface{}, fileq <-chan string, wg *sync.WaitGroup, wl *logger.WorkerLogger) {
	wl.Log(workerId, "is ready for work (records per tf=%d)", CLI.filesPerRecord)

	var status *jpegToTfRecordStatus
	for file := range fileq {
		// If the status is nil, and we have a file to process, grab
		// a file name from the factory and make a new status variable.
		if status == nil {
			fname := j2tfFactory.GetNextTfRecordName(CLI.recordPrefix)
			dstfp := strings.Join([]string{CLI.outputDir, fname}, string(filepath.Separator))
			status = newJpegToTfRecordStatus(workerId, dstfp)
		}

		wl.Log(workerId, "got file from queue: %s", file)

		// Once we have added this file to the record, tag it in the status.
		status.AddFile(file, commonMap)

		// If the status count has matched the number of nodes per shard,
		// we should finish up writing using the current writer and set
		// the status to nil so we grab a new file to write.
		if status.count == CLI.filesPerRecord {
			status.Flush()
			status = nil
		}

		// Job done, tag the workgroup and check for more work.
		wl.Log(workerId, "is now idle")
	}

	// If we are done with files to process, we need to close the writer
	// in the current status (if any), and proceed to tag the wait group
	// as complete.
	if status != nil {
		status.Flush()
		status = nil
	}
	wg.Done()
}

// queueFileForJpegToTfRecordJob is called on all files that are found by the
// appropriate file walker.  If the file's extension matches what we are looking
// for, it will queue it to the buffered channel of files to translate into a
// TFRecord.
func queueFileForJpegToTfRecordJob(fileq chan string, fp string) {
	if strings.ToLower(path.Ext(fp)) == ".jpeg" {
		// HACK: Ignore files with _empty_ in them
		if !strings.Contains(fp, "_empty_") {
			fileq <- fp
		}
	}
}

////////////////////////////////////////////////////////////////////////////////

func RunJob(nworkers int, args []string, wl *logger.WorkerLogger) error {
	var wg sync.WaitGroup

	// Parse arguments for this sub-command.
	fs := flag.NewFlagSet("JpegToTfRecord", flag.ExitOnError)
	fs.StringVar(&CLI.inputDir, "input", "", "input directory to read images from")
	fs.StringVar(&CLI.outputDir, "output", "", "output directory to read images from")
	fs.StringVar(&CLI.recordPrefix, "prefix", "", "tf record name prefix")
	fs.IntVar(&CLI.imageClassId, "image-class", 1002, "class to assign tf records to")
	fs.IntVar(&CLI.filesPerRecord, "shard-size", 1, "number of records per shard")
	fs.Parse(args)

	// Extract common args to set in the map based on any key-value params.
	commonMap := map[string]interface{}{}
	for _, keqv := range fs.Args() {
		items := strings.SplitN(keqv, "=", 2)
		if len(items) == 2 {
			commonMap[items[0]] = items[1]
		}
	}

	if len(CLI.inputDir) == 0 {
		return errors.New("specify input directory with --input")
	}
	if len(CLI.outputDir) == 0 {
		return errors.New("specify output directory with --output")
	}
	if !gcloud.IsBucketPath(CLI.outputDir) {
		if _, err := os.Stat(CLI.outputDir); os.IsNotExist(err) {
			if err := os.MkdirAll(CLI.outputDir, 0755); err != nil {
				return err
			}
		}
	}
	CLI.numWorkers = nworkers

	// Create a queue of files that we can keep ready for the worker to consume
	// since each worker will be aggregating a bunch of files.  Each job will
	// be responsible for creating its own output file in the output directory
	// and there will be a global mutex which controls the allowable file names.
	// Once all files have been loaded, or when the job count is hit for a job
	// worker, it is responsible for closing the record writer and asking for a
	// new output file provided there is more work to do.
	fileq := make(chan string, CLI.numWorkers)

	// Create the workers based on how many CPU cores the system has.
	for wId := 0; wId < CLI.numWorkers; wId++ {
		wg.Add(1)
		go jpegToTfRecordWorker(wId, commonMap, fileq, &wg, wl)
	}

	// TODO: SHABA_BOILERPLATE_ITERATOR
	// The below code is SUPER boilerplate! and can be abstracted out fi the
	// signature for the worker queue is identical (which it should be ... )
	if gcloud.IsBucketPath(CLI.inputDir) {
		bp, err := gcloud.NewBucketPath(CLI.inputDir)
		if err != nil {
			return err
		}

		wl.Status("Using google cloud APIs to access bucket: %s", bp.Bucket)
		bm, err := gcloud.GetBucketManager(bp.Bucket)
		if err != nil {
			return err
		}

		it, err := bm.GetBucketIterator(bp.Subpath)
		if err != nil {
			return err
		}

		c := 0
		for {
			attrs, err := it.Next()
			if err == iterator.Done {
				break
			} else {
				if err != nil {
					return err
				}
			}

			fp := "gs://" + strings.Join([]string{bp.Bucket, attrs.Name}, string(filepath.Separator))
			queueFileForJpegToTfRecordJob(fileq, fp)

			c += 1
			wl.Status("found %d files so far ...", c)
		}

	} else {
		wl.Status("Building file list ... (this can take a while on large mounted dirs) ...")
		err := filepath.Walk(CLI.inputDir, func(fp string, fi os.FileInfo, err error) error {
			if err != nil {
				wl.Status("Warning found error walking dir. Error: %s", err.Error())
			} else {
				queueFileForJpegToTfRecordJob(fileq, fp)
			}
			return nil
		})
		if err != nil {
			return err
		}
	}

	// We are done feeding work to the workers, close the fileq channel so that
	// they can end their routines.
	close(fileq)

	// Wait for all the records to finish getting written.
	wg.Wait()

	return nil
}

////////////////////////////////////////////////////////////////////////////////
<commit_msg>Add jpeg and jpg as allowed exts<commit_after>////////////////////////////////////////////////////////////////////////////////

package j2tf

////////////////////////////////////////////////////////////////////////////////

import (
	"errors"
	"flag"
	"fmt"
	"image/jpeg"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/recogni/sorten/gcloud"
	"github.com/recogni/sorten/logger"

	"github.com/recogni/tfutils"
	"google.golang.org/api/iterator"
)

////////////////////////////////////////////////////////////////////////////////

var (
	CLI = struct {
		// Command line args common to all commands
		inputDir       string
		outputDir      string
		recordPrefix   string
		filesPerRecord int
		imageClassId   int

		// Private stuff
		numWorkers int
	}{}
)

////////////////////////////////////////////////////////////////////////////////

type jpegToTfRecordNameFactory struct {
	sync.RWMutex
	recordCount int
}

func (jf *jpegToTfRecordNameFactory) GetNextTfRecordName(prefix string) string {
	jf.Lock()
	defer func() {
		jf.recordCount += 1
		jf.Unlock()
	}()

	return fmt.Sprintf("%s%05d.tfrecord", prefix, jf.recordCount)
}

// Global instance that is go-routine safe to access.  This is used
// by the various workers to fetch the appropriate name for the next
// record that they might want to write.  It is expected that a worker
// will only ask for a name once he finds at-least one record to write.
var j2tfFactory = &jpegToTfRecordNameFactory{recordCount: 0}

////////////////////////////////////////////////////////////////////////////////

// jpegToTfRecordStatus keeps track of a single workers status for a
// given running job.
type jpegToTfRecordStatus struct {
	writer     *tfutils.RecordWriter
	writerFile string   // path to the writer's file, if tfdst is local, this is tfdst
	workerId   int      // id of the worker that made this file
	tfdst      string   // output record path name
	count      int      // number of records that we have seen so far
	files      []string // files that we have converted so far
}

func newJpegToTfRecordStatus(wId int, dst string) *jpegToTfRecordStatus {
	writerOutFile := dst
	if gcloud.IsBucketPath(dst) {
		// Create a temporary file name for this worker to write to - if it already
		// exists, nuke the file so the writer starts again.
		writerOutFile = path.Join(os.TempDir(), fmt.Sprintf("worker_%d.tfrecord", wId))
	} else {
		// Local file - verify that its parent directory exists
		dir := path.Dir(writerOutFile)
		if _, err := os.Stat(dir); os.IsNotExist(err) {
			if err := os.MkdirAll(dir, 0755); err != nil {
				// TODO: This should be done from a mutex :)
				// TODO: Ignore this error for now
				// return err
			}
		}
	}

	w, err := tfutils.NewWriter(writerOutFile, nil)
	if err != nil {
		panic(err)
	}

	return &jpegToTfRecordStatus{
		writer:     w,
		writerFile: writerOutFile,
		workerId:   wId,
		tfdst:      dst,
		count:      0,
		files:      []string{},
	}
}

func (js *jpegToTfRecordStatus) Marshal() []byte {
	ret := fmt.Sprintf(`# Generated meta-data file
Worker ID:     %d
Record File:   %s
Num Files:     %d
File List:
`, js.workerId, js.tfdst, js.count)
	for _, f := range js.files {
		ret += f + "\n"
	}
	return []byte(ret)
}

func (js *jpegToTfRecordStatus) AddFile(file string, commonFeatures map[string]interface{}) error {
	sourceFile := file
	if gcloud.IsBucketPath(file) {
		sourceFile = path.Join(os.TempDir(), fmt.Sprintf("worker_%d.jpeg", js.workerId))
		bp, err := gcloud.NewBucketPath(file)
		if err != nil {
			return err
		}

		bm, err := gcloud.GetBucketManager(bp.Bucket)
		if err := bm.BucketDownloadFile(sourceFile, bp); err != nil {
			return err
		}
	}

	// Load the local jpeg image and examine its bounds.
	r, err := os.Open(sourceFile)
	if err != nil {
		return err
	}

	cfg, err := jpeg.DecodeConfig(r)
	if err != nil {
		return err
	}
	r.Close()

	bs, err := ioutil.ReadFile(sourceFile)
	if err != nil {
		return err
	}

	imageFeatures := map[string]interface{}{
		"image/colorspace":  "RGB",
		"image/width":       cfg.Width,
		"image/height":      cfg.Height,
		"image/channels":    3,
		"image/format":      "JPEG",
		"image/filename":    file,
		"image/encoded":     bs,
		"image/class/label": CLI.imageClassId,
	}
	for k, v := range commonFeatures {
		imageFeatures[k] = v
	}

	fs, err := tfutils.GetFeaturesFromMap(imageFeatures)
	if err != nil {
		return err
	}

	bs, err = tfutils.GetTFRecordStringForFeatures(fs)
	if err != nil {
		return err
	}

	js.writer.WriteRecord(bs)
	js.count += 1
	js.files = append(js.files, file)

	return nil
}

func (js *jpegToTfRecordStatus) Flush() error {
	if js == nil || js.count == 0 {
		return nil
	}

	// If the destination file is a bucket path, we upload the intermediate
	// file to the expected destination.
	ext := path.Ext(js.tfdst)
	if gcloud.IsBucketPath(js.tfdst) {
		bp, err := gcloud.NewBucketPath(js.tfdst)
		if err != nil {
			return err
		}

		bm, err := gcloud.GetBucketManager(bp.Bucket)
		if err != nil {
			return err
		}

		err = bm.BucketUploadFile(bp, js.writerFile)
		if err != nil {
			fmt.Printf("Warning, unable to upload file %s\n", js.tfdst)
			return err
		}

		metaFile := strings.Replace(js.tfdst, ext, ".metadata", -1)
		bp, err = gcloud.NewBucketPath(metaFile)
		if err != nil {
			return err
		}
		return bm.BucketUpload(bp, js.Marshal())
	} else {
		metaFile := strings.Replace(js.tfdst, ext, ".metadata", -1)
		ioutil.WriteFile(metaFile, js.Marshal(), 0600)
	}

	return nil
}

////////////////////////////////////////////////////////////////////////////////

// jpegToTfRecordWorker is ...
func jpegToTfRecordWorker(workerId int, commonMap map[string]interface{}, fileq <-chan string, wg *sync.WaitGroup, wl *logger.WorkerLogger) {
	wl.Log(workerId, "is ready for work (records per tf=%d)", CLI.filesPerRecord)

	var status *jpegToTfRecordStatus
	for file := range fileq {
		// If the status is nil, and we have a file to process, grab
		// a file name from the factory and make a new status variable.
		if status == nil {
			fname := j2tfFactory.GetNextTfRecordName(CLI.recordPrefix)
			dstfp := strings.Join([]string{CLI.outputDir, fname}, string(filepath.Separator))
			status = newJpegToTfRecordStatus(workerId, dstfp)
		}

		wl.Log(workerId, "got file from queue: %s", file)

		// Once we have added this file to the record, tag it in the status.
		status.AddFile(file, commonMap)

		// If the status count has matched the number of nodes per shard,
		// we should finish up writing using the current writer and set
		// the status to nil so we grab a new file to write.
		if status.count == CLI.filesPerRecord {
			status.Flush()
			status = nil
		}

		// Job done, tag the workgroup and check for more work.
		wl.Log(workerId, "is now idle")
	}

	// If we are done with files to process, we need to close the writer
	// in the current status (if any), and proceed to tag the wait group
	// as complete.
	if status != nil {
		status.Flush()
		status = nil
	}
	wg.Done()
}

// queueFileForJpegToTfRecordJob is called on all files that are found by the
// appropriate file walker.  If the file's extension matches what we are looking
// for, it will queue it to the buffered channel of files to translate into a
// TFRecord.
func queueFileForJpegToTfRecordJob(fileq chan string, fp string) {
	if strings.ToLower(path.Ext(fp)) == ".jpeg" || strings.ToLower(path.Ext(fp)) == ".jpg" {
		// HACK: Ignore files with _empty_ in them
		if !strings.Contains(fp, "_empty_") {
			fileq <- fp
		}
	}
}

////////////////////////////////////////////////////////////////////////////////

func RunJob(nworkers int, args []string, wl *logger.WorkerLogger) error {
	var wg sync.WaitGroup

	// Parse arguments for this sub-command.
	fs := flag.NewFlagSet("JpegToTfRecord", flag.ExitOnError)
	fs.StringVar(&CLI.inputDir, "input", "", "input directory to read images from")
	fs.StringVar(&CLI.outputDir, "output", "", "output directory to read images from")
	fs.StringVar(&CLI.recordPrefix, "prefix", "", "tf record name prefix")
	fs.IntVar(&CLI.imageClassId, "image-class", 1002, "class to assign tf records to")
	fs.IntVar(&CLI.filesPerRecord, "shard-size", 1, "number of records per shard")
	fs.Parse(args)

	// Extract common args to set in the map based on any key-value params.
	commonMap := map[string]interface{}{}
	for _, keqv := range fs.Args() {
		items := strings.SplitN(keqv, "=", 2)
		if len(items) == 2 {
			commonMap[items[0]] = items[1]
		}
	}

	if len(CLI.inputDir) == 0 {
		return errors.New("specify input directory with --input")
	}
	if len(CLI.outputDir) == 0 {
		return errors.New("specify output directory with --output")
	}
	if !gcloud.IsBucketPath(CLI.outputDir) {
		if _, err := os.Stat(CLI.outputDir); os.IsNotExist(err) {
			if err := os.MkdirAll(CLI.outputDir, 0755); err != nil {
				return err
			}
		}
	}
	CLI.numWorkers = nworkers

	// Create a queue of files that we can keep ready for the worker to consume
	// since each worker will be aggregating a bunch of files.  Each job will
	// be responsible for creating its own output file in the output directory
	// and there will be a global mutex which controls the allowable file names.
	// Once all files have been loaded, or when the job count is hit for a job
	// worker, it is responsible for closing the record writer and asking for a
	// new output file provided there is more work to do.
	fileq := make(chan string, CLI.numWorkers)

	// Create the workers based on how many CPU cores the system has.
	for wId := 0; wId < CLI.numWorkers; wId++ {
		wg.Add(1)
		go jpegToTfRecordWorker(wId, commonMap, fileq, &wg, wl)
	}

	// TODO: SHABA_BOILERPLATE_ITERATOR
	// The below code is SUPER boilerplate! and can be abstracted out fi the
	// signature for the worker queue is identical (which it should be ... )
	if gcloud.IsBucketPath(CLI.inputDir) {
		bp, err := gcloud.NewBucketPath(CLI.inputDir)
		if err != nil {
			return err
		}

		wl.Status("Using google cloud APIs to access bucket: %s", bp.Bucket)
		bm, err := gcloud.GetBucketManager(bp.Bucket)
		if err != nil {
			return err
		}

		it, err := bm.GetBucketIterator(bp.Subpath)
		if err != nil {
			return err
		}

		c := 0
		for {
			attrs, err := it.Next()
			if err == iterator.Done {
				break
			} else {
				if err != nil {
					return err
				}
			}

			fp := "gs://" + strings.Join([]string{bp.Bucket, attrs.Name}, string(filepath.Separator))
			queueFileForJpegToTfRecordJob(fileq, fp)

			c += 1
			wl.Status("found %d files so far ...", c)
		}

	} else {
		wl.Status("Building file list ... (this can take a while on large mounted dirs) ...")
		err := filepath.Walk(CLI.inputDir, func(fp string, fi os.FileInfo, err error) error {
			if err != nil {
				wl.Status("Warning found error walking dir. Error: %s", err.Error())
			} else {
				queueFileForJpegToTfRecordJob(fileq, fp)
			}
			return nil
		})
		if err != nil {
			return err
		}
	}

	// We are done feeding work to the workers, close the fileq channel so that
	// they can end their routines.
	close(fileq)

	// Wait for all the records to finish getting written.
	wg.Wait()

	return nil
}

////////////////////////////////////////////////////////////////////////////////
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/gopherjs/gopherjs/js"
	"github.com/jaracil/nxcli"
)

func main() {

	// Node.js
	if require := js.Global.Get("require"); require != js.Undefined {
		if ws := require.Invoke("ws"); ws != js.Undefined {
			fmt.Println("Running on Node.js with websocket module: 'ws'")

			if removeListener := ws.Get("prototype").Get("removeEventListener"); removeListener == js.Undefined {
				fmt.Println("Missing removeEventListener! Patching WebSocket library...")

				ws.Get("prototype").Set("removeEventListener", js.MakeFunc(func(this *js.Object, args []*js.Object) interface{} {
					this.Call("removeListener", args[0], args[1])
					return nil
				}))

				fmt.Println("OK")
			}

			js.Global.Set("WebSocket", ws)
		} else {
			fmt.Println("Running on Node.js but websocket module missing")
		}
	}

	dial := func(a string, cb func(interface{}, error)) {
		go func() {
			nc, e := nexus.Dial(a, nil)

			jsnc := js.MakeWrapper(nc)
			PatchNexusAsync(jsnc, nc)

			go cb(jsnc, e)
		}()
	}

	if js.Module == js.Undefined {
		// Browser
		js.Global.Set("Nexus", dial)
	} else {
		// Node.js
		js.Module.Get("exports").Set("dial", dial)
	}

	fmt.Println("Nexus Client loaded")
}
<commit_msg>Expose Nexus errors map<commit_after>package main

import (
	"fmt"

	"github.com/gopherjs/gopherjs/js"
	"github.com/jaracil/nxcli"
	"github.com/jaracil/nxcli/nxcore"
)

func main() {

	// Node.js
	if require := js.Global.Get("require"); require != js.Undefined {
		if ws := require.Invoke("ws"); ws != js.Undefined {
			fmt.Println("Running on Node.js with websocket module: 'ws'")

			if removeListener := ws.Get("prototype").Get("removeEventListener"); removeListener == js.Undefined {
				fmt.Println("Missing removeEventListener! Patching WebSocket library...")

				ws.Get("prototype").Set("removeEventListener", js.MakeFunc(func(this *js.Object, args []*js.Object) interface{} {
					this.Call("removeListener", args[0], args[1])
					return nil
				}))

				fmt.Println("OK")
			}

			js.Global.Set("WebSocket", ws)
		} else {
			fmt.Println("Running on Node.js but websocket module missing")
		}
	}

	dial := func(a string, cb func(interface{}, error)) {
		go func() {
			nc, e := nexus.Dial(a, nil)

			jsnc := js.MakeWrapper(nc)
			PatchNexusAsync(jsnc, nc)

			go cb(jsnc, e)
		}()
	}

	if js.Module == js.Undefined {
		// Browser
		js.Global.Set("Nexus", make(map[interface{}]interface{}))
		nexus := js.Global.Get("Nexus")
		nexus.Set("Dial", dial)
		nexus.Set("Errors", nxcore.ErrStr)
	} else {
		// Node.js
		js.Module.Get("exports").Set("dial", dial)
	}

	fmt.Println("Nexus Client loaded")
}
<|endoftext|>


<commit_before>package pluginproxy

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"text/template"

	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/util"
	"golang.org/x/oauth2/google"
)

//ApplyRoute should use the plugin route data to set auth headers and custom headers
func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route *plugins.AppPluginRoute, ds *m.DataSource) {
	proxyPath = strings.TrimPrefix(proxyPath, route.Path)

	data := templateData{
		JsonData:       ds.JsonData.Interface().(map[string]interface{}),
		SecureJsonData: ds.SecureJsonData.Decrypt(),
	}

	interpolatedURL, err := interpolateString(route.Url, data)
	if err != nil {
		logger.Error("Error interpolating proxy url", "error", err)
		return
	}

	routeURL, err := url.Parse(interpolatedURL)
	if err != nil {
		logger.Error("Error parsing plugin route url", "error", err)
		return
	}

	req.URL.Scheme = routeURL.Scheme
	req.URL.Host = routeURL.Host
	req.Host = routeURL.Host
	req.URL.Path = util.JoinUrlFragments(routeURL.Path, proxyPath)

	if err := addHeaders(&req.Header, route, data); err != nil {
		logger.Error("Failed to render plugin headers", "error", err)
	}

	tokenProvider := newAccessTokenProvider(ds, route)

	if route.TokenAuth != nil {
		if token, err := tokenProvider.getAccessToken(data); err != nil {
			logger.Error("Failed to get access token", "error", err)
		} else {
			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
		}
	}

	if route.JwtTokenAuth != nil && len(ds.SecureJsonData["privateKey"]) != 0 {
		if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil {
			logger.Error("Failed to get access token", "error", err)
		} else {
			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
		}
	}

	if req.Header.Get("Authorization") == "" && ds.Type == "stackdriver" {
		defaultCredentials, err := google.FindDefaultCredentials(ctx)
		projectName := ds.JsonData.Get("defaultProject").MustString()
		if projectName == "" {
			ds.JsonData.Set("defaultProject", defaultCredentials.ProjectID)
		}
		if err != nil {
			logger.Error("Failed to get default credentials", "error", err)
		} else {
			token, err := defaultCredentials.TokenSource.Token()
			if err != nil {
				logger.Error("Failed to get default access token", "error", err)
			} else {
				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken))
			}
		}
	}

	if req.Header.Get("Authorization") == "" && ds.Type == "stackdriver" {
		credentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read")
		projectName := ds.JsonData.Get("defaultProject").MustString()
		if projectName == "" {
			ds.JsonData.Set("defaultProject", credentials.ProjectID)
		}
		if err != nil {
			logger.Error("Failed to get access token", "error", err)
		} else {
			token, err := credentials.TokenSource.Token()
			if err != nil {
				logger.Error("Failed to get access token", "error", err)
			} else {
				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken))
			}
		}
	}

	logger.Info("Requesting", "url", req.URL.String())

}

func interpolateString(text string, data templateData) (string, error) {
	t, err := template.New("content").Parse(text)
	if err != nil {
		return "", fmt.Errorf("could not parse template %s", text)
	}

	var contentBuf bytes.Buffer
	err = t.Execute(&contentBuf, data)
	if err != nil {
		return "", fmt.Errorf("failed to execute template %s", text)
	}

	return contentBuf.String(), nil
}

func addHeaders(reqHeaders *http.Header, route *plugins.AppPluginRoute, data templateData) error {
	for _, header := range route.Headers {
		interpolated, err := interpolateString(header.Content, data)
		if err != nil {
			return err
		}
		reqHeaders.Add(header.Name, interpolated)
	}

	return nil
}
<commit_msg>stackdriver: wip - add scope and remove debug code<commit_after>package pluginproxy

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"text/template"

	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/util"
	"golang.org/x/oauth2/google"
)

//ApplyRoute should use the plugin route data to set auth headers and custom headers
func ApplyRoute(ctx context.Context, req *http.Request, proxyPath string, route *plugins.AppPluginRoute, ds *m.DataSource) {
	proxyPath = strings.TrimPrefix(proxyPath, route.Path)

	data := templateData{
		JsonData:       ds.JsonData.Interface().(map[string]interface{}),
		SecureJsonData: ds.SecureJsonData.Decrypt(),
	}

	interpolatedURL, err := interpolateString(route.Url, data)
	if err != nil {
		logger.Error("Error interpolating proxy url", "error", err)
		return
	}

	routeURL, err := url.Parse(interpolatedURL)
	if err != nil {
		logger.Error("Error parsing plugin route url", "error", err)
		return
	}

	req.URL.Scheme = routeURL.Scheme
	req.URL.Host = routeURL.Host
	req.Host = routeURL.Host
	req.URL.Path = util.JoinUrlFragments(routeURL.Path, proxyPath)

	if err := addHeaders(&req.Header, route, data); err != nil {
		logger.Error("Failed to render plugin headers", "error", err)
	}

	tokenProvider := newAccessTokenProvider(ds, route)

	if route.TokenAuth != nil {
		if token, err := tokenProvider.getAccessToken(data); err != nil {
			logger.Error("Failed to get access token", "error", err)
		} else {
			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
		}
	}

	if route.JwtTokenAuth != nil && len(ds.SecureJsonData["privateKey"]) != 0 {
		if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil {
			logger.Error("Failed to get access token", "error", err)
		} else {
			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
		}
	}

	if req.Header.Get("Authorization") == "" && ds.Type == "stackdriver" {
		defaultCredentials, err := google.FindDefaultCredentials(ctx, "https://www.googleapis.com/auth/monitoring.read")
		projectName := ds.JsonData.Get("defaultProject").MustString()
		if projectName == "" {
			ds.JsonData.Set("defaultProject", defaultCredentials.ProjectID)
		}
		if err != nil {
			logger.Error("Failed to get default credentials", "error", err)
		} else {
			token, err := defaultCredentials.TokenSource.Token()
			if err != nil {
				logger.Error("Failed to get default access token", "error", err)
			} else {
				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken))
			}
		}
	}

	logger.Info("Requesting", "url", req.URL.String())

}

func interpolateString(text string, data templateData) (string, error) {
	t, err := template.New("content").Parse(text)
	if err != nil {
		return "", fmt.Errorf("could not parse template %s", text)
	}

	var contentBuf bytes.Buffer
	err = t.Execute(&contentBuf, data)
	if err != nil {
		return "", fmt.Errorf("failed to execute template %s", text)
	}

	return contentBuf.String(), nil
}

func addHeaders(reqHeaders *http.Header, route *plugins.AppPluginRoute, data templateData) error {
	for _, header := range route.Headers {
		interpolated, err := interpolateString(header.Content, data)
		if err != nil {
			return err
		}
		reqHeaders.Add(header.Name, interpolated)
	}

	return nil
}
<|endoftext|>


<commit_before>package pluginproxy

import (
	"bytes"
	"context"
	"fmt"
	"html/template"
	"net/http"
	"net/url"
	"strings"

	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/util"
)

func applyRoute(ctx context.Context, req *http.Request, proxyPath string, route *plugins.AppPluginRoute, ds *m.DataSource) {
	proxyPath = strings.TrimPrefix(proxyPath, route.Path)

	data := templateData{
		JsonData:       ds.JsonData.Interface().(map[string]interface{}),
		SecureJsonData: ds.SecureJsonData.Decrypt(),
	}

	interpolatedURL, err := interpolateString(route.Url, data)
	if err != nil {
		logger.Error("Error interpolating proxy url", "error", err)
		return
	}

	routeURL, err := url.Parse(interpolatedURL)
	if err != nil {
		logger.Error("Error parsing plugin route url", "error", err)
		return
	}

	req.URL.Scheme = routeURL.Scheme
	req.URL.Host = routeURL.Host
	req.Host = routeURL.Host
	req.URL.Path = util.JoinUrlFragments(routeURL.Path, proxyPath)

	if err := addHeaders(&req.Header, route, data); err != nil {
		logger.Error("Failed to render plugin headers", "error", err)
	}

	tokenProvider := newAccessTokenProvider(ds.Id, route)

	if route.TokenAuth != nil {
		if token, err := tokenProvider.getAccessToken(data); err != nil {
			logger.Error("Failed to get access token", "error", err)
		} else {
			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
		}
	}

	if route.JwtTokenAuth != nil {
		if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil {
			logger.Error("Failed to get access token", "error", err)
		} else {
			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
		}
	}
	logger.Info("Requesting", "url", req.URL.String())

}

func interpolateString(text string, data templateData) (string, error) {
	t, err := template.New("content").Parse(text)
	if err != nil {
		return "", fmt.Errorf("could not parse template %s", text)
	}

	var contentBuf bytes.Buffer
	err = t.Execute(&contentBuf, data)
	if err != nil {
		return "", fmt.Errorf("failed to execute template %s", text)
	}

	return contentBuf.String(), nil
}

func addHeaders(reqHeaders *http.Header, route *plugins.AppPluginRoute, data templateData) error {
	for _, header := range route.Headers {
		interpolated, err := interpolateString(header.Content, data)
		if err != nil {
			return err
		}
		reqHeaders.Add(header.Name, interpolated)
	}

	return nil
}
<commit_msg>stackdriver: fix test<commit_after>package pluginproxy

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"text/template"

	m "github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/util"
)

func applyRoute(ctx context.Context, req *http.Request, proxyPath string, route *plugins.AppPluginRoute, ds *m.DataSource) {
	proxyPath = strings.TrimPrefix(proxyPath, route.Path)

	data := templateData{
		JsonData:       ds.JsonData.Interface().(map[string]interface{}),
		SecureJsonData: ds.SecureJsonData.Decrypt(),
	}

	interpolatedURL, err := interpolateString(route.Url, data)
	if err != nil {
		logger.Error("Error interpolating proxy url", "error", err)
		return
	}

	routeURL, err := url.Parse(interpolatedURL)
	if err != nil {
		logger.Error("Error parsing plugin route url", "error", err)
		return
	}

	req.URL.Scheme = routeURL.Scheme
	req.URL.Host = routeURL.Host
	req.Host = routeURL.Host
	req.URL.Path = util.JoinUrlFragments(routeURL.Path, proxyPath)

	if err := addHeaders(&req.Header, route, data); err != nil {
		logger.Error("Failed to render plugin headers", "error", err)
	}

	tokenProvider := newAccessTokenProvider(ds.Id, route)

	if route.TokenAuth != nil {
		if token, err := tokenProvider.getAccessToken(data); err != nil {
			logger.Error("Failed to get access token", "error", err)
		} else {
			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
		}
	}

	if route.JwtTokenAuth != nil {
		if token, err := tokenProvider.getJwtAccessToken(ctx, data); err != nil {
			logger.Error("Failed to get access token", "error", err)
		} else {
			req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
		}
	}
	logger.Info("Requesting", "url", req.URL.String())

}

func interpolateString(text string, data templateData) (string, error) {
	t, err := template.New("content").Parse(text)
	if err != nil {
		return "", fmt.Errorf("could not parse template %s", text)
	}

	var contentBuf bytes.Buffer
	err = t.Execute(&contentBuf, data)
	if err != nil {
		return "", fmt.Errorf("failed to execute template %s", text)
	}

	return contentBuf.String(), nil
}

func addHeaders(reqHeaders *http.Header, route *plugins.AppPluginRoute, data templateData) error {
	for _, header := range route.Headers {
		interpolated, err := interpolateString(header.Content, data)
		if err != nil {
			return err
		}
		reqHeaders.Add(header.Name, interpolated)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright (C) 2016 Red Hat, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"github.com/docker/machine/drivers/hyperv"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/jimmidyson/minishift/pkg/minikube/constants"
)

func createHypervHost(config MachineConfig) drivers.Driver {
	d := hyperv.NewDriver(constants.MachineName, constants.Minipath)
	d.Boot2DockerURL = config.MinikubeISO
	d.MemSize = config.Memory
	d.CPU = config.CPUs
	d.DiskSize = int(config.DiskSize)
	return d
}
<commit_msg>Add docker ssh user to hyperv config<commit_after>/*
Copyright (C) 2016 Red Hat, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"github.com/docker/machine/drivers/hyperv"
	"github.com/docker/machine/libmachine/drivers"
	"github.com/jimmidyson/minishift/pkg/minikube/constants"
)

func createHypervHost(config MachineConfig) drivers.Driver {
	d := hyperv.NewDriver(constants.MachineName, constants.Minipath)
	d.Boot2DockerURL = config.MinikubeISO
	d.MemSize = config.Memory
	d.CPU = config.CPUs
	d.DiskSize = int(config.DiskSize)
	d.SSHUser = "docker"
	return d
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lint

import (
	"context"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"testing"

	"sigs.k8s.io/kustomize/kyaml/yaml"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/parser"
	v1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/version"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

var testSkaffoldYaml = `apiVersion: skaffold/v2beta21
kind: Config
build:
  artifacts:
    - image: test-app
`

var invalidSkaffoldYaml = `invalid{\{\Yaml}`

func TestGetSkaffoldYamlsLintResults(t *testing.T) {
	tests := []struct {
		description            string
		moduleAndSkaffoldYamls map[string]string
		profiles               []string
		module                 []string
		shouldErr              bool
		err                    error
		expected               map[string]*[]Result
	}{
		{
			description:            "get all skaffold yaml lint rules for 2 skaffold yaml files",
			moduleAndSkaffoldYamls: map[string]string{"cfg0": testSkaffoldYaml, "cfg1": testSkaffoldYaml},
			expected: map[string]*[]Result{
				"cfg0": {
					{
						Rule: &Rule{
							Filter: YamlFieldFilter{
								Filter: yaml.FieldMatcher{Name: "apiVersion", StringRegexValue: fmt.Sprintf("[^%s]", version.Get().ConfigVersion)},
							},
							RuleID:   SkaffoldYamlAPIVersionOutOfDate,
							RuleType: YamlFieldLintRule,
							Explanation: fmt.Sprintf("Found 'apiVersion' field with value that is not the latest skaffold apiVersion. Modify the apiVersion to the latest version: `apiVersion: %s` "+
								"or run the 'skaffold fix' command to have skaffold upgrade this for you.", version.Get().ConfigVersion),
						},
						Line:   1,
						Column: 12,
					},
				},
				"cfg1": {
					{
						Rule: &Rule{
							Filter: YamlFieldFilter{
								Filter: yaml.FieldMatcher{Name: "apiVersion", StringRegexValue: fmt.Sprintf("[^%s]", version.Get().ConfigVersion)},
							},
							RuleID:   SkaffoldYamlAPIVersionOutOfDate,
							RuleType: YamlFieldLintRule,
							Explanation: fmt.Sprintf("Found 'apiVersion' field with value that is not the latest skaffold apiVersion. Modify the apiVersion to the latest version: `apiVersion: %s` "+
								"or run the 'skaffold fix' command to have skaffold upgrade this for you.", version.Get().ConfigVersion),
						},
						Line:   1,
						Column: 12,
					},
				},
			},
		},
		{
			description:            "get all skaffold yaml lint rules for one module",
			moduleAndSkaffoldYamls: map[string]string{"cfg0": testSkaffoldYaml, "cfg1": testSkaffoldYaml},
			module:                 []string{"cfg0"},
			expected: map[string]*[]Result{
				"cfg0": {
					{
						Rule: &Rule{
							Filter: YamlFieldFilter{
								Filter: yaml.FieldMatcher{Name: "apiVersion", StringRegexValue: fmt.Sprintf("[^%s]", version.Get().ConfigVersion)},
							},
							RuleID:   SkaffoldYamlAPIVersionOutOfDate,
							RuleType: YamlFieldLintRule,
							Explanation: fmt.Sprintf("Found 'apiVersion' field with value that is not the latest skaffold apiVersion. Modify the apiVersion to the latest version: `apiVersion: %s` "+
								"or run the 'skaffold fix' command to have skaffold upgrade this for you.", version.Get().ConfigVersion),
						},
						Line:   1,
						Column: 12,
					},
				},
			},
		},
		{
			description:            "invalid skaffold yaml file",
			moduleAndSkaffoldYamls: map[string]string{"cfg0": invalidSkaffoldYaml},
			shouldErr:              true,
		},
	}

	for _, test := range tests {
		testutil.Run(t, test.description, func(t *testutil.T) {
			tmpdir := t.TempDir()
			configSet := parser.SkaffoldConfigSet{}
			for module, skaffoldyamlText := range test.moduleAndSkaffoldYamls {
				fp := filepath.Join(tmpdir, fmt.Sprintf("%s.yaml", module))
				err := ioutil.WriteFile(fp, []byte(skaffoldyamlText), 0644)
				if err != nil {
					t.Fatalf("error creating skaffold.yaml file with name %s: %v", fp, err)
				}
				configSet = append(configSet, &parser.SkaffoldConfigEntry{SkaffoldConfig: &v1.SkaffoldConfig{
					Metadata: v1.Metadata{Name: module},
				},
					SourceFile: fp,
				})
				// test overwrites file paths for expected SkaffoldYamlRules as they are made dynamically
				results := test.expected[module]
				if results == nil {
					continue
				}
				for i := range *results {
					(*results)[i].AbsFilePath = configSet[len(configSet)-1].SourceFile
					(*results)[i].RelFilePath = configSet[len(configSet)-1].SourceFile
				}
			}
			t.Override(&realWorkDir, func() (string, error) {
				return "", nil
			})
			t.Override(&getConfigSet, func(_ context.Context, opts config.SkaffoldOptions) (parser.SkaffoldConfigSet, error) {
				// mock profile activation
				var set parser.SkaffoldConfigSet
				for _, c := range configSet {
					if len(opts.ConfigurationFilter) > 0 && !util.StrSliceContains(opts.ConfigurationFilter, c.Metadata.Name) {
						continue
					}
					for _, pName := range opts.Profiles {
						for _, profile := range c.Profiles {
							if profile.Name != pName {
								continue
							}
							c.Test = profile.Test
						}
					}
					set = append(set, c)
				}
				return set, test.err
			})
			results, err := GetSkaffoldYamlsLintResults(context.Background(), Options{
				OutFormat: "json", Modules: test.module, Profiles: test.profiles})
			t.CheckError(test.shouldErr, err)
			expectedResults := &[]Result{}

			// this is done to enforce result order
			for i := 0; i < len(test.expected); i++ {
				*expectedResults = append(*expectedResults, *test.expected[fmt.Sprintf("cfg%d", i)]...)
			}
			if !test.shouldErr {
				t.CheckDeepEqual(expectedResults, results)
			}
		})
	}
}
<commit_msg>fix: update stale util.StringSlice use to new stringslice lib (#6752)<commit_after>/*
Copyright 2021 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package lint

import (
	"context"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"testing"

	"sigs.k8s.io/kustomize/kyaml/yaml"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/config"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/parser"
	v1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util/stringslice"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/version"
	"github.com/GoogleContainerTools/skaffold/testutil"
)

var testSkaffoldYaml = `apiVersion: skaffold/v2beta21
kind: Config
build:
  artifacts:
    - image: test-app
`

var invalidSkaffoldYaml = `invalid{\{\Yaml}`

func TestGetSkaffoldYamlsLintResults(t *testing.T) {
	tests := []struct {
		description            string
		moduleAndSkaffoldYamls map[string]string
		profiles               []string
		module                 []string
		shouldErr              bool
		err                    error
		expected               map[string]*[]Result
	}{
		{
			description:            "get all skaffold yaml lint rules for 2 skaffold yaml files",
			moduleAndSkaffoldYamls: map[string]string{"cfg0": testSkaffoldYaml, "cfg1": testSkaffoldYaml},
			expected: map[string]*[]Result{
				"cfg0": {
					{
						Rule: &Rule{
							Filter: YamlFieldFilter{
								Filter: yaml.FieldMatcher{Name: "apiVersion", StringRegexValue: fmt.Sprintf("[^%s]", version.Get().ConfigVersion)},
							},
							RuleID:   SkaffoldYamlAPIVersionOutOfDate,
							RuleType: YamlFieldLintRule,
							Explanation: fmt.Sprintf("Found 'apiVersion' field with value that is not the latest skaffold apiVersion. Modify the apiVersion to the latest version: `apiVersion: %s` "+
								"or run the 'skaffold fix' command to have skaffold upgrade this for you.", version.Get().ConfigVersion),
						},
						Line:   1,
						Column: 12,
					},
				},
				"cfg1": {
					{
						Rule: &Rule{
							Filter: YamlFieldFilter{
								Filter: yaml.FieldMatcher{Name: "apiVersion", StringRegexValue: fmt.Sprintf("[^%s]", version.Get().ConfigVersion)},
							},
							RuleID:   SkaffoldYamlAPIVersionOutOfDate,
							RuleType: YamlFieldLintRule,
							Explanation: fmt.Sprintf("Found 'apiVersion' field with value that is not the latest skaffold apiVersion. Modify the apiVersion to the latest version: `apiVersion: %s` "+
								"or run the 'skaffold fix' command to have skaffold upgrade this for you.", version.Get().ConfigVersion),
						},
						Line:   1,
						Column: 12,
					},
				},
			},
		},
		{
			description:            "get all skaffold yaml lint rules for one module",
			moduleAndSkaffoldYamls: map[string]string{"cfg0": testSkaffoldYaml, "cfg1": testSkaffoldYaml},
			module:                 []string{"cfg0"},
			expected: map[string]*[]Result{
				"cfg0": {
					{
						Rule: &Rule{
							Filter: YamlFieldFilter{
								Filter: yaml.FieldMatcher{Name: "apiVersion", StringRegexValue: fmt.Sprintf("[^%s]", version.Get().ConfigVersion)},
							},
							RuleID:   SkaffoldYamlAPIVersionOutOfDate,
							RuleType: YamlFieldLintRule,
							Explanation: fmt.Sprintf("Found 'apiVersion' field with value that is not the latest skaffold apiVersion. Modify the apiVersion to the latest version: `apiVersion: %s` "+
								"or run the 'skaffold fix' command to have skaffold upgrade this for you.", version.Get().ConfigVersion),
						},
						Line:   1,
						Column: 12,
					},
				},
			},
		},
		{
			description:            "invalid skaffold yaml file",
			moduleAndSkaffoldYamls: map[string]string{"cfg0": invalidSkaffoldYaml},
			shouldErr:              true,
		},
	}

	for _, test := range tests {
		testutil.Run(t, test.description, func(t *testutil.T) {
			tmpdir := t.TempDir()
			configSet := parser.SkaffoldConfigSet{}
			for module, skaffoldyamlText := range test.moduleAndSkaffoldYamls {
				fp := filepath.Join(tmpdir, fmt.Sprintf("%s.yaml", module))
				err := ioutil.WriteFile(fp, []byte(skaffoldyamlText), 0644)
				if err != nil {
					t.Fatalf("error creating skaffold.yaml file with name %s: %v", fp, err)
				}
				configSet = append(configSet, &parser.SkaffoldConfigEntry{SkaffoldConfig: &v1.SkaffoldConfig{
					Metadata: v1.Metadata{Name: module},
				},
					SourceFile: fp,
				})
				// test overwrites file paths for expected SkaffoldYamlRules as they are made dynamically
				results := test.expected[module]
				if results == nil {
					continue
				}
				for i := range *results {
					(*results)[i].AbsFilePath = configSet[len(configSet)-1].SourceFile
					(*results)[i].RelFilePath = configSet[len(configSet)-1].SourceFile
				}
			}
			t.Override(&realWorkDir, func() (string, error) {
				return "", nil
			})
			t.Override(&getConfigSet, func(_ context.Context, opts config.SkaffoldOptions) (parser.SkaffoldConfigSet, error) {
				// mock profile activation
				var set parser.SkaffoldConfigSet
				for _, c := range configSet {
					if len(opts.ConfigurationFilter) > 0 && !stringslice.Contains(opts.ConfigurationFilter, c.Metadata.Name) {
						continue
					}
					for _, pName := range opts.Profiles {
						for _, profile := range c.Profiles {
							if profile.Name != pName {
								continue
							}
							c.Test = profile.Test
						}
					}
					set = append(set, c)
				}
				return set, test.err
			})
			results, err := GetSkaffoldYamlsLintResults(context.Background(), Options{
				OutFormat: "json", Modules: test.module, Profiles: test.profiles})
			t.CheckError(test.shouldErr, err)
			expectedResults := &[]Result{}

			// this is done to enforce result order
			for i := 0; i < len(test.expected); i++ {
				*expectedResults = append(*expectedResults, *test.expected[fmt.Sprintf("cfg%d", i)]...)
			}
			if !test.shouldErr {
				t.CheckDeepEqual(expectedResults, results)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package worker

import (
	"github.com/Matir/webborer/logging"
	"github.com/Matir/webborer/util"
	"github.com/Matir/webborer/workqueue"
	"golang.org/x/net/html"
	"io"
	"net/http"
	"net/url"
	"strings"
)

const (
  maxHTMLWorkerSize = 10*1024*1024
)

type HTMLWorker struct {
	// Function to add future work
	adder workqueue.QueueAddFunc
}

func NewHTMLWorker(adder workqueue.QueueAddFunc) *HTMLWorker {
	return &HTMLWorker{adder: adder}
}

// Work on this response
func (w *HTMLWorker) Handle(URL *url.URL, body io.Reader) {
  limitedBody := io.LimitReader(body, maxHTMLWorkerSize)
	links := w.GetLinks(limitedBody)
	foundURLs := make([]*url.URL, 0, len(links))
	for _, l := range links {
		u, err := url.Parse(l)
		if err != nil {
			logging.Logf(logging.LogInfo, "Error parsing URL (%s): %s", l, err.Error())
			continue
		}
		resolved := URL.ResolveReference(u)
		foundURLs = append(foundURLs, resolved)
		// Include parents of the found URL.
		// Worker will remove duplicates
		foundURLs = append(foundURLs, util.GetParentPaths(resolved)...)
	}
	w.adder(foundURLs...)
}

// Check if this response can be handled by this worker
func (*HTMLWorker) Eligible(resp *http.Response) bool {
	ct := resp.Header.Get("Content-type")
	if strings.ToLower(ct) != "text/html" {
		return false
	}
	// ContentLength is often -1, indicating unknown, so we'll try to parse those
	return resp.ContentLength == -1 || (
	  resp.ContentLength > 0 && resp.ContentLength < maxHTMLWorkerSize)
}

// Get the links for the body.
func (*HTMLWorker) GetLinks(body io.Reader) []string {
	tree, err := html.Parse(body)
	if err != nil {
		logging.Logf(logging.LogInfo, "Unable to parse HTML document: %s", err.Error())
		return nil
	}
	links := collectElementAttributes(tree, "a", "href")
	links = append(links, collectElementAttributes(tree, "img", "src")...)
	links = append(links, collectElementAttributes(tree, "script", "src")...)
	links = append(links, collectElementAttributes(tree, "style", "src")...)
	return util.DedupeStrings(links)
}

func getElementsByTagName(root *html.Node, name string) []*html.Node {
	results := make([]*html.Node, 0)
	var handleNode func(*html.Node)
	handleNode = func(node *html.Node) {
		if node.Type == html.ElementNode && strings.ToLower(node.Data) == name {
			results = append(results, node)
		}
		for n := node.FirstChild; n != nil; n = n.NextSibling {
			handleNode(n)
		}
	}
	handleNode(root)
	return results
}

func getElementAttribute(node *html.Node, attrName string) *string {
	for _, a := range node.Attr {
		if strings.ToLower(a.Key) == attrName {
			return &a.Val
		}
	}
	return nil
}

func collectElementAttributes(root *html.Node, tagName, attrName string) []string {
	results := make([]string, 0)
	for _, el := range getElementsByTagName(root, tagName) {
		if val := getElementAttribute(el, attrName); val != nil {
			results = append(results, *val)
		}
	}
	return results
}
<commit_msg>Fix gofmt.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package worker

import (
	"github.com/Matir/webborer/logging"
	"github.com/Matir/webborer/util"
	"github.com/Matir/webborer/workqueue"
	"golang.org/x/net/html"
	"io"
	"net/http"
	"net/url"
	"strings"
)

const (
	maxHTMLWorkerSize = 10 * 1024 * 1024
)

type HTMLWorker struct {
	// Function to add future work
	adder workqueue.QueueAddFunc
}

func NewHTMLWorker(adder workqueue.QueueAddFunc) *HTMLWorker {
	return &HTMLWorker{adder: adder}
}

// Work on this response
func (w *HTMLWorker) Handle(URL *url.URL, body io.Reader) {
	limitedBody := io.LimitReader(body, maxHTMLWorkerSize)
	links := w.GetLinks(limitedBody)
	foundURLs := make([]*url.URL, 0, len(links))
	for _, l := range links {
		u, err := url.Parse(l)
		if err != nil {
			logging.Logf(logging.LogInfo, "Error parsing URL (%s): %s", l, err.Error())
			continue
		}
		resolved := URL.ResolveReference(u)
		foundURLs = append(foundURLs, resolved)
		// Include parents of the found URL.
		// Worker will remove duplicates
		foundURLs = append(foundURLs, util.GetParentPaths(resolved)...)
	}
	w.adder(foundURLs...)
}

// Check if this response can be handled by this worker
func (*HTMLWorker) Eligible(resp *http.Response) bool {
	ct := resp.Header.Get("Content-type")
	if strings.ToLower(ct) != "text/html" {
		return false
	}
	// ContentLength is often -1, indicating unknown, so we'll try to parse those
	return resp.ContentLength == -1 || (resp.ContentLength > 0 && resp.ContentLength < maxHTMLWorkerSize)
}

// Get the links for the body.
func (*HTMLWorker) GetLinks(body io.Reader) []string {
	tree, err := html.Parse(body)
	if err != nil {
		logging.Logf(logging.LogInfo, "Unable to parse HTML document: %s", err.Error())
		return nil
	}
	links := collectElementAttributes(tree, "a", "href")
	links = append(links, collectElementAttributes(tree, "img", "src")...)
	links = append(links, collectElementAttributes(tree, "script", "src")...)
	links = append(links, collectElementAttributes(tree, "style", "src")...)
	return util.DedupeStrings(links)
}

func getElementsByTagName(root *html.Node, name string) []*html.Node {
	results := make([]*html.Node, 0)
	var handleNode func(*html.Node)
	handleNode = func(node *html.Node) {
		if node.Type == html.ElementNode && strings.ToLower(node.Data) == name {
			results = append(results, node)
		}
		for n := node.FirstChild; n != nil; n = n.NextSibling {
			handleNode(n)
		}
	}
	handleNode(root)
	return results
}

func getElementAttribute(node *html.Node, attrName string) *string {
	for _, a := range node.Attr {
		if strings.ToLower(a.Key) == attrName {
			return &a.Val
		}
	}
	return nil
}

func collectElementAttributes(root *html.Node, tagName, attrName string) []string {
	results := make([]string, 0)
	for _, el := range getElementsByTagName(root, tagName) {
		if val := getElementAttribute(el, attrName); val != nil {
			results = append(results, *val)
		}
	}
	return results
}
<|endoftext|>


<commit_before>package sudoku

//TODO: register this with techniques.go
type xywingTechnique struct {
	*basicSolveTechnique
}

func (self *xywingTechnique) humanLikelihood(step *SolveStep) float64 {
	//TODO: reason about what the proper value should be for this.
	return self.difficultyHelper(60.0)
}

func (self *xywingTechnique) Description(step *SolveStep) string {
	return "TODO: IMPLEMENT THIS"
}

func (self *xywingTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {

	getter := grid.queue().DefaultGetter()

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		pivot := getter.GetSmallerThan(3)

		if pivot == nil {
			break
		}

		pivotCell := pivot.(*Cell)

		possibilities := pivotCell.Possibilities()

		if len(possibilities) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		//OK, this is our pivot cell. Let's consider its two possibilities to
		//be X and Y.

		x := possibilities[0]
		y := possibilities[1]

		//We want to find two neighbors of the pivot cell that have two
		//possibilities that are XZ and YZ. So we're looking for neighbors
		//that have two possibilities, one of which is either x or y.

		xList := pivotCell.Neighbors().FilterByPossible(x).FilterByNumPossibilities(2)
		yList := pivotCell.Neighbors().FilterByPossible(y).FilterByNumPossibilities(2)

		//Now, we'll check for each possible value of Z
		for z := 0; z < DIM; z++ {
			//z can't be either x or y, so don't do that work
			if z == x || z == y {
				continue
			}

			zFilteredXList := xList.FilterByPossible(z)
			zFilteredYList := yList.FilterByPossible(z)

			//If there area any items left, we've found candidates.
			if len(zFilteredXList) == 0 || len(zFilteredYList) == 0 {
				//TOOD: we can save a tiny bit of work by checking if
				//zFilteredXList is len() 0 before calculating zFilteredYList
				continue
			}

			//Okay, now the two-up combinations of all cells in the
			//two lists are all candidates. Often it will only be 1
			//cell in each list, but we'll general case this.
			for _, xCell := range zFilteredXList {
				for _, yCell := range zFilteredYList {
					//xCell, yCell is a possible pointing pair group.

					//find cells that are in both neighbor lists
					intersection := xCell.Neighbors().toCellSet().intersection(yCell.Neighbors().toCellSet()).toSlice(grid)

					//TODO: consider if we actually need to remove all of these cells;
					//it might never be able to be in the list anyway.
					intersection = intersection.RemoveCells(CellSlice{pivotCell, xCell, yCell})

					//TODO: consider chunking up this list of affectedCells by
					//the block the cell is in; this fits more closely with
					//how humans actually apply the technique in practice.
					//(Hmm, possibly there should be chunk and unchunked
					//variants?)
					affectedCells := intersection.FilterByPossible(z)

					//Okay, we have a candidate step. Is it useful?
					step := &SolveStep{self,
						affectedCells,
						IntSlice{z},
						CellSlice{pivotCell, xCell, yCell},
						IntSlice{x, y, z},
						nil,
					}

					if step.IsUseful(grid) {
						select {
						case results <- step:
						case <-done:
							return
						}
					}

				}
			}

		}

	}
}
<commit_msg>TESTS FAIL. PANIC. Early out of XYWing if affected cells is nil.<commit_after>package sudoku

//TODO: register this with techniques.go
type xywingTechnique struct {
	*basicSolveTechnique
}

func (self *xywingTechnique) humanLikelihood(step *SolveStep) float64 {
	//TODO: reason about what the proper value should be for this.
	return self.difficultyHelper(60.0)
}

func (self *xywingTechnique) Description(step *SolveStep) string {
	return "TODO: IMPLEMENT THIS"
}

func (self *xywingTechnique) Find(grid *Grid, results chan *SolveStep, done chan bool) {

	getter := grid.queue().DefaultGetter()

	for {

		//Check if it's time to stop.
		select {
		case <-done:
			return
		default:
		}

		pivot := getter.GetSmallerThan(3)

		if pivot == nil {
			break
		}

		pivotCell := pivot.(*Cell)

		possibilities := pivotCell.Possibilities()

		if len(possibilities) != 2 {
			//We found one with 1 possibility, which isn't interesting for us--nakedSingle should do that one.
			continue
		}

		//OK, this is our pivot cell. Let's consider its two possibilities to
		//be X and Y.

		x := possibilities[0]
		y := possibilities[1]

		//We want to find two neighbors of the pivot cell that have two
		//possibilities that are XZ and YZ. So we're looking for neighbors
		//that have two possibilities, one of which is either x or y.

		xList := pivotCell.Neighbors().FilterByPossible(x).FilterByNumPossibilities(2)
		yList := pivotCell.Neighbors().FilterByPossible(y).FilterByNumPossibilities(2)

		//Now, we'll check for each possible value of Z
		for z := 0; z < DIM; z++ {
			//z can't be either x or y, so don't do that work
			if z == x || z == y {
				continue
			}

			zFilteredXList := xList.FilterByPossible(z)
			zFilteredYList := yList.FilterByPossible(z)

			//If there area any items left, we've found candidates.
			if len(zFilteredXList) == 0 || len(zFilteredYList) == 0 {
				//TOOD: we can save a tiny bit of work by checking if
				//zFilteredXList is len() 0 before calculating zFilteredYList
				continue
			}

			//Okay, now the two-up combinations of all cells in the
			//two lists are all candidates. Often it will only be 1
			//cell in each list, but we'll general case this.
			for _, xCell := range zFilteredXList {
				for _, yCell := range zFilteredYList {
					//xCell, yCell is a possible pointing pair group.

					//find cells that are in both neighbor lists
					intersection := xCell.Neighbors().toCellSet().intersection(yCell.Neighbors().toCellSet()).toSlice(grid)

					//TODO: consider if we actually need to remove all of these cells;
					//it might never be able to be in the list anyway.
					intersection = intersection.RemoveCells(CellSlice{pivotCell, xCell, yCell})

					//TODO: consider chunking up this list of affectedCells by
					//the block the cell is in; this fits more closely with
					//how humans actually apply the technique in practice.
					//(Hmm, possibly there should be chunk and unchunked
					//variants?)
					affectedCells := intersection.FilterByPossible(z)

					if len(affectedCells) == 0 {
						continue
					}

					//Okay, we have a candidate step. Is it useful?
					step := &SolveStep{self,
						affectedCells,
						IntSlice{z},
						CellSlice{pivotCell, xCell, yCell},
						IntSlice{x, y, z},
						nil,
					}

					if step.IsUseful(grid) {
						select {
						case results <- step:
						case <-done:
							return
						}
					}

				}
			}

		}

	}
}
<|endoftext|>