<commit_before>package queue

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/innovate-technologies/Dispatch/dispatchd/config"
	"github.com/innovate-technologies/Dispatch/dispatchd/unit"
	"github.com/innovate-technologies/Dispatch/dispatchd/unit/template"

	"strconv"

	etcd "github.com/coreos/etcd/client"
	"golang.org/x/net/context"
)

var (
	ctx     = context.Background()
	etcdAPI etcd.KeysAPI
	// Config is a pointer need to be set to the main configuration
	Config     *config.ConfigurationInfo
	queueMutex = &sync.Mutex{}
)

// Run checks for waiting units and assigns them
func Run() {
	setUpEtcd()
	importExisting()
	go watchQueue()
	go checkQueue() // make sure to not forget the unsatisfiable
}

// AddUnit adds a unit to the queue
func AddUnit(name string) {
	if etcdAPI == nil {
		setUpEtcd()
	}
	etcdAPI.Set(ctx, fmt.Sprintf("/dispatch/queue/%s/%s", Config.Zone, name), name, &etcd.SetOptions{})
}

func checkQueue() {
	for {
		time.Sleep(5 * time.Second)
		importExisting()
	}
}

func importExisting() {
	queueMutex.Lock()
	response, err := etcdAPI.Get(ctx, fmt.Sprintf("/dispatch/queue/%s/", Config.Zone), &etcd.GetOptions{})
	if err != nil {
		return
	}
	for _, node := range response.Node.Nodes {
		go assignUnit(node.Value)
	}
	queueMutex.Unlock()
}

func watchQueue() {
	w := etcdAPI.Watcher(fmt.Sprintf("/dispatch/queue/%s/", Config.Zone), &etcd.WatcherOptions{Recursive: true})
	for {
		r, err := w.Next(ctx)
		if err != nil {
			go watchQueue()
			return
		}
		queueMutex.Lock()
		if r.Action == "set" {
			go assignUnit(r.Node.Value)
		}
		queueMutex.Unlock()
	}
}

func assignUnit(name string) {
	fmt.Println(name)
	newUnit := unit.NewFromEtcd(name)
	machine := getMachineForUnitConstraints(newUnit)
	if machine != "" {
		assignUnitToMachine(name, machine)
	}
}

func getMachineForUnitConstraints(u unit.Unit) string {
	// contraints := u.Constraints
	ports := u.Ports
	var unitTemplate template.Template
	if u.Template != "" {
		unitTemplate = template.NewFromEtcd(u.Template)
	}

	// TO DO implement constraints
	machinesForLoad := map[string]float64{}
	response, err := etcdAPI.Get(ctx, fmt.Sprintf("/dispatch/machines/%s/", Config.Zone), &etcd.GetOptions{Recursive: true})
	if err != nil {
		fmt.Println(err)
		return "" //not important for now
	}
	for _, machine := range response.Node.Nodes {
		machineNameParts := strings.Split(machine.Key, "/")

		machineName := machineNameParts[len(machineNameParts)-1]
		var load float64
		unitNames := []string{}

		for _, key := range machine.Nodes {
			keyParts := strings.Split(key.Key, "/")
			if keyParts[len(keyParts)-1] == "load" {
				load, _ = strconv.ParseFloat(key.Value, 64)
			}
			if keyParts[len(keyParts)-1] == "units" {
				for _, unit := range key.Nodes {
					unitNames = append(unitNames, unit.Value)
				}
			}
		}

		goCount := 0
		unitChan := make(chan unit.Unit)
		units := []unit.Unit{}
		for _, unitName := range unitNames {
			go getUnit(unitName, unitChan)
			goCount++
		}
		for goCount > 0 {
			unit := <-unitChan
			units = append(units, unit)
			goCount--
		}
		allPortsAvailable := true
		var numSameTemplate int64
	L:
		for _, unit := range units {
			// check template
			if unit.Template == u.Template {
				numSameTemplate++
			}

			// check ports
			for _, unitPort := range unit.Ports {
				for _, port := range ports {
					if unitPort == port {
						allPortsAvailable = false
						break L
					}
				}
			}
		}
		if allPortsAvailable && (numSameTemplate < unitTemplate.MaxPerMachine || unitTemplate.MaxPerMachine == 0) { // check ports and template constraints
			machinesForLoad[machineName] = load
		}
	}
	isCompared := false
	lowestLoad := 0.0
	lowestLoadMachine := ""
	for machine, load := range machinesForLoad {
		if load < lowestLoad || !isCompared {
			lowestLoad = load
			lowestLoadMachine = machine
			isCompared = true
		}
	}
	return lowestLoadMachine
}

func getUnit(name string, out chan unit.Unit) {
	out <- unit.NewFromEtcd(name)
}

func assignUnitToMachine(unit, machine string) {
	etcdAPI.Set(ctx, fmt.Sprintf("/dispatch/machines/%s/%s/units/%s", Config.Zone, machine, unit), unit, &etcd.SetOptions{})
	etcdAPI.Delete(ctx, fmt.Sprintf("/dispatch/queue/%s/%s", Config.Zone, unit), &etcd.DeleteOptions{})
}

func setUpEtcd() {
	c, err := etcd.New(etcd.Config{
		Endpoints:               []string{Config.EtcdAddress},
		Transport:               etcd.DefaultTransport,
		HeaderTimeoutPerRequest: 10 * time.Second,
	})
	if err != nil {
		panic(err)
	}
	etcdAPI = etcd.NewKeysAPI(c)
}
<commit_msg>add machine to /machine on queue assign<commit_after>package queue

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/innovate-technologies/Dispatch/dispatchd/config"
	"github.com/innovate-technologies/Dispatch/dispatchd/unit"
	"github.com/innovate-technologies/Dispatch/dispatchd/unit/template"

	"strconv"

	etcd "github.com/coreos/etcd/client"
	"golang.org/x/net/context"
)

var (
	ctx     = context.Background()
	etcdAPI etcd.KeysAPI
	// Config is a pointer need to be set to the main configuration
	Config     *config.ConfigurationInfo
	queueMutex = &sync.Mutex{}
)

// Run checks for waiting units and assigns them
func Run() {
	setUpEtcd()
	importExisting()
	go watchQueue()
	go checkQueue() // make sure to not forget the unsatisfiable
}

// AddUnit adds a unit to the queue
func AddUnit(name string) {
	if etcdAPI == nil {
		setUpEtcd()
	}
	etcdAPI.Set(ctx, fmt.Sprintf("/dispatch/queue/%s/%s", Config.Zone, name), name, &etcd.SetOptions{})
	etcdAPI.Set(ctx, fmt.Sprintf("/dispatch/units/%s/%s/machine", Config.Zone, name), "", &etcd.SetOptions{})
}

func checkQueue() {
	for {
		time.Sleep(5 * time.Second)
		importExisting()
	}
}

func importExisting() {
	queueMutex.Lock()
	response, err := etcdAPI.Get(ctx, fmt.Sprintf("/dispatch/queue/%s/", Config.Zone), &etcd.GetOptions{})
	if err != nil {
		return
	}
	for _, node := range response.Node.Nodes {
		go assignUnit(node.Value)
	}
	queueMutex.Unlock()
}

func watchQueue() {
	w := etcdAPI.Watcher(fmt.Sprintf("/dispatch/queue/%s/", Config.Zone), &etcd.WatcherOptions{Recursive: true})
	for {
		r, err := w.Next(ctx)
		if err != nil {
			go watchQueue()
			return
		}
		queueMutex.Lock()
		if r.Action == "set" {
			go assignUnit(r.Node.Value)
		}
		queueMutex.Unlock()
	}
}

func assignUnit(name string) {
	fmt.Println(name)
	newUnit := unit.NewFromEtcd(name)
	machine := getMachineForUnitConstraints(newUnit)
	if machine != "" {
		assignUnitToMachine(name, machine)
	}
}

func getMachineForUnitConstraints(u unit.Unit) string {
	// contraints := u.Constraints
	ports := u.Ports
	var unitTemplate template.Template
	if u.Template != "" {
		unitTemplate = template.NewFromEtcd(u.Template)
	}

	// TO DO implement constraints
	machinesForLoad := map[string]float64{}
	response, err := etcdAPI.Get(ctx, fmt.Sprintf("/dispatch/machines/%s/", Config.Zone), &etcd.GetOptions{Recursive: true})
	if err != nil {
		fmt.Println(err)
		return "" //not important for now
	}
	for _, machine := range response.Node.Nodes {
		machineNameParts := strings.Split(machine.Key, "/")

		machineName := machineNameParts[len(machineNameParts)-1]
		var load float64
		unitNames := []string{}

		for _, key := range machine.Nodes {
			keyParts := strings.Split(key.Key, "/")
			if keyParts[len(keyParts)-1] == "load" {
				load, _ = strconv.ParseFloat(key.Value, 64)
			}
			if keyParts[len(keyParts)-1] == "units" {
				for _, unit := range key.Nodes {
					unitNames = append(unitNames, unit.Value)
				}
			}
		}

		goCount := 0
		unitChan := make(chan unit.Unit)
		units := []unit.Unit{}
		for _, unitName := range unitNames {
			go getUnit(unitName, unitChan)
			goCount++
		}
		for goCount > 0 {
			unit := <-unitChan
			units = append(units, unit)
			goCount--
		}
		allPortsAvailable := true
		var numSameTemplate int64
	L:
		for _, unit := range units {
			// check template
			if unit.Template == u.Template {
				numSameTemplate++
			}

			// check ports
			for _, unitPort := range unit.Ports {
				for _, port := range ports {
					if unitPort == port {
						allPortsAvailable = false
						break L
					}
				}
			}
		}
		if allPortsAvailable && (numSameTemplate < unitTemplate.MaxPerMachine || unitTemplate.MaxPerMachine == 0) { // check ports and template constraints
			machinesForLoad[machineName] = load
		}
	}
	isCompared := false
	lowestLoad := 0.0
	lowestLoadMachine := ""
	for machine, load := range machinesForLoad {
		if load < lowestLoad || !isCompared {
			lowestLoad = load
			lowestLoadMachine = machine
			isCompared = true
		}
	}
	return lowestLoadMachine
}

func getUnit(name string, out chan unit.Unit) {
	out <- unit.NewFromEtcd(name)
}

func assignUnitToMachine(unit, machine string) {
	etcdAPI.Set(ctx, fmt.Sprintf("/dispatch/machines/%s/%s/units/%s", Config.Zone, machine, unit), unit, &etcd.SetOptions{})
	etcdAPI.Set(ctx, fmt.Sprintf("/dispatch/units/%s/%s/machine", Config.Zone, unit), machine, &etcd.SetOptions{})
	etcdAPI.Delete(ctx, fmt.Sprintf("/dispatch/queue/%s/%s", Config.Zone, unit), &etcd.DeleteOptions{})
}

func setUpEtcd() {
	c, err := etcd.New(etcd.Config{
		Endpoints:               []string{Config.EtcdAddress},
		Transport:               etcd.DefaultTransport,
		HeaderTimeoutPerRequest: 10 * time.Second,
	})
	if err != nil {
		panic(err)
	}
	etcdAPI = etcd.NewKeysAPI(c)
}
<|endoftext|>


<commit_before>// Copyright 2014 The Sporting Exchange Limited. All rights reserved.
// Use of this source code is governed by a free license that can be
// found in the LICENSE file.

package collect

import (
	"expvar"
	"log"
	"runtime"
	"strings"
	"sync"
	"time"

	"opentsp.org/contrib/collect-netscaler/nitro"
	"opentsp.org/internal/tsdb"
)

var (
	statCycleMillis = expvar.NewInt("collect.CycleMillis")
)

// collector represents a collector for a given subsystem.
type collector interface {
	Subsystem() string // "lbvserver", "ssl", etc.
}

// statsCollector is a collector that calls a member of the "stat" family of
// functions in the Nitro API.
type statsCollector interface {
	CollectStats(emitFn, *nitro.ResponseStat)
}

// configCollector is a collector that calls a member of the "config" family of
// functions in the Nitro API.
type configCollector interface {
	CollectConfig(emitFn, *nitro.ResponseConfig)
}

// emitFn queues a data point for emission.
type emitFn func(string, interface{})

var Client *nitro.Client

// collect emits data points based on the provided collector.
func collect(emit emitFn, c collector) {
	switch cc := c.(type) {
	default:
		log.Panicf("unsupported collector type: %T", c)

	case statsCollector:
		resp, err := Client.Stat.Get(c.Subsystem())
		if err != nil {
			log.Print(err)
			return
		}
		logPanics(func() {
			cc.CollectStats(emit, resp)
		})

	case configCollector:
		resp, err := Client.Config.Get(c.Subsystem())
		if err != nil {
			log.Print(err)
			return
		}
		logPanics(func() {
			cc.CollectConfig(emit, resp)
		})
	}
}

// Loop loops indefinitely, running all collectors at the provided interval,
// and writing to w the resulting data points.
func Loop(w chan *tsdb.Point, interval time.Duration) {
	tick := tsdb.Tick(interval)
	t := time.Now()
	for ; ; t = <-tick {
		start := time.Now()

		emit := newEmitter(w, t)
		var wg sync.WaitGroup
		for _, c := range collectors {
			c := c
			go func() {
				collect(emit, c)
				wg.Done()
			}()
			wg.Add(1)
		}
		wg.Wait()

		statCycleMillis.Add(time.Since(start).Nanoseconds() / 1e6)
	}
}

// newEmitter returns a function that emits data points for the provided
// time instant.
func newEmitter(w chan *tsdb.Point, timestamp time.Time) emitFn {
	return func(series string, value interface{}) {
		if value == nil {
			panic("zero value")
		}
		series = "netscaler." + series
		id := strings.Fields(strings.Replace(series, "=", " ", -1))
		p, err := tsdb.NewPoint(timestamp, value, id[0], id[1:]...)
		if err != nil {
			panic(err)
		}
		w <- p
	}
}

// collectors is a list of all registered collectors.
var collectors []collector

func register(c collector) {
	collectors = append(collectors, c)
}

func registerStatFunc(subsystem string, fn func(emitFn, *nitro.ResponseStat)) {
	register(statFunc{
		subsystem: subsystem,
		fn:        fn,
	})
}

func registerConfigFunc(subsystem string, fn func(emitFn, *nitro.ResponseConfig)) {
	register(configFunc{
		subsystem: subsystem,
		fn:        fn,
	})
}

// statFunc is an adapter to allow the use of ordinary functions as stats
// collectors.
type statFunc struct {
	subsystem string
	fn        func(emitFn, *nitro.ResponseStat)
}

func (sf statFunc) Subsystem() string {
	return sf.subsystem
}

func (sf statFunc) CollectStats(emit emitFn, r *nitro.ResponseStat) {
	sf.fn(emit, r)
}

// configFunc is an adapter to allow the use of ordinary functions as config
// collectors.
type configFunc struct {
	subsystem string
	fn        func(emitFn, *nitro.ResponseConfig)
}

func (cf configFunc) Subsystem() string {
	return cf.subsystem
}

func (cf configFunc) CollectConfig(emit emitFn, r *nitro.ResponseConfig) {
	cf.fn(emit, r)
}

func logPanics(fn func()) {
	defer func() {
		if err := recover(); err != nil {
			const size = 4096
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			log.Printf("handler panic: %v\n%s", err, buf)
		}
	}()
	fn()
}
<commit_msg>Add nil checking logic to NewEmitter<commit_after>// Copyright 2014 The Sporting Exchange Limited. All rights reserved.
// Use of this source code is governed by a free license that can be
// found in the LICENSE file.

package collect

import (
	"expvar"
	"log"
	"runtime"
	"strings"
	"sync"
	"time"

	"opentsp.org/contrib/collect-netscaler/nitro"
	"opentsp.org/internal/tsdb"
)

var (
	statCycleMillis = expvar.NewInt("collect.CycleMillis")
)

// collector represents a collector for a given subsystem.
type collector interface {
	Subsystem() string // "lbvserver", "ssl", etc.
}

// statsCollector is a collector that calls a member of the "stat" family of
// functions in the Nitro API.
type statsCollector interface {
	CollectStats(emitFn, *nitro.ResponseStat)
}

// configCollector is a collector that calls a member of the "config" family of
// functions in the Nitro API.
type configCollector interface {
	CollectConfig(emitFn, *nitro.ResponseConfig)
}

// emitFn queues a data point for emission.
type emitFn func(string, interface{})

var Client *nitro.Client

// collect emits data points based on the provided collector.
func collect(emit emitFn, c collector) {
	switch cc := c.(type) {
	default:
		log.Panicf("unsupported collector type: %T", c)

	case statsCollector:
		resp, err := Client.Stat.Get(c.Subsystem())
		if err != nil {
			log.Print(err)
			return
		}
		logPanics(func() {
			cc.CollectStats(emit, resp)
		})

	case configCollector:
		resp, err := Client.Config.Get(c.Subsystem())
		if err != nil {
			log.Print(err)
			return
		}
		logPanics(func() {
			cc.CollectConfig(emit, resp)
		})
	}
}

// Loop loops indefinitely, running all collectors at the provided interval,
// and writing to w the resulting data points.
func Loop(w chan *tsdb.Point, interval time.Duration) {
	tick := tsdb.Tick(interval)
	t := time.Now()
	for ; ; t = <-tick {
		start := time.Now()

		emit := newEmitter(w, t)
		var wg sync.WaitGroup
		for _, c := range collectors {
			c := c
			go func() {
				collect(emit, c)
				wg.Done()
			}()
			wg.Add(1)
		}
		wg.Wait()

		statCycleMillis.Add(time.Since(start).Nanoseconds() / 1e6)
	}
}

// newEmitter returns a function that emits data points for the provided
// time instant.
func newEmitter(w chan *tsdb.Point, timestamp time.Time) emitFn {
	return func(series string, value interface{}) {
		var v interface{}
		if value == nil {
			panic("zero value")
		}
		switch u := value.(type) {
		case uint64, int64, int, float64:
			v = u
		case *uint64:
			if u != nil {
				v = *u
			} else {
				return
			}
		case *float64:
			if u != nil {
				v = *u
			} else {
				return
			}
		default:
			log.Panicf("unsupported type: %T", value)
		}
		series = "netscaler." + series
		id := strings.Fields(strings.Replace(series, "=", " ", -1))
		p, err := tsdb.NewPoint(timestamp, v, id[0], id[1:]...)
		if err != nil {
			panic(err)
		}
		w <- p
	}
}

// collectors is a list of all registered collectors.
var collectors []collector

func register(c collector) {
	collectors = append(collectors, c)
}

func registerStatFunc(subsystem string, fn func(emitFn, *nitro.ResponseStat)) {
	register(statFunc{
		subsystem: subsystem,
		fn:        fn,
	})
}

func registerConfigFunc(subsystem string, fn func(emitFn, *nitro.ResponseConfig)) {
	register(configFunc{
		subsystem: subsystem,
		fn:        fn,
	})
}

// statFunc is an adapter to allow the use of ordinary functions as stats
// collectors.
type statFunc struct {
	subsystem string
	fn        func(emitFn, *nitro.ResponseStat)
}

func (sf statFunc) Subsystem() string {
	return sf.subsystem
}

func (sf statFunc) CollectStats(emit emitFn, r *nitro.ResponseStat) {
	sf.fn(emit, r)
}

// configFunc is an adapter to allow the use of ordinary functions as config
// collectors.
type configFunc struct {
	subsystem string
	fn        func(emitFn, *nitro.ResponseConfig)
}

func (cf configFunc) Subsystem() string {
	return cf.subsystem
}

func (cf configFunc) CollectConfig(emit emitFn, r *nitro.ResponseConfig) {
	cf.fn(emit, r)
}

func logPanics(fn func()) {
	defer func() {
		if err := recover(); err != nil {
			const size = 4096
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			log.Printf("handler panic: %v\n%s", err, buf)
		}
	}()
	fn()
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	networking "knative.dev/networking/pkg/apis/networking"
	"knative.dev/pkg/apis"
	duckv1 "knative.dev/pkg/apis/duck/v1"
	"knative.dev/pkg/kmeta"
)

// +genclient
// +genreconciler:krshapedlogic=true
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// ServerlessService is a proxy for the K8s service objects containing the
// endpoints for the revision, whether those are endpoints of the activator or
// revision pods.
// See: https://knative.page.link/naxz for details.
type ServerlessService struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`

	// Spec is the desired state of the ServerlessService.
	// More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
	// +optional
	Spec ServerlessServiceSpec `json:"spec,omitempty"`

	// Status is the current state of the ServerlessService.
	// More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
	// +optional
	Status ServerlessServiceStatus `json:"status,omitempty"`
}

// Verify that ServerlessService adheres to the appropriate interfaces.
var (
	// Check that ServerlessService may be validated and defaulted.
	_ apis.Validatable = (*ServerlessService)(nil)
	_ apis.Defaultable = (*ServerlessService)(nil)

	// Check that we can create OwnerReferences to a ServerlessService.
	_ kmeta.OwnerRefable = (*ServerlessService)(nil)

	// Check that the type conforms to the duck Knative Resource shape.
	_ duckv1.KRShaped = (*ServerlessService)(nil)
)

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// ServerlessServiceList is a collection of ServerlessService.
type ServerlessServiceList struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`

	// Items is the list of ServerlessService.
	Items []ServerlessService `json:"items"`
}

// ServerlessServiceOperationMode is an enumeration of the modes of operation
// for the ServerlessService.
type ServerlessServiceOperationMode string

const (
	// SKSOperationModeServe is reserved for the state when revision
	// pods are serving using traffic.
	SKSOperationModeServe ServerlessServiceOperationMode = "Serve"

	// SKSOperationModeProxy is reserved for the state when activator
	// pods are serving using traffic.
	SKSOperationModeProxy ServerlessServiceOperationMode = "Proxy"
)

// ServerlessServiceSpec describes the ServerlessService.
type ServerlessServiceSpec struct {
	// Mode describes the mode of operation of the ServerlessService.
	Mode ServerlessServiceOperationMode `json:"mode,omitempty"`

	// ObjectRef defines the resource that this ServerlessService
	// is responsible for making "serverless".
	ObjectRef corev1.ObjectReference `json:"objectRef"`

	// The application-layer protocol. Matches `RevisionProtocolType` set on the owning pa/revision.
	// serving imports networking, so just use string.
	ProtocolType networking.ProtocolType

	// NumActivators contains number of Activators that this revision should be
	// assigned.
	// O means — assign all.
	NumActivators int32 `json:"numActivators,omitempty"`
}

// ServerlessServiceStatus describes the current state of the ServerlessService.
type ServerlessServiceStatus struct {
	duckv1.Status `json:",inline"`

	// ServiceName holds the name of a core K8s Service resource that
	// load balances over the pods backing this Revision (activator or revision).
	// +optional
	ServiceName string `json:"serviceName,omitempty"`

	// PrivateServiceName holds the name of a core K8s Service resource that
	// load balances over the user service pods backing this Revision.
	// +optional
	PrivateServiceName string `json:"privateServiceName,omitempty"`
}

// ConditionType represents a ServerlessService condition value
const (
	// ServerlessServiceConditionReady is set when the ingress networking setting is
	// configured and it has a load balancer address.
	ServerlessServiceConditionReady = apis.ConditionReady

	// ServerlessServiceConditionEndspointsPopulated is set when the ServerlessService's underlying
	// Revision K8s Service has been populated with endpoints.
	ServerlessServiceConditionEndspointsPopulated apis.ConditionType = "EndpointsPopulated"

	// ActivatorEndpointsPopulated is an informational status that reports
	// when the revision is backed by activator points. This might happen even if
	// revision is active (no pods yet created) or even when it has healthy pods
	// (e.g. due to target burst capacity settings).
	ActivatorEndpointsPopulated apis.ConditionType = "ActivatorEndpointsPopulated"
)

// GetStatus retrieves the status of the ServerlessService. Implements the KRShaped interface.
func (ss *ServerlessService) GetStatus() *duckv1.Status {
	return &ss.Status.Status
}
<commit_msg>Add missing JSON tag to protocol type (#400)<commit_after>/*
Copyright 2019 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	networking "knative.dev/networking/pkg/apis/networking"
	"knative.dev/pkg/apis"
	duckv1 "knative.dev/pkg/apis/duck/v1"
	"knative.dev/pkg/kmeta"
)

// +genclient
// +genreconciler:krshapedlogic=true
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// ServerlessService is a proxy for the K8s service objects containing the
// endpoints for the revision, whether those are endpoints of the activator or
// revision pods.
// See: https://knative.page.link/naxz for details.
type ServerlessService struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`

	// Spec is the desired state of the ServerlessService.
	// More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
	// +optional
	Spec ServerlessServiceSpec `json:"spec,omitempty"`

	// Status is the current state of the ServerlessService.
	// More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
	// +optional
	Status ServerlessServiceStatus `json:"status,omitempty"`
}

// Verify that ServerlessService adheres to the appropriate interfaces.
var (
	// Check that ServerlessService may be validated and defaulted.
	_ apis.Validatable = (*ServerlessService)(nil)
	_ apis.Defaultable = (*ServerlessService)(nil)

	// Check that we can create OwnerReferences to a ServerlessService.
	_ kmeta.OwnerRefable = (*ServerlessService)(nil)

	// Check that the type conforms to the duck Knative Resource shape.
	_ duckv1.KRShaped = (*ServerlessService)(nil)
)

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// ServerlessServiceList is a collection of ServerlessService.
type ServerlessServiceList struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`

	// Items is the list of ServerlessService.
	Items []ServerlessService `json:"items"`
}

// ServerlessServiceOperationMode is an enumeration of the modes of operation
// for the ServerlessService.
type ServerlessServiceOperationMode string

const (
	// SKSOperationModeServe is reserved for the state when revision
	// pods are serving using traffic.
	SKSOperationModeServe ServerlessServiceOperationMode = "Serve"

	// SKSOperationModeProxy is reserved for the state when activator
	// pods are serving using traffic.
	SKSOperationModeProxy ServerlessServiceOperationMode = "Proxy"
)

// ServerlessServiceSpec describes the ServerlessService.
type ServerlessServiceSpec struct {
	// Mode describes the mode of operation of the ServerlessService.
	Mode ServerlessServiceOperationMode `json:"mode,omitempty"`

	// ObjectRef defines the resource that this ServerlessService
	// is responsible for making "serverless".
	ObjectRef corev1.ObjectReference `json:"objectRef"`

	// The application-layer protocol. Matches `RevisionProtocolType` set on the owning pa/revision.
	// serving imports networking, so just use string.
	ProtocolType networking.ProtocolType `json:"protocolType"`

	// NumActivators contains number of Activators that this revision should be
	// assigned.
	// O means — assign all.
	NumActivators int32 `json:"numActivators,omitempty"`
}

// ServerlessServiceStatus describes the current state of the ServerlessService.
type ServerlessServiceStatus struct {
	duckv1.Status `json:",inline"`

	// ServiceName holds the name of a core K8s Service resource that
	// load balances over the pods backing this Revision (activator or revision).
	// +optional
	ServiceName string `json:"serviceName,omitempty"`

	// PrivateServiceName holds the name of a core K8s Service resource that
	// load balances over the user service pods backing this Revision.
	// +optional
	PrivateServiceName string `json:"privateServiceName,omitempty"`
}

// ConditionType represents a ServerlessService condition value
const (
	// ServerlessServiceConditionReady is set when the ingress networking setting is
	// configured and it has a load balancer address.
	ServerlessServiceConditionReady = apis.ConditionReady

	// ServerlessServiceConditionEndspointsPopulated is set when the ServerlessService's underlying
	// Revision K8s Service has been populated with endpoints.
	ServerlessServiceConditionEndspointsPopulated apis.ConditionType = "EndpointsPopulated"

	// ActivatorEndpointsPopulated is an informational status that reports
	// when the revision is backed by activator points. This might happen even if
	// revision is active (no pods yet created) or even when it has healthy pods
	// (e.g. due to target burst capacity settings).
	ActivatorEndpointsPopulated apis.ConditionType = "ActivatorEndpointsPopulated"
)

// GetStatus retrieves the status of the ServerlessService. Implements the KRShaped interface.
func (ss *ServerlessService) GetStatus() *duckv1.Status {
	return &ss.Status.Status
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"

	"github.com/turing-complete/laboratory/src/internal/command"
	"github.com/turing-complete/laboratory/src/internal/config"
	"github.com/turing-complete/laboratory/src/internal/database"
	"github.com/turing-complete/laboratory/src/internal/solver"
	"github.com/turing-complete/laboratory/src/internal/system"
	"github.com/turing-complete/laboratory/src/internal/target"
)

var (
	outputFile = flag.String("o", "", "an output file (required)")
)

func main() {
	command.Run(function)
}

func function(config *config.Config) error {
	output, err := database.Create(*outputFile)
	if err != nil {
		return err
	}
	defer output.Close()

	system, err := system.New(&config.System)
	if err != nil {
		return err
	}

	target, err := target.New(system, &config.Target)
	if err != nil {
		return err
	}

	solver, err := solver.New(target, &config.Solver)
	if err != nil {
		return err
	}

	log.Println(system)
	log.Println(target)
	log.Println("Constructing a surrogate...")

	solution := solver.Compute(target)
	log.Println(solution)
	if err := output.Put("solution", *solution); err != nil {
		return err
	}

	return nil
}
<commit_msg>A stylistic adjustment<commit_after>package main

import (
	"flag"
	"log"

	"github.com/turing-complete/laboratory/src/internal/command"
	"github.com/turing-complete/laboratory/src/internal/config"
	"github.com/turing-complete/laboratory/src/internal/database"
	"github.com/turing-complete/laboratory/src/internal/solver"
	"github.com/turing-complete/laboratory/src/internal/system"
	"github.com/turing-complete/laboratory/src/internal/target"
)

var (
	outputFile = flag.String("o", "", "an output file (required)")
)

func main() {
	command.Run(function)
}

func function(config *config.Config) error {
	output, err := database.Create(*outputFile)
	if err != nil {
		return err
	}
	defer output.Close()

	system, err := system.New(&config.System)
	if err != nil {
		return err
	}

	target, err := target.New(system, &config.Target)
	if err != nil {
		return err
	}

	solver, err := solver.New(target, &config.Solver)
	if err != nil {
		return err
	}

	log.Println(system)
	log.Println(target)
	log.Println("Constructing a surrogate...")

	solution := solver.Compute(target)
	log.Println(solution)

	if err := output.Put("solution", *solution); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package webhook

import (
	"encoding/json"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/andygrunwald/go-jira"
	"github.com/matrix-org/go-neb/database"
	"github.com/matrix-org/go-neb/errors"
	"github.com/matrix-org/go-neb/realms/jira"
	"net/http"
	"strings"
)

type jiraWebhook struct {
	Name    string   `json:"name"`
	URL     string   `json:"url"`
	Events  []string `json:"events"`
	Filter  string   `json:"jqlFilter"`
	Exclude bool     `json:"excludeIssueDetails"`
	// These fields are populated on GET
	Enabled bool `json:"enabled"`
}

type Event struct {
	WebhookEvent string     `json:"webhookEvent"`
	Timestamp    int64      `json:"timestamp"`
	User         jira.User  `json:"user"`
	Issue        jira.Issue `json:"issue"`
}

// RegisterHook checks to see if this user is allowed to track the given projects and then tracks them.
func RegisterHook(jrealm *realms.JIRARealm, projects []string, userID, webhookEndpointURL string) error {
	// Tracking means that a webhook may need to be created on the remote JIRA installation.
	// We need to make sure that the user has permission to do this. If they don't, it may still be okay if
	// there is an existing webhook set up for this installation by someone else, *PROVIDED* that the projects
	// they wish to monitor are "public" (accessible by not logged in users).
	//
	// The methodology for this is as follows:
	//  - If they don't have a JIRA token for the remote install, fail.
	//  - Try to GET /webhooks. If this succeeds:
	//      * The user is an admin (only admins can GET webhooks)
	//      * If there is a NEB webhook already then return success.
	//      * Else create the webhook and then return success (if creation fails then fail).
	//  - Else:
	//      * The user is NOT an admin.
	//      * Are ALL the projects in the config public? If yes:
	//         - Is there an existing config for this remote JIRA installation? If yes:
	//              * Another user has setup a webhook. We can't check if the webhook is still alive though,
	//                return success.
	//         - Else:
	//              * There is no existing NEB webhook for this JIRA installation. The user cannot create a
	//                webhook to the JIRA installation, so fail.
	//      * Else:
	//         - There are private projects in the config and the user isn't an admin, so fail.
	logger := log.WithFields(log.Fields{
		"realm_id": jrealm.ID(),
		"jira_url": jrealm.JIRAEndpoint,
		"user_id":  userID,
	})
	cli, err := jrealm.JIRAClient(userID, false)
	if err != nil {
		logger.WithError(err).Print("No JIRA client exists")
		return err // no OAuth token on this JIRA endpoint
	}
	wh, httpErr := getWebhook(cli, webhookEndpointURL)
	if httpErr != nil {
		if httpErr.Code != 403 {
			logger.WithError(httpErr).Print("Failed to GET webhook")
			return httpErr
		}
		// User is not a JIRA admin (cannot GET webhooks)
		// The only way this is going to end well for this request is if all the projects
		// are PUBLIC. That is, they can be accessed directly without an access token.
		httpErr = checkProjectsArePublic(jrealm, projects, userID)
		if httpErr != nil {
			logger.WithError(httpErr).Print("Failed to assert that all projects are public")
			return httpErr
		}

		// All projects that wish to be tracked are public, but the user cannot create
		// webhooks. The only way this will work is if we already have a webhook for this
		// JIRA endpoint.
		if !jrealm.HasWebhook {
			logger.Print("No webhook exists for this realm.")
			return fmt.Errorf("Not authorised to create webhook: not an admin.")
		}
		return nil
	}

	// The user is probably an admin (can query webhooks endpoint)

	if wh != nil {
		logger.Print("Webhook already exists")
		return nil // we already have a NEB webhook :D
	}
	return createWebhook(jrealm, webhookEndpointURL, userID)
}

// OnReceiveRequest is called when JIRA hits NEB with an update.
// Returns the project key and webhook event, or an error.
func OnReceiveRequest(req *http.Request) (string, *Event, *errors.HTTPError) {
	// extract the JIRA webhook event JSON
	defer req.Body.Close()
	var whe Event
	err := json.NewDecoder(req.Body).Decode(&whe)
	if err != nil {
		return "", nil, &errors.HTTPError{err, "Failed to parse request JSON", 400}
	}

	if err != nil {
		return "", nil, &errors.HTTPError{err, "Failed to parse JIRA URL", 400}
	}
	projKey := strings.Split(whe.Issue.Key, "-")[0]
	projKey = strings.ToUpper(projKey)
	return projKey, &whe, nil
}

func createWebhook(jrealm *realms.JIRARealm, webhookEndpointURL, userID string) error {
	cli, err := jrealm.JIRAClient(userID, false)

	req, err := cli.NewRequest("POST", "rest/webhooks/1.0/webhook", jiraWebhook{
		Name:    "Go-NEB",
		URL:     webhookEndpointURL,
		Events:  []string{"jira:issue_created", "jira:issue_deleted", "jira:issue_updated"},
		Filter:  "",
		Exclude: false,
	})
	if err != nil {
		return err
	}
	res, err := cli.Do(req, nil)
	if err != nil {
		return err
	}
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return fmt.Errorf("Creating webhook returned HTTP %d", res.StatusCode)
	}
	log.WithFields(log.Fields{
		"status_code": res.StatusCode,
		"realm_id":    jrealm.ID(),
		"jira_url":    jrealm.JIRAEndpoint,
	}).Print("Created webhook")

	// mark this on the realm and persist it.
	jrealm.HasWebhook = true
	_, err = database.GetServiceDB().StoreAuthRealm(jrealm)
	return err
}

func getWebhook(cli *jira.Client, webhookEndpointURL string) (*jiraWebhook, *errors.HTTPError) {
	req, err := cli.NewRequest("GET", "rest/webhooks/1.0/webhook", nil)
	if err != nil {
		return nil, &errors.HTTPError{err, "Failed to prepare webhook request", 500}
	}
	var webhookList []jiraWebhook
	res, err := cli.Do(req, &webhookList)
	if err != nil {
		return nil, &errors.HTTPError{err, "Failed to query webhooks", 502}
	}
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return nil, &errors.HTTPError{
			err,
			fmt.Sprintf("Querying webhook returned HTTP %d", res.StatusCode),
			403,
		}
	}
	log.Print("Retrieved ", len(webhookList), " webhooks")
	var nebWH *jiraWebhook
	for _, wh := range webhookList {
		if wh.URL == webhookEndpointURL {
			nebWH = &wh
			break
		}
	}
	return nebWH, nil
}

func checkProjectsArePublic(jrealm *realms.JIRARealm, projects []string, userID string) *errors.HTTPError {
	publicCli, err := jrealm.JIRAClient("", true)
	if err != nil {
		return &errors.HTTPError{err, "Cannot create public JIRA client", 500}
	}
	for _, projectKey := range projects {
		// check you can query this project with a public client
		req, err := publicCli.NewRequest("GET", "rest/api/2/project/"+projectKey, nil)
		if err != nil {
			return &errors.HTTPError{err, "Failed to create project URL", 500}
		}
		res, err := publicCli.Do(req, nil)
		if err != nil {
			return &errors.HTTPError{err, fmt.Sprintf("Failed to query project %s", projectKey), 500}
		}
		if res.StatusCode < 200 || res.StatusCode >= 300 {
			return &errors.HTTPError{err, fmt.Sprintf("Project %s is not public. (HTTP %d)", projectKey, res.StatusCode), 403}
		}
	}
	return nil
}
<commit_msg>Comments<commit_after>package webhook

import (
	"encoding/json"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/andygrunwald/go-jira"
	"github.com/matrix-org/go-neb/database"
	"github.com/matrix-org/go-neb/errors"
	"github.com/matrix-org/go-neb/realms/jira"
	"net/http"
	"strings"
)

type jiraWebhook struct {
	Name    string   `json:"name"`
	URL     string   `json:"url"`
	Events  []string `json:"events"`
	Filter  string   `json:"jqlFilter"`
	Exclude bool     `json:"excludeIssueDetails"`
	// These fields are populated on GET
	Enabled bool `json:"enabled"`
}

// Event represents an incoming JIRA webhook event
type Event struct {
	WebhookEvent string     `json:"webhookEvent"`
	Timestamp    int64      `json:"timestamp"`
	User         jira.User  `json:"user"`
	Issue        jira.Issue `json:"issue"`
}

// RegisterHook checks to see if this user is allowed to track the given projects and then tracks them.
func RegisterHook(jrealm *realms.JIRARealm, projects []string, userID, webhookEndpointURL string) error {
	// Tracking means that a webhook may need to be created on the remote JIRA installation.
	// We need to make sure that the user has permission to do this. If they don't, it may still be okay if
	// there is an existing webhook set up for this installation by someone else, *PROVIDED* that the projects
	// they wish to monitor are "public" (accessible by not logged in users).
	//
	// The methodology for this is as follows:
	//  - If they don't have a JIRA token for the remote install, fail.
	//  - Try to GET /webhooks. If this succeeds:
	//      * The user is an admin (only admins can GET webhooks)
	//      * If there is a NEB webhook already then return success.
	//      * Else create the webhook and then return success (if creation fails then fail).
	//  - Else:
	//      * The user is NOT an admin.
	//      * Are ALL the projects in the config public? If yes:
	//         - Is there an existing config for this remote JIRA installation? If yes:
	//              * Another user has setup a webhook. We can't check if the webhook is still alive though,
	//                return success.
	//         - Else:
	//              * There is no existing NEB webhook for this JIRA installation. The user cannot create a
	//                webhook to the JIRA installation, so fail.
	//      * Else:
	//         - There are private projects in the config and the user isn't an admin, so fail.
	logger := log.WithFields(log.Fields{
		"realm_id": jrealm.ID(),
		"jira_url": jrealm.JIRAEndpoint,
		"user_id":  userID,
	})
	cli, err := jrealm.JIRAClient(userID, false)
	if err != nil {
		logger.WithError(err).Print("No JIRA client exists")
		return err // no OAuth token on this JIRA endpoint
	}
	wh, httpErr := getWebhook(cli, webhookEndpointURL)
	if httpErr != nil {
		if httpErr.Code != 403 {
			logger.WithError(httpErr).Print("Failed to GET webhook")
			return httpErr
		}
		// User is not a JIRA admin (cannot GET webhooks)
		// The only way this is going to end well for this request is if all the projects
		// are PUBLIC. That is, they can be accessed directly without an access token.
		httpErr = checkProjectsArePublic(jrealm, projects, userID)
		if httpErr != nil {
			logger.WithError(httpErr).Print("Failed to assert that all projects are public")
			return httpErr
		}

		// All projects that wish to be tracked are public, but the user cannot create
		// webhooks. The only way this will work is if we already have a webhook for this
		// JIRA endpoint.
		if !jrealm.HasWebhook {
			logger.Print("No webhook exists for this realm.")
			return fmt.Errorf("Not authorised to create webhook: not an admin.")
		}
		return nil
	}

	// The user is probably an admin (can query webhooks endpoint)

	if wh != nil {
		logger.Print("Webhook already exists")
		return nil // we already have a NEB webhook :D
	}
	return createWebhook(jrealm, webhookEndpointURL, userID)
}

// OnReceiveRequest is called when JIRA hits NEB with an update.
// Returns the project key and webhook event, or an error.
func OnReceiveRequest(req *http.Request) (string, *Event, *errors.HTTPError) {
	// extract the JIRA webhook event JSON
	defer req.Body.Close()
	var whe Event
	err := json.NewDecoder(req.Body).Decode(&whe)
	if err != nil {
		return "", nil, &errors.HTTPError{err, "Failed to parse request JSON", 400}
	}

	if err != nil {
		return "", nil, &errors.HTTPError{err, "Failed to parse JIRA URL", 400}
	}
	projKey := strings.Split(whe.Issue.Key, "-")[0]
	projKey = strings.ToUpper(projKey)
	return projKey, &whe, nil
}

func createWebhook(jrealm *realms.JIRARealm, webhookEndpointURL, userID string) error {
	cli, err := jrealm.JIRAClient(userID, false)

	req, err := cli.NewRequest("POST", "rest/webhooks/1.0/webhook", jiraWebhook{
		Name:    "Go-NEB",
		URL:     webhookEndpointURL,
		Events:  []string{"jira:issue_created", "jira:issue_deleted", "jira:issue_updated"},
		Filter:  "",
		Exclude: false,
	})
	if err != nil {
		return err
	}
	res, err := cli.Do(req, nil)
	if err != nil {
		return err
	}
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return fmt.Errorf("Creating webhook returned HTTP %d", res.StatusCode)
	}
	log.WithFields(log.Fields{
		"status_code": res.StatusCode,
		"realm_id":    jrealm.ID(),
		"jira_url":    jrealm.JIRAEndpoint,
	}).Print("Created webhook")

	// mark this on the realm and persist it.
	jrealm.HasWebhook = true
	_, err = database.GetServiceDB().StoreAuthRealm(jrealm)
	return err
}

func getWebhook(cli *jira.Client, webhookEndpointURL string) (*jiraWebhook, *errors.HTTPError) {
	req, err := cli.NewRequest("GET", "rest/webhooks/1.0/webhook", nil)
	if err != nil {
		return nil, &errors.HTTPError{err, "Failed to prepare webhook request", 500}
	}
	var webhookList []jiraWebhook
	res, err := cli.Do(req, &webhookList)
	if err != nil {
		return nil, &errors.HTTPError{err, "Failed to query webhooks", 502}
	}
	if res.StatusCode < 200 || res.StatusCode >= 300 {
		return nil, &errors.HTTPError{
			err,
			fmt.Sprintf("Querying webhook returned HTTP %d", res.StatusCode),
			403,
		}
	}
	log.Print("Retrieved ", len(webhookList), " webhooks")
	var nebWH *jiraWebhook
	for _, wh := range webhookList {
		if wh.URL == webhookEndpointURL {
			nebWH = &wh
			break
		}
	}
	return nebWH, nil
}

func checkProjectsArePublic(jrealm *realms.JIRARealm, projects []string, userID string) *errors.HTTPError {
	publicCli, err := jrealm.JIRAClient("", true)
	if err != nil {
		return &errors.HTTPError{err, "Cannot create public JIRA client", 500}
	}
	for _, projectKey := range projects {
		// check you can query this project with a public client
		req, err := publicCli.NewRequest("GET", "rest/api/2/project/"+projectKey, nil)
		if err != nil {
			return &errors.HTTPError{err, "Failed to create project URL", 500}
		}
		res, err := publicCli.Do(req, nil)
		if err != nil {
			return &errors.HTTPError{err, fmt.Sprintf("Failed to query project %s", projectKey), 500}
		}
		if res.StatusCode < 200 || res.StatusCode >= 300 {
			return &errors.HTTPError{err, fmt.Sprintf("Project %s is not public. (HTTP %d)", projectKey, res.StatusCode), 403}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package handlers

import (
	"bytes"
	"context"
	"crypto/tls"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"

	"golang.org/x/crypto/acme/autocert"
	"golang.org/x/oauth2"

	gh "github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	lru "github.com/hashicorp/golang-lru"
	"github.com/play-with-docker/play-with-docker/config"
	"github.com/play-with-docker/play-with-docker/event"
	"github.com/play-with-docker/play-with-docker/pwd"
	"github.com/play-with-docker/play-with-docker/pwd/types"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/urfave/negroni"
	oauth2FB "golang.org/x/oauth2/facebook"
	oauth2Github "golang.org/x/oauth2/github"
)

var core pwd.PWDApi
var e event.EventApi
var landings = map[string][]byte{}

var latencyHistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{
	Name:    "pwd_handlers_duration_ms",
	Help:    "How long it took to process a specific handler, in a specific host",
	Buckets: []float64{300, 1200, 5000},
}, []string{"action"})

type HandlerExtender func(h *mux.Router)

func init() {
	prometheus.MustRegister(latencyHistogramVec)

}

func Bootstrap(c pwd.PWDApi, ev event.EventApi) {
	core = c
	e = ev
}

func Register(extend HandlerExtender) {
	initPlaygrounds()

	r := mux.NewRouter()
	corsRouter := mux.NewRouter()

	corsHandler := gh.CORS(gh.AllowCredentials(), gh.AllowedHeaders([]string{"x-requested-with", "content-type"}), gh.AllowedMethods([]string{"GET", "POST", "HEAD", "DELETE"}), gh.AllowedOriginValidator(func(origin string) bool {
		if strings.Contains(origin, "localhost") ||
			strings.HasSuffix(origin, "play-with-docker.com") ||
			strings.HasSuffix(origin, "play-with-kubernetes.com") ||
			strings.HasSuffix(origin, "play-with-moby.com") {
			return true
		}
		return false
	}), gh.AllowedOrigins([]string{}))

	// Specific routes
	r.HandleFunc("/ping", Ping).Methods("GET")
	corsRouter.HandleFunc("/instances/images", GetInstanceImages).Methods("GET")
	corsRouter.HandleFunc("/sessions/{sessionId}", GetSession).Methods("GET")
	corsRouter.HandleFunc("/sessions/{sessionId}", CloseSession).Methods("DELETE")
	corsRouter.HandleFunc("/sessions/{sessionId}/setup", SessionSetup).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances", NewInstance).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/uploads", FileUpload).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}", DeleteInstance).Methods("DELETE")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/exec", Exec).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/fstree", fsTree).Methods("GET")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/file", file).Methods("GET")

	r.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/editor", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "www/editor.html")
	})

	r.HandleFunc("/ooc", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "./www/ooc.html")
	}).Methods("GET")
	r.HandleFunc("/503", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "./www/503.html")
	}).Methods("GET")
	r.HandleFunc("/p/{sessionId}", Home).Methods("GET")
	r.PathPrefix("/assets").Handler(http.FileServer(http.Dir("./www")))
	r.HandleFunc("/robots.txt", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "www/robots.txt")
	})

	corsRouter.HandleFunc("/sessions/{sessionId}/ws/", WSH)
	r.Handle("/metrics", promhttp.Handler())

	// Generic routes
	r.HandleFunc("/", Landing).Methods("GET")

	corsRouter.HandleFunc("/users/me", LoggedInUser).Methods("GET")
	r.HandleFunc("/users/{userId:^(?me)}", GetUser).Methods("GET")
	r.HandleFunc("/oauth/providers", ListProviders).Methods("GET")
	r.HandleFunc("/oauth/providers/{provider}/login", Login).Methods("GET")
	r.HandleFunc("/oauth/providers/{provider}/callback", LoginCallback).Methods("GET")
	r.HandleFunc("/playgrounds", NewPlayground).Methods("PUT")
	r.HandleFunc("/playgrounds", ListPlaygrounds).Methods("GET")
	r.HandleFunc("/my/playground", GetCurrentPlayground).Methods("GET")

	corsRouter.HandleFunc("/", NewSession).Methods("POST")

	if extend != nil {
		extend(corsRouter)
	}

	n := negroni.Classic()

	r.PathPrefix("/").Handler(negroni.New(negroni.Wrap(corsHandler(corsRouter))))
	n.UseHandler(r)

	httpServer := http.Server{
		Addr:              "0.0.0.0:" + config.PortNumber,
		Handler:           n,
		IdleTimeout:       30 * time.Second,
		ReadHeaderTimeout: 5 * time.Second,
	}

	if config.UseLetsEncrypt {
		domainCache, err := lru.New(5000)
		if err != nil {
			log.Fatalf("Could not start domain cache. Got: %v", err)
		}
		certManager := autocert.Manager{
			Prompt: autocert.AcceptTOS,
			HostPolicy: func(ctx context.Context, host string) error {
				if _, found := domainCache.Get(host); !found {
					if playground := core.PlaygroundFindByDomain(host); playground == nil {
						return fmt.Errorf("Playground for domain %s was not found", host)
					}
					domainCache.Add(host, true)
				}
				return nil
			},
			Cache: autocert.DirCache(config.LetsEncryptCertsDir),
		}

		httpServer.TLSConfig = &tls.Config{
			GetCertificate: certManager.GetCertificate,
		}

		go func() {
			rr := mux.NewRouter()
			rr.HandleFunc("/ping", Ping).Methods("GET")
			rr.Handle("/metrics", promhttp.Handler())
			rr.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
				target := fmt.Sprintf("https://%s%s", r.Host, r.URL.Path)
				if len(r.URL.RawQuery) > 0 {
					target += "?" + r.URL.RawQuery
				}
				http.Redirect(rw, r, target, http.StatusMovedPermanently)
			})
			nr := negroni.Classic()
			nr.UseHandler(rr)
			log.Println("Starting redirect server")
			redirectServer := http.Server{
				Addr:              "0.0.0.0:3001",
				Handler:           certManager.HTTPHandler(nr),
				IdleTimeout:       30 * time.Second,
				ReadHeaderTimeout: 5 * time.Second,
			}
			log.Fatal(redirectServer.ListenAndServe())
		}()

		log.Println("Listening on port " + config.PortNumber)
		log.Fatal(httpServer.ListenAndServeTLS("", ""))
	} else {
		log.Println("Listening on port " + config.PortNumber)
		log.Fatal(httpServer.ListenAndServe())
	}
}

func initPlaygrounds() {
	pgs, err := core.PlaygroundList()
	if err != nil {
		log.Fatal("Error getting playgrounds for initialization")
	}

	for _, p := range pgs {
		initAssets(p)
		initOauthProviders(p)
	}
}

func initAssets(p *types.Playground) {
	if p.AssetsDir == "" {
		p.AssetsDir = "default"
	}

	var b bytes.Buffer
	t, err := template.New("landing.html").Delims("[[", "]]").ParseFiles(fmt.Sprintf("./www/%s/landing.html", p.AssetsDir))
	if err != nil {
		log.Fatalf("Error parsing template %v", err)
	}
	if err := t.Execute(&b, struct{ SegmentId string }{config.SegmentId}); err != nil {
		log.Fatalf("Error executing template %v", err)
	}
	landingBytes, err := ioutil.ReadAll(&b)
	if err != nil {
		log.Fatalf("Error reading template bytes %v", err)
	}
	landings[p.Id] = landingBytes
}

func initOauthProviders(p *types.Playground) {
	config.Providers[p.Id] = map[string]*oauth2.Config{}

	if p.GithubClientID != "" && p.GithubClientSecret != "" {
		conf := &oauth2.Config{
			ClientID:     p.GithubClientID,
			ClientSecret: p.GithubClientSecret,
			Scopes:       []string{"user:email"},
			Endpoint:     oauth2Github.Endpoint,
		}

		config.Providers[p.Id]["github"] = conf
	}
	if p.FacebookClientID != "" && p.FacebookClientSecret != "" {
		conf := &oauth2.Config{
			ClientID:     p.FacebookClientID,
			ClientSecret: p.FacebookClientSecret,
			Scopes:       []string{"email", "public_profile"},
			Endpoint:     oauth2FB.Endpoint,
		}

		config.Providers[p.Id]["facebook"] = conf
	}
	if p.DockerClientID != "" && p.DockerClientSecret != "" {
		oauth2.RegisterBrokenAuthHeaderProvider(".id.docker.com")
		conf := &oauth2.Config{
			ClientID:     p.DockerClientID,
			ClientSecret: p.DockerClientSecret,
			Scopes:       []string{"openid"},
			Endpoint: oauth2.Endpoint{
				AuthURL:  "https://id.docker.com/id/oauth/authorize/",
				TokenURL: "https://id.docker.com/id/oauth/token",
			},
		}

		config.Providers[p.Id]["docker"] = conf
	}
}
<commit_msg>Add docker.com CORS support<commit_after>package handlers

import (
	"bytes"
	"context"
	"crypto/tls"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"time"

	"golang.org/x/crypto/acme/autocert"
	"golang.org/x/oauth2"

	gh "github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	lru "github.com/hashicorp/golang-lru"
	"github.com/play-with-docker/play-with-docker/config"
	"github.com/play-with-docker/play-with-docker/event"
	"github.com/play-with-docker/play-with-docker/pwd"
	"github.com/play-with-docker/play-with-docker/pwd/types"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/urfave/negroni"
	oauth2FB "golang.org/x/oauth2/facebook"
	oauth2Github "golang.org/x/oauth2/github"
)

var core pwd.PWDApi
var e event.EventApi
var landings = map[string][]byte{}

var latencyHistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{
	Name:    "pwd_handlers_duration_ms",
	Help:    "How long it took to process a specific handler, in a specific host",
	Buckets: []float64{300, 1200, 5000},
}, []string{"action"})

type HandlerExtender func(h *mux.Router)

func init() {
	prometheus.MustRegister(latencyHistogramVec)

}

func Bootstrap(c pwd.PWDApi, ev event.EventApi) {
	core = c
	e = ev
}

func Register(extend HandlerExtender) {
	initPlaygrounds()

	r := mux.NewRouter()
	corsRouter := mux.NewRouter()

	corsHandler := gh.CORS(gh.AllowCredentials(), gh.AllowedHeaders([]string{"x-requested-with", "content-type"}), gh.AllowedMethods([]string{"GET", "POST", "HEAD", "DELETE"}), gh.AllowedOriginValidator(func(origin string) bool {
		if strings.Contains(origin, "localhost") ||
			strings.HasSuffix(origin, "play-with-docker.com") ||
			strings.HasSuffix(origin, "play-with-kubernetes.com") ||
			strings.HasSuffix(origin, "docker.com") ||
			strings.HasSuffix(origin, "play-with-moby.com") {
			return true
		}
		return false
	}), gh.AllowedOrigins([]string{}))

	// Specific routes
	r.HandleFunc("/ping", Ping).Methods("GET")
	corsRouter.HandleFunc("/instances/images", GetInstanceImages).Methods("GET")
	corsRouter.HandleFunc("/sessions/{sessionId}", GetSession).Methods("GET")
	corsRouter.HandleFunc("/sessions/{sessionId}", CloseSession).Methods("DELETE")
	corsRouter.HandleFunc("/sessions/{sessionId}/setup", SessionSetup).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances", NewInstance).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/uploads", FileUpload).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}", DeleteInstance).Methods("DELETE")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/exec", Exec).Methods("POST")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/fstree", fsTree).Methods("GET")
	corsRouter.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/file", file).Methods("GET")

	r.HandleFunc("/sessions/{sessionId}/instances/{instanceName}/editor", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "www/editor.html")
	})

	r.HandleFunc("/ooc", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "./www/ooc.html")
	}).Methods("GET")
	r.HandleFunc("/503", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "./www/503.html")
	}).Methods("GET")
	r.HandleFunc("/p/{sessionId}", Home).Methods("GET")
	r.PathPrefix("/assets").Handler(http.FileServer(http.Dir("./www")))
	r.HandleFunc("/robots.txt", func(rw http.ResponseWriter, r *http.Request) {
		http.ServeFile(rw, r, "www/robots.txt")
	})

	corsRouter.HandleFunc("/sessions/{sessionId}/ws/", WSH)
	r.Handle("/metrics", promhttp.Handler())

	// Generic routes
	r.HandleFunc("/", Landing).Methods("GET")

	corsRouter.HandleFunc("/users/me", LoggedInUser).Methods("GET")
	r.HandleFunc("/users/{userId:^(?me)}", GetUser).Methods("GET")
	r.HandleFunc("/oauth/providers", ListProviders).Methods("GET")
	r.HandleFunc("/oauth/providers/{provider}/login", Login).Methods("GET")
	r.HandleFunc("/oauth/providers/{provider}/callback", LoginCallback).Methods("GET")
	r.HandleFunc("/playgrounds", NewPlayground).Methods("PUT")
	r.HandleFunc("/playgrounds", ListPlaygrounds).Methods("GET")
	r.HandleFunc("/my/playground", GetCurrentPlayground).Methods("GET")

	corsRouter.HandleFunc("/", NewSession).Methods("POST")

	if extend != nil {
		extend(corsRouter)
	}

	n := negroni.Classic()

	r.PathPrefix("/").Handler(negroni.New(negroni.Wrap(corsHandler(corsRouter))))
	n.UseHandler(r)

	httpServer := http.Server{
		Addr:              "0.0.0.0:" + config.PortNumber,
		Handler:           n,
		IdleTimeout:       30 * time.Second,
		ReadHeaderTimeout: 5 * time.Second,
	}

	if config.UseLetsEncrypt {
		domainCache, err := lru.New(5000)
		if err != nil {
			log.Fatalf("Could not start domain cache. Got: %v", err)
		}
		certManager := autocert.Manager{
			Prompt: autocert.AcceptTOS,
			HostPolicy: func(ctx context.Context, host string) error {
				if _, found := domainCache.Get(host); !found {
					if playground := core.PlaygroundFindByDomain(host); playground == nil {
						return fmt.Errorf("Playground for domain %s was not found", host)
					}
					domainCache.Add(host, true)
				}
				return nil
			},
			Cache: autocert.DirCache(config.LetsEncryptCertsDir),
		}

		httpServer.TLSConfig = &tls.Config{
			GetCertificate: certManager.GetCertificate,
		}

		go func() {
			rr := mux.NewRouter()
			rr.HandleFunc("/ping", Ping).Methods("GET")
			rr.Handle("/metrics", promhttp.Handler())
			rr.HandleFunc("/", func(rw http.ResponseWriter, r *http.Request) {
				target := fmt.Sprintf("https://%s%s", r.Host, r.URL.Path)
				if len(r.URL.RawQuery) > 0 {
					target += "?" + r.URL.RawQuery
				}
				http.Redirect(rw, r, target, http.StatusMovedPermanently)
			})
			nr := negroni.Classic()
			nr.UseHandler(rr)
			log.Println("Starting redirect server")
			redirectServer := http.Server{
				Addr:              "0.0.0.0:3001",
				Handler:           certManager.HTTPHandler(nr),
				IdleTimeout:       30 * time.Second,
				ReadHeaderTimeout: 5 * time.Second,
			}
			log.Fatal(redirectServer.ListenAndServe())
		}()

		log.Println("Listening on port " + config.PortNumber)
		log.Fatal(httpServer.ListenAndServeTLS("", ""))
	} else {
		log.Println("Listening on port " + config.PortNumber)
		log.Fatal(httpServer.ListenAndServe())
	}
}

func initPlaygrounds() {
	pgs, err := core.PlaygroundList()
	if err != nil {
		log.Fatal("Error getting playgrounds for initialization")
	}

	for _, p := range pgs {
		initAssets(p)
		initOauthProviders(p)
	}
}

func initAssets(p *types.Playground) {
	if p.AssetsDir == "" {
		p.AssetsDir = "default"
	}

	var b bytes.Buffer
	t, err := template.New("landing.html").Delims("[[", "]]").ParseFiles(fmt.Sprintf("./www/%s/landing.html", p.AssetsDir))
	if err != nil {
		log.Fatalf("Error parsing template %v", err)
	}
	if err := t.Execute(&b, struct{ SegmentId string }{config.SegmentId}); err != nil {
		log.Fatalf("Error executing template %v", err)
	}
	landingBytes, err := ioutil.ReadAll(&b)
	if err != nil {
		log.Fatalf("Error reading template bytes %v", err)
	}
	landings[p.Id] = landingBytes
}

func initOauthProviders(p *types.Playground) {
	config.Providers[p.Id] = map[string]*oauth2.Config{}

	if p.GithubClientID != "" && p.GithubClientSecret != "" {
		conf := &oauth2.Config{
			ClientID:     p.GithubClientID,
			ClientSecret: p.GithubClientSecret,
			Scopes:       []string{"user:email"},
			Endpoint:     oauth2Github.Endpoint,
		}

		config.Providers[p.Id]["github"] = conf
	}
	if p.FacebookClientID != "" && p.FacebookClientSecret != "" {
		conf := &oauth2.Config{
			ClientID:     p.FacebookClientID,
			ClientSecret: p.FacebookClientSecret,
			Scopes:       []string{"email", "public_profile"},
			Endpoint:     oauth2FB.Endpoint,
		}

		config.Providers[p.Id]["facebook"] = conf
	}
	if p.DockerClientID != "" && p.DockerClientSecret != "" {
		oauth2.RegisterBrokenAuthHeaderProvider(".id.docker.com")
		conf := &oauth2.Config{
			ClientID:     p.DockerClientID,
			ClientSecret: p.DockerClientSecret,
			Scopes:       []string{"openid"},
			Endpoint: oauth2.Endpoint{
				AuthURL:  "https://id.docker.com/id/oauth/authorize/",
				TokenURL: "https://id.docker.com/id/oauth/token",
			},
		}

		config.Providers[p.Id]["docker"] = conf
	}
}
<|endoftext|>


<commit_before>package frontmatter

import (
	"bytes"
	"encoding/json"
	"errors"
	"log"
	"reflect"
	"sort"
	"strconv"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/BurntSushi/toml"
	"github.com/hacdias/caddy-filemanager/utils/variables"

	"github.com/spf13/cast"
)

const (
	mainName   = "#MAIN#"
	objectType = "object"
	arrayType  = "array"
)

var mainTitle = ""

// Pretty creates a new FrontMatter object
func Pretty(content []byte) (*Content, string, error) {
	data, err := Unmarshal(content)

	if err != nil {
		return &Content{}, "", err
	}

	kind := reflect.ValueOf(data).Kind()

	if kind.String() == "invalid" {
		return &Content{}, "", nil
	}

	object := new(Block)
	object.Type = objectType
	object.Name = mainName

	if kind == reflect.Map {
		object.Type = objectType
	} else if kind == reflect.Slice || kind == reflect.Array {
		object.Type = arrayType
	}

	return rawToPretty(data, object), mainTitle, nil
}

// Unmarshal returns the data of the frontmatter
func Unmarshal(content []byte) (interface{}, error) {
	mark := rune(content[0])
	var data interface{}

	switch mark {
	case '-':
		// If it's YAML
		if err := yaml.Unmarshal(content, &data); err != nil {
			return nil, err
		}
	case '+':
		// If it's TOML
		content = bytes.Replace(content, []byte("+"), []byte(""), -1)
		if _, err := toml.Decode(string(content), &data); err != nil {
			return nil, err
		}
	case '{', '[':
		// If it's JSON
		if err := json.Unmarshal(content, &data); err != nil {
			return nil, err
		}
	default:
		return nil, errors.New("Invalid frontmatter type.")
	}

	return data, nil
}

// Content is the block content
type Content struct {
	Other   interface{}
	Fields  []*Block
	Arrays  []*Block
	Objects []*Block
}

// Block is a block
type Block struct {
	Name     string
	Title    string
	Type     string
	HTMLType string
	Content  *Content
	Parent   *Block
}

func rawToPretty(config interface{}, parent *Block) *Content {
	objects := []*Block{}
	arrays := []*Block{}
	fields := []*Block{}

	cnf := map[string]interface{}{}
	kind := reflect.TypeOf(config)

	switch kind {
	case reflect.TypeOf(map[interface{}]interface{}{}):
		for key, value := range config.(map[interface{}]interface{}) {
			cnf[key.(string)] = value
		}
	case reflect.TypeOf([]map[string]interface{}{}):
		for index, value := range config.([]map[string]interface{}) {
			cnf[strconv.Itoa(index)] = value
		}
	case reflect.TypeOf([]map[interface{}]interface{}{}):
		for index, value := range config.([]map[interface{}]interface{}) {
			cnf[strconv.Itoa(index)] = value
		}
	case reflect.TypeOf([]interface{}{}):
		for index, value := range config.([]interface{}) {
			cnf[strconv.Itoa(index)] = value
		}
	default:
		cnf = config.(map[string]interface{})
	}

	for name, element := range cnf {
		if variables.IsMap(element) {
			objects = append(objects, handleObjects(element, parent, name))
		} else if variables.IsSlice(element) {
			arrays = append(arrays, handleArrays(element, parent, name))
		} else {
			if name == "title" && parent.Name == mainName {
				mainTitle = element.(string)
			}
			fields = append(fields, handleFlatValues(element, parent, name))
		}
	}

	sort.Sort(sortByTitle(fields))
	sort.Sort(sortByTitle(arrays))
	sort.Sort(sortByTitle(objects))
	return &Content{
		Fields:  fields,
		Arrays:  arrays,
		Objects: objects,
	}
}

type sortByTitle []*Block

func (f sortByTitle) Len() int      { return len(f) }
func (f sortByTitle) Swap(i, j int) { f[i], f[j] = f[j], f[i] }
func (f sortByTitle) Less(i, j int) bool {
	return strings.ToLower(f[i].Name) < strings.ToLower(f[j].Name)
}

func handleObjects(content interface{}, parent *Block, name string) *Block {
	c := new(Block)
	c.Parent = parent
	c.Type = objectType
	c.Title = name

	if parent.Name == mainName {
		c.Name = c.Title
	} else if parent.Type == arrayType {
		c.Name = parent.Name + "[" + name + "]"
	} else {
		c.Name = parent.Name + "." + c.Title
	}

	c.Content = rawToPretty(content, c)
	return c
}

func handleArrays(content interface{}, parent *Block, name string) *Block {
	c := new(Block)
	c.Parent = parent
	c.Type = arrayType
	c.Title = name

	if parent.Name == mainName {
		c.Name = name
	} else {
		c.Name = parent.Name + "." + name
	}

	c.Content = rawToPretty(content, c)
	return c
}

func handleFlatValues(content interface{}, parent *Block, name string) *Block {
	c := new(Block)
	c.Parent = parent

	switch content.(type) {
	case bool:
		c.Type = "boolean"
	case int, float32, float64:
		c.Type = "number"
	default:
		c.Type = "string"
	}

	c.Content = &Content{Other: content}

	switch strings.ToLower(name) {
	case "description":
		c.HTMLType = "textarea"
	case "date", "publishdate":
		c.HTMLType = "datetime"
		c.Content = &Content{Other: cast.ToTime(content)}
	default:
		c.HTMLType = "text"
	}

	if parent.Type == arrayType {
		c.Name = parent.Name + "[]"
		c.Title = content.(string)
	} else if parent.Type == objectType {
		c.Title = name
		c.Name = parent.Name + "." + name

		if parent.Name == mainName {
			c.Name = name
		}
	} else {
		log.Panic("Parent type not allowed in handleFlatValues.")
	}

	return c
}
<commit_msg>update<commit_after>package frontmatter

import (
	"bytes"
	"encoding/json"
	"errors"
	"log"
	"reflect"
	"sort"
	"strconv"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/BurntSushi/toml"
	"github.com/hacdias/caddy-filemanager/utils/variables"

	"github.com/spf13/cast"
)

const (
	mainName   = "#MAIN#"
	objectType = "object"
	arrayType  = "array"
)

var mainTitle = ""

// Pretty creates a new FrontMatter object
func Pretty(content []byte) (*Content, string, error) {
	data, err := Unmarshal(content)

	if err != nil {
		return &Content{}, "", err
	}

	kind := reflect.ValueOf(data).Kind()

	if kind == reflect.Invalid {
		return &Content{}, "", nil
	}

	object := new(Block)
	object.Type = objectType
	object.Name = mainName

	if kind == reflect.Map {
		object.Type = objectType
	} else if kind == reflect.Slice || kind == reflect.Array {
		object.Type = arrayType
	}

	return rawToPretty(data, object), mainTitle, nil
}

// Unmarshal returns the data of the frontmatter
func Unmarshal(content []byte) (interface{}, error) {
	mark := rune(content[0])
	var data interface{}

	switch mark {
	case '-':
		// If it's YAML
		if err := yaml.Unmarshal(content, &data); err != nil {
			return nil, err
		}
	case '+':
		// If it's TOML
		content = bytes.Replace(content, []byte("+"), []byte(""), -1)
		if _, err := toml.Decode(string(content), &data); err != nil {
			return nil, err
		}
	case '{', '[':
		// If it's JSON
		if err := json.Unmarshal(content, &data); err != nil {
			return nil, err
		}
	default:
		return nil, errors.New("Invalid frontmatter type.")
	}

	return data, nil
}

// Content is the block content
type Content struct {
	Other   interface{}
	Fields  []*Block
	Arrays  []*Block
	Objects []*Block
}

// Block is a block
type Block struct {
	Name     string
	Title    string
	Type     string
	HTMLType string
	Content  *Content
	Parent   *Block
}

func rawToPretty(config interface{}, parent *Block) *Content {
	objects := []*Block{}
	arrays := []*Block{}
	fields := []*Block{}

	cnf := map[string]interface{}{}
	kind := reflect.TypeOf(config)

	switch kind {
	case reflect.TypeOf(map[interface{}]interface{}{}):
		for key, value := range config.(map[interface{}]interface{}) {
			cnf[key.(string)] = value
		}
	case reflect.TypeOf([]map[string]interface{}{}):
		for index, value := range config.([]map[string]interface{}) {
			cnf[strconv.Itoa(index)] = value
		}
	case reflect.TypeOf([]map[interface{}]interface{}{}):
		for index, value := range config.([]map[interface{}]interface{}) {
			cnf[strconv.Itoa(index)] = value
		}
	case reflect.TypeOf([]interface{}{}):
		for index, value := range config.([]interface{}) {
			cnf[strconv.Itoa(index)] = value
		}
	default:
		cnf = config.(map[string]interface{})
	}

	for name, element := range cnf {
		if variables.IsMap(element) {
			objects = append(objects, handleObjects(element, parent, name))
		} else if variables.IsSlice(element) {
			arrays = append(arrays, handleArrays(element, parent, name))
		} else {
			if name == "title" && parent.Name == mainName {
				mainTitle = element.(string)
			}
			fields = append(fields, handleFlatValues(element, parent, name))
		}
	}

	sort.Sort(sortByTitle(fields))
	sort.Sort(sortByTitle(arrays))
	sort.Sort(sortByTitle(objects))
	return &Content{
		Fields:  fields,
		Arrays:  arrays,
		Objects: objects,
	}
}

type sortByTitle []*Block

func (f sortByTitle) Len() int      { return len(f) }
func (f sortByTitle) Swap(i, j int) { f[i], f[j] = f[j], f[i] }
func (f sortByTitle) Less(i, j int) bool {
	return strings.ToLower(f[i].Name) < strings.ToLower(f[j].Name)
}

func handleObjects(content interface{}, parent *Block, name string) *Block {
	c := new(Block)
	c.Parent = parent
	c.Type = objectType
	c.Title = name

	if parent.Name == mainName {
		c.Name = c.Title
	} else if parent.Type == arrayType {
		c.Name = parent.Name + "[" + name + "]"
	} else {
		c.Name = parent.Name + "." + c.Title
	}

	c.Content = rawToPretty(content, c)
	return c
}

func handleArrays(content interface{}, parent *Block, name string) *Block {
	c := new(Block)
	c.Parent = parent
	c.Type = arrayType
	c.Title = name

	if parent.Name == mainName {
		c.Name = name
	} else {
		c.Name = parent.Name + "." + name
	}

	c.Content = rawToPretty(content, c)
	return c
}

func handleFlatValues(content interface{}, parent *Block, name string) *Block {
	c := new(Block)
	c.Parent = parent

	switch content.(type) {
	case bool:
		c.Type = "boolean"
	case int, float32, float64:
		c.Type = "number"
	default:
		c.Type = "string"
	}

	c.Content = &Content{Other: content}

	switch strings.ToLower(name) {
	case "description":
		c.HTMLType = "textarea"
	case "date", "publishdate":
		c.HTMLType = "datetime"
		c.Content = &Content{Other: cast.ToTime(content)}
	default:
		c.HTMLType = "text"
	}

	if parent.Type == arrayType {
		c.Name = parent.Name + "[]"
		c.Title = content.(string)
	} else if parent.Type == objectType {
		c.Title = name
		c.Name = parent.Name + "." + name

		if parent.Name == mainName {
			c.Name = name
		}
	} else {
		log.Panic("Parent type not allowed in handleFlatValues.")
	}

	return c
}
<|endoftext|>


<commit_before>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package swarming

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strings"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/luci/luci-go/client/logdog/annotee"
	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/logdog/types"
	miloProto "github.com/luci/luci-go/common/proto/milo"
	"github.com/luci/luci-go/common/transport"
	"golang.org/x/net/context"

	"github.com/luci/luci-go/appengine/cmd/milo/resp"
	"github.com/luci/luci-go/appengine/gaeauth/client"
)

func resolveServer(server string) string {
	// TODO(hinoka): configure this map in luci-config
	if server == "" || server == "default" || server == "dev" {
		return "chromium-swarm-dev.appspot.com"
	} else if server == "prod" {
		return "chromium-swarm.appspot.com"
	} else {
		return server
	}
}

// swarmingIDs that beging with "debug:" wil redirect to json found in
// /testdata/
func getSwarmingLog(server string, swarmingID string, c context.Context) ([]byte, error) {
	// Fetch the debug file instead.
	if strings.HasPrefix(swarmingID, "debug:") {
		filename := strings.Join(
			[]string{"testdata", swarmingID[6:]}, "/")
		b, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, err
		}
		return b, nil
	}

	swarmingURL := fmt.Sprintf(
		"https://%s/_ah/api/swarming/v1/task/%s/stdout",
		resolveServer(server), swarmingID)
	client := transport.GetClient(client.UseServiceAccountTransport(c,
		[]string{"https://www.googleapis.com/auth/userinfo.email"}, nil))
	resp, err := client.Get(swarmingURL)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("Failed to fetch %s, status code %d", swarmingURL, resp.StatusCode)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Decode the JSON and extract the actual log.
	sm := map[string]*string{}
	if err := json.Unmarshal(body, &sm); err != nil {
		return nil, err
	}

	// Decode the data using annotee.
	if output, ok := sm["output"]; ok {
		return []byte(*output), nil
	}
	return nil, fmt.Errorf("Swarming response did not contain output\n%s", body)
}

// TODO(hinoka): This should go in a more generic file, when milo has more
// than one page.
func getNavi(swarmingID string, URL string) *resp.Navigation {
	navi := &resp.Navigation{}
	navi.PageTitle = &resp.Link{
		Label: swarmingID,
		URL:   URL,
	}
	navi.SiteTitle = &resp.Link{
		Label: "Milo",
		URL:   "/",
	}
	return navi
}

// Given a logdog/milo step, translate it to a BuildComponent struct.
func miloBuildStep(
	url string, anno *miloProto.Step, name string) *resp.BuildComponent {
	comp := &resp.BuildComponent{}
	asc := anno.GetStepComponent()
	comp.Label = asc.Name
	switch asc.Status {
	case miloProto.Status_RUNNING:
		comp.Status = resp.Running

	case miloProto.Status_SUCCESS:
		comp.Status = resp.Success

	case miloProto.Status_FAILURE:
		if anno.GetFailureDetails() != nil {
			switch anno.GetFailureDetails().Type {
			case miloProto.FailureDetails_INFRA:
				comp.Status = resp.InfraFailure

			case miloProto.FailureDetails_DM_DEPENDENCY_FAILED:
				comp.Status = resp.DependencyFailure

			default:
				comp.Status = resp.Failure
			}
		} else {
			comp.Status = resp.Failure
		}

	case miloProto.Status_EXCEPTION:
		comp.Status = resp.InfraFailure

		// Missing the case of waiting on unfinished dependency...
	default:
		comp.Status = resp.NotRun
	}
	// Sub link is for one link per log that isn't stdio.
	for _, link := range asc.GetOtherLinks() {
		lds := link.GetLogdogStream()
		if lds == nil {
			continue // DNE???
		}
		shortName := lds.Name[5 : len(lds.Name)-2]
		if strings.HasSuffix(lds.Name, "annotations") || strings.HasSuffix(lds.Name, "stdio") {
			// Skip the special ones.
			continue
		}
		newLink := &resp.Link{
			Label: shortName,
			URL:   strings.Join([]string{url, lds.Name}, "/"),
		}
		comp.SubLink = append(comp.SubLink, newLink)
	}

	// Main link is a link to the stdio.
	comp.MainLink = &resp.Link{
		Label: "stdio",
		URL:   strings.Join([]string{url, name, "stdio"}, "/"),
	}

	// This should always be a step.
	comp.Type = resp.Step

	// This should always be 0
	comp.LevelsDeep = 0

	// Timeswamapts
	comp.Started = asc.Started.Time().Format(time.RFC3339)

	// This should be the exact same thing.
	comp.Text = asc.Text

	return comp
}

// Takes a butler client and return a fully populated milo build.
func buildFromClient(c context.Context, swarmingID string, url string, s *memoryClient) (*resp.MiloBuild, error) {
	// Build the basic page response.
	build := &resp.MiloBuild{}
	build.Navi = getNavi(swarmingID, url)
	build.CurrentTime = clock.Now(c).String()

	// Now Fetch the main annotation of the build.
	mainAnno := &miloProto.Step{}
	proto.Unmarshal(s.stream["annotations"].dg, mainAnno)

	// Now fill in each of the step components.
	// TODO(hinoka): This is totes cachable.
	for _, name := range mainAnno.SubstepLogdogNameBase {
		anno := &miloProto.Step{}
		fullname := strings.Join([]string{name, "annotations"}, "/")
		proto.Unmarshal(s.stream[fullname].dg, anno)
		build.Components = append(build.Components, miloBuildStep(url, anno, name))
	}

	// Take care of properties
	propGroup := &resp.PropertyGroup{
		GroupName: "Main",
	}
	for _, prop := range mainAnno.GetStepComponent().Property {
		propGroup.Property = append(propGroup.Property, &resp.Property{
			Key:   prop.Name,
			Value: prop.Value,
		})
	}
	build.PropertyGroup = append(build.PropertyGroup, propGroup)

	// And we're done!
	return build, nil
}

// Takes in an annotated log and returns a fully populated memory client.
func clientFromAnnotatedLog(ctx context.Context, log []byte) (*memoryClient, error) {
	c := &memoryClient{}
	p := annotee.Processor{
		Context:                ctx,
		Client:                 c,
		MetadataUpdateInterval: time.Hour * 24, // Neverrrrrr send incr updates.
	}
	is := annotee.Stream{
		Reader:           bytes.NewBuffer(log),
		Name:             types.StreamName("stdio"),
		Annotate:         true,
		StripAnnotations: true,
	}
	// If this ever has more than one stream then memoryClient needs to become
	// goroutine safe
	if err := p.RunStreams([]*annotee.Stream{&is}); err != nil {
		return nil, err
	}
	return c, nil
}

func swarmingBuildImpl(c context.Context, URL string, server string, id string) (*resp.MiloBuild, error) {
	// Fetch the data from Swarming
	body, err := getSwarmingLog(server, id, c)
	if err != nil {
		return nil, err
	}

	// Decode the data using annotee.
	client, err := clientFromAnnotatedLog(c, body)
	if err != nil {
		return nil, err
	}

	return buildFromClient(c, id, URL, client)
}
<commit_msg>Milo: Add warning for annotation bug<commit_after>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package swarming

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strings"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/luci/luci-go/client/logdog/annotee"
	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/logdog/types"
	"github.com/luci/luci-go/common/logging"
	miloProto "github.com/luci/luci-go/common/proto/milo"
	"github.com/luci/luci-go/common/transport"
	"golang.org/x/net/context"

	"github.com/luci/luci-go/appengine/cmd/milo/resp"
	"github.com/luci/luci-go/appengine/gaeauth/client"
)

func resolveServer(server string) string {
	// TODO(hinoka): configure this map in luci-config
	if server == "" || server == "default" || server == "dev" {
		return "chromium-swarm-dev.appspot.com"
	} else if server == "prod" {
		return "chromium-swarm.appspot.com"
	} else {
		return server
	}
}

// swarmingIDs that beging with "debug:" wil redirect to json found in
// /testdata/
func getSwarmingLog(server string, swarmingID string, c context.Context) ([]byte, error) {
	// Fetch the debug file instead.
	if strings.HasPrefix(swarmingID, "debug:") {
		filename := strings.Join(
			[]string{"testdata", swarmingID[6:]}, "/")
		b, err := ioutil.ReadFile(filename)
		if err != nil {
			return nil, err
		}
		return b, nil
	}

	swarmingURL := fmt.Sprintf(
		"https://%s/_ah/api/swarming/v1/task/%s/stdout",
		resolveServer(server), swarmingID)
	client := transport.GetClient(client.UseServiceAccountTransport(c,
		[]string{"https://www.googleapis.com/auth/userinfo.email"}, nil))
	resp, err := client.Get(swarmingURL)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("Failed to fetch %s, status code %d", swarmingURL, resp.StatusCode)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Decode the JSON and extract the actual log.
	sm := map[string]*string{}
	if err := json.Unmarshal(body, &sm); err != nil {
		return nil, err
	}

	// Decode the data using annotee.
	if output, ok := sm["output"]; ok {
		return []byte(*output), nil
	}
	return nil, fmt.Errorf("Swarming response did not contain output\n%s", body)
}

// TODO(hinoka): This should go in a more generic file, when milo has more
// than one page.
func getNavi(swarmingID string, URL string) *resp.Navigation {
	navi := &resp.Navigation{}
	navi.PageTitle = &resp.Link{
		Label: swarmingID,
		URL:   URL,
	}
	navi.SiteTitle = &resp.Link{
		Label: "Milo",
		URL:   "/",
	}
	return navi
}

// Given a logdog/milo step, translate it to a BuildComponent struct.
func miloBuildStep(
	c context.Context, url string, anno *miloProto.Step, name string) *resp.BuildComponent {
	comp := &resp.BuildComponent{}
	asc := anno.GetStepComponent()
	comp.Label = asc.Name
	switch asc.Status {
	case miloProto.Status_RUNNING:
		comp.Status = resp.Running

	case miloProto.Status_SUCCESS:
		comp.Status = resp.Success

	case miloProto.Status_FAILURE:
		if anno.GetFailureDetails() != nil {
			switch anno.GetFailureDetails().Type {
			case miloProto.FailureDetails_INFRA:
				comp.Status = resp.InfraFailure

			case miloProto.FailureDetails_DM_DEPENDENCY_FAILED:
				comp.Status = resp.DependencyFailure

			default:
				comp.Status = resp.Failure
			}
		} else {
			comp.Status = resp.Failure
		}

	case miloProto.Status_EXCEPTION:
		comp.Status = resp.InfraFailure

		// Missing the case of waiting on unfinished dependency...
	default:
		comp.Status = resp.NotRun
	}
	// Sub link is for one link per log that isn't stdio.
	for _, link := range asc.GetOtherLinks() {
		lds := link.GetLogdogStream()
		if lds == nil {
			logging.Warningf(c, "Warning: %v of %v has an empty logdog stream.", link, asc)
			continue // DNE???
		}
		shortName := lds.Name[5 : len(lds.Name)-2]
		if strings.HasSuffix(lds.Name, "annotations") || strings.HasSuffix(lds.Name, "stdio") {
			// Skip the special ones.
			continue
		}
		newLink := &resp.Link{
			Label: shortName,
			URL:   strings.Join([]string{url, lds.Name}, "/"),
		}
		comp.SubLink = append(comp.SubLink, newLink)
	}

	// Main link is a link to the stdio.
	comp.MainLink = &resp.Link{
		Label: "stdio",
		URL:   strings.Join([]string{url, name, "stdio"}, "/"),
	}

	// This should always be a step.
	comp.Type = resp.Step

	// This should always be 0
	comp.LevelsDeep = 0

	// Timeswamapts
	comp.Started = asc.Started.Time().Format(time.RFC3339)

	// This should be the exact same thing.
	comp.Text = asc.Text

	return comp
}

// Takes a butler client and return a fully populated milo build.
func buildFromClient(c context.Context, swarmingID string, url string, s *memoryClient) (*resp.MiloBuild, error) {
	// Build the basic page response.
	build := &resp.MiloBuild{}
	build.Navi = getNavi(swarmingID, url)
	build.CurrentTime = clock.Now(c).String()

	// Now Fetch the main annotation of the build.
	mainAnno := &miloProto.Step{}
	proto.Unmarshal(s.stream["annotations"].dg, mainAnno)

	// Now fill in each of the step components.
	// TODO(hinoka): This is totes cachable.
	for _, name := range mainAnno.SubstepLogdogNameBase {
		anno := &miloProto.Step{}
		fullname := strings.Join([]string{name, "annotations"}, "/")
		proto.Unmarshal(s.stream[fullname].dg, anno)
		build.Components = append(build.Components, miloBuildStep(c, url, anno, name))
	}

	// Take care of properties
	propGroup := &resp.PropertyGroup{
		GroupName: "Main",
	}
	for _, prop := range mainAnno.GetStepComponent().Property {
		propGroup.Property = append(propGroup.Property, &resp.Property{
			Key:   prop.Name,
			Value: prop.Value,
		})
	}
	build.PropertyGroup = append(build.PropertyGroup, propGroup)

	// And we're done!
	return build, nil
}

// Takes in an annotated log and returns a fully populated memory client.
func clientFromAnnotatedLog(ctx context.Context, log []byte) (*memoryClient, error) {
	c := &memoryClient{}
	p := annotee.Processor{
		Context:                ctx,
		Client:                 c,
		MetadataUpdateInterval: time.Hour * 24, // Neverrrrrr send incr updates.
	}
	is := annotee.Stream{
		Reader:           bytes.NewBuffer(log),
		Name:             types.StreamName("stdio"),
		Annotate:         true,
		StripAnnotations: true,
	}
	// If this ever has more than one stream then memoryClient needs to become
	// goroutine safe
	if err := p.RunStreams([]*annotee.Stream{&is}); err != nil {
		return nil, err
	}
	return c, nil
}

func swarmingBuildImpl(c context.Context, URL string, server string, id string) (*resp.MiloBuild, error) {
	// Fetch the data from Swarming
	body, err := getSwarmingLog(server, id, c)
	if err != nil {
		return nil, err
	}

	// Decode the data using annotee.
	client, err := clientFromAnnotatedLog(c, body)
	if err != nil {
		return nil, err
	}

	return buildFromClient(c, id, URL, client)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"os/exec"
)

func main() {
	if _, err := exec.LookPath("go-bindata"); err != nil {
		fmt.Println("Cannot find go-bindata executable in path")
		fmt.Println("Maybe you need: go get github.com/elazarl/go-bindata-assetfs/...")
		os.Exit(1)
	}
	cmd := exec.Command("go-bindata", os.Args[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		os.Exit(1)
	}
	in, err := os.Open("bindata.go")
	if err != nil {
		fmt.Fprintln(os.Stderr, "Cannot read 'bindata.go'", err)
		return
	}
	out, err := os.Create("bindata_assetfs.go")
	if err != nil {
		fmt.Fprintln(os.Stderr, "Cannot write 'bindata_assetfs.go'", err)
		return
	}
	r := bufio.NewReader(in)
	done := false
	for line, isPrefix, err := r.ReadLine(); err == nil; line, isPrefix, err = r.ReadLine() {
		line = append(line, '\n')
		if _, err := out.Write(line); err != nil {
			fmt.Fprintln(os.Stderr, "Cannot write to 'bindata_assetfs.go'", err)
			return
		}
		if !done && !isPrefix && bytes.HasPrefix(line, []byte("import (")) {
			fmt.Fprintln(out, "\t\"github.com/elazarl/go-bindata-assetfs\"")
			done = true
		}
	}
	fmt.Fprintln(out, `
func assetFS() *assetfs.AssetFS {
	for k := range _bintree.Children {
		return &assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir, Prefix: k}
	}
	panic("unreachable")
}`)
	// Close files BEFORE remove calls (don't use defer).
	in.Close()
	out.Close()
	if err := os.Remove("bindata.go"); err != nil {
		fmt.Fprintln(os.Stderr, "Cannot remove bindata_assetfs.go", err)
	}
}
<commit_msg>wrong filename in error message<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"os/exec"
)

const outfile = "bindata.go"

func main() {
	if _, err := exec.LookPath("go-bindata"); err != nil {
		fmt.Println("Cannot find go-bindata executable in path")
		fmt.Println("Maybe you need: go get github.com/elazarl/go-bindata-assetfs/...")
		os.Exit(1)
	}
	cmd := exec.Command("go-bindata", os.Args[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		os.Exit(1)
	}
	in, err := os.Open(outfile)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Cannot read 'bindata.go'", err)
		return
	}
	out, err := os.Create("bindata_assetfs.go")
	if err != nil {
		fmt.Fprintln(os.Stderr, "Cannot write 'bindata_assetfs.go'", err)
		return
	}
	r := bufio.NewReader(in)
	done := false
	for line, isPrefix, err := r.ReadLine(); err == nil; line, isPrefix, err = r.ReadLine() {
		line = append(line, '\n')
		if _, err := out.Write(line); err != nil {
			fmt.Fprintln(os.Stderr, "Cannot write to 'bindata_assetfs.go'", err)
			return
		}
		if !done && !isPrefix && bytes.HasPrefix(line, []byte("import (")) {
			fmt.Fprintln(out, "\t\"github.com/elazarl/go-bindata-assetfs\"")
			done = true
		}
	}
	fmt.Fprintln(out, `
func assetFS() *assetfs.AssetFS {
	for k := range _bintree.Children {
		return &assetfs.AssetFS{Asset: Asset, AssetDir: AssetDir, Prefix: k}
	}
	panic("unreachable")
}`)
	// Close files BEFORE remove calls (don't use defer).
	in.Close()
	out.Close()
	if err := os.Remove(outfile); err != nil {
		fmt.Fprintln(os.Stderr, "Cannot remove", outfile, err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"fmt"
	"log"

	"github.com/dynport/gocli"
)

func main() {
	if e := run(); e != nil {
		log.Fatal(e)
	}
}

var BUILD_INFO string

func run() error {
	b, e := base64.StdEncoding.DecodeString(BUILD_INFO)
	if e != nil {
		return e
	}
	fmt.Println(string(b))
	_ = gocli.Red("test")
	return nil
}
<commit_msg>add clever comment<commit_after>package main

import (
	"encoding/base64"
	"fmt"
	"log"

	"github.com/dynport/gocli"
)

func main() {
	if e := run(); e != nil {
		log.Fatal(e)
	}
}

var BUILD_INFO string

func run() error {
	b, e := base64.StdEncoding.DecodeString(BUILD_INFO)
	if e != nil {
		return e
	}
	fmt.Println(string(b))
	_ = gocli.Red("test") // just to add external dependencies
	return nil
}
<|endoftext|>