<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/clock"
	"k8s.io/klog"
)

// ExpirationCache implements the store interface
//	1. All entries are automatically time stamped on insert
//		a. The key is computed based off the original item/keyFunc
//		b. The value inserted under that key is the timestamped item
//	2. Expiration happens lazily on read based on the expiration policy
//      a. No item can be inserted into the store while we're expiring
//		   *any* item in the cache.
//	3. Time-stamps are stripped off unexpired entries before return
// Note that the ExpirationCache is inherently slower than a normal
// threadSafeStore because it takes a write lock every time it checks if
// an item has expired.
type ExpirationCache struct {
	cacheStorage     ThreadSafeStore
	keyFunc          KeyFunc
	clock            clock.Clock
	expirationPolicy ExpirationPolicy
	// expirationLock is a write lock used to guarantee that we don't clobber
	// newly inserted objects because of a stale expiration timestamp comparison
	expirationLock sync.Mutex
}

// ExpirationPolicy dictates when an object expires. Currently only abstracted out
// so unittests don't rely on the system clock.
type ExpirationPolicy interface {
	IsExpired(obj *timestampedEntry) bool
}

// TTLPolicy implements a ttl based ExpirationPolicy.
type TTLPolicy struct {
	//	 >0: Expire entries with an age > ttl
	//	<=0: Don't expire any entry
	Ttl time.Duration

	// Clock used to calculate ttl expiration
	Clock clock.Clock
}

// IsExpired returns true if the given object is older than the ttl, or it can't
// determine its age.
func (p *TTLPolicy) IsExpired(obj *timestampedEntry) bool {
	return p.Ttl > 0 && p.Clock.Since(obj.timestamp) > p.Ttl
}

// timestampedEntry is the only type allowed in a ExpirationCache.
type timestampedEntry struct {
	obj       interface{}
	timestamp time.Time
}

// getTimestampedEntry returns the timestampedEntry stored under the given key.
func (c *ExpirationCache) getTimestampedEntry(key string) (*timestampedEntry, bool) {
	item, _ := c.cacheStorage.Get(key)
	if tsEntry, ok := item.(*timestampedEntry); ok {
		return tsEntry, true
	}
	return nil, false
}

// getOrExpire retrieves the object from the timestampedEntry if and only if it hasn't
// already expired. It holds a write lock across deletion.
func (c *ExpirationCache) getOrExpire(key string) (interface{}, bool) {
	// Prevent all inserts from the time we deem an item as "expired" to when we
	// delete it, so an un-expired item doesn't sneak in under the same key, just
	// before the Delete.
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	timestampedItem, exists := c.getTimestampedEntry(key)
	if !exists {
		return nil, false
	}
	if c.expirationPolicy.IsExpired(timestampedItem) {
		klog.V(4).Infof("Entry %v: %+v has expired", key, timestampedItem.obj)
		c.cacheStorage.Delete(key)
		return nil, false
	}
	return timestampedItem.obj, true
}

// GetByKey returns the item stored under the key, or sets exists=false.
func (c *ExpirationCache) GetByKey(key string) (interface{}, bool, error) {
	obj, exists := c.getOrExpire(key)
	return obj, exists, nil
}

// Get returns unexpired items. It purges the cache of expired items in the
// process.
func (c *ExpirationCache) Get(obj interface{}) (interface{}, bool, error) {
	key, err := c.keyFunc(obj)
	if err != nil {
		return nil, false, KeyError{obj, err}
	}
	obj, exists := c.getOrExpire(key)
	return obj, exists, nil
}

// List retrieves a list of unexpired items. It purges the cache of expired
// items in the process.
func (c *ExpirationCache) List() []interface{} {
	items := c.cacheStorage.List()

	list := make([]interface{}, 0, len(items))
	for _, item := range items {
		obj := item.(*timestampedEntry).obj
		if key, err := c.keyFunc(obj); err != nil {
			list = append(list, obj)
		} else if obj, exists := c.getOrExpire(key); exists {
			list = append(list, obj)
		}
	}
	return list
}

// ListKeys returns a list of all keys in the expiration cache.
func (c *ExpirationCache) ListKeys() []string {
	return c.cacheStorage.ListKeys()
}

// Add timestamps an item and inserts it into the cache, overwriting entries
// that might exist under the same key.
func (c *ExpirationCache) Add(obj interface{}) error {
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()

	key, err := c.keyFunc(obj)
	if err != nil {
		return KeyError{obj, err}
	}
	c.cacheStorage.Add(key, &timestampedEntry{obj, c.clock.Now()})
	return nil
}

// Update has not been implemented yet for lack of a use case, so this method
// simply calls `Add`. This effectively refreshes the timestamp.
func (c *ExpirationCache) Update(obj interface{}) error {
	return c.Add(obj)
}

// Delete removes an item from the cache.
func (c *ExpirationCache) Delete(obj interface{}) error {
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	key, err := c.keyFunc(obj)
	if err != nil {
		return KeyError{obj, err}
	}
	c.cacheStorage.Delete(key)
	return nil
}

// Replace will convert all items in the given list to TimestampedEntries
// before attempting the replace operation. The replace operation will
// delete the contents of the ExpirationCache `c`.
func (c *ExpirationCache) Replace(list []interface{}, resourceVersion string) error {
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	items := make(map[string]interface{}, len(list))
	ts := c.clock.Now()
	for _, item := range list {
		key, err := c.keyFunc(item)
		if err != nil {
			return KeyError{item, err}
		}
		items[key] = &timestampedEntry{item, ts}
	}
	c.cacheStorage.Replace(items, resourceVersion)
	return nil
}

// Resync will touch all objects to put them into the processing queue
func (c *ExpirationCache) Resync() error {
	return c.cacheStorage.Resync()
}

// NewTTLStore creates and returns a ExpirationCache with a TTLPolicy
func NewTTLStore(keyFunc KeyFunc, ttl time.Duration) Store {
	return &ExpirationCache{
		cacheStorage:     NewThreadSafeStore(Indexers{}, Indices{}),
		keyFunc:          keyFunc,
		clock:            clock.RealClock{},
		expirationPolicy: &TTLPolicy{ttl, clock.RealClock{}},
	}
}
<commit_msg>Reduce the duration where expirationLock is held for expiration_cache<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"sync"
	"time"

	"k8s.io/apimachinery/pkg/util/clock"
	"k8s.io/klog"
)

// ExpirationCache implements the store interface
//	1. All entries are automatically time stamped on insert
//		a. The key is computed based off the original item/keyFunc
//		b. The value inserted under that key is the timestamped item
//	2. Expiration happens lazily on read based on the expiration policy
//      a. No item can be inserted into the store while we're expiring
//		   *any* item in the cache.
//	3. Time-stamps are stripped off unexpired entries before return
// Note that the ExpirationCache is inherently slower than a normal
// threadSafeStore because it takes a write lock every time it checks if
// an item has expired.
type ExpirationCache struct {
	cacheStorage     ThreadSafeStore
	keyFunc          KeyFunc
	clock            clock.Clock
	expirationPolicy ExpirationPolicy
	// expirationLock is a write lock used to guarantee that we don't clobber
	// newly inserted objects because of a stale expiration timestamp comparison
	expirationLock sync.Mutex
}

// ExpirationPolicy dictates when an object expires. Currently only abstracted out
// so unittests don't rely on the system clock.
type ExpirationPolicy interface {
	IsExpired(obj *timestampedEntry) bool
}

// TTLPolicy implements a ttl based ExpirationPolicy.
type TTLPolicy struct {
	//	 >0: Expire entries with an age > ttl
	//	<=0: Don't expire any entry
	Ttl time.Duration

	// Clock used to calculate ttl expiration
	Clock clock.Clock
}

// IsExpired returns true if the given object is older than the ttl, or it can't
// determine its age.
func (p *TTLPolicy) IsExpired(obj *timestampedEntry) bool {
	return p.Ttl > 0 && p.Clock.Since(obj.timestamp) > p.Ttl
}

// timestampedEntry is the only type allowed in a ExpirationCache.
type timestampedEntry struct {
	obj       interface{}
	timestamp time.Time
}

// getTimestampedEntry returns the timestampedEntry stored under the given key.
func (c *ExpirationCache) getTimestampedEntry(key string) (*timestampedEntry, bool) {
	item, _ := c.cacheStorage.Get(key)
	if tsEntry, ok := item.(*timestampedEntry); ok {
		return tsEntry, true
	}
	return nil, false
}

// getOrExpire retrieves the object from the timestampedEntry if and only if it hasn't
// already expired. It holds a write lock across deletion.
func (c *ExpirationCache) getOrExpire(key string) (interface{}, bool) {
	// Prevent all inserts from the time we deem an item as "expired" to when we
	// delete it, so an un-expired item doesn't sneak in under the same key, just
	// before the Delete.
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	timestampedItem, exists := c.getTimestampedEntry(key)
	if !exists {
		return nil, false
	}
	if c.expirationPolicy.IsExpired(timestampedItem) {
		klog.V(4).Infof("Entry %v: %+v has expired", key, timestampedItem.obj)
		c.cacheStorage.Delete(key)
		return nil, false
	}
	return timestampedItem.obj, true
}

// GetByKey returns the item stored under the key, or sets exists=false.
func (c *ExpirationCache) GetByKey(key string) (interface{}, bool, error) {
	obj, exists := c.getOrExpire(key)
	return obj, exists, nil
}

// Get returns unexpired items. It purges the cache of expired items in the
// process.
func (c *ExpirationCache) Get(obj interface{}) (interface{}, bool, error) {
	key, err := c.keyFunc(obj)
	if err != nil {
		return nil, false, KeyError{obj, err}
	}
	obj, exists := c.getOrExpire(key)
	return obj, exists, nil
}

// List retrieves a list of unexpired items. It purges the cache of expired
// items in the process.
func (c *ExpirationCache) List() []interface{} {
	items := c.cacheStorage.List()

	list := make([]interface{}, 0, len(items))
	for _, item := range items {
		obj := item.(*timestampedEntry).obj
		if key, err := c.keyFunc(obj); err != nil {
			list = append(list, obj)
		} else if obj, exists := c.getOrExpire(key); exists {
			list = append(list, obj)
		}
	}
	return list
}

// ListKeys returns a list of all keys in the expiration cache.
func (c *ExpirationCache) ListKeys() []string {
	return c.cacheStorage.ListKeys()
}

// Add timestamps an item and inserts it into the cache, overwriting entries
// that might exist under the same key.
func (c *ExpirationCache) Add(obj interface{}) error {
	key, err := c.keyFunc(obj)
	if err != nil {
		return KeyError{obj, err}
	}
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()

	c.cacheStorage.Add(key, &timestampedEntry{obj, c.clock.Now()})
	return nil
}

// Update has not been implemented yet for lack of a use case, so this method
// simply calls `Add`. This effectively refreshes the timestamp.
func (c *ExpirationCache) Update(obj interface{}) error {
	return c.Add(obj)
}

// Delete removes an item from the cache.
func (c *ExpirationCache) Delete(obj interface{}) error {
	key, err := c.keyFunc(obj)
	if err != nil {
		return KeyError{obj, err}
	}
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	c.cacheStorage.Delete(key)
	return nil
}

// Replace will convert all items in the given list to TimestampedEntries
// before attempting the replace operation. The replace operation will
// delete the contents of the ExpirationCache `c`.
func (c *ExpirationCache) Replace(list []interface{}, resourceVersion string) error {
	items := make(map[string]interface{}, len(list))
	ts := c.clock.Now()
	for _, item := range list {
		key, err := c.keyFunc(item)
		if err != nil {
			return KeyError{item, err}
		}
		items[key] = &timestampedEntry{item, ts}
	}
	c.expirationLock.Lock()
	defer c.expirationLock.Unlock()
	c.cacheStorage.Replace(items, resourceVersion)
	return nil
}

// Resync will touch all objects to put them into the processing queue
func (c *ExpirationCache) Resync() error {
	return c.cacheStorage.Resync()
}

// NewTTLStore creates and returns a ExpirationCache with a TTLPolicy
func NewTTLStore(keyFunc KeyFunc, ttl time.Duration) Store {
	return &ExpirationCache{
		cacheStorage:     NewThreadSafeStore(Indexers{}, Indices{}),
		keyFunc:          keyFunc,
		clock:            clock.RealClock{},
		expirationPolicy: &TTLPolicy{ttl, clock.RealClock{}},
	}
}
<|endoftext|>


<commit_before>package csaupgrade_test

import (
	"reflect"
	"testing"

	"github.com/google/go-cmp/cmp"
	"k8s.io/apimachinery/pkg/api/equality"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/conversion"
	"k8s.io/apimachinery/pkg/util/yaml"
	"k8s.io/client-go/util/csaupgrade"
)

var avoidTimestampEqualities = func() conversion.Equalities {
	var eqs = equality.Semantic.Copy()

	err := eqs.AddFunc(
		func(a, b metav1.ManagedFieldsEntry) bool {
			// Two objects' managed fields are equivalent if, ignoring timestamp,
			//	the objects are deeply equal.
			a.Time = nil
			b.Time = nil
			return reflect.DeepEqual(a, b)
		},
	)

	if err != nil {
		panic(err)
	}

	return eqs
}()

func TestUpgradeCSA(t *testing.T) {
	// Initial object has managed fields from using CSA
	originalYAML := []byte(`
apiVersion: v1
data:
  key: value
  legacy: unused
kind: ConfigMap
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}}
  creationTimestamp: "2022-08-22T23:08:23Z"
  managedFields:
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:data:
        .: {}
        f:key: {}
        f:legacy: {}
      f:metadata:
        f:annotations:
          .: {}
          f:kubectl.kubernetes.io/last-applied-configuration: {}
    manager: kubectl-client-side-apply
    operation: Update
    time: "2022-08-22T23:08:23Z"
  name: test
  namespace: default
  resourceVersion: "12502"
  uid: 1f186675-58e6-4d7b-8bc5-7ece581e3013
`)
	initialObject := unstructured.Unstructured{}
	err := yaml.Unmarshal(originalYAML, &initialObject)
	if err != nil {
		t.Fatal(err)
	}

	upgraded, err := csaupgrade.UpgradeManagedFields(&initialObject, "kubectl-client-side-apply", "kubectl", "")
	if err != nil {
		t.Fatal(err)
	}

	expectedYAML := []byte(`
apiVersion: v1
data:
  key: value
  legacy: unused
kind: ConfigMap
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}}
  creationTimestamp: "2022-08-22T23:08:23Z"
  managedFields:
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:data:
        .: {}
        f:key: {}
        f:legacy: {}
      f:metadata:
        f:annotations:
          .: {}
          f:kubectl.kubernetes.io/last-applied-configuration: {}
    manager: kubectl
    operation: Apply
    time: "2022-08-22T23:08:23Z"
  name: test
  namespace: default
  resourceVersion: "12502"
  uid: 1f186675-58e6-4d7b-8bc5-7ece581e3013
`)

	expectedObject := unstructured.Unstructured{}
	err = yaml.Unmarshal(expectedYAML, &expectedObject)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(&expectedObject, upgraded) {
		t.Fatal(cmp.Diff(&expectedObject, upgraded))
	}
}
<commit_msg>remove  unused code<commit_after>package csaupgrade_test

import (
	"reflect"
	"testing"

	"github.com/google/go-cmp/cmp"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/util/yaml"
	"k8s.io/client-go/util/csaupgrade"
)

func TestUpgradeCSA(t *testing.T) {
	// Initial object has managed fields from using CSA
	originalYAML := []byte(`
apiVersion: v1
data:
  key: value
  legacy: unused
kind: ConfigMap
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}}
  creationTimestamp: "2022-08-22T23:08:23Z"
  managedFields:
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:data:
        .: {}
        f:key: {}
        f:legacy: {}
      f:metadata:
        f:annotations:
          .: {}
          f:kubectl.kubernetes.io/last-applied-configuration: {}
    manager: kubectl-client-side-apply
    operation: Update
    time: "2022-08-22T23:08:23Z"
  name: test
  namespace: default
  resourceVersion: "12502"
  uid: 1f186675-58e6-4d7b-8bc5-7ece581e3013
`)
	initialObject := unstructured.Unstructured{}
	err := yaml.Unmarshal(originalYAML, &initialObject)
	if err != nil {
		t.Fatal(err)
	}

	upgraded, err := csaupgrade.UpgradeManagedFields(&initialObject, "kubectl-client-side-apply", "kubectl", "")
	if err != nil {
		t.Fatal(err)
	}

	expectedYAML := []byte(`
apiVersion: v1
data:
  key: value
  legacy: unused
kind: ConfigMap
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","data":{"key":"value","legacy":"unused"},"kind":"ConfigMap","metadata":{"annotations":{},"name":"test","namespace":"default"}}
  creationTimestamp: "2022-08-22T23:08:23Z"
  managedFields:
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:data:
        .: {}
        f:key: {}
        f:legacy: {}
      f:metadata:
        f:annotations:
          .: {}
          f:kubectl.kubernetes.io/last-applied-configuration: {}
    manager: kubectl
    operation: Apply
    time: "2022-08-22T23:08:23Z"
  name: test
  namespace: default
  resourceVersion: "12502"
  uid: 1f186675-58e6-4d7b-8bc5-7ece581e3013
`)

	expectedObject := unstructured.Unstructured{}
	err = yaml.Unmarshal(expectedYAML, &expectedObject)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(&expectedObject, upgraded) {
		t.Fatal(cmp.Diff(&expectedObject, upgraded))
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/fs"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/repository"
	"labix.org/v2/mgo/bson"
	"strings"
)

var fsystem fs.Fs

func filesystem() fs.Fs {
	if fsystem == nil {
		fsystem = fs.OsFs{}
	}
	return fsystem
}

// runCmd executes commands and log the given stdout and stderror.
func runCmd(cmd string, args ...string) (string, error) {
	out := bytes.Buffer{}
	err := executor().Execute(cmd, args, nil, &out, &out)
	log.Printf("running the cmd: %s with the args: %s", cmd, args)
	return out.String(), err
}

// container represents an docker container with the given name.
type container struct {
	name string
	id   string
}

// newContainer creates a new container in docker and stores it on database
func newContainer(app provision.App) *container {
	appName := app.GetName()
	c := &container{name: appName}
	id, err := c.create(app.GetPlatform(), repository.GetReadOnlyUrl(appName))
	if err != nil {
		log.Printf("Error creating container %s", appName)
		log.Printf("Error was: %s", err.Error())
		return c
	}
	c.id = id
	u := provision.Unit{
		Name:       app.GetName(),
		AppName:    app.GetName(),
		Type:       app.GetPlatform(),
		Machine:    0,
		InstanceId: app.GetName(),
		Status:     provision.StatusCreating,
		Ip:         "",
	}
	if err := collection().Insert(u); err != nil {
		log.Print(err)
		return c // should rollback
	}
	return c
}

// ip returns the ip for the container.
func (c *container) ip() (string, error) {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		return "", err
	}
	log.Printf("Getting ipaddress to instance %s", c.id)
	instanceJson, err := runCmd(docker, "inspect", c.id)
	if err != nil {
		msg := "error(%s) trying to inspect docker instance(%s) to get ipaddress"
		log.Printf(msg, err)
		return "", errors.New(msg)
	}
	var result map[string]interface{}
	if err := json.Unmarshal([]byte(instanceJson), &result); err != nil {
		msg := "error(%s) parsing json from docker when trying to get ipaddress"
		log.Printf(msg, err)
		return "", errors.New(msg)
	}
	if ns, ok := result["NetworkSettings"]; !ok || ns == nil {
		msg := "Error when getting container information. NetworkSettings is missing."
		log.Printf(msg)
		return "", errors.New(msg)
	}
	networkSettings := result["NetworkSettings"].(map[string]interface{})
	instanceIp := networkSettings["IpAddress"].(string)
	if instanceIp == "" {
		msg := "error: Can't get ipaddress..."
		log.Print(msg)
		return "", errors.New(msg)
	}
	log.Printf("Instance IpAddress: %s", instanceIp)
	return instanceIp, nil
}

// create creates a docker container with base template by default.
//
// It receives the application's platform in order to choose the correct
// docker image and the repository to pass to the script that will take
// care of the deploy.
func (c *container) create(platform, repository string) (string, error) {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		return "", err
	}
	repoNamespace, err := config.GetString("docker:repository-namespace")
	if err != nil {
		return "", err
	}
	cmd, err := config.GetString("docker:deploy-cmd")
	if err != nil {
		return "", err
	}
	cmd = fmt.Sprintf("%s %s", cmd, repository)                //replace with app's repo url
	imageName := fmt.Sprintf("%s/%s", repoNamespace, platform) // replace python with app's platform
	args := []string{"run", "-d", imageName, cmd}
	id, err := runCmd(docker, args...)
	id = strings.Replace(id, "\n", "", -1)
	log.Printf("docker id=%s", id)
	return id, err
}

// start starts a docker container.
func (c *container) start() error {
	// it isn't necessary to start a docker container after docker run.
	return nil
}

// stop stops a docker container.
func (c *container) stop() error {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		return err
	}
	//TODO: better error handling
	log.Printf("trying to stop instance %s", c.id)
	output, err := runCmd(docker, "stop", c.id)
	log.Printf("docker stop=%s", output)
	return err
}

// remove removes a docker container.
func (c *container) remove() error {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		return err
	}
	//TODO: better error handling
	//TODO: Remove host's nginx route
	log.Printf("trying to remove container %s", c.id)
	_, err = runCmd(docker, "rm", c.id)
	return err
}

// image represents a docker image.
type image struct {
	Name string
	Id   string
}

// repositoryName returns the image repository name for a given image.
//
// Repository is a docker concept, the image actually does not have a name,
// it has a repository, that is a composed name, e.g.: tsuru/base.
// Tsuru will always use a namespace, defined in tsuru.conf.
// Additionally, tsuru will use the application's name to do that composition.
func (img *image) repositoryName() string {
	repoNamespace, err := config.GetString("docker:repository-namespace")
	if err != nil {
		log.Printf("Tsuru is misconfigured. docker:repository-namespace config is missing.")
		return ""
	}
	return fmt.Sprintf("%s/%s", repoNamespace, img.Name)
}

// commit commits an image in docker
//
// This is another docker concept, in order to generate an image from a container
// one must commit it.
func (img *image) commit(cId string) (string, error) {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		log.Printf("Tsuru is misconfigured. docker:binary config is missing.")
		return "", err
	}
	log.Printf("attempting to commit image from container %s", cId)
	rName := img.repositoryName()
	id, err := runCmd(docker, "commit", cId, rName)
	if err != nil {
		log.Printf("Could not commit docker image: %s", err.Error())
		return "", err
	}
	img.Id = strings.Replace(id, "\n", "", -1)
	if err := imagesCollection().Insert(&img); err != nil {
		log.Printf("Could not store image information %s", err.Error())
		return "", err
	}
	return img.Id, nil
}

// remove removes an image from docker registry
func (img *image) remove() error {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		log.Printf("Tsuru is misconfigured. docker:binary config is missing.")
		return err
	}
	log.Printf("attempting to remove image %s from docker", img.repositoryName())
	_, err = runCmd(docker, "rmi", img.Id)
	if err != nil {
		log.Printf("Could not remove image %s from docker: %s", img.Id, err.Error())
		return err
	}
	err = imagesCollection().Remove(bson.M{"name": img.Name})
	if err != nil {
		log.Printf("Could not remove image %s from mongo: %s", img.Id, err.Error())
		return err
	}
	return nil
}
<commit_msg>provision/docker/docker.go: removed todo comment<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/fs"
	"github.com/globocom/tsuru/log"
	"github.com/globocom/tsuru/provision"
	"github.com/globocom/tsuru/repository"
	"labix.org/v2/mgo/bson"
	"strings"
)

var fsystem fs.Fs

func filesystem() fs.Fs {
	if fsystem == nil {
		fsystem = fs.OsFs{}
	}
	return fsystem
}

// runCmd executes commands and log the given stdout and stderror.
func runCmd(cmd string, args ...string) (string, error) {
	out := bytes.Buffer{}
	err := executor().Execute(cmd, args, nil, &out, &out)
	log.Printf("running the cmd: %s with the args: %s", cmd, args)
	return out.String(), err
}

// container represents an docker container with the given name.
type container struct {
	name string
	id   string
}

// newContainer creates a new container in docker and stores it on database
func newContainer(app provision.App) *container {
	appName := app.GetName()
	c := &container{name: appName}
	id, err := c.create(app.GetPlatform(), repository.GetReadOnlyUrl(appName))
	if err != nil {
		log.Printf("Error creating container %s", appName)
		log.Printf("Error was: %s", err.Error())
		return c
	}
	c.id = id
	u := provision.Unit{
		Name:       app.GetName(),
		AppName:    app.GetName(),
		Type:       app.GetPlatform(),
		Machine:    0,
		InstanceId: app.GetName(),
		Status:     provision.StatusCreating,
		Ip:         "",
	}
	if err := collection().Insert(u); err != nil {
		log.Print(err)
		return c // should rollback
	}
	return c
}

// ip returns the ip for the container.
func (c *container) ip() (string, error) {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		return "", err
	}
	log.Printf("Getting ipaddress to instance %s", c.id)
	instanceJson, err := runCmd(docker, "inspect", c.id)
	if err != nil {
		msg := "error(%s) trying to inspect docker instance(%s) to get ipaddress"
		log.Printf(msg, err)
		return "", errors.New(msg)
	}
	var result map[string]interface{}
	if err := json.Unmarshal([]byte(instanceJson), &result); err != nil {
		msg := "error(%s) parsing json from docker when trying to get ipaddress"
		log.Printf(msg, err)
		return "", errors.New(msg)
	}
	if ns, ok := result["NetworkSettings"]; !ok || ns == nil {
		msg := "Error when getting container information. NetworkSettings is missing."
		log.Printf(msg)
		return "", errors.New(msg)
	}
	networkSettings := result["NetworkSettings"].(map[string]interface{})
	instanceIp := networkSettings["IpAddress"].(string)
	if instanceIp == "" {
		msg := "error: Can't get ipaddress..."
		log.Print(msg)
		return "", errors.New(msg)
	}
	log.Printf("Instance IpAddress: %s", instanceIp)
	return instanceIp, nil
}

// create creates a docker container with base template by default.
//
// It receives the application's platform in order to choose the correct
// docker image and the repository to pass to the script that will take
// care of the deploy.
func (c *container) create(platform, repository string) (string, error) {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		return "", err
	}
	repoNamespace, err := config.GetString("docker:repository-namespace")
	if err != nil {
		return "", err
	}
	cmd, err := config.GetString("docker:deploy-cmd")
	if err != nil {
		return "", err
	}
	cmd = fmt.Sprintf("%s %s", cmd, repository)
	imageName := fmt.Sprintf("%s/%s", repoNamespace, platform)
	args := []string{"run", "-d", imageName, cmd}
	id, err := runCmd(docker, args...)
	id = strings.Replace(id, "\n", "", -1)
	log.Printf("docker id=%s", id)
	return id, err
}

// start starts a docker container.
func (c *container) start() error {
	// it isn't necessary to start a docker container after docker run.
	return nil
}

// stop stops a docker container.
func (c *container) stop() error {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		return err
	}
	//TODO: better error handling
	log.Printf("trying to stop instance %s", c.id)
	output, err := runCmd(docker, "stop", c.id)
	log.Printf("docker stop=%s", output)
	return err
}

// remove removes a docker container.
func (c *container) remove() error {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		return err
	}
	//TODO: better error handling
	//TODO: Remove host's nginx route
	log.Printf("trying to remove container %s", c.id)
	_, err = runCmd(docker, "rm", c.id)
	return err
}

// image represents a docker image.
type image struct {
	Name string
	Id   string
}

// repositoryName returns the image repository name for a given image.
//
// Repository is a docker concept, the image actually does not have a name,
// it has a repository, that is a composed name, e.g.: tsuru/base.
// Tsuru will always use a namespace, defined in tsuru.conf.
// Additionally, tsuru will use the application's name to do that composition.
func (img *image) repositoryName() string {
	repoNamespace, err := config.GetString("docker:repository-namespace")
	if err != nil {
		log.Printf("Tsuru is misconfigured. docker:repository-namespace config is missing.")
		return ""
	}
	return fmt.Sprintf("%s/%s", repoNamespace, img.Name)
}

// commit commits an image in docker
//
// This is another docker concept, in order to generate an image from a container
// one must commit it.
func (img *image) commit(cId string) (string, error) {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		log.Printf("Tsuru is misconfigured. docker:binary config is missing.")
		return "", err
	}
	log.Printf("attempting to commit image from container %s", cId)
	rName := img.repositoryName()
	id, err := runCmd(docker, "commit", cId, rName)
	if err != nil {
		log.Printf("Could not commit docker image: %s", err.Error())
		return "", err
	}
	img.Id = strings.Replace(id, "\n", "", -1)
	if err := imagesCollection().Insert(&img); err != nil {
		log.Printf("Could not store image information %s", err.Error())
		return "", err
	}
	return img.Id, nil
}

// remove removes an image from docker registry
func (img *image) remove() error {
	docker, err := config.GetString("docker:binary")
	if err != nil {
		log.Printf("Tsuru is misconfigured. docker:binary config is missing.")
		return err
	}
	log.Printf("attempting to remove image %s from docker", img.repositoryName())
	_, err = runCmd(docker, "rmi", img.Id)
	if err != nil {
		log.Printf("Could not remove image %s from docker: %s", img.Id, err.Error())
		return err
	}
	err = imagesCollection().Remove(bson.M{"name": img.Name})
	if err != nil {
		log.Printf("Could not remove image %s from mongo: %s", img.Id, err.Error())
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package bootstrap

import (
	"fmt"
	"time"

	"github.com/flynn/flynn/pkg/cluster"
)

type WaitHostsAction struct{}

func init() {
	Register("wait-hosts", &WaitHostsAction{})
}

func (a *WaitHostsAction) Run(s *State) error {
	const waitInterval = 500 * time.Millisecond

	hosts := make(map[*cluster.Host]struct{}, len(s.Hosts))
	for _, h := range s.Hosts {
		hosts[h] = struct{}{}
	}

	timeout := time.After(s.HostTimeout)
	up := 0
outer:
	for {
		for h := range hosts {
			status, err := h.GetStatus()
			if err != nil {
				continue
			}
			if status.Network != nil && status.Network.Subnet != "" && status.Discoverd != nil && status.Discoverd.URL != "" {
				delete(hosts, h)
				up++
			}
		}
		if up >= s.MinHosts {
			break outer
		}

		select {
		case <-timeout:
			msg := "bootstrap: timed out waiting for hosts to come up\n\nThe following hosts were unreachable:\n"
			for host := range hosts {
				msg += "\t" + host.Addr() + "\n"
			}
			msg += "\n"
			return fmt.Errorf(msg)
		case <-time.After(waitInterval):
		}
	}
	return nil
}
<commit_msg>bootstrap: Wait for discoverd registration in wait-hosts<commit_after>package bootstrap

import (
	"fmt"
	"time"

	discoverd "github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/cluster"
)

type WaitHostsAction struct{}

func init() {
	Register("wait-hosts", &WaitHostsAction{})
}

func (a *WaitHostsAction) Run(s *State) error {
	const waitInterval = 500 * time.Millisecond

	hosts := make(map[*cluster.Host]struct{}, len(s.Hosts))
	for _, h := range s.Hosts {
		hosts[h] = struct{}{}
	}

	timeout := time.After(s.HostTimeout)
	up := 0
outer:
	for {
		var instances []*discoverd.Instance
		disc, err := s.DiscoverdClient()
		if err != nil {
			goto wait
		}
		instances, err = disc.Service("flynn-host").Instances()
		if err != nil {
			goto wait
		}

		for h := range hosts {
			status, err := h.GetStatus()
			if err != nil {
				continue
			}
			if status.Network != nil && status.Network.Subnet != "" && status.Discoverd != nil && status.Discoverd.URL != "" {
				for _, inst := range instances {
					if inst.Addr == h.Addr() {
						delete(hosts, h)
						up++
						break
					}
				}
			}
		}
		if up >= s.MinHosts {
			break outer
		}

	wait:
		select {
		case <-timeout:
			if err != nil {
				return err
			}
			msg := "bootstrap: timed out waiting for hosts to come up\n\nThe following hosts were unreachable:\n"
			for host := range hosts {
				msg += "\t" + host.Addr() + "\n"
			}
			msg += "\n"
			return fmt.Errorf(msg)
		case <-time.After(waitInterval):
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package projects

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/ion-channel/ionic/aliases"
	"github.com/ion-channel/ionic/tags"
)

const (
	validEmailRegex  = `^[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,}$`
	validGitURIRegex = `^(?:(?:http|ftp|gopher|mailto|mid|cid|news|nntp|prospero|telnet|rlogin|tn3270|wais|svn|git|rsync)+\+ssh\:\/\/|git\+https?:\/\/|git\@|(?:http|ftp|gopher|mailto|mid|cid|news|nntp|prospero|telnet|rlogin|tn3270|wais|svn|git|rsync|ssh|file)+s?:\/\/)[^\s]+$`
)

var (
	// ErrInvalidProject is returned when a given project does not pass the
	// standards for a project
	ErrInvalidProject = fmt.Errorf("project has invalid fields")
)

//Project is a representation of a project within the Ion Channel system
type Project struct {
	ID             *string         `json:"id,omitempty"`
	TeamID         *string         `json:"team_id,omitempty"`
	RulesetID      *string         `json:"ruleset_id,omitempty"`
	Name           *string         `json:"name,omitempty"`
	Type           *string         `json:"type,omitempty"`
	Source         *string         `json:"source,omitempty"`
	Branch         *string         `json:"branch,omitempty"`
	Description    *string         `json:"description,omitempty"`
	Active         bool            `json:"active"`
	ChatChannel    string          `json:"chat_channel"`
	CreatedAt      time.Time       `json:"created_at"`
	UpdatedAt      time.Time       `json:"updated_at"`
	DeployKey      string          `json:"deploy_key"`
	Monitor        bool            `json:"should_monitor"`
	POCName        string          `json:"poc_name"`
	POCEmail       string          `json:"poc_email"`
	Username       string          `json:"username"`
	Password       string          `json:"password"`
	KeyFingerprint string          `json:"key_fingerprint"`
	Aliases        []aliases.Alias `json:"aliases"`
	Tags           []tags.Tag      `json:"tags"`
}

// Validate takes an http client and returns a slice of fields as a string and
// an error. The fields will be a list of fields that did not pass the
// validation. An error will only be returned if any of the fields fail their
// validation.
func (p *Project) Validate(client *http.Client) (map[string]string, error) {
	invalidFields := make(map[string]string)
	var projErr error

	if p.ID == nil {
		invalidFields["id"] = "missing id"
		projErr = ErrInvalidProject
	}

	if p.TeamID == nil {
		invalidFields["team_id"] = "missing team id"
		projErr = ErrInvalidProject
	}

	if p.RulesetID == nil {
		invalidFields["ruleset_id"] = "missing ruleset id"
		projErr = ErrInvalidProject
	}

	if p.Name == nil {
		invalidFields["name"] = "missing name"
		projErr = ErrInvalidProject
	}

	if p.Type == nil {
		invalidFields["type"] = "missing type"
		projErr = ErrInvalidProject
	}

	if p.Source == nil {
		invalidFields["source"] = "missing source"
		projErr = ErrInvalidProject
	}

	if p.Branch == nil && p.Type != nil && strings.ToLower(*p.Type) == "git" {
		invalidFields["branch"] = "missing branch"
		projErr = ErrInvalidProject
	}

	if p.Description == nil {
		invalidFields["description"] = "missing description"
		projErr = ErrInvalidProject
	}

	r := regexp.MustCompile(validEmailRegex)
	if p.POCEmail != "" && !r.MatchString(p.POCEmail) {
		invalidFields["poc_email"] = "invalid email supplied"
		projErr = ErrInvalidProject
	}

	isFinger, err := regexp.MatchString("[a-f0-9]{2}\\:[a-f0-9]{2}\\:[a-f0-9]{2}\\:", p.DeployKey)
	if err != nil {
		return nil, fmt.Errorf("failed to detect deploy key fingerprint: %v", err.Error())
	}

	if isFinger {
		p.DeployKey = ""
	}

	block, rest := pem.Decode([]byte(p.DeployKey))
	if block != nil {
		pkey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
		if err != nil {
			invalidFields["deploy_key"] = "must be a valid ssh key"
			projErr = ErrInvalidProject
		} else {
			err = pkey.Validate()
			if err != nil {
				invalidFields["deploy_key"] = "must be a valid ssh key"
				projErr = ErrInvalidProject
			}
		}
	}

	if block == nil && rest != nil && string(rest) != "" {
		invalidFields["deploy_key"] = "must be a valid ssh key"
		projErr = ErrInvalidProject
	}

	if p.Type != nil {
		switch strings.ToLower(*p.Type) {
		case "artifact":
			u, err := url.Parse(*p.Source)
			if err != nil {
				invalidFields["source"] = fmt.Sprintf("source must be a valid url: %v", err.Error())
				projErr = ErrInvalidProject
			}

			if u != nil {
				res, err := client.Head(u.String())
				if err != nil {
					invalidFields["source"] = "source failed to return a response"
					projErr = ErrInvalidProject
				}

				if res != nil && res.StatusCode == http.StatusNotFound {
					invalidFields["source"] = "source returned a not found"
					projErr = ErrInvalidProject
				}
			}
		case "git", "svn":
			r := regexp.MustCompile(validGitURIRegex)
			if p.Source != nil && !r.MatchString(*p.Source) {
				invalidFields["source"] = "source must be a valid uri"
				projErr = ErrInvalidProject
			}
		}
	}

	return invalidFields, projErr
}
<commit_msg>adding field to project to represent private<commit_after>package projects

import (
	"crypto/x509"
	"encoding/pem"
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/ion-channel/ionic/aliases"
	"github.com/ion-channel/ionic/tags"
)

const (
	validEmailRegex  = `^[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,}$`
	validGitURIRegex = `^(?:(?:http|ftp|gopher|mailto|mid|cid|news|nntp|prospero|telnet|rlogin|tn3270|wais|svn|git|rsync)+\+ssh\:\/\/|git\+https?:\/\/|git\@|(?:http|ftp|gopher|mailto|mid|cid|news|nntp|prospero|telnet|rlogin|tn3270|wais|svn|git|rsync|ssh|file)+s?:\/\/)[^\s]+$`
)

var (
	// ErrInvalidProject is returned when a given project does not pass the
	// standards for a project
	ErrInvalidProject = fmt.Errorf("project has invalid fields")
)

//Project is a representation of a project within the Ion Channel system
type Project struct {
	ID             *string         `json:"id,omitempty"`
	TeamID         *string         `json:"team_id,omitempty"`
	RulesetID      *string         `json:"ruleset_id,omitempty"`
	Name           *string         `json:"name,omitempty"`
	Type           *string         `json:"type,omitempty"`
	Source         *string         `json:"source,omitempty"`
	Branch         *string         `json:"branch,omitempty"`
	Description    *string         `json:"description,omitempty"`
	Active         bool            `json:"active"`
	ChatChannel    string          `json:"chat_channel"`
	CreatedAt      time.Time       `json:"created_at"`
	UpdatedAt      time.Time       `json:"updated_at"`
	DeployKey      string          `json:"deploy_key"`
	Monitor        bool            `json:"should_monitor"`
	POCName        string          `json:"poc_name"`
	POCEmail       string          `json:"poc_email"`
	Username       string          `json:"username"`
	Password       string          `json:"password"`
	KeyFingerprint string          `json:"key_fingerprint"`
	Private        bool            `json:"private"`
	Aliases        []aliases.Alias `json:"aliases"`
	Tags           []tags.Tag      `json:"tags"`
}

// Validate takes an http client and returns a slice of fields as a string and
// an error. The fields will be a list of fields that did not pass the
// validation. An error will only be returned if any of the fields fail their
// validation.
func (p *Project) Validate(client *http.Client) (map[string]string, error) {
	invalidFields := make(map[string]string)
	var projErr error

	if p.ID == nil {
		invalidFields["id"] = "missing id"
		projErr = ErrInvalidProject
	}

	if p.TeamID == nil {
		invalidFields["team_id"] = "missing team id"
		projErr = ErrInvalidProject
	}

	if p.RulesetID == nil {
		invalidFields["ruleset_id"] = "missing ruleset id"
		projErr = ErrInvalidProject
	}

	if p.Name == nil {
		invalidFields["name"] = "missing name"
		projErr = ErrInvalidProject
	}

	if p.Type == nil {
		invalidFields["type"] = "missing type"
		projErr = ErrInvalidProject
	}

	if p.Source == nil {
		invalidFields["source"] = "missing source"
		projErr = ErrInvalidProject
	}

	if p.Branch == nil && p.Type != nil && strings.ToLower(*p.Type) == "git" {
		invalidFields["branch"] = "missing branch"
		projErr = ErrInvalidProject
	}

	if p.Description == nil {
		invalidFields["description"] = "missing description"
		projErr = ErrInvalidProject
	}

	r := regexp.MustCompile(validEmailRegex)
	if p.POCEmail != "" && !r.MatchString(p.POCEmail) {
		invalidFields["poc_email"] = "invalid email supplied"
		projErr = ErrInvalidProject
	}

	isFinger, err := regexp.MatchString("[a-f0-9]{2}\\:[a-f0-9]{2}\\:[a-f0-9]{2}\\:", p.DeployKey)
	if err != nil {
		return nil, fmt.Errorf("failed to detect deploy key fingerprint: %v", err.Error())
	}

	if isFinger {
		p.DeployKey = ""
	}

	block, rest := pem.Decode([]byte(p.DeployKey))
	if block != nil {
		pkey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
		if err != nil {
			invalidFields["deploy_key"] = "must be a valid ssh key"
			projErr = ErrInvalidProject
		} else {
			err = pkey.Validate()
			if err != nil {
				invalidFields["deploy_key"] = "must be a valid ssh key"
				projErr = ErrInvalidProject
			}
		}
	}

	if block == nil && rest != nil && string(rest) != "" {
		invalidFields["deploy_key"] = "must be a valid ssh key"
		projErr = ErrInvalidProject
	}

	if p.Type != nil {
		switch strings.ToLower(*p.Type) {
		case "artifact":
			u, err := url.Parse(*p.Source)
			if err != nil {
				invalidFields["source"] = fmt.Sprintf("source must be a valid url: %v", err.Error())
				projErr = ErrInvalidProject
			}

			if u != nil {
				res, err := client.Head(u.String())
				if err != nil {
					invalidFields["source"] = "source failed to return a response"
					projErr = ErrInvalidProject
				}

				if res != nil && res.StatusCode == http.StatusNotFound {
					invalidFields["source"] = "source returned a not found"
					projErr = ErrInvalidProject
				}
			}
		case "git", "svn":
			r := regexp.MustCompile(validGitURIRegex)
			if p.Source != nil && !r.MatchString(*p.Source) {
				invalidFields["source"] = "source must be a valid uri"
				projErr = ErrInvalidProject
			}
		}
	}

	return invalidFields, projErr
}
<|endoftext|>


<commit_before>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * promise/functions.go                                   *
 *                                                        *
 * some functions of promise for Go.                      *
 *                                                        *
 * LastModified: Aug 14, 2016                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package promise

import (
	"errors"
	"sync/atomic"
	"time"
)

// Create creates a Promise object containing the result of asynchronously
// calling computation.
//
// If calling computation returns error, the returned Promise is rejected with
// the error.
//
// If calling computation returns a Promise object, completion of the created
// Promise will wait until the returned Promise completes, and will then
// complete with the same result.
//
// If calling computation returns a non-Promise value, the returned Promise is
// completed with that value.
func Create(computation func() (interface{}, error)) Promise {
	promise := New()
	go call(promise, computation)
	return promise
}

// Sync creates a Promise object containing the result of immediately calling
// computation.
//
// If calling computation returns error, the returned Promise is rejected with
// the error.
//
// If calling computation returns a Promise object, completion of the created
// Promise will wait until the returned Promise completes, and will then
// complete with the same result.
//
// If calling computation returns a non-Promise value, the returned Promise is
// completed with that value.
func Sync(computation func() (interface{}, error)) Promise {
	promise := New()
	call(promise, computation)
	return promise
}

// Delayed creates a Promise object with the given value after a delay.
//
// If the value is a Callable function, it will be executed after the given
// duration has passed, and the Promise object is completed with the result.
func Delayed(duration time.Duration, value interface{}) Promise {
	promise := New()
	go func() {
		time.Sleep(duration)
		switch computation := value.(type) {
		case func() (interface{}, error):
			call(promise, computation)
		case func():
			call1(promise, computation)
		default:
			promise.Resolve(value)
		}
	}()
	return promise
}

// All function returns a promise that resolves when all of the promises in the
// iterable argument have resolved, or rejects with the reason of the first
// passed promise that rejects.
func All(iterable ...interface{}) Promise {
	count := int64(len(iterable))
	if count == 0 {
		return Resolve(nil)
	}
	result := make([]interface{}, count)
	promise := New()
	for index, value := range iterable {
		ToPromise(value).Then(
			allHandler(promise, result, index, &count),
			promise.Reject)
	}
	return promise
}

func allHandler(promise Promise, result []interface{}, index int, count *int64) func(value interface{}) {
	return func(value interface{}) {
		result[index] = value
		if atomic.AddInt64(count, -1) == 0 {
			promise.Resolve(result)
		}
	}
}

// Race function returns a promise that resolves or rejects as soon as one of
// the promises in the iterable resolves or rejects, with the value or reason
// from that promise.
func Race(iterable ...interface{}) Promise {
	promise := New()
	for _, value := range iterable {
		ToPromise(value).Fill(promise)
	}
	return promise
}

// Any function is a competitive race that allows one winner.
//
// The returned promise will fulfill as soon as any one of the input promises
// fulfills, with the value of the fulfilled input promise.
//
// Or reject with a IllegalArgumentError if the input iterable is empty--i.e. it
// is impossible to have one winner.
//
// Or reject with an iterable of all the rejection reasons, if the input
// iterable is non-empty, and all input promises reject.
func Any(iterable ...interface{}) Promise {
	count := int64(len(iterable))
	if count == 0 {
		return Reject(IllegalArgumentError("any(): iterable must not be empty"))
	}
	promise := New()
	for _, value := range iterable {
		ToPromise(value).Then(promise.Resolve, anyHandler(promise, &count))
	}
	return promise
}

func anyHandler(promise Promise, count *int64) func() {
	return func() {
		if atomic.AddInt64(count, -1) == 0 {
			promise.Reject(errors.New("any(): all promises failed"))
		}
	}
}

// Each function executes a provided function once per element of iterable.
//
// The callback parameter is a function to execute for each element:
//
//		func(index int, value interface{})
//
// index: The index of the current element being processed.
//
// value: The current element being processed.
//
// If any of the promises in iterable is rejected, the callback will not be
// executed. the returned promise will be rejected with the rejection reason
// of the first promise that was rejected.
func Each(callback func(int, interface{}), iterable ...interface{}) Promise {
	return All(iterable...).Then(func(a interface{}) {
		if a != nil {
			each(callback, a.([]interface{}))
		}
	})
}

func each(callback func(int, interface{}), iterable []interface{}) {
	for index, value := range iterable {
		callback(index, value)
	}
}

// Every function tests whether all elements in the iterable pass the test
// implemented by the provided function.
//
// The callback parameter is a function to test for each element:
//
//		func(index int, value interface{}) bool
//
// index: The index of the current element being processed.
//
// value: The current element being processed.
//
// If any of the promises in iterable is rejected, the callback will not be
// executed. the returned promise will be rejected with the rejection reason
// of the first promise that was rejected.
//
// The returned promise will fulfill with false if the callback function returns
// false for any iterable element immediately. Otherwise, if callback returned
// a true value for all elements, Every will return promise fulfill with true.
//
// If iterable is empty, The returned promise will fulfill with true.
func Every(callback func(int, interface{}) bool, iterable ...interface{}) Promise {
	return All(iterable...).Then(func(a interface{}) (interface{}, error) {
		if a == nil {
			return true, nil
		}
		return every(callback, a.([]interface{})), nil
	})
}

func every(callback func(int, interface{}) bool, iterable []interface{}) bool {
	for index, value := range iterable {
		if !callback(index, value) {
			return false
		}
	}
	return true
}

// Some function tests whether some element in the iterable passes the test
// implemented by the provided function.
//
// The callback parameter is a function to test for each element:
//
//		func(index int, value interface{}) bool
//
// index: The index of the current element being processed.
//
// value: The current element being processed.
//
// If any of the promises in iterable is rejected, the callback will not be
// executed. the returned promise will be rejected with the rejection reason
// of the first promise that was rejected.
//
// The returned promise will fulfill with true if the callback function returns
// true for any iterable element immediately. Otherwise, if callback returned a
// false value for all elements, Some will return promise fulfill with false.
//
// If iterable is empty, The returned promise will fulfill with false.
func Some(callback func(int, interface{}) bool, iterable ...interface{}) Promise {
	return All(iterable...).Then(func(a interface{}) (interface{}, error) {
		if a == nil {
			return false, nil
		}
		return some(callback, a.([]interface{})), nil
	})
}

func some(callback func(int, interface{}) bool, iterable []interface{}) bool {
	for index, value := range iterable {
		if callback(index, value) {
			return true
		}
	}
	return false
}

// Filter function returns a promise fulfill with a slice that has all elements
// pass the test implemented by the provided function.
//
// The callback parameter is a function to test for each element:
//
//		func(index int, value interface{}) bool
//
// index: The index of the current element being processed.
//
// value: The current element being processed.
//
// If any of the promises in iterable is rejected, the callback will not be
// executed. the returned promise will be rejected with the rejection reason
// of the first promise that was rejected.
func Filter(callback func(int, interface{}) bool, iterable ...interface{}) Promise {
	return All(iterable...).Then(func(a interface{}) (interface{}, error) {
		if a == nil {
			return nil, nil
		}
		return filter(callback, a.([]interface{})), nil
	})
}

func filter(callback func(int, interface{}) bool, iterable []interface{}) []interface{} {
	result := make([]interface{}, 0, len(iterable))
	for index, value := range iterable {
		if callback(index, value) {
			result = append(result, value)
		}
	}
	return result
}

// Map function returns a promise fulfill with a slice that has the results of
// calling a provided function on every element in iterable.
//
// The callback parameter produces an element of the new slice:
//
//		func(index int, value interface{}) interface{}
//
// index: The index of the current element being processed.
//
// value: The current element being processed.
//
// If any of the promises in iterable is rejected, the callback will not be
// executed. the returned promise will be rejected with the rejection reason
// of the first promise that was rejected.
func Map(callback func(int, interface{}) interface{}, iterable ...interface{}) Promise {
	return All(iterable...).Then(func(a interface{}) (interface{}, error) {
		if a == nil {
			return nil, nil
		}
		iterable := a.([]interface{})
		result := make([]interface{}, len(iterable))
		for index, value := range iterable {
			result[index] = callback(index, value)
		}
		return result, nil
	})
}
<commit_msg>Added Reduce function.<commit_after>/**********************************************************\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: http://www.hprose.com/                 |
|                   http://www.hprose.org/                 |
|                                                          |
\**********************************************************/
/**********************************************************\
 *                                                        *
 * promise/functions.go                                   *
 *                                                        *
 * some functions of promise for Go.                      *
 *                                                        *
 * LastModified: Aug 14, 2016                             *
 * Author: Ma Bingyao <andot@hprose.com>                  *
 *                                                        *
\**********************************************************/

package promise

import (
	"errors"
	"sync/atomic"
	"time"
)

// Create creates a Promise object containing the result of asynchronously
// calling computation.
//
// If calling computation returns error, the returned Promise is rejected with
// the error.
//
// If calling computation returns a Promise object, completion of the created
// Promise will wait until the returned Promise completes, and will then
// complete with the same result.
//
// If calling computation returns a non-Promise value, the returned Promise is
// completed with that value.
func Create(computation func() (interface{}, error)) Promise {
	promise := New()
	go call(promise, computation)
	return promise
}

// Sync creates a Promise object containing the result of immediately calling
// computation.
//
// If calling computation returns error, the returned Promise is rejected with
// the error.
//
// If calling computation returns a Promise object, completion of the created
// Promise will wait until the returned Promise completes, and will then
// complete with the same result.
//
// If calling computation returns a non-Promise value, the returned Promise is
// completed with that value.
func Sync(computation func() (interface{}, error)) Promise {
	promise := New()
	call(promise, computation)
	return promise
}

// Delayed creates a Promise object with the given value after a delay.
//
// If the value is a Callable function, it will be executed after the given
// duration has passed, and the Promise object is completed with the result.
func Delayed(duration time.Duration, value interface{}) Promise {
	promise := New()
	go func() {
		time.Sleep(duration)
		switch computation := value.(type) {
		case func() (interface{}, error):
			call(promise, computation)
		case func():
			call1(promise, computation)
		default:
			promise.Resolve(value)
		}
	}()
	return promise
}

// All function returns a promise that resolves when all of the promises in the
// iterable argument have resolved, or rejects with the reason of the first
// passed promise that rejects.
func All(iterable ...interface{}) Promise {
	count := int64(len(iterable))
	if count == 0 {
		return Resolve(nil)
	}
	result := make([]interface{}, count)
	promise := New()
	for index, value := range iterable {
		ToPromise(value).Then(
			allHandler(promise, result, index, &count),
			promise.Reject)
	}
	return promise
}

func allHandler(promise Promise, result []interface{}, index int, count *int64) func(value interface{}) {
	return func(value interface{}) {
		result[index] = value
		if atomic.AddInt64(count, -1) == 0 {
			promise.Resolve(result)
		}
	}
}

// Race function returns a promise that resolves or rejects as soon as one of
// the promises in the iterable resolves or rejects, with the value or reason
// from that promise.
func Race(iterable ...interface{}) Promise {
	promise := New()
	for _, value := range iterable {
		ToPromise(value).Fill(promise)
	}
	return promise
}

// Any function is a competitive race that allows one winner.
//
// The returned promise will fulfill as soon as any one of the input promises
// fulfills, with the value of the fulfilled input promise.
//
// Or reject with a IllegalArgumentError if the input iterable is empty--i.e. it
// is impossible to have one winner.
//
// Or reject with an iterable of all the rejection reasons, if the input
// iterable is non-empty, and all input promises reject.
func Any(iterable ...interface{}) Promise {
	count := int64(len(iterable))
	if count == 0 {
		return Reject(IllegalArgumentError("Any(): iterable must not be empty"))
	}
	promise := New()
	for _, value := range iterable {
		ToPromise(value).Then(promise.Resolve, anyHandler(promise, &count))
	}
	return promise
}

func anyHandler(promise Promise, count *int64) func() {
	return func() {
		if atomic.AddInt64(count, -1) == 0 {
			promise.Reject(errors.New("Any(): all promises failed"))
		}
	}
}

// Each function executes a provided function once per element of iterable.
//
// The callback parameter is a function to execute for each element:
//
//		func(index int, value interface{})
//
// index: The index of the current element being processed.
//
// value: The current element being processed.
//
// If any of the promises in iterable is rejected, the callback will not be
// executed. the returned promise will be rejected with the rejection reason
// of the first promise that was rejected.
func Each(callback func(int, interface{}), iterable ...interface{}) Promise {
	return All(iterable...).Then(func(a interface{}) {
		if a != nil {
			each(callback, a.([]interface{}))
		}
	})
}

func each(callback func(int, interface{}), iterable []interface{}) {
	for index, value := range iterable {
		callback(index, value)
	}
}

// Every function tests whether all elements in the iterable pass the test
// implemented by the provided function.
//
// The callback parameter is a function to test for each element:
//
//		func(index int, value interface{}) bool
//
// index: The index of the current element being processed.
//
// value: The current element being processed.
//
// If any of the promises in iterable is rejected, the callback will not be
// executed. the returned promise will be rejected with the rejection reason
// of the first promise that was rejected.
//
// The returned promise will fulfill with false if the callback function returns
// false for any iterable element immediately. Otherwise, if callback returned
// a true value for all elements, Every will return promise fulfill with true.
//
// If iterable is empty, The returned promise will fulfill with true.
func Every(callback func(int, interface{}) bool, iterable ...interface{}) Promise {
	return All(iterable...).Then(func(a interface{}) (interface{}, error) {
		if a == nil {
			return true, nil
		}
		return every(callback, a.([]interface{})), nil
	})
}

func every(callback func(int, interface{}) bool, iterable []interface{}) bool {
	for index, value := range iterable {
		if !callback(index, value) {
			return false
		}
	}
	return true
}

// Some function tests whether some element in the iterable passes the test
// implemented by the provided function.
//
// The callback parameter is a function to test for each element:
//
//		func(index int, value interface{}) bool
//
// index: The index of the current element being processed.
//
// value: The current element being processed.
//
// If any of the promises in iterable is rejected, the callback will not be
// executed. the returned promise will be rejected with the rejection reason
// of the first promise that was rejected.
//
// The returned promise will fulfill with true if the callback function returns
// true for any iterable element immediately. Otherwise, if callback returned a
// false value for all elements, Some will return promise fulfill with false.
//
// If iterable is empty, The returned promise will fulfill with false.
func Some(callback func(int, interface{}) bool, iterable ...interface{}) Promise {
	return All(iterable...).Then(func(a interface{}) (interface{}, error) {
		if a == nil {
			return false, nil
		}
		return some(callback, a.([]interface{})), nil
	})
}

func some(callback func(int, interface{}) bool, iterable []interface{}) bool {
	for index, value := range iterable {
		if callback(index, value) {
			return true
		}
	}
	return false
}

// Filter function returns a promise fulfill with a slice that has all elements
// pass the test implemented by the provided function.
//
// The callback parameter is a function to test for each element:
//
//		func(index int, value interface{}) bool
//
// index: The index of the current element being processed.
//
// value: The current element being processed.
//
// If any of the promises in iterable is rejected, the callback will not be
// executed. the returned promise will be rejected with the rejection reason
// of the first promise that was rejected.
func Filter(callback func(int, interface{}) bool, iterable ...interface{}) Promise {
	return All(iterable...).Then(func(a interface{}) (interface{}, error) {
		if a == nil {
			return nil, nil
		}
		return filter(callback, a.([]interface{})), nil
	})
}

func filter(callback func(int, interface{}) bool, iterable []interface{}) []interface{} {
	result := make([]interface{}, 0, len(iterable))
	for index, value := range iterable {
		if callback(index, value) {
			result = append(result, value)
		}
	}
	return result
}

// Map function returns a promise fulfill with a slice that has the results of
// calling a provided function on every element in iterable.
//
// The callback parameter produces an element of the new slice:
//
//		func(index int, value interface{}) interface{}
//
// index: The index of the current element being processed.
//
// value: The current element being processed.
//
// If any of the promises in iterable is rejected, the callback will not be
// executed. the returned promise will be rejected with the rejection reason
// of the first promise that was rejected.
func Map(callback func(int, interface{}) interface{}, iterable ...interface{}) Promise {
	return All(iterable...).Then(func(a interface{}) (interface{}, error) {
		if a == nil {
			return nil, nil
		}
		iterable := a.([]interface{})
		result := make([]interface{}, len(iterable))
		for index, value := range iterable {
			result[index] = callback(index, value)
		}
		return result, nil
	})
}

// Reduce function applies a function against an accumulator and each value of
// the iterable (from left-to-right) to reduce it to a single value.
//
// The callback parameter executes on each value in the iterable, taking three
// arguments:
//
//     func(index int, value interface{}, prev interface{}) interface{}
//
// index: The index of the current element being processed.
//
// value: The current element being processed.
//
//  prev: The value previously returned in the last invocation of the callback.
//
// The first time the callback is called, prev will be equal to the first value
// in the iterable and value will be equal to the second.
//
// If any of the promises in iterable is rejected, the callback will not be
// executed. the returned promise will be rejected with the rejection reason
// of the first promise that was rejected.
//
// If iterable is empty, the returned promise will be rejected with a
// IllegalArgumentError.
func Reduce(callback func(int, interface{}, interface{}) interface{}, iterable ...interface{}) Promise {
	return All(iterable...).Then(func(a interface{}) (interface{}, error) {
		if a == nil {
			return nil, IllegalArgumentError("Reduce(): iterable must not be empty")
		}
		return reduce(callback, a.([]interface{})), nil
	})
}

func reduce(callback func(int, interface{}, interface{}) interface{}, iterable []interface{}) interface{} {
	count := len(iterable)
	result := iterable[0]
	for index := 1; index < count; index++ {
		result = callback(index, iterable[index], result)
	}
	return result
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"reflect"
	"regexp"
	"strings"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema"
	"github.com/pkg/errors"
	blackfriday "gopkg.in/russross/blackfriday.v2"
)

const (
	version7  = "http://json-schema-org/draft-07/schema#"
	defPrefix = "#/definitions/"
)

type schemaGenerator struct {
	strict bool
}

type Schema struct {
	*Definition
	Version     string                 `json:"$schema,omitempty"`
	Definitions map[string]*Definition `json:"definitions,omitempty"`
}

type Definition struct {
	Ref                  string                 `json:"$ref,omitempty"`
	Items                *Definition            `json:"items,omitempty"`
	Required             []string               `json:"required,omitempty"`
	Properties           map[string]*Definition `json:"properties,omitempty"`
	PreferredOrder       []string               `json:"preferredOrder,omitempty"`
	AdditionalProperties interface{}            `json:"additionalProperties,omitempty"`
	Type                 string                 `json:"type,omitempty"`
	AnyOf                []*Definition          `json:"anyOf,omitempty"`
	Description          string                 `json:"description,omitempty"`
	HTMLDescription      string                 `json:"x-intellij-html-description,omitempty"`
	Default              interface{}            `json:"default,omitempty"`
	Examples             []string               `json:"examples,omitempty"`
}

func main() {
	if _, err := generateSchemas(".", false); err != nil {
		panic(err)
	}
}

func generateSchemas(root string, dryRun bool) (bool, error) {
	same := true

	for i, version := range schema.SchemaVersions {
		apiVersion := strings.TrimPrefix(version.APIVersion, "skaffold/")

		folder := apiVersion
		strict := false
		if i == len(schema.SchemaVersions)-1 {
			folder = "latest"
			strict = true
		}

		input := fmt.Sprintf("%s/pkg/skaffold/schema/%s/config.go", root, folder)
		output := fmt.Sprintf("%s/docs/content/en/schemas/%s.json", root, apiVersion)

		generator := schemaGenerator{
			strict: strict,
		}

		buf, err := generator.Apply(input)
		if err != nil {
			return false, errors.Wrapf(err, "unable to generate schema for version %s", version.APIVersion)
		}

		var current []byte
		if _, err := os.Stat(output); err == nil {
			var err error
			current, err = ioutil.ReadFile(output)
			if err != nil {
				return false, errors.Wrapf(err, "unable to read existing schema for version %s", version.APIVersion)
			}
		} else if !os.IsNotExist(err) {
			return false, errors.Wrapf(err, "unable to check that file exists %s", output)
		}

		if string(current) != string(buf) {
			same = false
		}

		if !dryRun {
			ioutil.WriteFile(output, buf, os.ModePerm)
		}
	}

	return same, nil
}

func yamlFieldName(field *ast.Field) string {
	tag := strings.Replace(field.Tag.Value, "`", "", -1)
	tags := reflect.StructTag(tag)
	yamlTag := tags.Get("yaml")

	return strings.Split(yamlTag, ",")[0]
}

func setTypeOrRef(def *Definition, typeName string) {
	switch typeName {
	case "string":
		def.Type = typeName
	case "bool":
		def.Type = "boolean"
	case "int", "int64":
		def.Type = "number"
	default:
		def.Ref = defPrefix + typeName
	}
}

func (g *schemaGenerator) newDefinition(name string, t ast.Expr, comment string) *Definition {
	def := &Definition{}

	switch tt := t.(type) {
	case *ast.Ident:
		typeName := tt.Name
		setTypeOrRef(def, typeName)

		switch typeName {
		case "string":
			// def.Default = "\"\""
		case "bool":
			def.Default = "false"
		case "int", "int64":
			// def.Default = "0"
		}

	case *ast.StarExpr:
		if ident, ok := tt.X.(*ast.Ident); ok {
			typeName := ident.Name
			setTypeOrRef(def, typeName)
		} else if _, ok := tt.X.(*ast.SelectorExpr); ok {
			def.Type = "object"
		}

	case *ast.ArrayType:
		def.Type = "array"
		def.Items = g.newDefinition("", tt.Elt, "")
		if def.Items.Ref == "" {
			def.Default = "[]"
		}

	case *ast.MapType:
		def.Type = "object"
		def.Default = "{}"
		def.AdditionalProperties = g.newDefinition("", tt.Value, "")

	case *ast.StructType:
		for _, field := range tt.Fields.List {
			yamlName := yamlFieldName(field)

			if strings.Contains(field.Tag.Value, "inline") {
				def.AnyOf = append(def.AnyOf, &Definition{
					Ref: defPrefix + field.Type.(*ast.Ident).Name,
				})
				continue
			}

			if yamlName == "" || yamlName == "-" {
				continue
			}

			if strings.Contains(field.Tag.Value, "required") {
				def.Required = append(def.Required, yamlName)
			}

			if def.Properties == nil {
				def.Properties = make(map[string]*Definition)
			}

			def.PreferredOrder = append(def.PreferredOrder, yamlName)
			def.Properties[yamlName] = g.newDefinition(field.Names[0].Name, field.Type, field.Doc.Text())
			def.AdditionalProperties = false
		}
	}

	description := strings.TrimSpace(strings.Replace(comment, "\n", " ", -1))

	// Extract default value
	if m := regexp.MustCompile("(.*)Defaults to `(.*)`").FindStringSubmatch(description); m != nil {
		description = strings.TrimSpace(m[1])
		def.Default = m[2]
	}

	// Extract example
	if m := regexp.MustCompile("(.*)For example: `(.*)`").FindStringSubmatch(description); m != nil {
		description = strings.TrimSpace(m[1])
		def.Examples = []string{m[2]}
	}

	// Remove type prefix
	description = strings.TrimPrefix(description, name+" is the ")
	description = strings.TrimPrefix(description, name+" is ")
	description = strings.TrimPrefix(description, name+" are the ")
	description = strings.TrimPrefix(description, name+" are ")
	description = strings.TrimPrefix(description, name+" lists ")
	description = strings.TrimPrefix(description, name+" ")

	def.Description = description
	if g.strict && name != "" && description == "" {
		panic(fmt.Sprintf("no description on field %s", name))
	}

	// Convert to HTML
	html := string(blackfriday.Run([]byte(description), blackfriday.WithNoExtensions()))
	html = strings.Replace(html, "
", "", -1)
	html = strings.Replace(html, "
", "", -1)
	def.HTMLDescription = html

	return def
}

func (g *schemaGenerator) Apply(inputPath string) ([]byte, error) {
	fset := token.NewFileSet()
	node, err := parser.ParseFile(fset, inputPath, nil, parser.ParseComments)
	if err != nil {
		return nil, err
	}

	var preferredOrder []string
	definitions := make(map[string]*Definition)

	for _, i := range node.Decls {
		declaration, ok := i.(*ast.GenDecl)
		if !ok {
			continue
		}

		for _, spec := range declaration.Specs {
			typeSpec, ok := spec.(*ast.TypeSpec)
			if !ok {
				continue
			}

			name := typeSpec.Name.Name
			preferredOrder = append(preferredOrder, name)
			definitions[name] = g.newDefinition(name, typeSpec.Type, declaration.Doc.Text())
		}
	}

	// Inline anyOfs
	for _, k := range preferredOrder {
		def := definitions[k]
		if len(def.AnyOf) == 0 {
			continue
		}

		var options []*Definition
		options = append(options, &Definition{
			Properties:           def.Properties,
			PreferredOrder:       def.PreferredOrder,
			AdditionalProperties: false,
		})

		for _, anyOf := range def.AnyOf {
			ref := strings.TrimPrefix(anyOf.Ref, defPrefix)
			referenced := definitions[ref]

			for _, key := range referenced.PreferredOrder {
				var preferredOrder []string
				choice := make(map[string]*Definition)

				if len(def.Properties) > 0 {
					for _, pkey := range def.PreferredOrder {
						preferredOrder = append(preferredOrder, pkey)
						choice[pkey] = def.Properties[pkey]
					}
				}

				preferredOrder = append(preferredOrder, key)
				choice[key] = referenced.Properties[key]

				options = append(options, &Definition{
					Properties:           choice,
					PreferredOrder:       preferredOrder,
					AdditionalProperties: false,
				})
			}
		}

		def.Properties = nil
		def.PreferredOrder = nil
		def.AdditionalProperties = nil
		def.AnyOf = options
	}

	schema := Schema{
		Version: version7,
		Definition: &Definition{
			Type: "object",
			AnyOf: []*Definition{{
				Ref: defPrefix + "SkaffoldPipeline",
			}},
		},
		Definitions: definitions,
	}

	return toJSON(schema)
}

// Make sure HTML description are not encoded
func toJSON(v interface{}) ([]byte, error) {
	var buf bytes.Buffer
	encoder := json.NewEncoder(&buf)
	encoder.SetEscapeHTML(false)
	encoder.SetIndent("", "  ")

	if err := encoder.Encode(v); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}
<commit_msg>Simplify code<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"reflect"
	"regexp"
	"strings"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema"
	"github.com/pkg/errors"
	blackfriday "gopkg.in/russross/blackfriday.v2"
)

const (
	version7  = "http://json-schema-org/draft-07/schema#"
	defPrefix = "#/definitions/"
)

var (
	regexpDefaults = regexp.MustCompile("(.*)Defaults to `(.*)`")
	regexpExample  = regexp.MustCompile("(.*)For example: `(.*)`")
	pTags          = regexp.MustCompile("(
)|(
)")
)

type schemaGenerator struct {
	strict bool
}

type Schema struct {
	*Definition
	Version     string                 `json:"$schema,omitempty"`
	Definitions map[string]*Definition `json:"definitions,omitempty"`
}

type Definition struct {
	Ref                  string                 `json:"$ref,omitempty"`
	Items                *Definition            `json:"items,omitempty"`
	Required             []string               `json:"required,omitempty"`
	Properties           map[string]*Definition `json:"properties,omitempty"`
	PreferredOrder       []string               `json:"preferredOrder,omitempty"`
	AdditionalProperties interface{}            `json:"additionalProperties,omitempty"`
	Type                 string                 `json:"type,omitempty"`
	AnyOf                []*Definition          `json:"anyOf,omitempty"`
	Description          string                 `json:"description,omitempty"`
	HTMLDescription      string                 `json:"x-intellij-html-description,omitempty"`
	Default              interface{}            `json:"default,omitempty"`
	Examples             []string               `json:"examples,omitempty"`
}

func main() {
	if _, err := generateSchemas(".", false); err != nil {
		panic(err)
	}
}

func generateSchemas(root string, dryRun bool) (bool, error) {
	same := true

	for i, version := range schema.SchemaVersions {
		apiVersion := strings.TrimPrefix(version.APIVersion, "skaffold/")

		folder := apiVersion
		strict := false
		if i == len(schema.SchemaVersions)-1 {
			folder = "latest"
			strict = true
		}

		input := fmt.Sprintf("%s/pkg/skaffold/schema/%s/config.go", root, folder)
		output := fmt.Sprintf("%s/docs/content/en/schemas/%s.json", root, apiVersion)

		generator := schemaGenerator{
			strict: strict,
		}

		buf, err := generator.Apply(input)
		if err != nil {
			return false, errors.Wrapf(err, "unable to generate schema for version %s", version.APIVersion)
		}

		var current []byte
		if _, err := os.Stat(output); err == nil {
			var err error
			current, err = ioutil.ReadFile(output)
			if err != nil {
				return false, errors.Wrapf(err, "unable to read existing schema for version %s", version.APIVersion)
			}
		} else if !os.IsNotExist(err) {
			return false, errors.Wrapf(err, "unable to check that file exists %s", output)
		}

		if string(current) != string(buf) {
			same = false
		}

		if !dryRun {
			ioutil.WriteFile(output, buf, os.ModePerm)
		}
	}

	return same, nil
}

func yamlFieldName(field *ast.Field) string {
	tag := strings.Replace(field.Tag.Value, "`", "", -1)
	tags := reflect.StructTag(tag)
	yamlTag := tags.Get("yaml")

	return strings.Split(yamlTag, ",")[0]
}

func setTypeOrRef(def *Definition, typeName string) {
	switch typeName {
	case "string":
		def.Type = typeName
	case "bool":
		def.Type = "boolean"
	case "int", "int64":
		def.Type = "number"
	default:
		def.Ref = defPrefix + typeName
	}
}

func (g *schemaGenerator) newDefinition(name string, t ast.Expr, comment string) *Definition {
	def := &Definition{}

	switch tt := t.(type) {
	case *ast.Ident:
		typeName := tt.Name
		setTypeOrRef(def, typeName)

		switch typeName {
		case "string":
			// def.Default = "\"\""
		case "bool":
			def.Default = "false"
		case "int", "int64":
			// def.Default = "0"
		}

	case *ast.StarExpr:
		if ident, ok := tt.X.(*ast.Ident); ok {
			typeName := ident.Name
			setTypeOrRef(def, typeName)
		} else if _, ok := tt.X.(*ast.SelectorExpr); ok {
			def.Type = "object"
		}

	case *ast.ArrayType:
		def.Type = "array"
		def.Items = g.newDefinition("", tt.Elt, "")
		if def.Items.Ref == "" {
			def.Default = "[]"
		}

	case *ast.MapType:
		def.Type = "object"
		def.Default = "{}"
		def.AdditionalProperties = g.newDefinition("", tt.Value, "")

	case *ast.StructType:
		for _, field := range tt.Fields.List {
			yamlName := yamlFieldName(field)

			if strings.Contains(field.Tag.Value, "inline") {
				def.AnyOf = append(def.AnyOf, &Definition{
					Ref: defPrefix + field.Type.(*ast.Ident).Name,
				})
				continue
			}

			if yamlName == "" || yamlName == "-" {
				continue
			}

			if strings.Contains(field.Tag.Value, "required") {
				def.Required = append(def.Required, yamlName)
			}

			if def.Properties == nil {
				def.Properties = make(map[string]*Definition)
			}

			def.PreferredOrder = append(def.PreferredOrder, yamlName)
			def.Properties[yamlName] = g.newDefinition(field.Names[0].Name, field.Type, field.Doc.Text())
			def.AdditionalProperties = false
		}
	}

	description := strings.TrimSpace(strings.Replace(comment, "\n", " ", -1))

	// Extract default value
	if m := regexpDefaults.FindStringSubmatch(description); m != nil {
		description = strings.TrimSpace(m[1])
		def.Default = m[2]
	}

	// Extract example
	if m := regexpExample.FindStringSubmatch(description); m != nil {
		description = strings.TrimSpace(m[1])
		def.Examples = []string{m[2]}
	}

	// Remove type prefix
	if m := regexp.MustCompile("^" + name + " ((is (the )?)|(are (the )?)|(lists ))?(.*)").FindStringSubmatch(description); m != nil {
		description = m[7]
	}

	if g.strict && name != "" && description == "" {
		panic(fmt.Sprintf("no description on field %s", name))
	}
	def.Description = description

	// Convert to HTML
	html := string(blackfriday.Run([]byte(description), blackfriday.WithNoExtensions()))
	def.HTMLDescription = pTags.ReplaceAllString(html, "")

	return def
}

func (g *schemaGenerator) Apply(inputPath string) ([]byte, error) {
	fset := token.NewFileSet()
	node, err := parser.ParseFile(fset, inputPath, nil, parser.ParseComments)
	if err != nil {
		return nil, err
	}

	var preferredOrder []string
	definitions := make(map[string]*Definition)

	for _, i := range node.Decls {
		declaration, ok := i.(*ast.GenDecl)
		if !ok {
			continue
		}

		for _, spec := range declaration.Specs {
			typeSpec, ok := spec.(*ast.TypeSpec)
			if !ok {
				continue
			}

			name := typeSpec.Name.Name
			preferredOrder = append(preferredOrder, name)
			definitions[name] = g.newDefinition(name, typeSpec.Type, declaration.Doc.Text())
		}
	}

	// Inline anyOfs
	for _, k := range preferredOrder {
		def := definitions[k]
		if len(def.AnyOf) == 0 {
			continue
		}

		var options []*Definition
		options = append(options, &Definition{
			Properties:           def.Properties,
			PreferredOrder:       def.PreferredOrder,
			AdditionalProperties: false,
		})

		for _, anyOf := range def.AnyOf {
			ref := strings.TrimPrefix(anyOf.Ref, defPrefix)
			referenced := definitions[ref]

			for _, key := range referenced.PreferredOrder {
				var preferredOrder []string
				choice := make(map[string]*Definition)

				if len(def.Properties) > 0 {
					for _, pkey := range def.PreferredOrder {
						preferredOrder = append(preferredOrder, pkey)
						choice[pkey] = def.Properties[pkey]
					}
				}

				preferredOrder = append(preferredOrder, key)
				choice[key] = referenced.Properties[key]

				options = append(options, &Definition{
					Properties:           choice,
					PreferredOrder:       preferredOrder,
					AdditionalProperties: false,
				})
			}
		}

		def.Properties = nil
		def.PreferredOrder = nil
		def.AdditionalProperties = nil
		def.AnyOf = options
	}

	schema := Schema{
		Version: version7,
		Definition: &Definition{
			Type: "object",
			AnyOf: []*Definition{{
				Ref: defPrefix + "SkaffoldPipeline",
			}},
		},
		Definitions: definitions,
	}

	return toJSON(schema)
}

// Make sure HTML description are not encoded
func toJSON(v interface{}) ([]byte, error) {
	var buf bytes.Buffer
	encoder := json.NewEncoder(&buf)
	encoder.SetEscapeHTML(false)
	encoder.SetIndent("", "  ")

	if err := encoder.Encode(v); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}
<|endoftext|>