<commit_before>package inet

//#define LONG_SIZE sizeof(long)
//#define INT_SIZE sizeof(int)
//#define SHORT_SIZE sizeof(short)
import "C"
import (
	"encoding/binary"
	"unsafe"
)

const (
	HOST_SHORT_SIZE = C.SHORT_SIZE
	HOST_INT_SIZE   = C.INT_SIZE
	HOST_LONG_SIZE  = C.LONG_SIZE
)

var (
	IsBigEndian    bool
	IsLittleEndian bool
	HostByteOrder  binary.ByteOrder
	// host to host
	Short    func([]byte) uint16
	PutShort func([]byte, uint16)
	Int      func([]byte) uint32
	PutInt   func([]byte, uint32)
	Long     func([]byte) uint64
	PutLong  func([]byte, uint64)

	// (nh)s
	NToHS    func([]byte) uint16
	PutNToHS func([]byte, uint16)
	HToNS    func([]byte) uint16
	HToNSFS  func(uint16) uint16
	PutHToNS func([]byte, uint16)

	// (nh)i
	NToHI    func([]byte) uint32
	PutNToHI func([]byte, uint32)
	HToNI    func([]byte) uint32
	HToNIFI  func(uint32) uint32
	PutHToNI func([]byte, uint32)

	// (nh)l
	NToHL    func([]byte) uint64
	PutNToHL func([]byte, uint64)
	HToNL    func([]byte) uint64
	HToNLFL  func(uint64) uint64
	PutHToNL func([]byte, uint64)
)

func init() {
	isBE := bigEndian()
	if isBE {
		IsBigEndian = true
		IsLittleEndian = false
		HostByteOrder = binary.BigEndian
	} else {
		IsBigEndian = false
		IsLittleEndian = true
		HostByteOrder = binary.LittleEndian
	}

	// network uses BigEndian
	NToHS = binary.BigEndian.Uint16
	PutHToNS = binary.BigEndian.PutUint16

	HToNS = HostByteOrder.Uint16
	PutNToHS = HostByteOrder.PutUint16
	Short = HostByteOrder.Uint16
	PutShort = HostByteOrder.PutUint16
	if isBE {
		HToNSFS = _beSFS
		HToNIFI = _beIFI
		HToNLFL = _beLFL
	} else {
		HToNSFS = _beToLeSFS
		HToNLFL = _beToLeLFL
		HToNIFI = _beToLeIFI
	}
	if HOST_INT_SIZE == 4 {
		Int = HostByteOrder.Uint32
		PutInt = HostByteOrder.PutUint32

		PutHToNI = binary.BigEndian.PutUint32
		HToNI = HostByteOrder.Uint32
		PutNToHI = HostByteOrder.PutUint32
		NToHI = binary.BigEndian.Uint32
	} else {
		Int = func(b []byte) uint32 {
			return uint32(HostByteOrder.Uint64(b))
		}
		PutInt = func(b []byte, v uint32) {
			HostByteOrder.PutUint64(b, uint64(v))
		}
		PutHToNI = func(b []byte, v uint32) {
			binary.BigEndian.PutUint64(b, uint64(v))
		}
		HToNI = func(b []byte) uint32 {
			return uint32(HostByteOrder.Uint64(b))
		}
		PutNToHI = func(b []byte, v uint32) {
			HostByteOrder.PutUint64(b, uint64(v))
		}
		NToHI = func(b []byte) uint32 {
			return uint32(binary.BigEndian.Uint64(b))
		}
	}
	if HOST_LONG_SIZE == 4 {
		Long = func(b []byte) uint64 {
			return uint64(HostByteOrder.Uint32(b))
		}
		PutLong = func(b []byte, v uint64) {
			HostByteOrder.PutUint32(b, uint32(v))
		}
		PutHToNL = func(b []byte, v uint64) {
			binary.BigEndian.PutUint32(b, uint32(v))
		}
		HToNL = func(b []byte) uint64 {
			return uint64(HostByteOrder.Uint32(b))
		}
		PutNToHL = func(b []byte, v uint64) {
			HostByteOrder.PutUint32(b, uint32(v))
		}
		NToHL = func(b []byte) uint64 {
			return uint64(binary.BigEndian.Uint32(b))
		}
	} else {
		Long = HostByteOrder.Uint64
		PutLong = HostByteOrder.PutUint64
		PutHToNL = binary.BigEndian.PutUint64
		HToNL = HostByteOrder.Uint64
		PutNToHL = HostByteOrder.PutUint64
		NToHL = binary.BigEndian.Uint64
	}

}

func bigEndian() (ret bool) {
	var i int = 0x1
	bs := (*[int(unsafe.Sizeof(0))]byte)(unsafe.Pointer(&i))
	if bs[0] == 0 {
		return true
	} else {
		return false
	}

}

func _beSFS(v uint16) uint16 {
	return v
}

func _beIFI(v uint32) uint32 {
	return v
}

func _beLFL(v uint64) uint64 {
	return v
}

func _beToLeSFS(v uint16) uint16 {
	return (v&0xff)<<8 |
		(v&0xff00)>>8
}

func _beToLeIFI(v uint32) uint32 {
	return (v&0xff)<<24 |
		(v&0xff00)<<8 |
		(v&0xff0000)>>8 |
		(v&0xff000000)>>24
}

func _beToLeLFL(v uint64) uint64 {
	return (v&0xff)<<56 |
		(v&0xff00)<<40 |
		(v&0xff0000)<<24 |
		(v&0xff000000)<<8 |
		(v&0xff00000000)>>8 |
		(v&0xff0000000000)>>24 |
		(v&0xff000000000000)>>40 |
		(v&0xff00000000000000)>>56
}
<commit_msg>arranging for sense<commit_after>package inet

//#define LONG_SIZE sizeof(long)
//#define INT_SIZE sizeof(int)
//#define SHORT_SIZE sizeof(short)
import "C"
import (
	"encoding/binary"
	"unsafe"
)

const (
	HOST_SHORT_SIZE = C.SHORT_SIZE
	HOST_INT_SIZE   = C.INT_SIZE
	HOST_LONG_SIZE  = C.LONG_SIZE
)

var (
	IsBigEndian    bool
	IsLittleEndian bool
	HostByteOrder  binary.ByteOrder
	// host to host
	Short    func([]byte) uint16
	PutShort func([]byte, uint16)
	Int      func([]byte) uint32
	PutInt   func([]byte, uint32)
	Long     func([]byte) uint64
	PutLong  func([]byte, uint64)

	// (nh)s
	NToHS    func([]byte) uint16
	PutNToHS func([]byte, uint16)
	HToNS    func([]byte) uint16
	HToNSFS  func(uint16) uint16
	PutHToNS func([]byte, uint16)

	// (nh)i
	NToHI    func([]byte) uint32
	PutNToHI func([]byte, uint32)
	HToNI    func([]byte) uint32
	HToNIFI  func(uint32) uint32
	PutHToNI func([]byte, uint32)

	// (nh)l
	NToHL    func([]byte) uint64
	PutNToHL func([]byte, uint64)
	HToNL    func([]byte) uint64
	HToNLFL  func(uint64) uint64
	PutHToNL func([]byte, uint64)
)

func init() {
	isBE := bigEndian()
	if isBE {
		IsBigEndian = true
		IsLittleEndian = false
		HostByteOrder = binary.BigEndian
	} else {
		IsBigEndian = false
		IsLittleEndian = true
		HostByteOrder = binary.LittleEndian
	}
	Short = HostByteOrder.Uint16
	PutShort = HostByteOrder.PutUint16

	PutHToNS = binary.BigEndian.PutUint16
	HToNS = HostByteOrder.Uint16
	PutNToHS = HostByteOrder.PutUint16
	NToHS = binary.BigEndian.Uint16
	if isBE {
		HToNSFS = _beSFS
		HToNIFI = _beIFI
		HToNLFL = _beLFL
	} else {
		HToNSFS = _beToLeSFS
		HToNLFL = _beToLeLFL
		HToNIFI = _beToLeIFI
	}
	if HOST_INT_SIZE == 4 {
		Int = HostByteOrder.Uint32
		PutInt = HostByteOrder.PutUint32

		PutHToNI = binary.BigEndian.PutUint32
		HToNI = HostByteOrder.Uint32
		PutNToHI = HostByteOrder.PutUint32
		NToHI = binary.BigEndian.Uint32
	} else {
		Int = func(b []byte) uint32 {
			return uint32(HostByteOrder.Uint64(b))
		}
		PutInt = func(b []byte, v uint32) {
			HostByteOrder.PutUint64(b, uint64(v))
		}
		PutHToNI = func(b []byte, v uint32) {
			binary.BigEndian.PutUint64(b, uint64(v))
		}
		HToNI = func(b []byte) uint32 {
			return uint32(HostByteOrder.Uint64(b))
		}
		PutNToHI = func(b []byte, v uint32) {
			HostByteOrder.PutUint64(b, uint64(v))
		}
		NToHI = func(b []byte) uint32 {
			return uint32(binary.BigEndian.Uint64(b))
		}
	}
	if HOST_LONG_SIZE == 4 {
		Long = func(b []byte) uint64 {
			return uint64(HostByteOrder.Uint32(b))
		}
		PutLong = func(b []byte, v uint64) {
			HostByteOrder.PutUint32(b, uint32(v))
		}
		PutHToNL = func(b []byte, v uint64) {
			binary.BigEndian.PutUint32(b, uint32(v))
		}
		HToNL = func(b []byte) uint64 {
			return uint64(HostByteOrder.Uint32(b))
		}
		PutNToHL = func(b []byte, v uint64) {
			HostByteOrder.PutUint32(b, uint32(v))
		}
		NToHL = func(b []byte) uint64 {
			return uint64(binary.BigEndian.Uint32(b))
		}
	} else {
		Long = HostByteOrder.Uint64
		PutLong = HostByteOrder.PutUint64
		PutHToNL = binary.BigEndian.PutUint64
		HToNL = HostByteOrder.Uint64
		PutNToHL = HostByteOrder.PutUint64
		NToHL = binary.BigEndian.Uint64
	}

}

func bigEndian() (ret bool) {
	var i int = 0x1
	bs := (*[int(unsafe.Sizeof(0))]byte)(unsafe.Pointer(&i))
	if bs[0] == 0 {
		return true
	} else {
		return false
	}

}

func _beSFS(v uint16) uint16 {
	return v
}

func _beIFI(v uint32) uint32 {
	return v
}

func _beLFL(v uint64) uint64 {
	return v
}

func _beToLeSFS(v uint16) uint16 {
	return (v&0xff)<<8 |
		(v&0xff00)>>8
}

func _beToLeIFI(v uint32) uint32 {
	return (v&0xff)<<24 |
		(v&0xff00)<<8 |
		(v&0xff0000)>>8 |
		(v&0xff000000)>>24
}

func _beToLeLFL(v uint64) uint64 {
	return (v&0xff)<<56 |
		(v&0xff00)<<40 |
		(v&0xff0000)<<24 |
		(v&0xff000000)<<8 |
		(v&0xff00000000)>>8 |
		(v&0xff0000000000)>>24 |
		(v&0xff000000000000)>>40 |
		(v&0xff00000000000000)>>56
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"net/http"
	"sync"

	"github.com/Dataman-Cloud/swan/types"
	log "github.com/Sirupsen/logrus"
)

func (r *Router) purge(w http.ResponseWriter, req *http.Request) {
	apps, err := r.db.ListApps()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	tokenBucket := make(chan struct{}, 50) // TODO(nmg): delete step, make it configurable

	for _, app := range apps {
		go func(app *types.Application) {
			var (
				hasError = false
				wg       sync.WaitGroup
			)

			wg.Add(len(app.Tasks))
			for _, task := range app.Tasks {
				tokenBucket <- struct{}{}

				go func(task *types.Task, appId string) {
					defer func() {
						wg.Done()
						<-tokenBucket
					}()

					if err := r.driver.KillTask(task.ID, task.AgentId); err != nil {
						log.Errorf("Kill task %s got error: %v", task.ID, err)

						hasError = true

						task.OpStatus = fmt.Sprintf("kill task error: %v", err)
						if err = r.db.UpdateTask(appId, task); err != nil {
							log.Errorf("update task %s got error: %v", task.Name, err)
						}

						return
					}

					if err := r.db.DeleteTask(task.ID); err != nil {
						log.Errorf("Delete task %s got error: %v", task.ID, err)

						hasError = true

						task.OpStatus = fmt.Sprintf("delete task error: %v", err)
						if err = r.db.UpdateTask(appId, task); err != nil {
							log.Errorf("update task %s got error: %v", task.Name, err)
						}

						return
					}

				}(task, app.ID)
			}

			wg.Wait()

			close(tokenBucket)

			if hasError {
				log.Errorf("Delete some tasks of app %s got error.", app.ID)
				return
			}

			if err := r.db.DeleteApp(app.ID); err != nil {
				log.Error("Delete app %s got error: %v", app.ID, err)
				return
			}

		}(app)
	}

	writeJSON(w, http.StatusNoContent, "")
}
<commit_msg>fixed purge for close token-bucket after all tasks removed<commit_after>package api

import (
	"fmt"
	"net/http"
	"sync"

	"github.com/Dataman-Cloud/swan/types"
	log "github.com/Sirupsen/logrus"
)

func (r *Router) purge(w http.ResponseWriter, req *http.Request) {
	apps, err := r.db.ListApps()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	tokenBucket := make(chan struct{}, 50) // TODO(nmg): delete step, make it configurable

	go func() {
		var all sync.WaitGroup

		for _, app := range apps {
			var (
				hasError = false
				wg       sync.WaitGroup
			)

			wg.Add(len(app.Tasks))
			for _, task := range app.Tasks {
				tokenBucket <- struct{}{}

				go func(task *types.Task, appId string) {
					all.Add(1)

					defer func() {
						wg.Done()
						all.Done()
						<-tokenBucket
					}()

					if err := r.driver.KillTask(task.ID, task.AgentId); err != nil {
						log.Errorf("Kill task %s got error: %v", task.ID, err)

						hasError = true

						task.OpStatus = fmt.Sprintf("kill task error: %v", err)
						if err = r.db.UpdateTask(appId, task); err != nil {
							log.Errorf("update task %s got error: %v", task.Name, err)
						}

						return
					}

					if err := r.db.DeleteTask(task.ID); err != nil {
						log.Errorf("Delete task %s got error: %v", task.ID, err)

						hasError = true

						task.OpStatus = fmt.Sprintf("delete task error: %v", err)
						if err = r.db.UpdateTask(appId, task); err != nil {
							log.Errorf("update task %s got error: %v", task.Name, err)
						}

						return
					}

				}(task, app.ID)
			}

			wg.Wait()

			if hasError {
				log.Errorf("Delete some tasks of app %s got error.", app.ID)
				return
			}

			if err := r.db.DeleteApp(app.ID); err != nil {
				log.Error("Delete app %s got error: %v", app.ID, err)
				return
			}

		}

		all.Wait()

		close(tokenBucket)
	}()

	writeJSON(w, http.StatusNoContent, "")
}
<|endoftext|>


<commit_before><commit_msg>add archetype to agent config<commit_after><|endoftext|>


<commit_before>package main

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"github.com/Boostport/kubernetes-vault/common"
	"github.com/Sirupsen/logrus"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/vault/api"
	"github.com/pkg/errors"
	"io/ioutil"
	"math/big"
	"net"
	"net/http"
	"os"
	"time"
)

type authToken struct {
	ClientToken   string `json:"clientToken"`
	Accessor      string `json:"accessor"`
	LeaseDuration int    `json:"leaseDuration"`
	Renewable     bool   `json:"renewable"`
}

func main() {

	logger := logrus.New()
	logger.Level = logrus.DebugLevel

	roleId := os.Getenv("VAULT_ROLE_ID")

	if roleId == "" {
		logger.Fatal("The VAULT_ROLE_ID environment variable must be set.")
	}

	timeoutStr := os.Getenv("TIMEOUT")

	var (
		timeout time.Duration
		err     error
	)

	if timeoutStr == "" {
		timeout = 5 * time.Minute
	} else {

		timeout, err = time.ParseDuration(timeoutStr)

		if err != nil {
			logger.Fatalf("Invalid timeout (%s): %s", timeoutStr, err)
		}
	}

	tokenPath := os.Getenv("TOKEN_PATH")

	if tokenPath == "" {
		tokenPath = "/var/run/secrets/boostport.com/vault-token"
	}

	ip, err := common.ExternalIP()

	if err != nil {
		logger.Fatalf("Error looking up external ip for container: %s", err)
	}

	certificate, err := generateCertificate(ip, timeout)

	if err != nil {
		logger.Fatalf("Could not generate certificate: %s", err)
	}

	result := make(chan common.WrappedSecretId)

	go startHTTPServer(certificate, logger, result)

	for {
		select {
		case wrappedSecretId := <-result:

			authToken, err := processWrappedSecretId(wrappedSecretId, roleId)

			if err != nil {
				logger.Debugf("Could not get auth token: %s", err)
				os.Exit(1)
			}

			b, err := json.Marshal(authToken)

			if err != nil {
				logger.Debugf("Could not marshal auth token to JSON: %s", err)
				os.Exit(1)
			}

			err = ioutil.WriteFile(tokenPath, b, 0444)

			if err != nil {
				logger.Debugf("Could not write auth token to path (%s): %s", tokenPath, err)
				os.Exit(1)
			}

			logger.Debug("Successfully created the vault token. Exiting.")
			os.Exit(0)

		case <-time.After(timeout):
			logger.Info("Failed to create vault auth token because we timed out before receiving the secret_id. Exiting.")
			os.Exit(1)
		}
	}

}

func startHTTPServer(certificate tls.Certificate, logger *logrus.Logger, wrappedSecretId chan<- common.WrappedSecretId) {
	tlsConfig := &tls.Config{
		Certificates: []tls.Certificate{certificate},
	}

	tlsConfig.BuildNameToCertificate()

	mux := http.NewServeMux()

	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {

		if req.URL.Path != "/" {
			w.WriteHeader(http.StatusNotFound)
			return
		}

		if req.Method == "POST" {

			decoder := json.NewDecoder(req.Body)

			var wrappedSecret common.WrappedSecretId

			err := decoder.Decode(&wrappedSecret)

			if err != nil {
				logger.Debugf("Error decoding wrapped secret: %s", err)
				w.WriteHeader(http.StatusBadRequest)
				w.Write([]byte("Could not decode wrapped secret."))
				return
			}

			wrappedSecretId <- wrappedSecret
			w.WriteHeader(http.StatusOK)
			return

		} else {
			logger.Debugf("The / endpoint only support POSTs")
			w.WriteHeader(http.StatusMethodNotAllowed)
			w.Write([]byte("The / endpoint only support POSTs"))
		}

	})

	server := &http.Server{
		Handler:   mux,
		Addr:      fmt.Sprintf(":%d", common.InitContainerPort),
		TLSConfig: tlsConfig,
	}

	server.ListenAndServeTLS("", "")
}

func processWrappedSecretId(wrappedSecretId common.WrappedSecretId, roleId string) (authToken, error) {

	if err := wrappedSecretId.Validate(); err != nil {
		return authToken{}, errors.Wrap(err, "could not validate wrapped secret_id")
	}

	client, err := api.NewClient(&api.Config{Address: wrappedSecretId.VaultAddr, HttpClient: cleanhttp.DefaultPooledClient()})

	client.SetToken(wrappedSecretId.Token)

	if err != nil {
		return authToken{}, errors.Wrap(err, "could not create vault client")
	}

	secret, err := client.Logical().Unwrap("")

	if err != nil {
		return authToken{}, errors.Wrap(err, "error unwrapping secret_id")
	}

	secretId, ok := secret.Data["secret_id"]

	if !ok {
		return authToken{}, errors.New("Wrapped response is missing secret_id")
	}

	token, err := client.Logical().Write("auth/approle/login", map[string]interface{}{
		"role_id":   roleId,
		"secret_id": secretId,
	})

	if err != nil {
		return authToken{}, errors.Wrap(err, "could not log in using role_id and secret_id")
	}

	secretAuth := token.Auth

	return authToken{
		ClientToken:   secretAuth.ClientToken,
		Accessor:      secretAuth.Accessor,
		LeaseDuration: secretAuth.LeaseDuration,
		Renewable:     secretAuth.Renewable,
	}, nil
}

func generateCertificate(ip net.IP, duration time.Duration) (tls.Certificate, error) {

	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)

	if err != nil {
		return tls.Certificate{}, errors.Wrap(err, "could not generate ECDSA key.")
	}

	notBefore := time.Now()

	notAfter := notBefore.Add(duration)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)

	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)

	if err != nil {
		return tls.Certificate{}, errors.Wrap(err, "failed to generate serial number")
	}

	template := x509.Certificate{
		SerialNumber:          serialNumber,
		NotBefore:             notBefore,
		NotAfter:              notAfter,
		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
		IPAddresses:           []net.IP{ip},
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)

	if err != nil {
		return tls.Certificate{}, errors.Wrap(err, "could not generate certificate")
	}

	certPem := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})

	b, err := x509.MarshalECPrivateKey(priv)

	if err != nil {
		return tls.Certificate{}, errors.Wrap(err, "could not marshal ECDSA private key")
	}

	keyPem := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: b})

	cert, err := tls.X509KeyPair(certPem, keyPem)

	if err != nil {
		return tls.Certificate{}, errors.Wrap(err, "could not parse PEM certificate and private key")
	}

	return cert, nil
}
<commit_msg>Add the vault server address to token information written by the init container.<commit_after>package main

import (
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"github.com/Boostport/kubernetes-vault/common"
	"github.com/Sirupsen/logrus"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/hashicorp/vault/api"
	"github.com/pkg/errors"
	"io/ioutil"
	"math/big"
	"net"
	"net/http"
	"os"
	"time"
)

type authToken struct {
	ClientToken   string `json:"clientToken"`
	Accessor      string `json:"accessor"`
	LeaseDuration int    `json:"leaseDuration"`
	Renewable     bool   `json:"renewable"`
	VaultAddr     string `json:"vaultAddr"`
}

func main() {

	logger := logrus.New()
	logger.Level = logrus.DebugLevel

	roleId := os.Getenv("VAULT_ROLE_ID")

	if roleId == "" {
		logger.Fatal("The VAULT_ROLE_ID environment variable must be set.")
	}

	timeoutStr := os.Getenv("TIMEOUT")

	var (
		timeout time.Duration
		err     error
	)

	if timeoutStr == "" {
		timeout = 5 * time.Minute
	} else {

		timeout, err = time.ParseDuration(timeoutStr)

		if err != nil {
			logger.Fatalf("Invalid timeout (%s): %s", timeoutStr, err)
		}
	}

	tokenPath := os.Getenv("TOKEN_PATH")

	if tokenPath == "" {
		tokenPath = "/var/run/secrets/boostport.com/vault-token"
	}

	ip, err := common.ExternalIP()

	if err != nil {
		logger.Fatalf("Error looking up external ip for container: %s", err)
	}

	certificate, err := generateCertificate(ip, timeout)

	if err != nil {
		logger.Fatalf("Could not generate certificate: %s", err)
	}

	result := make(chan common.WrappedSecretId)

	go startHTTPServer(certificate, logger, result)

	for {
		select {
		case wrappedSecretId := <-result:

			authToken, err := processWrappedSecretId(wrappedSecretId, roleId)

			if err != nil {
				logger.Debugf("Could not get auth token: %s", err)
				os.Exit(1)
			}

			b, err := json.Marshal(authToken)

			if err != nil {
				logger.Debugf("Could not marshal auth token to JSON: %s", err)
				os.Exit(1)
			}

			err = ioutil.WriteFile(tokenPath, b, 0444)

			if err != nil {
				logger.Debugf("Could not write auth token to path (%s): %s", tokenPath, err)
				os.Exit(1)
			}

			logger.Debug("Successfully created the vault token. Exiting.")
			os.Exit(0)

		case <-time.After(timeout):
			logger.Info("Failed to create vault auth token because we timed out before receiving the secret_id. Exiting.")
			os.Exit(1)
		}
	}

}

func startHTTPServer(certificate tls.Certificate, logger *logrus.Logger, wrappedSecretId chan<- common.WrappedSecretId) {
	tlsConfig := &tls.Config{
		Certificates: []tls.Certificate{certificate},
	}

	tlsConfig.BuildNameToCertificate()

	mux := http.NewServeMux()

	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {

		if req.URL.Path != "/" {
			w.WriteHeader(http.StatusNotFound)
			return
		}

		if req.Method == "POST" {

			decoder := json.NewDecoder(req.Body)

			var wrappedSecret common.WrappedSecretId

			err := decoder.Decode(&wrappedSecret)

			if err != nil {
				logger.Debugf("Error decoding wrapped secret: %s", err)
				w.WriteHeader(http.StatusBadRequest)
				w.Write([]byte("Could not decode wrapped secret."))
				return
			}

			wrappedSecretId <- wrappedSecret
			w.WriteHeader(http.StatusOK)
			return

		} else {
			logger.Debugf("The / endpoint only support POSTs")
			w.WriteHeader(http.StatusMethodNotAllowed)
			w.Write([]byte("The / endpoint only support POSTs"))
		}

	})

	server := &http.Server{
		Handler:   mux,
		Addr:      fmt.Sprintf(":%d", common.InitContainerPort),
		TLSConfig: tlsConfig,
	}

	server.ListenAndServeTLS("", "")
}

func processWrappedSecretId(wrappedSecretId common.WrappedSecretId, roleId string) (authToken, error) {

	if err := wrappedSecretId.Validate(); err != nil {
		return authToken{}, errors.Wrap(err, "could not validate wrapped secret_id")
	}

	client, err := api.NewClient(&api.Config{Address: wrappedSecretId.VaultAddr, HttpClient: cleanhttp.DefaultPooledClient()})

	client.SetToken(wrappedSecretId.Token)

	if err != nil {
		return authToken{}, errors.Wrap(err, "could not create vault client")
	}

	secret, err := client.Logical().Unwrap("")

	if err != nil {
		return authToken{}, errors.Wrap(err, "error unwrapping secret_id")
	}

	secretId, ok := secret.Data["secret_id"]

	if !ok {
		return authToken{}, errors.New("Wrapped response is missing secret_id")
	}

	token, err := client.Logical().Write("auth/approle/login", map[string]interface{}{
		"role_id":   roleId,
		"secret_id": secretId,
	})

	if err != nil {
		return authToken{}, errors.Wrap(err, "could not log in using role_id and secret_id")
	}

	secretAuth := token.Auth

	return authToken{
		ClientToken:   secretAuth.ClientToken,
		Accessor:      secretAuth.Accessor,
		LeaseDuration: secretAuth.LeaseDuration,
		Renewable:     secretAuth.Renewable,
		VaultAddr:     wrappedSecretId.VaultAddr,
	}, nil
}

func generateCertificate(ip net.IP, duration time.Duration) (tls.Certificate, error) {

	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)

	if err != nil {
		return tls.Certificate{}, errors.Wrap(err, "could not generate ECDSA key.")
	}

	notBefore := time.Now()

	notAfter := notBefore.Add(duration)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)

	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)

	if err != nil {
		return tls.Certificate{}, errors.Wrap(err, "failed to generate serial number")
	}

	template := x509.Certificate{
		SerialNumber:          serialNumber,
		NotBefore:             notBefore,
		NotAfter:              notAfter,
		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
		IPAddresses:           []net.IP{ip},
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)

	if err != nil {
		return tls.Certificate{}, errors.Wrap(err, "could not generate certificate")
	}

	certPem := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})

	b, err := x509.MarshalECPrivateKey(priv)

	if err != nil {
		return tls.Certificate{}, errors.Wrap(err, "could not marshal ECDSA private key")
	}

	keyPem := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: b})

	cert, err := tls.X509KeyPair(certPem, keyPem)

	if err != nil {
		return tls.Certificate{}, errors.Wrap(err, "could not parse PEM certificate and private key")
	}

	return cert, nil
}
<|endoftext|>


<commit_before>
package core

import ("fmt"
	"os"
	"strconv"
	"github.com/jackyb/go-sdl2/sdl"
	"github.com/op/go-nanomsg"
	"github.com/fire/go-ogre3d")

func InitCore() {
	sdl.Init(sdl.INIT_EVERYTHING)
	window := sdl.CreateWindow("es_core::SDL",
		sdl.WINDOWPOS_UNDEFINED,
		sdl.WINDOWPOS_UNDEFINED,
		800,
		600,
		sdl.WINDOW_SHOWN)
	if window == nil {
		panic(fmt.Sprintf("sdl.CreateWindow failed: %s\n", sdl.GetError()))
	}
	defer sdl.Quit()
	var info sdl.SysWMInfo
	if !window.GetWMInfo(&info) {
		panic(fmt.Sprintf("window.GetWMInfo failed.\n"))
	}
	// Parse and print info's version
	// Parse and print info's SYSWM_TYPE
	root := ogre.NewRoot("", "", "ogre.log")
	defer root.Destroy()
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	root.LoadPlugin(wd  + "/../frameworks/RenderSystem_GL.framework")
	renderers := root.GetAvailableRenderers()
	if renderers.RenderSystemListSize() != 1 {
		panic(fmt.Sprintf("Failed to initalize RendererRenderSystem_GL"))
	}
	root.SetRenderSystem(renderers.RenderSystemListGet(0))
	root.Initialise(false, "es_core::ogre")
	params := ogre.CreateNameValuePairList()
	params.AddPair("macAPI", "cocoa")
	cocoaInfo := info.GetCocoaInfo()
	windowString := strconv.FormatUint(uint64(*(*uint32)(cocoaInfo.Window)), 10)
	params.AddPair("parentWindowHandle", windowString)

	renderWindow := root.CreateRenderWindow("es_core::ogre", 800, 600, false, params)
	renderWindow.SetVisible(true)

	gameSocket, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.BUS)
        if err != nil {
                panic(err)
        }
        _, err = gameSocket.Bind("tcp://127.0.0.1:60206")
        if err != nil {
                panic(err)
        }

	renderSocket, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.BUS)
	if err != nil {
                panic(err)
        }
        _, err = renderSocket.Bind("tcp://127.0.0.1:60207")
        if err != nil {
                panic(err)
        }

	inputPub, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.PUB)
        if err != nil {
                panic(err)
        }
        _, err = inputPub.Bind("tcp://127.0.0.1:60208")
        if err != nil {
                panic(err)
        }

	inputPull, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.PULL)
        if err != nil {
                panic(err)
        }
        _, err = inputPull.Bind("tcp://127.0.0.1:60209")
        if err != nil {
                panic(err)
        }
	go gameThread()
	var renderThreadParams RenderThreadParams
	renderThreadParams.root = root
	renderThreadParams.window = window
	renderThreadParams.ogreWindow = renderWindow

	go renderThread(renderThreadParams)

}
<commit_msg>Code input state.<commit_after>
package core

import ("fmt"
	"os"
	"strconv"
	"github.com/jackyb/go-sdl2/sdl"
	"github.com/op/go-nanomsg"
	"github.com/fire/go-ogre3d")

type InputState struct {
	yawSens float32
	pitchSens float32
	orientationFactor float32 // +1/-1 easy switch between look around and manipulate something
	yaw float32 // degrees, modulo [-180,180] range
	pitch float32 // degrees, clamped [-90,90] range
	roll float32
	// orientation ogre.Quaternion // current orientation
}

func InitCore() {
	sdl.Init(sdl.INIT_EVERYTHING)
	window := sdl.CreateWindow("es_core::SDL",
		sdl.WINDOWPOS_UNDEFINED,
		sdl.WINDOWPOS_UNDEFINED,
		800,
		600,
		sdl.WINDOW_SHOWN)
	if window == nil {
		panic(fmt.Sprintf("sdl.CreateWindow failed: %s\n", sdl.GetError()))
	}
	defer sdl.Quit()
	var info sdl.SysWMInfo
	if !window.GetWMInfo(&info) {
		panic(fmt.Sprintf("window.GetWMInfo failed.\n"))
	}
	// Parse and print info's version
	// Parse and print info's SYSWM_TYPE
	root := ogre.NewRoot("", "", "ogre.log")
	defer root.Destroy()
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	root.LoadPlugin(wd  + "/../frameworks/RenderSystem_GL.framework")
	renderers := root.GetAvailableRenderers()
	if renderers.RenderSystemListSize() != 1 {
		panic(fmt.Sprintf("Failed to initalize RendererRenderSystem_GL"))
	}
	root.SetRenderSystem(renderers.RenderSystemListGet(0))
	root.Initialise(false, "es_core::ogre")
	params := ogre.CreateNameValuePairList()
	params.AddPair("macAPI", "cocoa")
	cocoaInfo := info.GetCocoaInfo()
	windowString := strconv.FormatUint(uint64(*(*uint32)(cocoaInfo.Window)), 10)
	params.AddPair("parentWindowHandle", windowString)

	renderWindow := root.CreateRenderWindow("es_core::ogre", 800, 600, false, params)
	renderWindow.SetVisible(true)

	nnGameSocket, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.BUS)
        if err != nil {
                panic(err)
        }
        _, err = nnGameSocket.Bind("tcp://127.0.0.1:60206")
        if err != nil {
                panic(err)
        }

	nnRenderSocket, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.BUS)
	if err != nil {
                panic(err)
        }
        _, err = nnRenderSocket.Bind("tcp://127.0.0.1:60207")
        if err != nil {
                panic(err)
        }

	nnInputPub, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.PUB)
        if err != nil {
                panic(err)
        }
        _, err = nnInputPub.Bind("tcp://127.0.0.1:60208")
        if err != nil {
                panic(err)
        }

	nnInputPull, err := nanomsg.NewSocket(nanomsg.AF_SP, nanomsg.PULL)
        if err != nil {
                panic(err)
        }
        _, err = nnInputPull.Bind("tcp://127.0.0.1:60209")
        if err != nil {
                panic(err)
        }
	go gameThread()
	var renderThreadParams RenderThreadParams
	renderThreadParams.root = root
	renderThreadParams.window = window
	renderThreadParams.ogreWindow = renderWindow

	go renderThread(renderThreadParams)

	window.SetGrab(true)
	sdl.SetRelativeMouseMode(true)

	shutdownRequested := false
	var is InputState
	is.yawSens = 0.1
	is.yaw = 0.0
	is.pitchSens = 0.1
	is.pitch = 0.0
	is.roll = 0.0
	is.orientationFactor = -1.0 // Look around config

	for !shutdownRequested {
		var inputPull string
		string, err := nnInputPull.RecvString()
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"path"
	"strings"

	"github.com/coreos/fleet/client"
	"github.com/coreos/fleet/job"
	"github.com/coreos/fleet/log"
	"github.com/coreos/fleet/pkg"
	"github.com/coreos/fleet/schema"
	"github.com/coreos/fleet/unit"

	gsunit "github.com/coreos/fleet/Godeps/_workspace/src/github.com/coreos/go-systemd/unit"
)

func wireUpUnitsResource(mux *http.ServeMux, prefix string, tokenLimit int, cAPI client.API) {
	base := path.Join(prefix, "units")
	ur := unitsResource{cAPI, base, uint16(tokenLimit)}
	mux.Handle(base, &ur)
	mux.Handle(base+"/", &ur)
}

type unitsResource struct {
	cAPI       client.API
	basePath   string
	tokenLimit uint16
}

func (ur *unitsResource) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if isCollectionPath(ur.basePath, req.URL.Path) {
		switch req.Method {
		case "GET":
			ur.list(rw, req)
		default:
			sendError(rw, http.StatusMethodNotAllowed, errors.New("only GET supported against this resource"))
		}
	} else if item, ok := isItemPath(ur.basePath, req.URL.Path); ok {
		switch req.Method {
		case "GET":
			ur.get(rw, req, item)
		case "DELETE":
			ur.destroy(rw, req, item)
		case "PUT":
			ur.set(rw, req, item)
		default:
			sendError(rw, http.StatusMethodNotAllowed, errors.New("only GET, PUT and DELETE supported against this resource"))
		}
	} else {
		sendError(rw, http.StatusNotFound, nil)
	}
}

func (ur *unitsResource) set(rw http.ResponseWriter, req *http.Request, item string) {
	if err := validateContentType(req); err != nil {
		sendError(rw, http.StatusUnsupportedMediaType, err)
		return
	}

	var su schema.Unit
	dec := json.NewDecoder(req.Body)
	err := dec.Decode(&su)
	if err != nil {
		sendError(rw, http.StatusBadRequest, fmt.Errorf("unable to decode body: %v", err))
		return
	}
	if su.Name == "" {
		su.Name = item
	}
	if item != su.Name {
		sendError(rw, http.StatusBadRequest, fmt.Errorf("name in URL %q differs from unit name in request body %q", item, su.Name))
		return
	}
	if err := ValidateName(su.Name); err != nil {
		sendError(rw, http.StatusBadRequest, err)
		return
	}

	eu, err := ur.cAPI.Unit(su.Name)
	if err != nil {
		log.Errorf("Failed fetching Unit(%s) from Registry: %v", su.Name, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	if eu == nil {
		if len(su.Options) == 0 {
			err := errors.New("unit does not exist and options field empty")
			sendError(rw, http.StatusConflict, err)
		} else if err := ValidateOptions(su.Options); err != nil {
			sendError(rw, http.StatusBadRequest, err)
		} else {
			ur.create(rw, su.Name, &su)
		}
		return
	}

	if len(su.DesiredState) == 0 {
		err := errors.New("must provide DesiredState to update existing unit")
		sendError(rw, http.StatusConflict, err)
		return
	}

	un := unit.NewUnitNameInfo(su.Name)
	if un.IsTemplate() && job.JobState(su.DesiredState) != job.JobStateInactive {
		err := fmt.Errorf("cannot activate template %q", su.Name)
		sendError(rw, http.StatusBadRequest, err)
		return
	}

	ur.update(rw, su.Name, su.DesiredState)
}

const (
	// These constants taken from systemd
	unitNameMax    = 256
	digits         = "0123456789"
	lowercase      = "abcdefghijklmnopqrstuvwxyz"
	uppercase      = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
	alphanumerical = digits + lowercase + uppercase
	validChars     = alphanumerical + `:-_.\@`
)

var validUnitTypes = pkg.NewUnsafeSet(
	"service",
	"socket",
	"busname",
	"target",
	"snapshot",
	"device",
	"mount",
	"automount",
	"swap",
	"timer",
	"path",
	"slice",
	"scope",
)

// ValidateName ensures that a given unit name is valid; if not, an error is
// returned describing the first issue encountered.
// systemd reference: `unit_name_is_valid` in `unit-name.c`
func ValidateName(name string) error {
	length := len(name)
	if length == 0 {
		return errors.New("unit name cannot be empty")
	}
	if length > unitNameMax {
		return fmt.Errorf("unit name exceeds maximum length (%d)", unitNameMax)
	}
	dot := strings.LastIndex(name, ".")
	if dot == -1 {
		return errors.New(`unit name must contain "."`)
	}
	if dot == length-1 {
		return errors.New(`unit name cannot end in "."`)
	}
	if suffix := name[dot+1:]; !validUnitTypes.Contains(suffix) {
		return fmt.Errorf("invalid unit type: %q", suffix)
	}
	for _, char := range name[:dot] {
		if !strings.ContainsRune(validChars, char) {
			return fmt.Errorf("invalid character %q in unit name", char)
		}
	}
	if strings.HasPrefix(name, "@") {
		return errors.New(`unit name cannot start in "@"`)
	}
	return nil
}

// ValidateOptions ensures that a set of UnitOptions is valid; if not, an error
// is returned detailing the issue encountered.  If there are several problems
// with a set of options, only the first is returned.
func ValidateOptions(opts []*schema.UnitOption) error {
	uf := schema.MapSchemaUnitOptionsToUnitFile(opts)
	// Sanity check using go-systemd's deserializer, which will do things
	// like check for excessive line lengths
	_, err := gsunit.Deserialize(gsunit.Serialize(uf.Options))
	if err != nil {
		return err
	}

	j := &job.Job{
		Unit: *uf,
	}
	conflicts := pkg.NewUnsafeSet(j.Conflicts()...)
	peers := pkg.NewUnsafeSet(j.Peers()...)
	for _, peer := range peers.Values() {
		for _, conflict := range conflicts.Values() {
			matched, _ := path.Match(conflict, peer)
			if matched {
				return fmt.Errorf("unresolvable requirements: peer %q matches conflict %q", peer, conflict)
			}
		}
	}
	hasPeers := peers.Length() != 0
	hasConflicts := conflicts.Length() != 0
	_, hasReqTarget := j.RequiredTarget()
	u := &job.Unit{
		Unit: *uf,
	}
	isGlobal := u.IsGlobal()

	switch {
	case hasReqTarget && hasPeers:
		return errors.New("MachineID cannot be used with Peers")
	case hasReqTarget && hasConflicts:
		return errors.New("MachineID cannot be used with Conflicts")
	case hasReqTarget && isGlobal:
		return errors.New("MachineID cannot be used with Global")
	case isGlobal && hasPeers:
		return errors.New("Global cannot be used with Peers")
	case isGlobal && hasConflicts:
		return errors.New("Global cannot be used with Conflicts")
	}

	return nil
}

func (ur *unitsResource) create(rw http.ResponseWriter, name string, u *schema.Unit) {
	if err := ur.cAPI.CreateUnit(u); err != nil {
		log.Errorf("Failed creating Unit(%s) in Registry: %v", u.Name, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	rw.WriteHeader(http.StatusCreated)
}

func (ur *unitsResource) update(rw http.ResponseWriter, item, ds string) {
	if err := ur.cAPI.SetUnitTargetState(item, ds); err != nil {
		log.Errorf("Failed setting target state of Unit(%s): %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	rw.WriteHeader(http.StatusNoContent)
}

func (ur *unitsResource) destroy(rw http.ResponseWriter, req *http.Request, item string) {
	u, err := ur.cAPI.Unit(item)
	if err != nil {
		log.Errorf("Failed fetching Unit(%s): %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	if u == nil {
		sendError(rw, http.StatusNotFound, errors.New("unit does not exist"))
		return
	}

	err = ur.cAPI.DestroyUnit(item)
	if err != nil {
		log.Errorf("Failed destroying Unit(%s): %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	rw.WriteHeader(http.StatusNoContent)
}

func (ur *unitsResource) get(rw http.ResponseWriter, req *http.Request, item string) {
	u, err := ur.cAPI.Unit(item)
	if err != nil {
		log.Errorf("Failed fetching Unit(%s) from Registry: %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	if u == nil {
		sendError(rw, http.StatusNotFound, errors.New("unit does not exist"))
		return
	}

	sendResponse(rw, http.StatusOK, *u)
}

func (ur *unitsResource) list(rw http.ResponseWriter, req *http.Request) {
	token, err := findNextPageToken(req.URL, ur.tokenLimit)
	if err != nil {
		sendError(rw, http.StatusBadRequest, err)
		return
	}

	if token == nil {
		def := DefaultPageToken(ur.tokenLimit)
		token = &def
	}

	page, err := getUnitPage(ur.cAPI, *token)
	if err != nil {
		log.Errorf("Failed fetching page of Units: %v", err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	sendResponse(rw, http.StatusOK, page)
}

func getUnitPage(cAPI client.API, tok PageToken) (*schema.UnitPage, error) {
	units, err := cAPI.Units()
	if err != nil {
		return nil, err
	}

	items, next := extractUnitPageData(units, tok)
	page := schema.UnitPage{
		Units: items,
	}

	if next != nil {
		page.NextPageToken = next.Encode()
	}

	return &page, nil
}

func extractUnitPageData(all []*schema.Unit, tok PageToken) (items []*schema.Unit, next *PageToken) {
	total := len(all)

	startIndex := int((tok.Page - 1) * tok.Limit)
	stopIndex := int(tok.Page * tok.Limit)

	if startIndex < total {
		if stopIndex > total {
			stopIndex = total
		} else {
			n := tok.Next()
			next = &n
		}

		items = all[startIndex:stopIndex]
	}

	return
}
<commit_msg>units: when creating units check if this is a new version<commit_after>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"path"
	"strings"

	"github.com/coreos/fleet/client"
	"github.com/coreos/fleet/job"
	"github.com/coreos/fleet/log"
	"github.com/coreos/fleet/pkg"
	"github.com/coreos/fleet/schema"
	"github.com/coreos/fleet/unit"

	gsunit "github.com/coreos/fleet/Godeps/_workspace/src/github.com/coreos/go-systemd/unit"
)

func wireUpUnitsResource(mux *http.ServeMux, prefix string, tokenLimit int, cAPI client.API) {
	base := path.Join(prefix, "units")
	ur := unitsResource{cAPI, base, uint16(tokenLimit)}
	mux.Handle(base, &ur)
	mux.Handle(base+"/", &ur)
}

type unitsResource struct {
	cAPI       client.API
	basePath   string
	tokenLimit uint16
}

func (ur *unitsResource) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	if isCollectionPath(ur.basePath, req.URL.Path) {
		switch req.Method {
		case "GET":
			ur.list(rw, req)
		default:
			sendError(rw, http.StatusMethodNotAllowed, errors.New("only GET supported against this resource"))
		}
	} else if item, ok := isItemPath(ur.basePath, req.URL.Path); ok {
		switch req.Method {
		case "GET":
			ur.get(rw, req, item)
		case "DELETE":
			ur.destroy(rw, req, item)
		case "PUT":
			ur.set(rw, req, item)
		default:
			sendError(rw, http.StatusMethodNotAllowed, errors.New("only GET, PUT and DELETE supported against this resource"))
		}
	} else {
		sendError(rw, http.StatusNotFound, nil)
	}
}

func (ur *unitsResource) set(rw http.ResponseWriter, req *http.Request, item string) {
	if err := validateContentType(req); err != nil {
		sendError(rw, http.StatusUnsupportedMediaType, err)
		return
	}

	var su schema.Unit
	dec := json.NewDecoder(req.Body)
	err := dec.Decode(&su)
	if err != nil {
		sendError(rw, http.StatusBadRequest, fmt.Errorf("unable to decode body: %v", err))
		return
	}
	if su.Name == "" {
		su.Name = item
	}
	if item != su.Name {
		sendError(rw, http.StatusBadRequest, fmt.Errorf("name in URL %q differs from unit name in request body %q", item, su.Name))
		return
	}
	if err := ValidateName(su.Name); err != nil {
		sendError(rw, http.StatusBadRequest, err)
		return
	}

	eu, err := ur.cAPI.Unit(su.Name)
	if err != nil {
		log.Errorf("Failed fetching Unit(%s) from Registry: %v", su.Name, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	newUnit := false
	if eu == nil {
		if len(su.Options) == 0 {
			err := errors.New("unit does not exist and options field empty")
			sendError(rw, http.StatusConflict, err)
			return
		} else if err := ValidateOptions(su.Options); err != nil {
			sendError(rw, http.StatusBadRequest, err)
			return
		} else {
			// New valid unit
			newUnit = true
		}
	} else if eu.Name == su.Name && len(su.Options) > 0 {
		// There is already a unit with the same name that
		// was submitted before. Check their hashes, if they do
		// not match then this is probably a new version which
		// needs its own new unit entry.
		// In the other case if su.Options == 0 then probably we
		// don't want to update the Unit options nor its content
		// but only set the target job state of the
		// corresponding unit, in this case just ignore.
		a := schema.MapSchemaUnitOptionsToUnitFile(su.Options)
		b := schema.MapSchemaUnitOptionsToUnitFile(eu.Options)
		newUnit = !unit.MatchUnitFiles(a, b)
	}

	if newUnit {
		ur.create(rw, su.Name, &su)
		return
	}

	if len(su.DesiredState) == 0 {
		err := errors.New("must provide DesiredState to update existing unit")
		sendError(rw, http.StatusConflict, err)
		return
	}

	un := unit.NewUnitNameInfo(su.Name)
	if un.IsTemplate() && job.JobState(su.DesiredState) != job.JobStateInactive {
		err := fmt.Errorf("cannot activate template %q", su.Name)
		sendError(rw, http.StatusBadRequest, err)
		return
	}

	ur.update(rw, su.Name, su.DesiredState)
}

const (
	// These constants taken from systemd
	unitNameMax    = 256
	digits         = "0123456789"
	lowercase      = "abcdefghijklmnopqrstuvwxyz"
	uppercase      = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
	alphanumerical = digits + lowercase + uppercase
	validChars     = alphanumerical + `:-_.\@`
)

var validUnitTypes = pkg.NewUnsafeSet(
	"service",
	"socket",
	"busname",
	"target",
	"snapshot",
	"device",
	"mount",
	"automount",
	"swap",
	"timer",
	"path",
	"slice",
	"scope",
)

// ValidateName ensures that a given unit name is valid; if not, an error is
// returned describing the first issue encountered.
// systemd reference: `unit_name_is_valid` in `unit-name.c`
func ValidateName(name string) error {
	length := len(name)
	if length == 0 {
		return errors.New("unit name cannot be empty")
	}
	if length > unitNameMax {
		return fmt.Errorf("unit name exceeds maximum length (%d)", unitNameMax)
	}
	dot := strings.LastIndex(name, ".")
	if dot == -1 {
		return errors.New(`unit name must contain "."`)
	}
	if dot == length-1 {
		return errors.New(`unit name cannot end in "."`)
	}
	if suffix := name[dot+1:]; !validUnitTypes.Contains(suffix) {
		return fmt.Errorf("invalid unit type: %q", suffix)
	}
	for _, char := range name[:dot] {
		if !strings.ContainsRune(validChars, char) {
			return fmt.Errorf("invalid character %q in unit name", char)
		}
	}
	if strings.HasPrefix(name, "@") {
		return errors.New(`unit name cannot start in "@"`)
	}
	return nil
}

// ValidateOptions ensures that a set of UnitOptions is valid; if not, an error
// is returned detailing the issue encountered.  If there are several problems
// with a set of options, only the first is returned.
func ValidateOptions(opts []*schema.UnitOption) error {
	uf := schema.MapSchemaUnitOptionsToUnitFile(opts)
	// Sanity check using go-systemd's deserializer, which will do things
	// like check for excessive line lengths
	_, err := gsunit.Deserialize(gsunit.Serialize(uf.Options))
	if err != nil {
		return err
	}

	j := &job.Job{
		Unit: *uf,
	}
	conflicts := pkg.NewUnsafeSet(j.Conflicts()...)
	peers := pkg.NewUnsafeSet(j.Peers()...)
	for _, peer := range peers.Values() {
		for _, conflict := range conflicts.Values() {
			matched, _ := path.Match(conflict, peer)
			if matched {
				return fmt.Errorf("unresolvable requirements: peer %q matches conflict %q", peer, conflict)
			}
		}
	}
	hasPeers := peers.Length() != 0
	hasConflicts := conflicts.Length() != 0
	_, hasReqTarget := j.RequiredTarget()
	u := &job.Unit{
		Unit: *uf,
	}
	isGlobal := u.IsGlobal()

	switch {
	case hasReqTarget && hasPeers:
		return errors.New("MachineID cannot be used with Peers")
	case hasReqTarget && hasConflicts:
		return errors.New("MachineID cannot be used with Conflicts")
	case hasReqTarget && isGlobal:
		return errors.New("MachineID cannot be used with Global")
	case isGlobal && hasPeers:
		return errors.New("Global cannot be used with Peers")
	case isGlobal && hasConflicts:
		return errors.New("Global cannot be used with Conflicts")
	}

	return nil
}

func (ur *unitsResource) create(rw http.ResponseWriter, name string, u *schema.Unit) {
	if err := ur.cAPI.CreateUnit(u); err != nil {
		log.Errorf("Failed creating Unit(%s) in Registry: %v", u.Name, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	rw.WriteHeader(http.StatusCreated)
}

func (ur *unitsResource) update(rw http.ResponseWriter, item, ds string) {
	if err := ur.cAPI.SetUnitTargetState(item, ds); err != nil {
		log.Errorf("Failed setting target state of Unit(%s): %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	rw.WriteHeader(http.StatusNoContent)
}

func (ur *unitsResource) destroy(rw http.ResponseWriter, req *http.Request, item string) {
	u, err := ur.cAPI.Unit(item)
	if err != nil {
		log.Errorf("Failed fetching Unit(%s): %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	if u == nil {
		sendError(rw, http.StatusNotFound, errors.New("unit does not exist"))
		return
	}

	err = ur.cAPI.DestroyUnit(item)
	if err != nil {
		log.Errorf("Failed destroying Unit(%s): %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	rw.WriteHeader(http.StatusNoContent)
}

func (ur *unitsResource) get(rw http.ResponseWriter, req *http.Request, item string) {
	u, err := ur.cAPI.Unit(item)
	if err != nil {
		log.Errorf("Failed fetching Unit(%s) from Registry: %v", item, err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	if u == nil {
		sendError(rw, http.StatusNotFound, errors.New("unit does not exist"))
		return
	}

	sendResponse(rw, http.StatusOK, *u)
}

func (ur *unitsResource) list(rw http.ResponseWriter, req *http.Request) {
	token, err := findNextPageToken(req.URL, ur.tokenLimit)
	if err != nil {
		sendError(rw, http.StatusBadRequest, err)
		return
	}

	if token == nil {
		def := DefaultPageToken(ur.tokenLimit)
		token = &def
	}

	page, err := getUnitPage(ur.cAPI, *token)
	if err != nil {
		log.Errorf("Failed fetching page of Units: %v", err)
		sendError(rw, http.StatusInternalServerError, nil)
		return
	}

	sendResponse(rw, http.StatusOK, page)
}

func getUnitPage(cAPI client.API, tok PageToken) (*schema.UnitPage, error) {
	units, err := cAPI.Units()
	if err != nil {
		return nil, err
	}

	items, next := extractUnitPageData(units, tok)
	page := schema.UnitPage{
		Units: items,
	}

	if next != nil {
		page.NextPageToken = next.Encode()
	}

	return &page, nil
}

func extractUnitPageData(all []*schema.Unit, tok PageToken) (items []*schema.Unit, next *PageToken) {
	total := len(all)

	startIndex := int((tok.Page - 1) * tok.Limit)
	stopIndex := int(tok.Page * tok.Limit)

	if startIndex < total {
		if stopIndex > total {
			stopIndex = total
		} else {
			n := tok.Next()
			next = &n
		}

		items = all[startIndex:stopIndex]
	}

	return
}
<|endoftext|>


<commit_before>package quic

import (
	"context"
	"io"
	"net"
	"time"

	"github.com/lucas-clemente/quic-go/internal/handshake"
	"github.com/lucas-clemente/quic-go/internal/protocol"
)

// The StreamID is the ID of a QUIC stream.
type StreamID = protocol.StreamID

// A VersionNumber is a QUIC version number.
type VersionNumber = protocol.VersionNumber

// A Cookie can be used to verify the ownership of the client address.
type Cookie struct {
	RemoteAddr string
	SentTime   time.Time
}

// ConnectionState records basic details about the QUIC connection.
type ConnectionState = handshake.ConnectionState

// An ErrorCode is an application-defined error code.
type ErrorCode = protocol.ApplicationErrorCode

// Stream is the interface implemented by QUIC streams
type Stream interface {
	// StreamID returns the stream ID.
	StreamID() StreamID
	// Read reads data from the stream.
	// Read can be made to time out and return a net.Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetReadDeadline.
	// If the stream was canceled by the peer, the error implements the StreamError
	// interface, and Canceled() == true.
	io.Reader
	// Write writes data to the stream.
	// Write can be made to time out and return a net.Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetWriteDeadline.
	// If the stream was canceled by the peer, the error implements the StreamError
	// interface, and Canceled() == true.
	io.Writer
	// Close closes the write-direction of the stream.
	// Future calls to Write are not permitted after calling Close.
	// It must not be called concurrently with Write.
	// It must not be called after calling CancelWrite.
	io.Closer
	// CancelWrite aborts sending on this stream.
	// Data already written, but not yet delivered to the peer is not guaranteed to be delivered reliably.
	// Write will unblock immediately, and future calls to Write will fail.
	// When called multiple times or after closing the stream it is a no-op.
	CancelWrite(ErrorCode)
	// CancelRead aborts receiving on this stream.
	// It will ask the peer to stop transmitting stream data.
	// Read will unblock immediately, and future Read calls will fail.
	// When called multiple times or after reading the io.EOF it is a no-op.
	CancelRead(ErrorCode)
	// The context is canceled as soon as the write-side of the stream is closed.
	// This happens when Close() is called, or when the stream is reset (either locally or remotely).
	// Warning: This API should not be considered stable and might change soon.
	Context() context.Context
	// SetReadDeadline sets the deadline for future Read calls and
	// any currently-blocked Read call.
	// A zero value for t means Read will not time out.
	SetReadDeadline(t time.Time) error
	// SetWriteDeadline sets the deadline for future Write calls
	// and any currently-blocked Write call.
	// Even if write times out, it may return n > 0, indicating that
	// some of the data was successfully written.
	// A zero value for t means Write will not time out.
	SetWriteDeadline(t time.Time) error
	// SetDeadline sets the read and write deadlines associated
	// with the connection. It is equivalent to calling both
	// SetReadDeadline and SetWriteDeadline.
	SetDeadline(t time.Time) error
}

// A ReceiveStream is a unidirectional Receive Stream.
type ReceiveStream interface {
	// see Stream.StreamID
	StreamID() StreamID
	// see Stream.Read
	io.Reader
	// see Stream.CancelRead
	CancelRead(ErrorCode)
	// see Stream.SetReadDealine
	SetReadDeadline(t time.Time) error
}

// A SendStream is a unidirectional Send Stream.
type SendStream interface {
	// see Stream.StreamID
	StreamID() StreamID
	// see Stream.Write
	io.Writer
	// see Stream.Close
	io.Closer
	// see Stream.CancelWrite
	CancelWrite(ErrorCode)
	// see Stream.Context
	Context() context.Context
	// see Stream.SetWriteDeadline
	SetWriteDeadline(t time.Time) error
}

// StreamError is returned by Read and Write when the peer cancels the stream.
type StreamError interface {
	error
	Canceled() bool
	ErrorCode() ErrorCode
}

// A Session is a QUIC connection between two peers.
type Session interface {
	// AcceptStream returns the next stream opened by the peer, blocking until one is available.
	AcceptStream() (Stream, error)
	// AcceptUniStream returns the next unidirectional stream opened by the peer, blocking until one is available.
	AcceptUniStream() (ReceiveStream, error)
	// OpenStream opens a new bidirectional QUIC stream.
	// There is no signaling to the peer about new streams:
	// The peer can only accept the stream after data has been sent on the stream.
	// If the error is non-nil, it satisfies the net.Error interface.
	// When reaching the peer's stream limit, err.Temporary() will be true.
	OpenStream() (Stream, error)
	// OpenStreamSync opens a new bidirectional QUIC stream.
	// It blocks until a new stream can be opened.
	// If the error is non-nil, it satisfies the net.Error interface.
	OpenStreamSync() (Stream, error)
	// OpenUniStream opens a new outgoing unidirectional QUIC stream.
	// If the error is non-nil, it satisfies the net.Error interface.
	// When reaching the peer's stream limit, Temporary() will be true.
	OpenUniStream() (SendStream, error)
	// OpenUniStreamSync opens a new outgoing unidirectional QUIC stream.
	// It blocks until a new stream can be opened.
	// If the error is non-nil, it satisfies the net.Error interface.
	OpenUniStreamSync() (SendStream, error)
	// LocalAddr returns the local address.
	LocalAddr() net.Addr
	// RemoteAddr returns the address of the peer.
	RemoteAddr() net.Addr
	// Close the connection.
	io.Closer
	// Close the connection with an error.
	// The error must not be nil.
	CloseWithError(ErrorCode, error) error
	// The context is cancelled when the session is closed.
	// Warning: This API should not be considered stable and might change soon.
	Context() context.Context
	// ConnectionState returns basic details about the QUIC connection.
	// Warning: This API should not be considered stable and might change soon.
	ConnectionState() ConnectionState
}

// Config contains all configuration data needed for a QUIC server or client.
type Config struct {
	// The QUIC versions that can be negotiated.
	// If not set, it uses all versions available.
	// Warning: This API should not be considered stable and will change soon.
	Versions []VersionNumber
	// The length of the connection ID in bytes.
	// It can be 0, or any value between 4 and 18.
	// If not set, the interpretation depends on where the Config is used:
	// If used for dialing an address, a 0 byte connection ID will be used.
	// If used for a server, or dialing on a packet conn, a 4 byte connection ID will be used.
	// When dialing on a packet conn, the ConnectionIDLength value must be the same for every Dial call.
	ConnectionIDLength int
	// HandshakeTimeout is the maximum duration that the cryptographic handshake may take.
	// If the timeout is exceeded, the connection is closed.
	// If this value is zero, the timeout is set to 10 seconds.
	HandshakeTimeout time.Duration
	// IdleTimeout is the maximum duration that may pass without any incoming network activity.
	// This value only applies after the handshake has completed.
	// If the timeout is exceeded, the connection is closed.
	// If this value is zero, the timeout is set to 30 seconds.
	IdleTimeout time.Duration
	// AcceptCookie determines if a Cookie is accepted.
	// It is called with cookie = nil if the client didn't send an Cookie.
	// If not set, it verifies that the address matches, and that the Cookie was issued within the last 24 hours.
	// This option is only valid for the server.
	AcceptCookie func(clientAddr net.Addr, cookie *Cookie) bool
	// MaxReceiveStreamFlowControlWindow is the maximum stream-level flow control window for receiving data.
	// If this value is zero, it will default to 1 MB for the server and 6 MB for the client.
	MaxReceiveStreamFlowControlWindow uint64
	// MaxReceiveConnectionFlowControlWindow is the connection-level flow control window for receiving data.
	// If this value is zero, it will default to 1.5 MB for the server and 15 MB for the client.
	MaxReceiveConnectionFlowControlWindow uint64
	// MaxIncomingStreams is the maximum number of concurrent bidirectional streams that a peer is allowed to open.
	// If not set, it will default to 100.
	// If set to a negative value, it doesn't allow any bidirectional streams.
	MaxIncomingStreams int
	// MaxIncomingUniStreams is the maximum number of concurrent unidirectional streams that a peer is allowed to open.
	// If not set, it will default to 100.
	// If set to a negative value, it doesn't allow any unidirectional streams.
	MaxIncomingUniStreams int
	// KeepAlive defines whether this peer will periodically send PING frames to keep the connection alive.
	KeepAlive bool
}

// A Listener for incoming QUIC connections
type Listener interface {
	// Close the server, sending CONNECTION_CLOSE frames to each peer.
	Close() error
	// Addr returns the local network addr that the server is listening on.
	Addr() net.Addr
	// Accept returns new sessions. It should be called in a loop.
	Accept() (Session, error)
}
<commit_msg>add documentation about timeout errors<commit_after>package quic

import (
	"context"
	"io"
	"net"
	"time"

	"github.com/lucas-clemente/quic-go/internal/handshake"
	"github.com/lucas-clemente/quic-go/internal/protocol"
)

// The StreamID is the ID of a QUIC stream.
type StreamID = protocol.StreamID

// A VersionNumber is a QUIC version number.
type VersionNumber = protocol.VersionNumber

// A Cookie can be used to verify the ownership of the client address.
type Cookie struct {
	RemoteAddr string
	SentTime   time.Time
}

// ConnectionState records basic details about the QUIC connection.
type ConnectionState = handshake.ConnectionState

// An ErrorCode is an application-defined error code.
type ErrorCode = protocol.ApplicationErrorCode

// Stream is the interface implemented by QUIC streams
type Stream interface {
	// StreamID returns the stream ID.
	StreamID() StreamID
	// Read reads data from the stream.
	// Read can be made to time out and return a net.Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetReadDeadline.
	// If the stream was canceled by the peer, the error implements the StreamError
	// interface, and Canceled() == true.
	// If the session was closed due to a timeout, the error satisfies
	// the net.Error interface, and Timeout() will be true.
	io.Reader
	// Write writes data to the stream.
	// Write can be made to time out and return a net.Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetWriteDeadline.
	// If the stream was canceled by the peer, the error implements the StreamError
	// interface, and Canceled() == true.
	// If the session was closed due to a timeout, the error satisfies
	// the net.Error interface, and Timeout() will be true.
	io.Writer
	// Close closes the write-direction of the stream.
	// Future calls to Write are not permitted after calling Close.
	// It must not be called concurrently with Write.
	// It must not be called after calling CancelWrite.
	io.Closer
	// CancelWrite aborts sending on this stream.
	// Data already written, but not yet delivered to the peer is not guaranteed to be delivered reliably.
	// Write will unblock immediately, and future calls to Write will fail.
	// When called multiple times or after closing the stream it is a no-op.
	CancelWrite(ErrorCode)
	// CancelRead aborts receiving on this stream.
	// It will ask the peer to stop transmitting stream data.
	// Read will unblock immediately, and future Read calls will fail.
	// When called multiple times or after reading the io.EOF it is a no-op.
	CancelRead(ErrorCode)
	// The context is canceled as soon as the write-side of the stream is closed.
	// This happens when Close() is called, or when the stream is reset (either locally or remotely).
	// Warning: This API should not be considered stable and might change soon.
	Context() context.Context
	// SetReadDeadline sets the deadline for future Read calls and
	// any currently-blocked Read call.
	// A zero value for t means Read will not time out.
	SetReadDeadline(t time.Time) error
	// SetWriteDeadline sets the deadline for future Write calls
	// and any currently-blocked Write call.
	// Even if write times out, it may return n > 0, indicating that
	// some of the data was successfully written.
	// A zero value for t means Write will not time out.
	SetWriteDeadline(t time.Time) error
	// SetDeadline sets the read and write deadlines associated
	// with the connection. It is equivalent to calling both
	// SetReadDeadline and SetWriteDeadline.
	SetDeadline(t time.Time) error
}

// A ReceiveStream is a unidirectional Receive Stream.
type ReceiveStream interface {
	// see Stream.StreamID
	StreamID() StreamID
	// see Stream.Read
	io.Reader
	// see Stream.CancelRead
	CancelRead(ErrorCode)
	// see Stream.SetReadDealine
	SetReadDeadline(t time.Time) error
}

// A SendStream is a unidirectional Send Stream.
type SendStream interface {
	// see Stream.StreamID
	StreamID() StreamID
	// see Stream.Write
	io.Writer
	// see Stream.Close
	io.Closer
	// see Stream.CancelWrite
	CancelWrite(ErrorCode)
	// see Stream.Context
	Context() context.Context
	// see Stream.SetWriteDeadline
	SetWriteDeadline(t time.Time) error
}

// StreamError is returned by Read and Write when the peer cancels the stream.
type StreamError interface {
	error
	Canceled() bool
	ErrorCode() ErrorCode
}

// A Session is a QUIC connection between two peers.
type Session interface {
	// AcceptStream returns the next stream opened by the peer, blocking until one is available.
	// If the session was closed due to a timeout, the error satisfies
	// the net.Error interface, and Timeout() will be true.
	AcceptStream() (Stream, error)
	// AcceptUniStream returns the next unidirectional stream opened by the peer, blocking until one is available.
	// If the session was closed due to a timeout, the error satisfies
	// the net.Error interface, and Timeout() will be true.
	AcceptUniStream() (ReceiveStream, error)
	// OpenStream opens a new bidirectional QUIC stream.
	// There is no signaling to the peer about new streams:
	// The peer can only accept the stream after data has been sent on the stream.
	// If the error is non-nil, it satisfies the net.Error interface.
	// When reaching the peer's stream limit, err.Temporary() will be true.
	// If the session was closed due to a timeout, Timeout() will be true.
	OpenStream() (Stream, error)
	// OpenStreamSync opens a new bidirectional QUIC stream.
	// It blocks until a new stream can be opened.
	// If the error is non-nil, it satisfies the net.Error interface.
	// If the session was closed due to a timeout, Timeout() will be true.
	OpenStreamSync() (Stream, error)
	// OpenUniStream opens a new outgoing unidirectional QUIC stream.
	// If the error is non-nil, it satisfies the net.Error interface.
	// When reaching the peer's stream limit, Temporary() will be true.
	// If the session was closed due to a timeout, Timeout() will be true.
	OpenUniStream() (SendStream, error)
	// OpenUniStreamSync opens a new outgoing unidirectional QUIC stream.
	// It blocks until a new stream can be opened.
	// If the error is non-nil, it satisfies the net.Error interface.
	// If the session was closed due to a timeout, Timeout() will be true.
	OpenUniStreamSync() (SendStream, error)
	// LocalAddr returns the local address.
	LocalAddr() net.Addr
	// RemoteAddr returns the address of the peer.
	RemoteAddr() net.Addr
	// Close the connection.
	io.Closer
	// Close the connection with an error.
	// The error must not be nil.
	CloseWithError(ErrorCode, error) error
	// The context is cancelled when the session is closed.
	// Warning: This API should not be considered stable and might change soon.
	Context() context.Context
	// ConnectionState returns basic details about the QUIC connection.
	// Warning: This API should not be considered stable and might change soon.
	ConnectionState() ConnectionState
}

// Config contains all configuration data needed for a QUIC server or client.
type Config struct {
	// The QUIC versions that can be negotiated.
	// If not set, it uses all versions available.
	// Warning: This API should not be considered stable and will change soon.
	Versions []VersionNumber
	// The length of the connection ID in bytes.
	// It can be 0, or any value between 4 and 18.
	// If not set, the interpretation depends on where the Config is used:
	// If used for dialing an address, a 0 byte connection ID will be used.
	// If used for a server, or dialing on a packet conn, a 4 byte connection ID will be used.
	// When dialing on a packet conn, the ConnectionIDLength value must be the same for every Dial call.
	ConnectionIDLength int
	// HandshakeTimeout is the maximum duration that the cryptographic handshake may take.
	// If the timeout is exceeded, the connection is closed.
	// If this value is zero, the timeout is set to 10 seconds.
	HandshakeTimeout time.Duration
	// IdleTimeout is the maximum duration that may pass without any incoming network activity.
	// This value only applies after the handshake has completed.
	// If the timeout is exceeded, the connection is closed.
	// If this value is zero, the timeout is set to 30 seconds.
	IdleTimeout time.Duration
	// AcceptCookie determines if a Cookie is accepted.
	// It is called with cookie = nil if the client didn't send an Cookie.
	// If not set, it verifies that the address matches, and that the Cookie was issued within the last 24 hours.
	// This option is only valid for the server.
	AcceptCookie func(clientAddr net.Addr, cookie *Cookie) bool
	// MaxReceiveStreamFlowControlWindow is the maximum stream-level flow control window for receiving data.
	// If this value is zero, it will default to 1 MB for the server and 6 MB for the client.
	MaxReceiveStreamFlowControlWindow uint64
	// MaxReceiveConnectionFlowControlWindow is the connection-level flow control window for receiving data.
	// If this value is zero, it will default to 1.5 MB for the server and 15 MB for the client.
	MaxReceiveConnectionFlowControlWindow uint64
	// MaxIncomingStreams is the maximum number of concurrent bidirectional streams that a peer is allowed to open.
	// If not set, it will default to 100.
	// If set to a negative value, it doesn't allow any bidirectional streams.
	MaxIncomingStreams int
	// MaxIncomingUniStreams is the maximum number of concurrent unidirectional streams that a peer is allowed to open.
	// If not set, it will default to 100.
	// If set to a negative value, it doesn't allow any unidirectional streams.
	MaxIncomingUniStreams int
	// KeepAlive defines whether this peer will periodically send PING frames to keep the connection alive.
	KeepAlive bool
}

// A Listener for incoming QUIC connections
type Listener interface {
	// Close the server, sending CONNECTION_CLOSE frames to each peer.
	Close() error
	// Addr returns the local network addr that the server is listening on.
	Addr() net.Addr
	// Accept returns new sessions. It should be called in a loop.
	Accept() (Session, error)
}
<|endoftext|>


<commit_before>// Package v1 provides lstags v1 API to be used both by the application
// itself and by external projects
package v1

import (
	"bufio"
	"fmt"
	log "github.com/sirupsen/logrus"
	"io"
	"runtime"
	"strings"
	"time"

	"github.com/ivanilves/lstags/api/v1/collection"
	dockerclient "github.com/ivanilves/lstags/docker/client"
	dockerconfig "github.com/ivanilves/lstags/docker/config"
	"github.com/ivanilves/lstags/repository"
	"github.com/ivanilves/lstags/tag"
	"github.com/ivanilves/lstags/tag/local"
	"github.com/ivanilves/lstags/tag/remote"
	"github.com/ivanilves/lstags/util/wait"
)

// Config holds API instance configuration
type Config struct {
	// DockerJSONConfigFile is a path to Docker JSON config file
	DockerJSONConfigFile string
	// ConcurrentRequests defines how much requests to registry we could run in parallel
	ConcurrentRequests int
	// WaitBetween defines how much we will wait between batches of requests (incl. pull and push)
	WaitBetween time.Duration
	// TraceRequests sets if we will print out registry HTTP request traces
	TraceRequests bool
	// RetryRequests defines how much retries we will do to the failed HTTP request
	RetryRequests int
	// RetryDelay defines how much we will wait between failed HTTP request and retry
	RetryDelay time.Duration
	// InsecureRegistryEx is a regex string to match insecure (non-HTTPS) registries
	InsecureRegistryEx string
	// VerboseLogging sets if we will print debug log messages
	VerboseLogging bool
}

// PushConfig holds push-specific configuration (where to push and with which prefix)
type PushConfig struct {
	// Prefix is prepended to the repository path while pushing to the registry
	Prefix string
	// Registry is an address of the Docker registry in which we push our images
	Registry string
	// UpdateChanged tells us if we will re-push (update/overwrite) images having same tag, but different digest
	UpdateChanged bool
}

// API represents configured application API instance,
// the main abstraction you are supposed to work with
type API struct {
	config       Config
	dockerClient *dockerclient.DockerClient
}

// fn gives the name of the calling function (e.g. enriches log.Debugf() output)
// + optionally attaches free form string labels (mainly to identify goroutines)
func fn(labels ...string) string {
	function, _, _, _ := runtime.Caller(1)

	longname := runtime.FuncForPC(function).Name()

	nameparts := strings.Split(longname, ".")
	shortname := nameparts[len(nameparts)-1]

	if labels == nil {
		return fmt.Sprintf("[%s()]", shortname)
	}

	return fmt.Sprintf("[%s():%s]", shortname, strings.Join(labels, ":"))
}

func getBatchedSlices(batchSize int, unbatched ...string) [][]string {
	batchedSlices := make([][]string, 0)

	index := 0

	for range unbatched {
		batchedSlice := make([]string, 0)

		for c := 0; c < batchSize; c++ {
			batchedSlice = append(batchedSlice, unbatched[index])

			index++

			if index == len(unbatched) {
				break
			}
		}

		batchedSlices = append(batchedSlices, batchedSlice)

		if index == len(unbatched) {
			break
		}
	}

	return batchedSlices
}

type rtags struct {
	ref  string
	tags []*tag.Tag
}

// CollectTags collects information on tags present in remote registry and [local] Docker daemon,
// makes required comparisons between them and spits organized info back as collection.Collection
func (api *API) CollectTags(refs ...string) (*collection.Collection, error) {
	if len(refs) == 0 {
		return nil, fmt.Errorf("no image references passed")
	}

	_, err := repository.ParseRefs(refs)
	if err != nil {
		return nil, err
	}

	tagc := make(chan rtags, len(refs))
	tags := make(map[string][]*tag.Tag)

	batchedSlicesOfRefs := getBatchedSlices(api.config.ConcurrentRequests, refs...)

	for bindex, brefs := range batchedSlicesOfRefs {
		log.Infof("BATCH %d of %d", bindex+1, len(batchedSlicesOfRefs))

		log.Debugf("%s references: %+v", fn(), brefs)

		repos, _ := repository.ParseRefs(brefs)
		for _, repo := range repos {
			log.Debugf("%s repository: %+v", fn(), repo)
		}

		done := make(chan error, len(repos))

		for _, repo := range repos {
			go func(repo *repository.Repository, done chan error) {
				log.Infof("ANALYZE %s", repo.Ref())

				username, password, _ := api.dockerClient.Config().GetCredentials(repo.Registry())

				remoteTags, err := remote.FetchTags(repo, username, password)
				if err != nil {
					done <- err
					return
				}
				log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags)

				localTags, _ := local.FetchTags(repo, api.dockerClient)

				log.Debugf("%s local tags: %+v", fn(repo.Ref()), localTags)

				sortedKeys, tagNames, joinedTags := tag.Join(
					remoteTags,
					localTags,
					repo.Tags(),
				)
				log.Debugf("%s sending joined tags: %+v", fn(repo.Ref()), joinedTags)

				tagc <- rtags{ref: repo.Ref(), tags: tag.Collect(sortedKeys, tagNames, joinedTags)}
				done <- nil

				log.Infof("FETCHED %s", repo.Ref())

				return
			}(repo, done)
		}

		if err := wait.Until(done); err != nil {
			return nil, err
		}

		time.Sleep(api.config.WaitBetween)
	}

	step := 1
	size := cap(tagc)
	for t := range tagc {
		log.Debugf("[%s] receiving tags: %+v", t.ref, t.tags)

		tags[t.ref] = t.tags

		if step >= size {
			close(tagc)
		}

		step++
	}

	log.Debugf("%s tags: %+v", fn(), tags)

	return collection.New(refs, tags)
}

func getPushPrefix(prefix, defaultPrefix string) string {
	if prefix == "" {
		return defaultPrefix
	}

	if prefix[0:1] != "/" {
		prefix = "/" + prefix
	}

	if prefix[len(prefix)-1:] != "/" {
		prefix = prefix + "/"
	}

	return prefix
}

// CollectPushTags blends passed collection with information fetched from [local] "push" registry,
// makes required comparisons between them and spits organized info back as collection.Collection
func (api *API) CollectPushTags(cn *collection.Collection, push PushConfig) (*collection.Collection, error) {
	log.Debugf(
		"%s collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)
	log.Debugf("%s push config: %+v", fn(), push)

	refs := make([]string, len(cn.Refs()))
	done := make(chan error, len(cn.Refs()))
	tagc := make(chan rtags, len(refs))
	tags := make(map[string][]*tag.Tag)

	for i, repo := range cn.Repos() {
		go func(repo *repository.Repository, i int, done chan error) {
			refs[i] = repo.Ref()

			pushRef := fmt.Sprintf(
				"%s%s~/.*/",
				push.Registry,
				getPushPrefix(push.Prefix, repo.PushPrefix())+repo.Path(),
			)

			log.Debugf("%s 'push' reference: %+v", fn(repo.Ref()), pushRef)

			pushRepo, _ := repository.ParseRef(pushRef)

			log.Infof("[PULL/PUSH] ANALYZE %s => %s", repo.Ref(), pushRef)

			username, password, _ := api.dockerClient.Config().GetCredentials(push.Registry)

			pushedTags, err := remote.FetchTags(pushRepo, username, password)
			if err != nil {
				if !strings.Contains(err.Error(), "404 Not Found") {
					done <- err
					return
				}

				log.Warnf("%s repo not found: %+s", fn(repo.Ref()), pushRef)

				pushedTags = make(map[string]*tag.Tag)
			}
			log.Debugf("%s pushed tags: %+v", fn(repo.Ref()), pushedTags)

			remoteTags := cn.TagMap(repo.Ref())
			log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags)

			sortedKeys, tagNames, joinedTags := tag.Join(
				remoteTags,
				pushedTags,
				repo.Tags(),
			)
			log.Debugf("%s joined tags: %+v", fn(repo.Ref()), joinedTags)

			tagsToPush := make([]*tag.Tag, 0)
			for _, key := range sortedKeys {
				name := tagNames[key]
				tg := joinedTags[name]

				if tg.NeedsPush(push.UpdateChanged) {
					tagsToPush = append(tagsToPush, tg)
				}
			}
			log.Debugf("%s sending 'push' tags: %+v", fn(repo.Ref()), tagsToPush)

			tagc <- rtags{ref: repo.Ref(), tags: tagsToPush}
			done <- nil

			return
		}(repo, i, done)

		time.Sleep(api.config.WaitBetween)
	}

	if err := wait.Until(done); err != nil {
		return nil, err
	}

	step := 1
	size := cap(tagc)
	for t := range tagc {
		log.Debugf("[%s] receiving 'push' tags: %+v", t.ref, t.tags)

		tags[t.ref] = t.tags

		if step >= size {
			close(tagc)
		}

		step++
	}

	log.Debugf("%s 'push' tags: %+v", fn(), tags)

	return collection.New(refs, tags)
}

// PullTags compares images from remote registry and Docker daemon and pulls
// images that match tag spec passed and are not present in Docker daemon.
func (api *API) PullTags(cn *collection.Collection) error {
	log.Debugf(
		"%s collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)

	done := make(chan error, cn.TagCount())

	for _, ref := range cn.Refs() {
		repo := cn.Repo(ref)
		tags := cn.Tags(ref)

		log.Debugf("%s repository: %+v", fn(), repo)
		for _, tg := range tags {
			log.Debugf("%s tag: %+v", fn(), tg)
		}

		go func(repo *repository.Repository, tags []*tag.Tag, done chan error) {
			for _, tg := range tags {
				if !tg.NeedsPull() {
					done <- nil
					continue
				}

				ref := repo.Name() + ":" + tg.Name()

				log.Infof("PULLING %s", ref)

				resp, err := api.dockerClient.Pull(ref)
				if err != nil {
					done <- err
					return
				}

				logDebugData(resp)

				done <- nil
			}
		}(repo, tags, done)

		time.Sleep(api.config.WaitBetween)
	}

	return wait.WithTolerance(done)
}

// PushTags compares images from remote and "push" (usually local) registries,
// pulls images that are present in remote registry, but are not in "push" one
// and then [re-]pushes them to the "push" registry.
func (api *API) PushTags(cn *collection.Collection, push PushConfig) error {
	log.Debugf(
		"%s 'push' collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)
	log.Debugf("%s push config: %+v", fn(), push)

	done := make(chan error, cn.TagCount())

	if cn.TagCount() == 0 {
		log.Infof("%s No tags to push", fn())
		return nil
	}

	for _, ref := range cn.Refs() {
		repo := cn.Repo(ref)
		tags := cn.Tags(ref)

		log.Debugf("%s repository: %+v", fn(), repo)
		for _, tg := range tags {
			log.Debugf("%s tag: %+v", fn(), tg)
		}

		go func(repo *repository.Repository, tags []*tag.Tag, done chan error) {
			for _, tg := range tags {
				srcRef := repo.Name() + ":" + tg.Name()
				dstRef := push.Registry + getPushPrefix(push.Prefix, repo.PushPrefix()) + repo.Path() + ":" + tg.Name()

				log.Infof("[PULL/PUSH] PUSHING %s => %s", srcRef, dstRef)

				pullResp, err := api.dockerClient.Pull(srcRef)
				if err != nil {
					done <- err
					return
				}
				logDebugData(pullResp)

				api.dockerClient.Tag(srcRef, dstRef)

				pushResp, err := api.dockerClient.Push(dstRef)
				if err != nil {
					done <- err
					return
				}
				logDebugData(pushResp)

				done <- err
			}
		}(repo, tags, done)

		time.Sleep(api.config.WaitBetween)
	}

	return wait.WithTolerance(done)
}

func logDebugData(data io.Reader) {
	scanner := bufio.NewScanner(data)
	for scanner.Scan() {
		log.Debug(scanner.Text())
	}
}

// New creates new instance of application API
func New(config Config) (*API, error) {
	if config.VerboseLogging {
		log.SetLevel(log.DebugLevel)
	}
	log.Debugf("%s API config: %+v", fn(), config)

	if config.ConcurrentRequests == 0 {
		config.ConcurrentRequests = 8
	}
	remote.ConcurrentRequests = config.ConcurrentRequests
	remote.WaitBetween = config.WaitBetween
	remote.TraceRequests = config.TraceRequests
	remote.RetryRequests = config.RetryRequests
	remote.RetryDelay = config.RetryDelay

	if config.InsecureRegistryEx != "" {
		repository.InsecureRegistryEx = config.InsecureRegistryEx
	}

	if config.DockerJSONConfigFile == "" {
		config.DockerJSONConfigFile = dockerconfig.DefaultDockerJSON
	}
	dockerConfig, err := dockerconfig.Load(config.DockerJSONConfigFile)
	if err != nil {
		return nil, err
	}
	dockerClient, err := dockerclient.New(dockerConfig)
	if err != nil {
		return nil, err
	}

	return &API{
		config:       config,
		dockerClient: dockerClient,
	}, nil
}
<commit_msg>refactor(api/v1): DRY the way we avoid concurrent map writes<commit_after>// Package v1 provides lstags v1 API to be used both by the application
// itself and by external projects
package v1

import (
	"bufio"
	"fmt"
	log "github.com/sirupsen/logrus"
	"io"
	"runtime"
	"strings"
	"time"

	"github.com/ivanilves/lstags/api/v1/collection"
	dockerclient "github.com/ivanilves/lstags/docker/client"
	dockerconfig "github.com/ivanilves/lstags/docker/config"
	"github.com/ivanilves/lstags/repository"
	"github.com/ivanilves/lstags/tag"
	"github.com/ivanilves/lstags/tag/local"
	"github.com/ivanilves/lstags/tag/remote"
	"github.com/ivanilves/lstags/util/wait"
)

// Config holds API instance configuration
type Config struct {
	// DockerJSONConfigFile is a path to Docker JSON config file
	DockerJSONConfigFile string
	// ConcurrentRequests defines how much requests to registry we could run in parallel
	ConcurrentRequests int
	// WaitBetween defines how much we will wait between batches of requests (incl. pull and push)
	WaitBetween time.Duration
	// TraceRequests sets if we will print out registry HTTP request traces
	TraceRequests bool
	// RetryRequests defines how much retries we will do to the failed HTTP request
	RetryRequests int
	// RetryDelay defines how much we will wait between failed HTTP request and retry
	RetryDelay time.Duration
	// InsecureRegistryEx is a regex string to match insecure (non-HTTPS) registries
	InsecureRegistryEx string
	// VerboseLogging sets if we will print debug log messages
	VerboseLogging bool
}

// PushConfig holds push-specific configuration (where to push and with which prefix)
type PushConfig struct {
	// Prefix is prepended to the repository path while pushing to the registry
	Prefix string
	// Registry is an address of the Docker registry in which we push our images
	Registry string
	// UpdateChanged tells us if we will re-push (update/overwrite) images having same tag, but different digest
	UpdateChanged bool
}

// API represents configured application API instance,
// the main abstraction you are supposed to work with
type API struct {
	config       Config
	dockerClient *dockerclient.DockerClient
}

// rtags is a structure to send collection of referenced tags using chan
type rtags struct {
	ref  string
	tags []*tag.Tag
}

// fn gives the name of the calling function (e.g. enriches log.Debugf() output)
// + optionally attaches free form string labels (mainly to identify goroutines)
func fn(labels ...string) string {
	function, _, _, _ := runtime.Caller(1)

	longname := runtime.FuncForPC(function).Name()

	nameparts := strings.Split(longname, ".")
	shortname := nameparts[len(nameparts)-1]

	if labels == nil {
		return fmt.Sprintf("[%s()]", shortname)
	}

	return fmt.Sprintf("[%s():%s]", shortname, strings.Join(labels, ":"))
}

func getBatchedSlices(batchSize int, unbatched ...string) [][]string {
	batchedSlices := make([][]string, 0)

	index := 0

	for range unbatched {
		batchedSlice := make([]string, 0)

		for c := 0; c < batchSize; c++ {
			batchedSlice = append(batchedSlice, unbatched[index])

			index++

			if index == len(unbatched) {
				break
			}
		}

		batchedSlices = append(batchedSlices, batchedSlice)

		if index == len(unbatched) {
			break
		}
	}

	return batchedSlices
}

func receiveTags(tagc chan rtags) map[string][]*tag.Tag {
	tags := make(map[string][]*tag.Tag)

	step := 1
	size := cap(tagc)
	for t := range tagc {
		log.Debugf("[%s] receiving tags: %+v", t.ref, t.tags)

		tags[t.ref] = t.tags

		if step >= size {
			close(tagc)
		}

		step++
	}

	return tags
}

// CollectTags collects information on tags present in remote registry and [local] Docker daemon,
// makes required comparisons between them and spits organized info back as collection.Collection
func (api *API) CollectTags(refs ...string) (*collection.Collection, error) {
	if len(refs) == 0 {
		return nil, fmt.Errorf("no image references passed")
	}

	_, err := repository.ParseRefs(refs)
	if err != nil {
		return nil, err
	}

	tagc := make(chan rtags, len(refs))

	batchedSlicesOfRefs := getBatchedSlices(api.config.ConcurrentRequests, refs...)

	for bindex, brefs := range batchedSlicesOfRefs {
		log.Infof("BATCH %d of %d", bindex+1, len(batchedSlicesOfRefs))

		log.Debugf("%s references: %+v", fn(), brefs)

		repos, _ := repository.ParseRefs(brefs)
		for _, repo := range repos {
			log.Debugf("%s repository: %+v", fn(), repo)
		}

		done := make(chan error, len(repos))

		for _, repo := range repos {
			go func(repo *repository.Repository, done chan error) {
				log.Infof("ANALYZE %s", repo.Ref())

				username, password, _ := api.dockerClient.Config().GetCredentials(repo.Registry())

				remoteTags, err := remote.FetchTags(repo, username, password)
				if err != nil {
					done <- err
					return
				}
				log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags)

				localTags, _ := local.FetchTags(repo, api.dockerClient)

				log.Debugf("%s local tags: %+v", fn(repo.Ref()), localTags)

				sortedKeys, tagNames, joinedTags := tag.Join(
					remoteTags,
					localTags,
					repo.Tags(),
				)
				log.Debugf("%s sending joined tags: %+v", fn(repo.Ref()), joinedTags)

				tagc <- rtags{ref: repo.Ref(), tags: tag.Collect(sortedKeys, tagNames, joinedTags)}
				done <- nil

				log.Infof("FETCHED %s", repo.Ref())

				return
			}(repo, done)
		}

		if err := wait.Until(done); err != nil {
			return nil, err
		}

		time.Sleep(api.config.WaitBetween)
	}

	tags := receiveTags(tagc)

	log.Debugf("%s tags: %+v", fn(), tags)

	return collection.New(refs, tags)
}

func getPushPrefix(prefix, defaultPrefix string) string {
	if prefix == "" {
		return defaultPrefix
	}

	if prefix[0:1] != "/" {
		prefix = "/" + prefix
	}

	if prefix[len(prefix)-1:] != "/" {
		prefix = prefix + "/"
	}

	return prefix
}

// CollectPushTags blends passed collection with information fetched from [local] "push" registry,
// makes required comparisons between them and spits organized info back as collection.Collection
func (api *API) CollectPushTags(cn *collection.Collection, push PushConfig) (*collection.Collection, error) {
	log.Debugf(
		"%s collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)
	log.Debugf("%s push config: %+v", fn(), push)

	refs := make([]string, len(cn.Refs()))
	done := make(chan error, len(cn.Refs()))
	tagc := make(chan rtags, len(refs))

	for i, repo := range cn.Repos() {
		go func(repo *repository.Repository, i int, done chan error) {
			refs[i] = repo.Ref()

			pushRef := fmt.Sprintf(
				"%s%s~/.*/",
				push.Registry,
				getPushPrefix(push.Prefix, repo.PushPrefix())+repo.Path(),
			)

			log.Debugf("%s 'push' reference: %+v", fn(repo.Ref()), pushRef)

			pushRepo, _ := repository.ParseRef(pushRef)

			log.Infof("[PULL/PUSH] ANALYZE %s => %s", repo.Ref(), pushRef)

			username, password, _ := api.dockerClient.Config().GetCredentials(push.Registry)

			pushedTags, err := remote.FetchTags(pushRepo, username, password)
			if err != nil {
				if !strings.Contains(err.Error(), "404 Not Found") {
					done <- err
					return
				}

				log.Warnf("%s repo not found: %+s", fn(repo.Ref()), pushRef)

				pushedTags = make(map[string]*tag.Tag)
			}
			log.Debugf("%s pushed tags: %+v", fn(repo.Ref()), pushedTags)

			remoteTags := cn.TagMap(repo.Ref())
			log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags)

			sortedKeys, tagNames, joinedTags := tag.Join(
				remoteTags,
				pushedTags,
				repo.Tags(),
			)
			log.Debugf("%s joined tags: %+v", fn(repo.Ref()), joinedTags)

			tagsToPush := make([]*tag.Tag, 0)
			for _, key := range sortedKeys {
				name := tagNames[key]
				tg := joinedTags[name]

				if tg.NeedsPush(push.UpdateChanged) {
					tagsToPush = append(tagsToPush, tg)
				}
			}
			log.Debugf("%s sending 'push' tags: %+v", fn(repo.Ref()), tagsToPush)

			tagc <- rtags{ref: repo.Ref(), tags: tagsToPush}
			done <- nil

			return
		}(repo, i, done)

		time.Sleep(api.config.WaitBetween)
	}

	if err := wait.Until(done); err != nil {
		return nil, err
	}

	tags := receiveTags(tagc)

	log.Debugf("%s 'push' tags: %+v", fn(), tags)

	return collection.New(refs, tags)
}

// PullTags compares images from remote registry and Docker daemon and pulls
// images that match tag spec passed and are not present in Docker daemon.
func (api *API) PullTags(cn *collection.Collection) error {
	log.Debugf(
		"%s collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)

	done := make(chan error, cn.TagCount())

	for _, ref := range cn.Refs() {
		repo := cn.Repo(ref)
		tags := cn.Tags(ref)

		log.Debugf("%s repository: %+v", fn(), repo)
		for _, tg := range tags {
			log.Debugf("%s tag: %+v", fn(), tg)
		}

		go func(repo *repository.Repository, tags []*tag.Tag, done chan error) {
			for _, tg := range tags {
				if !tg.NeedsPull() {
					done <- nil
					continue
				}

				ref := repo.Name() + ":" + tg.Name()

				log.Infof("PULLING %s", ref)

				resp, err := api.dockerClient.Pull(ref)
				if err != nil {
					done <- err
					return
				}

				logDebugData(resp)

				done <- nil
			}
		}(repo, tags, done)

		time.Sleep(api.config.WaitBetween)
	}

	return wait.WithTolerance(done)
}

// PushTags compares images from remote and "push" (usually local) registries,
// pulls images that are present in remote registry, but are not in "push" one
// and then [re-]pushes them to the "push" registry.
func (api *API) PushTags(cn *collection.Collection, push PushConfig) error {
	log.Debugf(
		"%s 'push' collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)
	log.Debugf("%s push config: %+v", fn(), push)

	done := make(chan error, cn.TagCount())

	if cn.TagCount() == 0 {
		log.Infof("%s No tags to push", fn())
		return nil
	}

	for _, ref := range cn.Refs() {
		repo := cn.Repo(ref)
		tags := cn.Tags(ref)

		log.Debugf("%s repository: %+v", fn(), repo)
		for _, tg := range tags {
			log.Debugf("%s tag: %+v", fn(), tg)
		}

		go func(repo *repository.Repository, tags []*tag.Tag, done chan error) {
			for _, tg := range tags {
				srcRef := repo.Name() + ":" + tg.Name()
				dstRef := push.Registry + getPushPrefix(push.Prefix, repo.PushPrefix()) + repo.Path() + ":" + tg.Name()

				log.Infof("[PULL/PUSH] PUSHING %s => %s", srcRef, dstRef)

				pullResp, err := api.dockerClient.Pull(srcRef)
				if err != nil {
					done <- err
					return
				}
				logDebugData(pullResp)

				api.dockerClient.Tag(srcRef, dstRef)

				pushResp, err := api.dockerClient.Push(dstRef)
				if err != nil {
					done <- err
					return
				}
				logDebugData(pushResp)

				done <- err
			}
		}(repo, tags, done)

		time.Sleep(api.config.WaitBetween)
	}

	return wait.WithTolerance(done)
}

func logDebugData(data io.Reader) {
	scanner := bufio.NewScanner(data)
	for scanner.Scan() {
		log.Debug(scanner.Text())
	}
}

// New creates new instance of application API
func New(config Config) (*API, error) {
	if config.VerboseLogging {
		log.SetLevel(log.DebugLevel)
	}
	log.Debugf("%s API config: %+v", fn(), config)

	if config.ConcurrentRequests == 0 {
		config.ConcurrentRequests = 8
	}
	remote.ConcurrentRequests = config.ConcurrentRequests
	remote.WaitBetween = config.WaitBetween
	remote.TraceRequests = config.TraceRequests
	remote.RetryRequests = config.RetryRequests
	remote.RetryDelay = config.RetryDelay

	if config.InsecureRegistryEx != "" {
		repository.InsecureRegistryEx = config.InsecureRegistryEx
	}

	if config.DockerJSONConfigFile == "" {
		config.DockerJSONConfigFile = dockerconfig.DefaultDockerJSON
	}
	dockerConfig, err := dockerconfig.Load(config.DockerJSONConfigFile)
	if err != nil {
		return nil, err
	}
	dockerClient, err := dockerclient.New(dockerConfig)
	if err != nil {
		return nil, err
	}

	return &API{
		config:       config,
		dockerClient: dockerClient,
	}, nil
}
<|endoftext|>


<commit_before>package app

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	Ref Reference
)

type Reference struct {
	RootPath string
	DBFile   string
	DB       *DBx
	IOs      []*I2Cx
	WebPort  int
	ButtonPressDelay time.Duration
}

func Print(format string, a ...interface{}) {
	fmt.Fprint(os.Stdout, "[", time.Now().String(), "] ")
	fmt.Fprintf(os.Stdout, format, a...)
}

func SetBit(n byte, pos byte) byte {
	n |= (1 << pos)
	return n
}
func ClearBit(n byte, pos byte) byte {
	var mask byte = ^(1 << pos)
	n &= mask
	return n
}
func ToggleBit(n byte, pos byte) byte {
	if HasBit(n, pos) {
		n = ClearBit(n, pos)
	} else {
		n = SetBit(n, pos)
	}
	return n
}
func HasBit(n byte, pos byte) bool {
	val := n & (1 << pos)
	return (val > 0)
}
func ConvertTo8BitBinaryString(num byte) string {
	bin := strconv.FormatInt(int64(num), 2)
	return strings.Repeat("0", 8-len(bin)) + bin
}
<commit_msg>gofmt'ed<commit_after>package app

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"
)

var (
	Ref Reference
)

type Reference struct {
	RootPath         string
	DBFile           string
	DB               *DBx
	IOs              []*I2Cx
	WebPort          int
	ButtonPressDelay time.Duration
}

func Print(format string, a ...interface{}) {
	fmt.Fprint(os.Stdout, "[", time.Now().String(), "] ")
	fmt.Fprintf(os.Stdout, format, a...)
}

func SetBit(n byte, pos byte) byte {
	n |= (1 << pos)
	return n
}
func ClearBit(n byte, pos byte) byte {
	var mask byte = ^(1 << pos)
	n &= mask
	return n
}
func ToggleBit(n byte, pos byte) byte {
	if HasBit(n, pos) {
		n = ClearBit(n, pos)
	} else {
		n = SetBit(n, pos)
	}
	return n
}
func HasBit(n byte, pos byte) bool {
	val := n & (1 << pos)
	return (val > 0)
}
func ConvertTo8BitBinaryString(num byte) string {
	bin := strconv.FormatInt(int64(num), 2)
	return strings.Repeat("0", 8-len(bin)) + bin
}
<|endoftext|>