<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The storage stage is responsible for partitioning disks, creating RAID
// arrays, formatting partitions, writing files, writing systemd units, and
// writing network units.

package disks

import (
	"fmt"
	"os/exec"

	"github.com/coreos/ignition/config/types"
	"github.com/coreos/ignition/internal/exec/stages"
	"github.com/coreos/ignition/internal/exec/util"
	"github.com/coreos/ignition/internal/log"
	"github.com/coreos/ignition/internal/sgdisk"
	"github.com/coreos/ignition/internal/systemd"
)

const (
	name = "disks"
)

func init() {
	stages.Register(creator{})
}

type creator struct{}

func (creator) Create(logger *log.Logger, root string) stages.Stage {
	return &stage{util.Util{
		DestDir: root,
		Logger:  logger,
	}}
}

func (creator) Name() string {
	return name
}

type stage struct {
	util.Util
}

func (stage) Name() string {
	return name
}

func (s stage) Run(config types.Config) bool {
	if err := s.createPartitions(config); err != nil {
		s.Logger.Crit("create partitions failed: %v", err)
		return false
	}

	if err := s.createRaids(config); err != nil {
		s.Logger.Crit("failed to create raids: %v", err)
		return false
	}

	if err := s.createFilesystems(config); err != nil {
		s.Logger.Crit("failed to create filesystems: %v", err)
		return false
	}

	return true
}

// waitOnDevices waits for the devices enumerated in devs as a logged operation
// using ctxt for the logging and systemd unit identity.
func (s stage) waitOnDevices(devs []string, ctxt string) error {
	if err := s.LogOp(
		func() error { return systemd.WaitOnDevices(devs, ctxt) },
		"waiting for devices %v", devs,
	); err != nil {
		return fmt.Errorf("failed to wait on %s devs: %v", ctxt, err)
	}
	return nil
}

// createPartitions creates the partitions described in config.Storage.Disks.
func (s stage) createPartitions(config types.Config) error {
	if len(config.Storage.Disks) == 0 {
		return nil
	}
	s.Logger.PushPrefix("createPartitions")
	defer s.Logger.PopPrefix()

	devs := []string{}
	for _, disk := range config.Storage.Disks {
		devs = append(devs, string(disk.Device))
	}

	if err := s.waitOnDevices(devs, "disks"); err != nil {
		return err
	}

	for _, dev := range config.Storage.Disks {
		err := s.Logger.LogOp(func() error {
			op := sgdisk.Begin(s.Logger, string(dev.Device))
			if dev.WipeTable {
				s.Logger.Info("wiping partition table requested on %q", dev.Device)
				op.WipeTable(true)
			}

			for _, part := range dev.Partitions {
				op.CreatePartition(sgdisk.Partition{
					Number:   part.Number,
					Length:   uint64(part.Size),
					Offset:   uint64(part.Start),
					Label:    string(part.Label),
					TypeGUID: string(part.TypeGUID),
				})
			}

			if err := op.Commit(); err != nil {
				return fmt.Errorf("commit failure: %v", err)
			}
			return nil
		}, "partitioning %q", dev.Device)
		if err != nil {
			return err
		}
	}

	return nil
}

// createRaids creates the raid arrays described in config.Storage.Arrays.
func (s stage) createRaids(config types.Config) error {
	if len(config.Storage.Arrays) == 0 {
		return nil
	}
	s.Logger.PushPrefix("createRaids")
	defer s.Logger.PopPrefix()

	devs := []string{}
	for _, array := range config.Storage.Arrays {
		for _, dev := range array.Devices {
			devs = append(devs, string(dev))
		}
	}

	if err := s.waitOnDevices(devs, "raids"); err != nil {
		return err
	}

	for _, md := range config.Storage.Arrays {
		// FIXME(vc): this is utterly flummoxed by a preexisting md.Name, the magic of device-resident md metadata really interferes with us.
		// It's as if what ignition really needs is to turn off automagic md probing/running before getting started.
		args := []string{
			"--create", md.Name,
			"--force",
			"--run",
			"--level", md.Level,
			"--raid-devices", fmt.Sprintf("%d", len(md.Devices)-md.Spares),
		}

		if md.Spares > 0 {
			args = append(args, "--spare-devices", fmt.Sprintf("%d", md.Spares))
		}

		for _, dev := range md.Devices {
			args = append(args, string(dev))
		}

		if err := s.Logger.LogCmd(
			exec.Command("/sbin/mdadm", args...),
			"creating %q", md.Name,
		); err != nil {
			return fmt.Errorf("mdadm failed: %v", err)
		}
	}

	return nil
}

// createFilesystems creates the filesystems described in config.Storage.Filesystems.
func (s stage) createFilesystems(config types.Config) error {
	fss := make([]types.FilesystemMount, 0, len(config.Storage.Filesystems))
	for _, fs := range config.Storage.Filesystems {
		if len(fs.Path) == 0 {
			fss = append(fss, *fs.Mount)
		}
	}

	if len(fss) == 0 {
		return nil
	}
	s.Logger.PushPrefix("createFilesystems")
	defer s.Logger.PopPrefix()

	devs := []string{}
	for _, fs := range fss {
		devs = append(devs, string(fs.Device))
	}

	if err := s.waitOnDevices(devs, "filesystems"); err != nil {
		return err
	}

	for _, fs := range fss {
		if err := s.createFilesystem(fs); err != nil {
			return err
		}
	}

	return nil
}

func (s stage) createFilesystem(fs types.FilesystemMount) error {
	if fs.Create == nil {
		return nil
	}

	mkfs := ""
	args := []string(fs.Create.Options)
	switch fs.Format {
	case "btrfs":
		mkfs = "/sbin/mkfs.btrfs"
		if fs.Create.Force {
			args = append(args, "--force")
		}
	case "ext4":
		mkfs = "/sbin/mkfs.ext4"
		if fs.Create.Force {
			args = append(args, "-F")
		}
	case "xfs":
		mkfs = "/sbin/mkfs.xfs"
		if fs.Create.Force {
			args = append(args, "-f")
		}
	default:
		return fmt.Errorf("unsupported filesystem format: %q", fs.Format)
	}

	args = append(args, string(fs.Device))
	if err := s.Logger.LogCmd(
		exec.Command(mkfs, args...),
		"creating %q filesystem on %q",
		fs.Format, string(fs.Device),
	); err != nil {
		return fmt.Errorf("failed to run %q: %v %v", mkfs, err, args)
	}

	return nil
}
<commit_msg>stages/disks: reword condition<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The storage stage is responsible for partitioning disks, creating RAID
// arrays, formatting partitions, writing files, writing systemd units, and
// writing network units.

package disks

import (
	"fmt"
	"os/exec"

	"github.com/coreos/ignition/config/types"
	"github.com/coreos/ignition/internal/exec/stages"
	"github.com/coreos/ignition/internal/exec/util"
	"github.com/coreos/ignition/internal/log"
	"github.com/coreos/ignition/internal/sgdisk"
	"github.com/coreos/ignition/internal/systemd"
)

const (
	name = "disks"
)

func init() {
	stages.Register(creator{})
}

type creator struct{}

func (creator) Create(logger *log.Logger, root string) stages.Stage {
	return &stage{util.Util{
		DestDir: root,
		Logger:  logger,
	}}
}

func (creator) Name() string {
	return name
}

type stage struct {
	util.Util
}

func (stage) Name() string {
	return name
}

func (s stage) Run(config types.Config) bool {
	if err := s.createPartitions(config); err != nil {
		s.Logger.Crit("create partitions failed: %v", err)
		return false
	}

	if err := s.createRaids(config); err != nil {
		s.Logger.Crit("failed to create raids: %v", err)
		return false
	}

	if err := s.createFilesystems(config); err != nil {
		s.Logger.Crit("failed to create filesystems: %v", err)
		return false
	}

	return true
}

// waitOnDevices waits for the devices enumerated in devs as a logged operation
// using ctxt for the logging and systemd unit identity.
func (s stage) waitOnDevices(devs []string, ctxt string) error {
	if err := s.LogOp(
		func() error { return systemd.WaitOnDevices(devs, ctxt) },
		"waiting for devices %v", devs,
	); err != nil {
		return fmt.Errorf("failed to wait on %s devs: %v", ctxt, err)
	}
	return nil
}

// createPartitions creates the partitions described in config.Storage.Disks.
func (s stage) createPartitions(config types.Config) error {
	if len(config.Storage.Disks) == 0 {
		return nil
	}
	s.Logger.PushPrefix("createPartitions")
	defer s.Logger.PopPrefix()

	devs := []string{}
	for _, disk := range config.Storage.Disks {
		devs = append(devs, string(disk.Device))
	}

	if err := s.waitOnDevices(devs, "disks"); err != nil {
		return err
	}

	for _, dev := range config.Storage.Disks {
		err := s.Logger.LogOp(func() error {
			op := sgdisk.Begin(s.Logger, string(dev.Device))
			if dev.WipeTable {
				s.Logger.Info("wiping partition table requested on %q", dev.Device)
				op.WipeTable(true)
			}

			for _, part := range dev.Partitions {
				op.CreatePartition(sgdisk.Partition{
					Number:   part.Number,
					Length:   uint64(part.Size),
					Offset:   uint64(part.Start),
					Label:    string(part.Label),
					TypeGUID: string(part.TypeGUID),
				})
			}

			if err := op.Commit(); err != nil {
				return fmt.Errorf("commit failure: %v", err)
			}
			return nil
		}, "partitioning %q", dev.Device)
		if err != nil {
			return err
		}
	}

	return nil
}

// createRaids creates the raid arrays described in config.Storage.Arrays.
func (s stage) createRaids(config types.Config) error {
	if len(config.Storage.Arrays) == 0 {
		return nil
	}
	s.Logger.PushPrefix("createRaids")
	defer s.Logger.PopPrefix()

	devs := []string{}
	for _, array := range config.Storage.Arrays {
		for _, dev := range array.Devices {
			devs = append(devs, string(dev))
		}
	}

	if err := s.waitOnDevices(devs, "raids"); err != nil {
		return err
	}

	for _, md := range config.Storage.Arrays {
		// FIXME(vc): this is utterly flummoxed by a preexisting md.Name, the magic of device-resident md metadata really interferes with us.
		// It's as if what ignition really needs is to turn off automagic md probing/running before getting started.
		args := []string{
			"--create", md.Name,
			"--force",
			"--run",
			"--level", md.Level,
			"--raid-devices", fmt.Sprintf("%d", len(md.Devices)-md.Spares),
		}

		if md.Spares > 0 {
			args = append(args, "--spare-devices", fmt.Sprintf("%d", md.Spares))
		}

		for _, dev := range md.Devices {
			args = append(args, string(dev))
		}

		if err := s.Logger.LogCmd(
			exec.Command("/sbin/mdadm", args...),
			"creating %q", md.Name,
		); err != nil {
			return fmt.Errorf("mdadm failed: %v", err)
		}
	}

	return nil
}

// createFilesystems creates the filesystems described in config.Storage.Filesystems.
func (s stage) createFilesystems(config types.Config) error {
	fss := make([]types.FilesystemMount, 0, len(config.Storage.Filesystems))
	for _, fs := range config.Storage.Filesystems {
		if fs.Mount != nil {
			fss = append(fss, *fs.Mount)
		}
	}

	if len(fss) == 0 {
		return nil
	}
	s.Logger.PushPrefix("createFilesystems")
	defer s.Logger.PopPrefix()

	devs := []string{}
	for _, fs := range fss {
		devs = append(devs, string(fs.Device))
	}

	if err := s.waitOnDevices(devs, "filesystems"); err != nil {
		return err
	}

	for _, fs := range fss {
		if err := s.createFilesystem(fs); err != nil {
			return err
		}
	}

	return nil
}

func (s stage) createFilesystem(fs types.FilesystemMount) error {
	if fs.Create == nil {
		return nil
	}

	mkfs := ""
	args := []string(fs.Create.Options)
	switch fs.Format {
	case "btrfs":
		mkfs = "/sbin/mkfs.btrfs"
		if fs.Create.Force {
			args = append(args, "--force")
		}
	case "ext4":
		mkfs = "/sbin/mkfs.ext4"
		if fs.Create.Force {
			args = append(args, "-F")
		}
	case "xfs":
		mkfs = "/sbin/mkfs.xfs"
		if fs.Create.Force {
			args = append(args, "-f")
		}
	default:
		return fmt.Errorf("unsupported filesystem format: %q", fs.Format)
	}

	args = append(args, string(fs.Device))
	if err := s.Logger.LogCmd(
		exec.Command(mkfs, args...),
		"creating %q filesystem on %q",
		fs.Format, string(fs.Device),
	); err != nil {
		return fmt.Errorf("failed to run %q: %v %v", mkfs, err, args)
	}

	return nil
}
<|endoftext|>


<commit_before>package tablestorageproxy

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"
)

type GoHaveStorage interface {
	GetKey() []byte
	GetAccount() string
}

type TableStorageProxy struct {
	goHaveStorage GoHaveStorage
	baseUrl string
}

func New(goHaveStorage GoHaveStorage) *TableStorageProxy {
	var tableStorageProxy TableStorageProxy

	tableStorageProxy.goHaveStorage = goHaveStorage
	tableStorageProxy.baseUrl = "https://"+goHaveStorage.GetAccount()+".table.core.windows.net/"

	return &tableStorageProxy
}

func (tableStorageProxy *TableStorageProxy) QueryTables() {
	tableStorageProxy.get("Tables")
}

func (tableStorageProxy *TableStorageProxy) QueryEntity(tableName string, partitionKey string, rowKey string, selects string) {
	client := &http.Client{}
	request, _ := http.NewRequest("GET", tableStorageProxy.baseUrl+tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29?$select="+selects, nil)
	request.Header.Set("Accept", "application/json;odata=nometadata")

	tableStorageProxy.executeRequest(request, client, tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29")
}

func (tableStorageProxy *TableStorageProxy) DeleteEntity(tableName string, partitionKey string, rowKey string) {
	tableStorageProxy.executeEntityRequest("DELETE",tableName, partitionKey, rowKey, nil, true)
}

func (tableStorageProxy *TableStorageProxy) UpdateEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("PUT",tableName, partitionKey, rowKey, json, true)
}

func (tableStorageProxy *TableStorageProxy) MergeEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("MERGE",tableName, partitionKey, rowKey, json, true)
}

func (tableStorageProxy *TableStorageProxy) InsertOrMergeEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("MERGE",tableName, partitionKey, rowKey, json, false)
}

func (tableStorageProxy *TableStorageProxy) InsertOrReplaceEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("PUT",tableName, partitionKey, rowKey, json, false)
}

func (tableStorageProxy *TableStorageProxy)  executeEntityRequest(httpVerb string, tableName string, partitionKey string, rowKey string, json []byte, useIfMatch bool) {
	client := &http.Client{}
	request, _ := http.NewRequest(httpVerb, tableStorageProxy.baseUrl+tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29",  bytes.NewBuffer(json))

	if json != nil {
		addPayloadHeaders(request, len(json))
	}

	if useIfMatch {
		request.Header.Set("If-Match", "*")
	}

	tableStorageProxy.executeRequest(request, client, tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29")
}

func (tableStorageProxy *TableStorageProxy) QueryEntities(tableName string, selects string, filter string, top string) {
	client := &http.Client{}
	request, _ := http.NewRequest("GET", tableStorageProxy.baseUrl+tableName +"?$filter="+filter + "&$select=" + selects+"&$top="+top, nil)
	request.Header.Set("Accept", "application/json;odata=nometadata")

	tableStorageProxy.executeRequest(request, client, tableName)
}

func (tableStorageProxy *TableStorageProxy) DeleteTable(tableName string) {
	target := "Tables%28%27" + tableName + "%27%29"

	client := &http.Client{}
	request, _ := http.NewRequest("DELETE", tableStorageProxy.baseUrl+target, nil)
	request.Header.Set("Content-Type", "application/atom+xml")

	tableStorageProxy.executeRequest(request, client, target)
}

type CreateTableArgs struct {
	TableName string
}

func (tableStorageProxy *TableStorageProxy) CreateTable(tableName string) {
	var createTableArgs CreateTableArgs
	createTableArgs.TableName = tableName

	json, _ := json.Marshal(createTableArgs)
	tableStorageProxy.postJson("Tables", json)
}

func (tableStorageProxy *TableStorageProxy) InsertEntity(tableName string, json []byte) {
	tableStorageProxy.postJson(tableName, json)
}

func (tableStorageProxy *TableStorageProxy) get(target string) {
	client := &http.Client{}
	request, _ := http.NewRequest("GET", tableStorageProxy.baseUrl+target, nil)
	request.Header.Set("Accept", "application/json;odata=nometadata")

	tableStorageProxy.executeRequest(request, client, target)
}

func (tableStorageProxy *TableStorageProxy) postJson(target string, json []byte) {
	client := &http.Client{}
	request, _ := http.NewRequest("POST", tableStorageProxy.baseUrl+target, bytes.NewBuffer(json))
	addPayloadHeaders(request, len(json))

	tableStorageProxy.executeRequest(request, client, target)
}

func addPayloadHeaders(request *http.Request, bodyLength int) {
	request.Header.Set("Content-Type", "application/json")
	request.Header.Set("Content-Length", string(bodyLength))
}

func (tableStorageProxy *TableStorageProxy) executeRequest(request *http.Request, client *http.Client, target string) {
	xmsdate, Authentication := tableStorageProxy.calculateDateAndAuthentication(target)

	request.Header.Set("x-ms-date", xmsdate)
	request.Header.Set("x-ms-version", "2013-08-15")
	request.Header.Set("Authorization", Authentication)

	requestDump, _ := httputil.DumpRequest(request, true)

	fmt.Printf("Request: %s\n", requestDump)

	response, _ := client.Do(request)

	responseDump, _ := httputil.DumpResponse(response, true)
	fmt.Printf("Response: %s\n", responseDump)
}

func (tableStorageProxy *TableStorageProxy) calculateDateAndAuthentication(target string) (string, string) {
	xmsdate := strings.Replace(time.Now().UTC().Add(-time.Minute).Format(time.RFC1123), "UTC", "GMT", -1)
	SignatureString := xmsdate + "\n/" + tableStorageProxy.goHaveStorage.GetAccount() + "/" + target
	Authentication := "SharedKeyLite " + tableStorageProxy.goHaveStorage.GetAccount() + ":" + computeHmac256(SignatureString, tableStorageProxy.goHaveStorage.GetKey())
	return xmsdate, Authentication
}

func computeHmac256(message string, key []byte) string {
	h := hmac.New(sha256.New, key)
	h.Write([]byte(message))
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
<commit_msg>Adding query string to get method<commit_after>package tablestorageproxy

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"
)

type GoHaveStorage interface {
	GetKey() []byte
	GetAccount() string
}

type TableStorageProxy struct {
	goHaveStorage GoHaveStorage
	baseUrl string
}

func New(goHaveStorage GoHaveStorage) *TableStorageProxy {
	var tableStorageProxy TableStorageProxy

	tableStorageProxy.goHaveStorage = goHaveStorage
	tableStorageProxy.baseUrl = "https://"+goHaveStorage.GetAccount()+".table.core.windows.net/"

	return &tableStorageProxy
}

func (tableStorageProxy *TableStorageProxy) QueryTables() {
	tableStorageProxy.get("Tables", "")
}

func (tableStorageProxy *TableStorageProxy) QueryEntity(tableName string, partitionKey string, rowKey string, selects string) {
	client := &http.Client{}
	request, _ := http.NewRequest("GET", tableStorageProxy.baseUrl+tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29?$select="+selects, nil)
	request.Header.Set("Accept", "application/json;odata=nometadata")

	tableStorageProxy.executeRequest(request, client, tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29")
}

func (tableStorageProxy *TableStorageProxy) DeleteEntity(tableName string, partitionKey string, rowKey string) {
	tableStorageProxy.executeEntityRequest("DELETE",tableName, partitionKey, rowKey, nil, true)
}

func (tableStorageProxy *TableStorageProxy) UpdateEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("PUT",tableName, partitionKey, rowKey, json, true)
}

func (tableStorageProxy *TableStorageProxy) MergeEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("MERGE",tableName, partitionKey, rowKey, json, true)
}

func (tableStorageProxy *TableStorageProxy) InsertOrMergeEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("MERGE",tableName, partitionKey, rowKey, json, false)
}

func (tableStorageProxy *TableStorageProxy) InsertOrReplaceEntity(tableName string, partitionKey string, rowKey string, json []byte) {
	tableStorageProxy.executeEntityRequest("PUT",tableName, partitionKey, rowKey, json, false)
}

func (tableStorageProxy *TableStorageProxy)  executeEntityRequest(httpVerb string, tableName string, partitionKey string, rowKey string, json []byte, useIfMatch bool) {
	client := &http.Client{}
	request, _ := http.NewRequest(httpVerb, tableStorageProxy.baseUrl+tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29",  bytes.NewBuffer(json))

	if json != nil {
		addPayloadHeaders(request, len(json))
	}

	if useIfMatch {
		request.Header.Set("If-Match", "*")
	}

	tableStorageProxy.executeRequest(request, client, tableName + "%28PartitionKey=%27" + partitionKey + "%27,RowKey=%27" + rowKey + "%27%29")
}

func (tableStorageProxy *TableStorageProxy) QueryEntities(tableName string, selects string, filter string, top string) {
	client := &http.Client{}
	request, _ := http.NewRequest("GET", tableStorageProxy.baseUrl+tableName +"?$filter="+filter + "&$select=" + selects+"&$top="+top, nil)
	request.Header.Set("Accept", "application/json;odata=nometadata")

	tableStorageProxy.executeRequest(request, client, tableName)
}

func (tableStorageProxy *TableStorageProxy) DeleteTable(tableName string) {
	target := "Tables%28%27" + tableName + "%27%29"

	client := &http.Client{}
	request, _ := http.NewRequest("DELETE", tableStorageProxy.baseUrl+target, nil)
	request.Header.Set("Content-Type", "application/atom+xml")

	tableStorageProxy.executeRequest(request, client, target)
}

type CreateTableArgs struct {
	TableName string
}

func (tableStorageProxy *TableStorageProxy) CreateTable(tableName string) {
	var createTableArgs CreateTableArgs
	createTableArgs.TableName = tableName

	json, _ := json.Marshal(createTableArgs)
	tableStorageProxy.postJson("Tables", json)
}

func (tableStorageProxy *TableStorageProxy) InsertEntity(tableName string, json []byte) {
	tableStorageProxy.postJson(tableName, json)
}

func (tableStorageProxy *TableStorageProxy) get(target string, query string) {
	client := &http.Client{}
	request, _ := http.NewRequest("GET", tableStorageProxy.baseUrl+target + query, nil)
	request.Header.Set("Accept", "application/json;odata=nometadata")

	tableStorageProxy.executeRequest(request, client, target)
}

func (tableStorageProxy *TableStorageProxy) postJson(target string, json []byte) {
	client := &http.Client{}
	request, _ := http.NewRequest("POST", tableStorageProxy.baseUrl+target, bytes.NewBuffer(json))
	addPayloadHeaders(request, len(json))

	tableStorageProxy.executeRequest(request, client, target)
}

func addPayloadHeaders(request *http.Request, bodyLength int) {
	request.Header.Set("Content-Type", "application/json")
	request.Header.Set("Content-Length", string(bodyLength))
}

func (tableStorageProxy *TableStorageProxy) executeRequest(request *http.Request, client *http.Client, target string) {
	xmsdate, Authentication := tableStorageProxy.calculateDateAndAuthentication(target)

	request.Header.Set("x-ms-date", xmsdate)
	request.Header.Set("x-ms-version", "2013-08-15")
	request.Header.Set("Authorization", Authentication)

	requestDump, _ := httputil.DumpRequest(request, true)

	fmt.Printf("Request: %s\n", requestDump)

	response, _ := client.Do(request)

	responseDump, _ := httputil.DumpResponse(response, true)
	fmt.Printf("Response: %s\n", responseDump)
}

func (tableStorageProxy *TableStorageProxy) calculateDateAndAuthentication(target string) (string, string) {
	xmsdate := strings.Replace(time.Now().UTC().Add(-time.Minute).Format(time.RFC1123), "UTC", "GMT", -1)
	SignatureString := xmsdate + "\n/" + tableStorageProxy.goHaveStorage.GetAccount() + "/" + target
	Authentication := "SharedKeyLite " + tableStorageProxy.goHaveStorage.GetAccount() + ":" + computeHmac256(SignatureString, tableStorageProxy.goHaveStorage.GetKey())
	return xmsdate, Authentication
}

func computeHmac256(message string, key []byte) string {
	h := hmac.New(sha256.New, key)
	h.Write([]byte(message))
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
<|endoftext|>


<commit_before>package infrastructure

import (
	"io/ioutil"

	"gopkg.in/yaml.v2"
)

// Configuration holds the values necessary to configure the application
type Configuration struct {
	Port         string   `yaml:"port"`
	ClientID     string   `yaml:"clientID"`
	ClientSecret string   `yaml:"clientSecret"`
	RedirectURI  string   `yaml:"redirectURI"`
	APIHost      string   `yaml:"apihost"`
	Scopes       []string `yaml:"scopes,flow"`
}

// GetConfiguration reads the file with the configuration and returns an struct
// with the fields
func GetConfiguration(path string) (*Configuration, error) {

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	conf := &Configuration{}

	err = yaml.Unmarshal(data, conf)
	if err != nil {
		return nil, err
	}

	return conf, nil

}
<commit_msg>Add Salt to configuration<commit_after>package infrastructure

import (
	"io/ioutil"

	"gopkg.in/yaml.v2"
)

// Configuration holds the values necessary to configure the application
type Configuration struct {
	Port         string   `yaml:"port"`
	ClientID     string   `yaml:"clientID"`
	ClientSecret string   `yaml:"clientSecret"`
	RedirectURI  string   `yaml:"redirectURI"`
	APIHost      string   `yaml:"apihost"`
	Salt         string   `yaml:"salt"`
	Scopes       []string `yaml:"scopes,flow"`
}

// GetConfiguration reads the file with the configuration and returns an struct
// with the fields
func GetConfiguration(path string) (*Configuration, error) {

	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}

	conf := &Configuration{}

	err = yaml.Unmarshal(data, conf)
	if err != nil {
		return nil, err
	}

	return conf, nil

}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"database/sql/driver"
	"errors"
	"time"

	"github.com/concourse/atc"
)

//go:generate counterfeiter . Conn

type Conn interface {
	Begin() (*sql.Tx, error)
	Close() error
	Driver() driver.Driver
	Exec(query string, args ...interface{}) (sql.Result, error)
	Ping() error
	Prepare(query string) (*sql.Stmt, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
	SetMaxIdleConns(n int)
	SetMaxOpenConns(n int)
}

type DB interface {
	SaveTeam(team Team) (SavedTeam, error)
	GetTeamByName(teamName string) (SavedTeam, error)

	GetBuild(buildID int) (Build, bool, error)
	GetBuildInputVersionedResouces(buildID int) (SavedVersionedResources, error)
	GetBuildOutputVersionedResouces(buildID int) (SavedVersionedResources, error)
	GetAllBuilds() ([]Build, error)
	GetAllStartedBuilds() ([]Build, error)

	CreatePipe(pipeGUID string, url string) error
	GetPipe(pipeGUID string) (Pipe, error)

	CreateOneOffBuild() (Build, error)

	LeaseBuildTracking(buildID int, interval time.Duration) (Lease, bool, error)
	LeaseBuildScheduling(buildID int, interval time.Duration) (Lease, bool, error)
	LeaseCacheInvalidation(interval time.Duration) (Lease, bool, error)

	StartBuild(buildID int, engineName, engineMetadata string) (bool, error)
	FinishBuild(buildID int, status Status) error
	ErrorBuild(buildID int, cause error) error

	SaveBuildInput(buildID int, input BuildInput) (SavedVersionedResource, error)
	SaveBuildOutput(buildID int, vr VersionedResource, explicit bool) (SavedVersionedResource, error)

	GetBuildEvents(buildID int, from uint) (EventSource, error)
	SaveBuildEvent(buildID int, event atc.Event) error

	SaveBuildEngineMetadata(buildID int, engineMetadata string) error

	AbortBuild(buildID int) error
	AbortNotifier(buildID int) (Notifier, error)

	Workers() ([]WorkerInfo, error) // auto-expires workers based on ttl
	GetWorker(workerName string) (WorkerInfo, bool, error)
	SaveWorker(WorkerInfo, time.Duration) error

	FindContainersByIdentifier(ContainerIdentifier) ([]Container, error)
	GetContainer(string) (Container, bool, error)
	CreateContainer(Container, time.Duration) error
	FindContainerByIdentifier(ContainerIdentifier) (Container, bool, error)
	UpdateExpiresAtOnContainer(handle string, ttl time.Duration) error
	ReapContainer(handle string) error

	DeleteContainer(string) error

	GetConfigByBuildID(buildID int) (atc.Config, ConfigVersion, error)

	InsertVolume(data Volume) error
	GetVolumes() ([]SavedVolume, error)
	SetVolumeTTL(SavedVolume, time.Duration) error
	GetVolumeTTL(volumeHandle string) (time.Duration, error)
}

//go:generate counterfeiter . Notifier

type Notifier interface {
	Notify() <-chan struct{}
	Close() error
}

//go:generate counterfeiter . PipelinesDB

type PipelinesDB interface {
	GetAllActivePipelines() ([]SavedPipeline, error)
	GetPipelineByName(pipelineName string) (SavedPipeline, error)

	OrderPipelines([]string) error
}

//go:generate counterfeiter . ConfigDB

type ConfigDB interface {
	GetConfig(pipelineName string) (atc.Config, ConfigVersion, error)
	SaveConfig(string, string, atc.Config, ConfigVersion, PipelinePausedState) (bool, error)
}

//ConfigVersion is a sequence identifier used for compare-and-swap
type ConfigVersion int

var ErrConfigComparisonFailed = errors.New("comparison with existing config failed during save")

//go:generate counterfeiter . Lock

type Lock interface {
	Release() error
}

var ErrEndOfBuildEventStream = errors.New("end of build event stream")
var ErrBuildEventStreamClosed = errors.New("build event stream closed")

//go:generate counterfeiter . EventSource

type EventSource interface {
	Next() (atc.Event, error)
	Close() error
}

type BuildInput struct {
	Name string

	VersionedResource

	FirstOccurrence bool
}

type BuildOutput struct {
	VersionedResource
}

type VersionHistory struct {
	VersionedResource SavedVersionedResource
	InputsTo          []*JobHistory
	OutputsOf         []*JobHistory
}

type JobHistory struct {
	JobName string
	Builds  []Build
}

type WorkerInfo struct {
	GardenAddr      string
	BaggageclaimURL string

	ActiveContainers int
	ResourceTypes    []atc.WorkerResourceType
	Platform         string
	Tags             []string
	Name             string
}

type SavedVolume struct {
	Volume

	ID        int
	ExpiresIn time.Duration
}

type Volume struct {
	WorkerName      string
	TTL             time.Duration
	Handle          string
	ResourceVersion atc.Version
	ResourceHash    string
}
<commit_msg>Fix compile error by updating DB interface<commit_after>package db

import (
	"database/sql"
	"database/sql/driver"
	"errors"
	"time"

	"github.com/concourse/atc"
)

//go:generate counterfeiter . Conn

type Conn interface {
	Begin() (*sql.Tx, error)
	Close() error
	Driver() driver.Driver
	Exec(query string, args ...interface{}) (sql.Result, error)
	Ping() error
	Prepare(query string) (*sql.Stmt, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
	QueryRow(query string, args ...interface{}) *sql.Row
	SetMaxIdleConns(n int)
	SetMaxOpenConns(n int)
}

type DB interface {
	SaveTeam(team Team) (SavedTeam, error)
	GetTeamByName(teamName string) (SavedTeam, error)

	GetBuild(buildID int) (Build, bool, error)
	GetBuildInputVersionedResouces(buildID int) (SavedVersionedResources, error)
	GetBuildOutputVersionedResouces(buildID int) (SavedVersionedResources, error)
	GetBuildResources(buildID int) ([]BuildInput, []BuildOutput, error)
	GetAllBuilds() ([]Build, error)
	GetAllStartedBuilds() ([]Build, error)

	CreatePipe(pipeGUID string, url string) error
	GetPipe(pipeGUID string) (Pipe, error)

	CreateOneOffBuild() (Build, error)

	LeaseBuildTracking(buildID int, interval time.Duration) (Lease, bool, error)
	LeaseBuildScheduling(buildID int, interval time.Duration) (Lease, bool, error)
	LeaseCacheInvalidation(interval time.Duration) (Lease, bool, error)

	StartBuild(buildID int, engineName, engineMetadata string) (bool, error)
	FinishBuild(buildID int, status Status) error
	ErrorBuild(buildID int, cause error) error

	SaveBuildInput(buildID int, input BuildInput) (SavedVersionedResource, error)
	SaveBuildOutput(buildID int, vr VersionedResource, explicit bool) (SavedVersionedResource, error)

	GetBuildEvents(buildID int, from uint) (EventSource, error)
	SaveBuildEvent(buildID int, event atc.Event) error

	SaveBuildEngineMetadata(buildID int, engineMetadata string) error

	AbortBuild(buildID int) error
	AbortNotifier(buildID int) (Notifier, error)

	Workers() ([]WorkerInfo, error) // auto-expires workers based on ttl
	GetWorker(workerName string) (WorkerInfo, bool, error)
	SaveWorker(WorkerInfo, time.Duration) error

	FindContainersByIdentifier(ContainerIdentifier) ([]Container, error)
	GetContainer(string) (Container, bool, error)
	CreateContainer(Container, time.Duration) error
	FindContainerByIdentifier(ContainerIdentifier) (Container, bool, error)
	UpdateExpiresAtOnContainer(handle string, ttl time.Duration) error
	ReapContainer(handle string) error

	DeleteContainer(string) error

	GetConfigByBuildID(buildID int) (atc.Config, ConfigVersion, error)

	InsertVolume(data Volume) error
	GetVolumes() ([]SavedVolume, error)
	SetVolumeTTL(SavedVolume, time.Duration) error
	GetVolumeTTL(volumeHandle string) (time.Duration, error)
}

//go:generate counterfeiter . Notifier

type Notifier interface {
	Notify() <-chan struct{}
	Close() error
}

//go:generate counterfeiter . PipelinesDB

type PipelinesDB interface {
	GetAllActivePipelines() ([]SavedPipeline, error)
	GetPipelineByName(pipelineName string) (SavedPipeline, error)

	OrderPipelines([]string) error
}

//go:generate counterfeiter . ConfigDB

type ConfigDB interface {
	GetConfig(pipelineName string) (atc.Config, ConfigVersion, error)
	SaveConfig(string, string, atc.Config, ConfigVersion, PipelinePausedState) (bool, error)
}

//ConfigVersion is a sequence identifier used for compare-and-swap
type ConfigVersion int

var ErrConfigComparisonFailed = errors.New("comparison with existing config failed during save")

//go:generate counterfeiter . Lock

type Lock interface {
	Release() error
}

var ErrEndOfBuildEventStream = errors.New("end of build event stream")
var ErrBuildEventStreamClosed = errors.New("build event stream closed")

//go:generate counterfeiter . EventSource

type EventSource interface {
	Next() (atc.Event, error)
	Close() error
}

type BuildInput struct {
	Name string

	VersionedResource

	FirstOccurrence bool
}

type BuildOutput struct {
	VersionedResource
}

type VersionHistory struct {
	VersionedResource SavedVersionedResource
	InputsTo          []*JobHistory
	OutputsOf         []*JobHistory
}

type JobHistory struct {
	JobName string
	Builds  []Build
}

type WorkerInfo struct {
	GardenAddr      string
	BaggageclaimURL string

	ActiveContainers int
	ResourceTypes    []atc.WorkerResourceType
	Platform         string
	Tags             []string
	Name             string
}

type SavedVolume struct {
	Volume

	ID        int
	ExpiresIn time.Duration
}

type Volume struct {
	WorkerName      string
	TTL             time.Duration
	Handle          string
	ResourceVersion atc.Version
	ResourceHash    string
}
<|endoftext|>


<commit_before>package rules

import (
	"testing"

	"github.com/smartystreets/assertions/should"
	"github.com/smartystreets/gunit"
)

func TestGameFixture(t *testing.T) {
	gunit.Run(new(GameFixture), t)
}

type GameFixture struct {
	*gunit.Fixture

	game *Game
}

func (this *GameFixture) Setup() {
	this.game = NewGame()
}

func (this *GameFixture) TestStartingGameConditions() {
	this.So(this.game.IsOver(), should.BeFalse)
	this.So(this.game.PlayerToMove(), should.Equal, White)
	this.So(this.game.FullMoveCount(), should.Equal, 1)
	this.So(this.game.HalfMoveCount(), should.Equal, 0)
	this.So(this.game.CanCastleKingside(White), should.BeTrue)
	this.So(this.game.CanCastleKingside(Black), should.BeTrue)
	this.So(this.game.CanCastleQueenside(White), should.BeTrue)
	this.So(this.game.CanCastleQueenside(Black), should.BeTrue)
	this.So(this.game.ExportFEN(), should.Equal, startingPositionFEN)
}

func (this *GameFixture) TestGameConditionsAfterFirstPawnMove() {
	err := this.game.Move(Move{From: ParseSquare("a2"), To: ParseSquare("a3"), Piece: WhitePawn})

	this.So(err, should.BeNil)
	this.So(this.game.IsOver(), should.BeFalse)
	this.So(this.game.PlayerToMove(), should.Equal, Black)
	this.So(this.game.FullMoveCount(), should.Equal, 1)
	this.So(this.game.HalfMoveCount(), should.Equal, 0) // pawn move
	this.So(this.game.ExportFEN(), should.Equal, positionAfter1A3)
}

func (this *GameFixture) TestLoadFEN() {
	const kingsOnBackRanks = "4k3/8/8/8/8/8/8/4K3 w - - 0 1"
	err := this.game.LoadFEN(kingsOnBackRanks)
	this.So(err, should.BeNil)
	this.So(this.game.ExportFEN(), should.Equal, kingsOnBackRanks)
}

func (this *GameFixture) TestLegalKingMoves() {
	this.game.MustLoadFEN("8/1k6/8/p7/P7/8/1K6/8 w - - 0 1") // white king on b2, surrounding squares empty
	moves := this.game.CalculateAvailableMoves()
	this.So(moves, should.HaveLength, 8)
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b2"), To: ParseSquare("a1")})
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b2"), To: ParseSquare("a2")})
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b2"), To: ParseSquare("a3")})
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b2"), To: ParseSquare("b1")})
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b2"), To: ParseSquare("b3")})
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b2"), To: ParseSquare("c1")})
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b2"), To: ParseSquare("c2")})
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b2"), To: ParseSquare("c3")})
}

func (this *GameFixture) TestLegalKingMoves_KingOnBottomEdge() {
	this.game.MustLoadFEN("8/1k6/8/p7/P7/8/8/1K6 w - - 0 1") // white king on b1
	moves := this.game.CalculateAvailableMoves()
	this.So(moves, should.HaveLength, 5)
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b1"), To: ParseSquare("a1")})
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b1"), To: ParseSquare("c1")})
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b1"), To: ParseSquare("a2")})
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b1"), To: ParseSquare("b2")})
	this.So(moves, should.Contain, Move{Piece: WhiteKing, From: ParseSquare("b1"), To: ParseSquare("c2")})
}

func (this *GameFixture) TestLegalKingMoves_KingOnTopEdge() {
	this.game.MustLoadFEN("1k6/8/8/p7/P7/8/8/1K6 b - - 0 1") // white king on b1
	moves := this.game.CalculateAvailableMoves()
	this.So(moves, should.HaveLength, 5)
	this.So(moves, should.Contain, Move{Piece: BlackKing, From: ParseSquare("b8"), To: ParseSquare("a8")})
	this.So(moves, should.Contain, Move{Piece: BlackKing, From: ParseSquare("b8"), To: ParseSquare("c8")})
	this.So(moves, should.Contain, Move{Piece: BlackKing, From: ParseSquare("b8"), To: ParseSquare("a7")})
	this.So(moves, should.Contain, Move{Piece: BlackKing, From: ParseSquare("b8"), To: ParseSquare("b7")})
	this.So(moves, should.Contain, Move{Piece: BlackKing, From: ParseSquare("b8"), To: ParseSquare("c7")})
}

func (this *GameFixture) TestLegalKingMoves_KingOnLeftEdge() {
	this.game.MustLoadFEN("8/k7/8/p7/P7/8/8/1K6 b - - 0 1") // white king on b1
	moves := this.game.CalculateAvailableMoves()
	this.So(moves, should.HaveLength, 5)
	this.So(moves, should.Contain, Move{Piece: BlackKing, From: ParseSquare("a7"), To: ParseSquare("a8")})
	this.So(moves, should.Contain, Move{Piece: BlackKing, From: ParseSquare("a7"), To: ParseSquare("a6")})
	this.So(moves, should.Contain, Move{Piece: BlackKing, From: ParseSquare("a7"), To: ParseSquare("b8")})
	this.So(moves, should.Contain, Move{Piece: BlackKing, From: ParseSquare("a7"), To: ParseSquare("b7")})
	this.So(moves, should.Contain, Move{Piece: BlackKing, From: ParseSquare("a7"), To: ParseSquare("b6")})
}

// TODO: Rook moves (Can land vertically or horizontally as far as first obstacle that is enemy (capture), is blocked by ally)
// TODO: Bishop moves: Can land diagonally as far as first obstacle that is enemy (capture), is blocked by ally
// TODO: Knight moves: Can jump over any piece to land on empty or enemy (capture)
// TODO: Queen moves: Combined movements of Bishop and Rook
// TODO: Legal pawn moves: advance 1 rank or optionally 2 ranks if on starting square as long as ending square is empty,
// TODO: Legal pawn captures: capture diagonally or en-passant if on its "5th" rank and eligible opposing pawn target exists
// TODO: Pawn promotions
// TODO: enforce castling limitations
// TODO: detect discovered check
// TODO: prevent illegal move into check
// TODO: detect checkmate
// TODO: detect stalemate
// TODO: detect draw by insufficient material
// TODO: detect three-fold repetition
// TODO: detect 50-move rule violation
// TODO: Load/Export PGN

func (this *GameFixture) TestLegalFirstMoves() {
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("a2"), To: ParseSquare("a3"), Piece: WhitePawn}, positionAfter1A3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("a2"), To: ParseSquare("a4"), Piece: WhitePawn}, positionAfter1A4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("b2"), To: ParseSquare("b3"), Piece: WhitePawn}, positionAfter1B3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("b2"), To: ParseSquare("b4"), Piece: WhitePawn}, positionAfter1B4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("c2"), To: ParseSquare("c3"), Piece: WhitePawn}, positionAfter1C3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("c2"), To: ParseSquare("c4"), Piece: WhitePawn}, positionAfter1C4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("d2"), To: ParseSquare("d3"), Piece: WhitePawn}, positionAfter1D3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("d2"), To: ParseSquare("d4"), Piece: WhitePawn}, positionAfter1D4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("e2"), To: ParseSquare("e3"), Piece: WhitePawn}, positionAfter1E3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("e2"), To: ParseSquare("e4"), Piece: WhitePawn}, positionAfter1E4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("f2"), To: ParseSquare("f3"), Piece: WhitePawn}, positionAfter1F3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("f2"), To: ParseSquare("f4"), Piece: WhitePawn}, positionAfter1F4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("g2"), To: ParseSquare("g3"), Piece: WhitePawn}, positionAfter1G3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("g2"), To: ParseSquare("g4"), Piece: WhitePawn}, positionAfter1G4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("h2"), To: ParseSquare("h3"), Piece: WhitePawn}, positionAfter1H3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("h2"), To: ParseSquare("h4"), Piece: WhitePawn}, positionAfter1H4)
}
func (this *GameFixture) assertFirstMoveSuccessful(move Move, expectedFEN string) {
	this.game.Reset()
	err := this.game.Move(move)
	this.So(err, should.BeNil)
	this.So(this.game.ExportFEN(), should.Equal, expectedFEN)
}

const positionAfter1A3 = "rnbqkbnr/pppppppp/8/8/8/P7/1PPPPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1A4 = "rnbqkbnr/pppppppp/8/8/P7/8/1PPPPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1B3 = "rnbqkbnr/pppppppp/8/8/8/1P6/P1PPPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1B4 = "rnbqkbnr/pppppppp/8/8/1P6/8/P1PPPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1C3 = "rnbqkbnr/pppppppp/8/8/8/2P5/PP1PPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1C4 = "rnbqkbnr/pppppppp/8/8/2P5/8/PP1PPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1D3 = "rnbqkbnr/pppppppp/8/8/8/3P4/PPP1PPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1D4 = "rnbqkbnr/pppppppp/8/8/3P4/8/PPP1PPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1E3 = "rnbqkbnr/pppppppp/8/8/8/4P3/PPPP1PPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1E4 = "rnbqkbnr/pppppppp/8/8/4P3/8/PPPP1PPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1F3 = "rnbqkbnr/pppppppp/8/8/8/5P2/PPPPP1PP/RNBQKBNR b KQkq - 0 1"
const positionAfter1F4 = "rnbqkbnr/pppppppp/8/8/5P2/8/PPPPP1PP/RNBQKBNR b KQkq - 0 1"
const positionAfter1G3 = "rnbqkbnr/pppppppp/8/8/8/6P1/PPPPPP1P/RNBQKBNR b KQkq - 0 1"
const positionAfter1G4 = "rnbqkbnr/pppppppp/8/8/6P1/8/PPPPPP1P/RNBQKBNR b KQkq - 0 1"
const positionAfter1H3 = "rnbqkbnr/pppppppp/8/8/8/7P/PPPPPPP1/RNBQKBNR b KQkq - 0 1"
const positionAfter1H4 = "rnbqkbnr/pppppppp/8/8/7P/8/PPPPPPP1/RNBQKBNR b KQkq - 0 1"
<commit_msg>Extract assertion helper.<commit_after>package rules

import (
	"testing"

	"github.com/smartystreets/assertions/should"
	"github.com/smartystreets/gunit"
)

func TestGameFixture(t *testing.T) {
	gunit.Run(new(GameFixture), t)
}

type GameFixture struct {
	*gunit.Fixture

	game *Game
}

func (this *GameFixture) Setup() {
	this.game = NewGame()
}

func (this *GameFixture) TestStartingGameConditions() {
	this.So(this.game.IsOver(), should.BeFalse)
	this.So(this.game.PlayerToMove(), should.Equal, White)
	this.So(this.game.FullMoveCount(), should.Equal, 1)
	this.So(this.game.HalfMoveCount(), should.Equal, 0)
	this.So(this.game.CanCastleKingside(White), should.BeTrue)
	this.So(this.game.CanCastleKingside(Black), should.BeTrue)
	this.So(this.game.CanCastleQueenside(White), should.BeTrue)
	this.So(this.game.CanCastleQueenside(Black), should.BeTrue)
	this.So(this.game.ExportFEN(), should.Equal, startingPositionFEN)
}

func (this *GameFixture) TestGameConditionsAfterFirstPawnMove() {
	err := this.game.Move(Move{From: ParseSquare("a2"), To: ParseSquare("a3"), Piece: WhitePawn})

	this.So(err, should.BeNil)
	this.So(this.game.IsOver(), should.BeFalse)
	this.So(this.game.PlayerToMove(), should.Equal, Black)
	this.So(this.game.FullMoveCount(), should.Equal, 1)
	this.So(this.game.HalfMoveCount(), should.Equal, 0) // pawn move
	this.So(this.game.ExportFEN(), should.Equal, positionAfter1A3)
}

func (this *GameFixture) TestLoadFEN() { // TODO: test many more pieces and scenarios
	const kingsOnBackRanks = "4k3/8/8/8/8/8/8/4K3 w - - 0 1"
	err := this.game.LoadFEN(kingsOnBackRanks)
	this.So(err, should.BeNil)
	this.So(this.game.ExportFEN(), should.Equal, kingsOnBackRanks)
}

func (this *GameFixture) assertPieceMoves(piece Piece, from string, targets []string) {
	moves := this.game.CalculateAvailableMoves() // TODO: possibly only inspect moves that correspond to the provided piece
	this.So(moves, should.HaveLength, len(targets))
	for _, to := range targets {
		this.So(moves, should.Contain, Move{Piece: piece, From: ParseSquare(from), To: ParseSquare(to)})
	}
}

func (this *GameFixture) TestLegalKingMoves() {
	this.game.MustLoadFEN("8/1k6/8/p7/P7/8/1K6/8 w - - 0 1") // white king on b2, surrounding squares empty
	this.assertPieceMoves(WhiteKing, "b2", []string{"a1", "a2", "a3", "b1", "b3", "c1", "c2", "c3"})
}

func (this *GameFixture) TestLegalKingMoves_KingOnBottomEdge() {
	this.game.MustLoadFEN("8/1k6/8/p7/P7/8/8/1K6 w - - 0 1") // white king on b1
	this.assertPieceMoves(WhiteKing, "b1", []string{"a1", "c1", "a2", "b2", "c2"})
}

func (this *GameFixture) TestLegalKingMoves_KingOnTopEdge() {
	this.game.MustLoadFEN("1k6/8/8/p7/P7/8/8/1K6 b - - 0 1") // black king on b8
	this.assertPieceMoves(BlackKing, "b8", []string{"a8", "c8", "a7", "b7", "c7"})
}

func (this *GameFixture) TestLegalKingMoves_KingOnLeftEdge() {
	this.game.MustLoadFEN("8/k7/8/p7/P7/8/8/1K6 b - - 0 1") // black king on a7
	this.assertPieceMoves(BlackKing, "a7", []string{"a8", "a6", "b8", "b7", "b6"})
}

// TODO: Rook moves (Can land vertically or horizontally as far as first obstacle that is enemy (capture), is blocked by ally)
// TODO: Bishop moves: Can land diagonally as far as first obstacle that is enemy (capture), is blocked by ally
// TODO: Knight moves: Can jump over any piece to land on empty or enemy (capture)
// TODO: Queen moves: Combined movements of Bishop and Rook
// TODO: Legal pawn moves: advance 1 rank or optionally 2 ranks if on starting square as long as ending square is empty,
// TODO: Legal pawn captures: capture diagonally or en-passant if on its "5th" rank and eligible opposing pawn target exists
// TODO: Pawn promotions
// TODO: enforce castling limitations
// TODO: detect discovered check
// TODO: prevent illegal move into check
// TODO: detect checkmate
// TODO: detect stalemate
// TODO: detect draw by insufficient material
// TODO: detect three-fold repetition
// TODO: detect 50-move rule violation
// TODO: Load/Export PGN

func (this *GameFixture) SkipTestLegalFirstMoves() {
	// TODO: assert that all these moves are generated by CalculateAvailableMoves from the starting position
	// depends on basic pawn movements and knight movement
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("a2"), To: ParseSquare("a3"), Piece: WhitePawn}, positionAfter1A3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("a2"), To: ParseSquare("a4"), Piece: WhitePawn}, positionAfter1A4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("b2"), To: ParseSquare("b3"), Piece: WhitePawn}, positionAfter1B3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("b2"), To: ParseSquare("b4"), Piece: WhitePawn}, positionAfter1B4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("c2"), To: ParseSquare("c3"), Piece: WhitePawn}, positionAfter1C3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("c2"), To: ParseSquare("c4"), Piece: WhitePawn}, positionAfter1C4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("d2"), To: ParseSquare("d3"), Piece: WhitePawn}, positionAfter1D3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("d2"), To: ParseSquare("d4"), Piece: WhitePawn}, positionAfter1D4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("e2"), To: ParseSquare("e3"), Piece: WhitePawn}, positionAfter1E3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("e2"), To: ParseSquare("e4"), Piece: WhitePawn}, positionAfter1E4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("f2"), To: ParseSquare("f3"), Piece: WhitePawn}, positionAfter1F3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("f2"), To: ParseSquare("f4"), Piece: WhitePawn}, positionAfter1F4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("g2"), To: ParseSquare("g3"), Piece: WhitePawn}, positionAfter1G3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("g2"), To: ParseSquare("g4"), Piece: WhitePawn}, positionAfter1G4)

	this.assertFirstMoveSuccessful(Move{From: ParseSquare("h2"), To: ParseSquare("h3"), Piece: WhitePawn}, positionAfter1H3)
	this.assertFirstMoveSuccessful(Move{From: ParseSquare("h2"), To: ParseSquare("h4"), Piece: WhitePawn}, positionAfter1H4)
}
func (this *GameFixture) assertFirstMoveSuccessful(move Move, expectedFEN string) {
	this.game.Reset()
	err := this.game.Move(move)
	this.So(err, should.BeNil)
	this.So(this.game.ExportFEN(), should.Equal, expectedFEN)
}

const positionAfter1A3 = "rnbqkbnr/pppppppp/8/8/8/P7/1PPPPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1A4 = "rnbqkbnr/pppppppp/8/8/P7/8/1PPPPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1B3 = "rnbqkbnr/pppppppp/8/8/8/1P6/P1PPPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1B4 = "rnbqkbnr/pppppppp/8/8/1P6/8/P1PPPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1C3 = "rnbqkbnr/pppppppp/8/8/8/2P5/PP1PPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1C4 = "rnbqkbnr/pppppppp/8/8/2P5/8/PP1PPPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1D3 = "rnbqkbnr/pppppppp/8/8/8/3P4/PPP1PPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1D4 = "rnbqkbnr/pppppppp/8/8/3P4/8/PPP1PPPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1E3 = "rnbqkbnr/pppppppp/8/8/8/4P3/PPPP1PPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1E4 = "rnbqkbnr/pppppppp/8/8/4P3/8/PPPP1PPP/RNBQKBNR b KQkq - 0 1"
const positionAfter1F3 = "rnbqkbnr/pppppppp/8/8/8/5P2/PPPPP1PP/RNBQKBNR b KQkq - 0 1"
const positionAfter1F4 = "rnbqkbnr/pppppppp/8/8/5P2/8/PPPPP1PP/RNBQKBNR b KQkq - 0 1"
const positionAfter1G3 = "rnbqkbnr/pppppppp/8/8/8/6P1/PPPPPP1P/RNBQKBNR b KQkq - 0 1"
const positionAfter1G4 = "rnbqkbnr/pppppppp/8/8/6P1/8/PPPPPP1P/RNBQKBNR b KQkq - 0 1"
const positionAfter1H3 = "rnbqkbnr/pppppppp/8/8/8/7P/PPPPPPP1/RNBQKBNR b KQkq - 0 1"
const positionAfter1H4 = "rnbqkbnr/pppppppp/8/8/7P/8/PPPPPPP1/RNBQKBNR b KQkq - 0 1"
<|endoftext|>


<commit_before>// Kimberly Burke
// G00269948
//https://golang.org/
package main

import "fmt"

func main() {
	fmt.Println("Hello, 世界")
}<commit_msg>In Japanese<commit_after>// Kimberly Burke
// G00269948
//https://golang.org/
package main

import "fmt"

func main() {
	fmt.Println("こんにちは, 世界")
}<|endoftext|>


<commit_before>package cart

import (
	"io"
	"log"
	"os"
	"time"
)

var (
	green        = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white        = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow       = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red          = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue         = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta      = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan         = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset        = string([]byte{27, 91, 48, 109})
	disableColor = false
)

const ENV_CART_MODE = "CART_MODE"

const (
	DebugMode   string = "debug"
	ReleaseMode string = "release"
)
const (
	debugCode = iota
	releaseCode
)

var DefaultWriter io.Writer = os.Stdout
var DefaultErrorWriter io.Writer = os.Stderr
var cartMode = debugCode

func init() {
	mode := os.Getenv(ENV_CART_MODE)
	if len(mode) == 0 {
		SetMode(DebugMode)
	} else {
		SetMode(mode)
	}
}

func SetMode(value string) {
	switch value {
	case DebugMode:
		cartMode = debugCode
	case ReleaseMode:
		cartMode = releaseCode
	default:
		panic("Cart mode unknown: " + value)
	}
}

func init() {
	log.SetFlags(0)
}

/*
IsDebugging returns true if the framework is running in debug mode.
Use SetMode(cart.Release) to switch to disable the debug mode.
*/
func IsDebugging() bool {
	return cartMode == debugCode
}

func debugPrint(format string, values ...interface{}) {
	if IsDebugging() {
		isTerm := true

		if _, ok := DefaultWriter.(*os.File); !ok || disableColor {
			isTerm = false
		}

		var yellowColor, resetColor string
		if isTerm {
			yellowColor = yellow
			resetColor = reset
		}
		now := time.Now().Format("2006-01-02 15:04:05")
		values = append([]interface{}{yellowColor, resetColor, now}, values...)
		log.Printf("%s[CART-DEBUG]%s %v "+format, values...)
	}
}

func debugWarning() {
	debugPrint(`[WARNING] Running in "debug" mode. Switch to "release" mode in production.
 - using env:	export CART_MODE=release
 - using code:	cart.SetMode(cart.ReleaseMode)
  ██████╗ █████╗ ██████╗ ████████╗
 ██╔════╝██╔══██╗██╔══██╗╚══██╔══╝
 ██║     ███████║██████╔╝   ██║
 ██║     ██╔══██║██╔══██╗   ██║
 ╚██████╗██║  ██║██║  ██║   ██║
  ╚═════╝╚═╝  ╚═╝╚═╝  ╚═╝   ╚═╝
`)
}

func debugError(err error) {
	if err != nil {
		if IsDebugging() {
			isTerm := true

			if _, ok := DefaultWriter.(*os.File); !ok || disableColor {
				isTerm = false
			}
			var redColor, resetColor string
			if isTerm {
				redColor = red
				resetColor = reset
			}
			now := time.Now().Format("2006-01-02 15:04:05")
			values := append([]interface{}{redColor, resetColor, now, err})
			log.Printf("%s[CART-ERROR]%s %v %v", values...)
		}
	}
}
<commit_msg>add easy log<commit_after>package cart

import (
	"io"
	"log"
	"os"
)

var (
	green        = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white        = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow       = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red          = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue         = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta      = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan         = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset        = string([]byte{27, 91, 48, 109})
	disableColor = false
)

const ENV_CART_MODE = "CART_MODE"

const (
	DebugMode   string = "debug"
	ReleaseMode string = "release"
)
const (
	debugCode = iota
	releaseCode
)

var DefaultWriter io.Writer = os.Stdout
var DefaultErrorWriter io.Writer = os.Stderr
var cartMode = debugCode

func init() {
	mode := os.Getenv(ENV_CART_MODE)
	if len(mode) == 0 {
		SetMode(DebugMode)
	} else {
		SetMode(mode)
	}
}

func SetMode(value string) {
	switch value {
	case DebugMode:
		cartMode = debugCode
	case ReleaseMode:
		cartMode = releaseCode
	default:
		panic("Cart mode unknown: " + value)
	}
}

func init() {
}

/*
IsDebugging returns true if the framework is running in debug mode.
Use SetMode(cart.Release) to switch to disable the debug mode.
*/
func IsDebugging() bool {
	return cartMode == debugCode
}

func debugPrint(format string, values ...interface{}) {
	if IsDebugging() {
		isTerm := true

		if _, ok := DefaultWriter.(*os.File); !ok || disableColor {
			isTerm = false
		}

		var yellowColor, resetColor string
		if isTerm {
			yellowColor = yellow
			resetColor = reset
		}
		values = append([]interface{}{yellowColor, resetColor}, values...)
		log.Printf("%s[CART-DEBUG]%s %v "+format, values...)
	}
}

func debugWarning() {
	debugPrint(`[WARNING] Running in "debug" mode. Switch to "release" mode in production.
 - using env:	export CART_MODE=release
 - using code:	cart.SetMode(cart.ReleaseMode)
  ██████╗ █████╗ ██████╗ ████████╗
 ██╔════╝██╔══██╗██╔══██╗╚══██╔══╝
 ██║     ███████║██████╔╝   ██║
 ██║     ██╔══██║██╔══██╗   ██║
 ╚██████╗██║  ██║██║  ██║   ██║
  ╚═════╝╚═╝  ╚═╝╚═╝  ╚═╝   ╚═╝
`)
}

func debugError(err error) {
	if err != nil {
		if IsDebugging() {
			isTerm := true

			if _, ok := DefaultWriter.(*os.File); !ok || disableColor {
				isTerm = false
			}
			var redColor, resetColor string
			if isTerm {
				redColor = red
				resetColor = reset
			}
			values := append([]interface{}{redColor, resetColor, err})
			log.Printf("%s[CART-ERROR]%s %v %v", values...)
		}
	}
}
<|endoftext|>


<commit_before>package karatsuba

import (
	//"fmt"
	"math/big"
)

const (
	THRESHOLD = 1000
)

func Multiply(x, y *big.Int) *big.Int {

	m := min(x.BitLen(), y.BitLen()) / 2

	if m < THRESHOLD {
		z := big.NewInt(0)
		return z.Mul(x, y)
	}

	return big.NewInt(0)
}

func split(x *big.Int, m uint) []*big.Int {
	left := bigint(int64(x.Uint64())).Rsh(x, m)

	t1 := bigint(left.Int64())
	t1 = t1.Lsh(t1, m)

	t3 := bigint(x.Int64())
	right := t3.Sub(t3, t1)

	return []*big.Int{left, right}
}

func bigint(n int64) *big.Int {
	return big.NewInt(int64(n))
}

func min(a, b int) int {
	if a > b {
		return b
	}
	return a
}
<commit_msg>Fixes for escapes. Totally premature...<commit_after>package karatsuba

import (
	//"fmt"
	"math/big"
)

const (
	THRESHOLD = 1000
)

func Multiply(x, y *big.Int) *big.Int {

	x1 := big.NewInt(x.Int64())
	y1 := big.NewInt(x.Int64())

	m := min(x1.BitLen(), y1.BitLen()) / 2

	if m < THRESHOLD {
		z := big.NewInt(0)
		return z.Mul(x1, y1)
	}

	return big.NewInt(0)
}

func split(x *big.Int, m uint) []*big.Int {
	x1 := big.NewInt(x.Int64())
	left := bigint(int64(x.Uint64())).Rsh(x1, m)

	t1 := bigint(left.Int64())
	t1 = t1.Lsh(t1, m)

	t3 := bigint(x.Int64())
	right := t3.Sub(t3, t1)

	return []*big.Int{left, right}
}

func bigint(n int64) *big.Int {
	return big.NewInt(n)
}

func min(a, b int) int {
	if a > b {
		return b
	}
	return a
}
<|endoftext|>


<commit_before>package cmd

import (
	"errors"
	"os"

	"github.com/spf13/cobra"
)

func init() {
	root.AddCommand(completionCmd)
}

// completionCmd represents the completion command
var completionCmd = &cobra.Command{
	Use:   "completion <bash|zsh>",
	Short: "Generate bash/zsh completion scripts",
	Long: `To load completion run

For zsh:
source <(faas completion zsh)

If you would like to use alias:
alias f=faas
compdef _faas f

For bash:
source <(faas completion bash)

`,
	ValidArgs: []string{"bash", "zsh"},
	Args:      cobra.ExactArgs(1),
	RunE: func(cmd *cobra.Command, args []string) (err error) {
		if len(args) < 1 {
			return errors.New("missing argument")
		}
		if args[0] == "bash" {
			err = root.GenBashCompletion(os.Stdout)
			return err
		}
		if args[0] == "zsh" {
			// manually edited script based on `root.GenZshCompletion(os.Stdout)`
			// unfortunately it doesn't support completion so well as for bash
			// some manual edits had to be done
			os.Stdout.WriteString(`
compdef _faas faas

function _faas {
  local -a commands

  _arguments -C \
    '--config[config file path]:file:_files' \
    '(-v --verbose)'{-v,--verbose}'[print verbose logs]' \
    "1: :->cmnds" \
    "*::arg:->args"

  case $state in
  cmnds)
    commands=(
      "completion:Generates bash/zsh completion scripts"
      "create:Create a Function"
      "delete:Delete deployed Function"
      "describe:Describe Function"
      "help:Help about any command"
      "list:Lists deployed Functions"
      "run:Run Function locally"
      "update:Update or create a deployed Function"
      "version:Print version"
    )
    _describe "command" commands
    ;;
  esac

  case "$words[1]" in
  completion)
    _faas_completion
    ;;
  create)
    _faas_create
    ;;
  delete)
    _faas_delete
    ;;
  describe)
    _faas_describe
    ;;
  help)
    _faas_help
    ;;
  list)
    _faas_list
    ;;
  run)
    _faas_run
    ;;
  update)
    _faas_update
    ;;
  version)
    _faas_version
    ;;
  esac
}

function _list_funs() {
    compadd $(faas list 2> /dev/null)
}

function _list_langs() {
		compadd node go quarkus
}

function _list_fmts() {
    compadd yaml xml json
}

function _list_regs() {
    local config="${HOME}/.docker/config.json"
    if command -v yq >/dev/null && test -f "$config";  then
		compadd $(jq -r ".auths | keys[] " "$config")
	fi
}

function _faas_completion {
  _arguments \
    '(-h --help)'{-h,--help}'[help for completion]' \
    '--config[config file path]:file:_files' \
    '(-v --verbose)'{-v,--verbose}'[print verbose logs]' \
    '1: :("bash" "zsh")'
}

function _faas_create {
  _arguments \
    '1:string:_list_langs' \
    '(-i --internal)'{-i,--internal}'[Create a cluster-local service without a publicly accessible route. $FAAS_INTERNAL]' \
    '(-l --local)'{-l,--local}'[create the service function locally only.]' \
    '(-n --name)'{-n,--name}'[optionally specify an explicit name for the serive, overriding path-derivation. $FAAS_NAME]:' \
    '(-s --namespace)'{-s,--namespace}'[namespace at image registry (usually username or org name). $FAAS_NAMESPACE]:' \
    '(-r --registry)'{-r,--registry}'[image registry (ex: quay.io). $FAAS_REGISTRY]:string:_list_regs' \
    '--config[config file path]:file:_files' \
    '(-v --verbose)'{-v,--verbose}'[print verbose logs]'
}

function _faas_delete {
  _arguments \
    '(-n --name)'{-n,--name}'[optionally specify an explicit name to remove, overriding path-derivation. $FAAS_NAME]:string:_list_funs' \
    '--config[config file path]:file:_files' \
    '(-v --verbose)'{-v,--verbose}'[print verbose logs]'
}

function _faas_describe {
  _arguments \
    '1:string:_list_funs' \
    '(-n --name)'{-n,--name}'[optionally specify an explicit name for the serive, overriding path-derivation. $FAAS_NAME]:string:_list_funs' \
    '(-o --output)'{-o,--output}'[optionally specify output format (yaml,xml,json).]:string:_list_fmts' \
    '--config[config file path]:file:_files' \
    '(-v --verbose)'{-v,--verbose}'[print verbose logs]'
}

function _faas_help {
  _arguments \
    '--config[config file path]:file:_files' \
    '(-v --verbose)'{-v,--verbose}'[print verbose logs]'
}

function _faas_list {
  _arguments \
    '--config[config file path]:file:_files' \
    '(-v --verbose)'{-v,--verbose}'[print verbose logs]'
}

function _faas_run {
  _arguments \
    '--config[config file path]:file:_files' \
    '(-v --verbose)'{-v,--verbose}'[print verbose logs]'
}

function _faas_update {
  _arguments \
    '--config[config file path]:file:_files' \
    '(-s --namespace)'{-s,--namespace}'[namespace at image registry (usually username or org name). $FAAS_NAMESPACE]:' \
    '(-r --registry)'{-r,--registry}'[image registry (ex: quay.io). $FAAS_REGISTRY]:string:_list_regs' \
    '(-v --verbose)'{-v,--verbose}'[print verbose logs]'
}

function _faas_version {
  _arguments \
    '--config[config file path]:file:_files' \
    '(-v --verbose)'{-v,--verbose}'[print verbose logs]'
}

`)
			return nil
		}
		return errors.New("unknown shell, only bash and zsh are supported")
	},
}
<commit_msg>fix: stop using manually edited completion<commit_after>package cmd

import (
	"errors"
	"os"

	"github.com/spf13/cobra"
)

func init() {
	root.AddCommand(completionCmd)
}

// completionCmd represents the completion command
var completionCmd = &cobra.Command{
	Use:   "completion <bash|zsh>",
	Short: "Generate bash/zsh completion scripts",
	Long: `To load completion run

For zsh:
source <(faas completion zsh)

If you would like to use alias:
alias f=faas
compdef _faas f

For bash:
source <(faas completion bash)

`,
	ValidArgs: []string{"bash", "zsh"},
	Args:      cobra.ExactArgs(1),
	RunE: func(cmd *cobra.Command, args []string) (err error) {
		if len(args) < 1 {
			return errors.New("missing argument")
		}
		if args[0] == "bash" {
			err = root.GenBashCompletion(os.Stdout)
			return err
		}
		if args[0] == "zsh" {
			err = root.GenZshCompletion(os.Stdout)
			return err
		}
		return errors.New("unknown shell, only bash and zsh are supported")
	},
}
<|endoftext|>


<commit_before>package cmd

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/spf13/cobra"
	"github.com/stormforger/cli/api/organisation"
	"github.com/stormforger/cli/api/testcase"
)

var completionCmd = &cobra.Command{
	Use:   "completion [bash|zsh|fish|powershell]",
	Short: "Generate completion script",
	Long: `To load completions:

Bash:

$ source <(yourprogram completion bash)

# To load completions for each session, execute once:
Linux:
  $ yourprogram completion bash > /etc/bash_completion.d/yourprogram
MacOS:
  $ yourprogram completion bash > /usr/local/etc/bash_completion.d/yourprogram

Zsh:

# If shell completion is not already enabled in your environment you will need
# to enable it.  You can execute the following once:

$ echo "autoload -U compinit; compinit" >> ~/.zshrc

# To load completions for each session, execute once:
$ yourprogram completion zsh > "${fpath[1]}/_yourprogram"

# You will need to start a new shell for this setup to take effect.

Fish:

$ yourprogram completion fish | source

# To load completions for each session, execute once:
$ yourprogram completion fish > ~/.config/fish/completions/yourprogram.fish
`,
	DisableFlagsInUseLine: true,
	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
	Args:                  cobra.ExactValidArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		switch args[0] {
		case "bash":
			cmd.Root().GenBashCompletion(os.Stdout)
		case "zsh":
			cmd.Root().GenZshCompletion(os.Stdout)
		case "fish":
			cmd.Root().GenFishCompletion(os.Stdout, true)
		case "powershell":
			cmd.Root().GenPowerShellCompletion(os.Stdout)
		}
	},
}

func init() {
	RootCmd.AddCommand(completionCmd)
}

func completeOrgaAndCase(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
	if strings.Contains(toComplete, "/") {
		return completionTestCases(toComplete), cobra.ShellCompDirectiveNoFileComp
	}
	return completionOrganisations(toComplete, "/"), cobra.ShellCompDirectiveNoFileComp | cobra.ShellCompDirectiveNoSpace
}

func completeOrga(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
	return completionOrganisations(toComplete, ""), cobra.ShellCompDirectiveNoFileComp
}

func completionOrganisations(toComplete, suffix string) []string {
	client := NewClient()

	success, result, err := client.ListOrganisations()
	if err != nil {
		log.Fatal(err)
	}
	if !success {
		log.Fatal(string(result))
	}

	items, err := organisation.Unmarshal(bytes.NewReader(result))
	if err != nil {
		log.Fatal(err)
	}

	out := []string{}
	for _, item := range items.Organisations {
		if toComplete == "" || strings.HasPrefix(item.Name, toComplete) {
			out = append(out, item.Name+suffix)
		}
	}

	return out
}

func completionTestCases(toComplete string) []string {
	client := NewClient()

	x := strings.Split(toComplete, "/")
	orgaName := x[0]
	tcPrefix := x[1]

	orgaUID := lookupOrganisationUID(client, orgaName)

	status, result, err := client.ListTestCases(orgaUID, "")
	if err != nil {
		log.Fatal(err)
	}

	if !status {
		fmt.Fprintln(os.Stderr, "Could not list test cases for "+orgaUID)
		fmt.Fprintln(os.Stderr, string(result))

		os.Exit(1)
	}

	items, err := testcase.Unmarshal(bytes.NewReader(result))
	if err != nil {
		log.Fatal(err)
	}

	out := []string{}
	for _, item := range items.TestCases {
		if strings.HasPrefix(item.Name, tcPrefix) {
			out = append(out, orgaName+"/"+item.Name)
		}
	}

	return out
}
<commit_msg>fix: Fix forge app name in completion docs (#284)<commit_after>package cmd

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/spf13/cobra"
	"github.com/stormforger/cli/api/organisation"
	"github.com/stormforger/cli/api/testcase"
)

var completionCmd = &cobra.Command{
	Use:   "completion [bash|zsh|fish|powershell]",
	Short: "Generate completion script",
	Long: `To load completions:

Bash:

$ source <(forge completion bash)

# To load completions for each session, execute once:
Linux:
  $ forge completion bash > /etc/bash_completion.d/forge
MacOS:
  $ forge completion bash > /usr/local/etc/bash_completion.d/forge

Zsh:

# If shell completion is not already enabled in your environment you will need
# to enable it.  You can execute the following once:

$ echo "autoload -U compinit; compinit" >> ~/.zshrc

# To load completions for each session, execute once:
$ forge completion zsh > "${fpath[1]}/_forge"

# You will need to start a new shell for this setup to take effect.

Fish:

$ forge completion fish | source

# To load completions for each session, execute once:
$ forge completion fish > ~/.config/fish/completions/forge.fish
`,
	DisableFlagsInUseLine: true,
	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
	Args:                  cobra.ExactValidArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		switch args[0] {
		case "bash":
			cmd.Root().GenBashCompletion(os.Stdout)
		case "zsh":
			cmd.Root().GenZshCompletion(os.Stdout)
		case "fish":
			cmd.Root().GenFishCompletion(os.Stdout, true)
		case "powershell":
			cmd.Root().GenPowerShellCompletion(os.Stdout)
		}
	},
}

func init() {
	RootCmd.AddCommand(completionCmd)
}

func completeOrgaAndCase(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
	if strings.Contains(toComplete, "/") {
		return completionTestCases(toComplete), cobra.ShellCompDirectiveNoFileComp
	}
	return completionOrganisations(toComplete, "/"), cobra.ShellCompDirectiveNoFileComp | cobra.ShellCompDirectiveNoSpace
}

func completeOrga(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
	return completionOrganisations(toComplete, ""), cobra.ShellCompDirectiveNoFileComp
}

func completionOrganisations(toComplete, suffix string) []string {
	client := NewClient()

	success, result, err := client.ListOrganisations()
	if err != nil {
		log.Fatal(err)
	}
	if !success {
		log.Fatal(string(result))
	}

	items, err := organisation.Unmarshal(bytes.NewReader(result))
	if err != nil {
		log.Fatal(err)
	}

	out := []string{}
	for _, item := range items.Organisations {
		if toComplete == "" || strings.HasPrefix(item.Name, toComplete) {
			out = append(out, item.Name+suffix)
		}
	}

	return out
}

func completionTestCases(toComplete string) []string {
	client := NewClient()

	x := strings.Split(toComplete, "/")
	orgaName := x[0]
	tcPrefix := x[1]

	orgaUID := lookupOrganisationUID(client, orgaName)

	status, result, err := client.ListTestCases(orgaUID, "")
	if err != nil {
		log.Fatal(err)
	}

	if !status {
		fmt.Fprintln(os.Stderr, "Could not list test cases for "+orgaUID)
		fmt.Fprintln(os.Stderr, string(result))

		os.Exit(1)
	}

	items, err := testcase.Unmarshal(bytes.NewReader(result))
	if err != nil {
		log.Fatal(err)
	}

	out := []string{}
	for _, item := range items.TestCases {
		if strings.HasPrefix(item.Name, tcPrefix) {
			out = append(out, orgaName+"/"+item.Name)
		}
	}

	return out
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
)

// htmlOutput reads the profile data from profile and generates an HTML
// coverage report, writing it to outfile. If outfile is empty,
// it writes the report to a temporary file and opens it in a web browser.
func htmlOutput(profile, outfile string) error {
	pf, err := os.Open(profile)
	if err != nil {
		return err
	}
	defer pf.Close()

	profiles, err := ParseProfiles(pf)
	if err != nil {
		return err
	}

	var d templateData

	for fn, profile := range profiles {
		if profile.Mode == "set" {
			d.Set = true
		}
		file, err := findFile(fn)
		if err != nil {
			return err
		}
		src, err := ioutil.ReadFile(file)
		if err != nil {
			return fmt.Errorf("can't read %q: %v", fn, err)
		}
		var buf bytes.Buffer
		err = htmlGen(&buf, src, profile.Boundaries(src))
		if err != nil {
			return err
		}
		d.Files = append(d.Files, &templateFile{
			Name: fn,
			Body: template.HTML(buf.String()),
		})
	}

	var out *os.File
	if outfile == "" {
		var dir string
		dir, err = ioutil.TempDir("", "cover")
		if err != nil {
			return err
		}
		out, err = os.Create(filepath.Join(dir, "coverage.html"))
	} else {
		out, err = os.Create(outfile)
	}
	err = htmlTemplate.Execute(out, d)
	if err == nil {
		err = out.Close()
	}
	if err != nil {
		return err
	}

	if outfile == "" {
		if !startBrowser("file://" + out.Name()) {
			fmt.Fprintf(os.Stderr, "HTML output written to %s\n", out.Name())
		}
	}

	return nil
}

// htmlGen generates an HTML coverage report with the provided filename,
// source code, and tokens, and writes it to the given Writer.
func htmlGen(w io.Writer, src []byte, boundaries []Boundary) error {
	dst := bufio.NewWriter(w)
	for i := range src {
		for len(boundaries) > 0 && boundaries[0].Offset == i {
			b := boundaries[0]
			if b.Start {
				n := 0
				if b.Count > 0 {
					n = int(math.Floor(b.Norm*9)) + 1
				}
				fmt.Fprintf(dst, `<span class="cov%v" title="%v">`, n, b.Count)
			} else {
				dst.WriteString("")
			}
			boundaries = boundaries[1:]
		}
		switch b := src[i]; b {
		case '>':
			dst.WriteString(">")
		case '<':
			dst.WriteString("<")
		case '&':
			dst.WriteString("&")
		case '\t':
			dst.WriteString("        ")
		default:
			dst.WriteByte(b)
		}
	}
	return dst.Flush()
}

// startBrowser tries to open the URL in a browser
// and reports whether it succeeds.
func startBrowser(url string) bool {
	// try to start the browser
	var args []string
	switch runtime.GOOS {
	case "darwin":
		args = []string{"open"}
	case "windows":
		args = []string{"cmd", "/c", "start"}
	default:
		args = []string{"xdg-open"}
	}
	cmd := exec.Command(args[0], append(args[1:], url)...)
	return cmd.Start() == nil
}

// rgb returns an rgb value for the specified coverage value
// between 0 (no coverage) and 10 (max coverage).
func rgb(n int) string {
	if n == 0 {
		return "rgb(192, 0, 0)" // Red
	}
	// Gradient from gray to green.
	r := 128 - 12*(n-1)
	g := 128 + 12*(n-1)
	b := 128 + 3*(n-1)
	return fmt.Sprintf("rgb(%v, %v, %v)", r, g, b)
}

// colors generates the CSS rules for coverage colors.
func colors() template.CSS {
	var buf bytes.Buffer
	for i := 0; i < 11; i++ {
		fmt.Fprintf(&buf, ".cov%v { color: %v }\n", i, rgb(i))
	}
	return template.CSS(buf.String())
}

var htmlTemplate = template.Must(template.New("html").Funcs(template.FuncMap{
	"colors": colors,
}).Parse(tmplHTML))

type templateData struct {
	Files []*templateFile
	Set   bool
}

type templateFile struct {
	Name string
	Body template.HTML
}

const tmplHTML = `
<!DOCTYPE html>

				<select id="files">
				{{range $i, $f := .Files}}
				<option value="file{{$i}}">{{$f.Name}}</option>
				{{end}}
				</select>

				not tracked
			{{if .Set}}
				not covered
				covered
			{{else}}
				no coverage
				low coverage
				*
				*
				*
				*
				*
				*
				*
				*
				high coverage
			{{end}}

		{{range $i, $f := .Files}}

{{$f.Body}}

		{{end}}

`
<commit_msg>go.tools/cmd/cover: add content-type meta tag to HTML output<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
)

// htmlOutput reads the profile data from profile and generates an HTML
// coverage report, writing it to outfile. If outfile is empty,
// it writes the report to a temporary file and opens it in a web browser.
func htmlOutput(profile, outfile string) error {
	pf, err := os.Open(profile)
	if err != nil {
		return err
	}
	defer pf.Close()

	profiles, err := ParseProfiles(pf)
	if err != nil {
		return err
	}

	var d templateData

	for fn, profile := range profiles {
		if profile.Mode == "set" {
			d.Set = true
		}
		file, err := findFile(fn)
		if err != nil {
			return err
		}
		src, err := ioutil.ReadFile(file)
		if err != nil {
			return fmt.Errorf("can't read %q: %v", fn, err)
		}
		var buf bytes.Buffer
		err = htmlGen(&buf, src, profile.Boundaries(src))
		if err != nil {
			return err
		}
		d.Files = append(d.Files, &templateFile{
			Name: fn,
			Body: template.HTML(buf.String()),
		})
	}

	var out *os.File
	if outfile == "" {
		var dir string
		dir, err = ioutil.TempDir("", "cover")
		if err != nil {
			return err
		}
		out, err = os.Create(filepath.Join(dir, "coverage.html"))
	} else {
		out, err = os.Create(outfile)
	}
	err = htmlTemplate.Execute(out, d)
	if err == nil {
		err = out.Close()
	}
	if err != nil {
		return err
	}

	if outfile == "" {
		if !startBrowser("file://" + out.Name()) {
			fmt.Fprintf(os.Stderr, "HTML output written to %s\n", out.Name())
		}
	}

	return nil
}

// htmlGen generates an HTML coverage report with the provided filename,
// source code, and tokens, and writes it to the given Writer.
func htmlGen(w io.Writer, src []byte, boundaries []Boundary) error {
	dst := bufio.NewWriter(w)
	for i := range src {
		for len(boundaries) > 0 && boundaries[0].Offset == i {
			b := boundaries[0]
			if b.Start {
				n := 0
				if b.Count > 0 {
					n = int(math.Floor(b.Norm*9)) + 1
				}
				fmt.Fprintf(dst, `<span class="cov%v" title="%v">`, n, b.Count)
			} else {
				dst.WriteString("")
			}
			boundaries = boundaries[1:]
		}
		switch b := src[i]; b {
		case '>':
			dst.WriteString(">")
		case '<':
			dst.WriteString("<")
		case '&':
			dst.WriteString("&")
		case '\t':
			dst.WriteString("        ")
		default:
			dst.WriteByte(b)
		}
	}
	return dst.Flush()
}

// startBrowser tries to open the URL in a browser
// and reports whether it succeeds.
func startBrowser(url string) bool {
	// try to start the browser
	var args []string
	switch runtime.GOOS {
	case "darwin":
		args = []string{"open"}
	case "windows":
		args = []string{"cmd", "/c", "start"}
	default:
		args = []string{"xdg-open"}
	}
	cmd := exec.Command(args[0], append(args[1:], url)...)
	return cmd.Start() == nil
}

// rgb returns an rgb value for the specified coverage value
// between 0 (no coverage) and 10 (max coverage).
func rgb(n int) string {
	if n == 0 {
		return "rgb(192, 0, 0)" // Red
	}
	// Gradient from gray to green.
	r := 128 - 12*(n-1)
	g := 128 + 12*(n-1)
	b := 128 + 3*(n-1)
	return fmt.Sprintf("rgb(%v, %v, %v)", r, g, b)
}

// colors generates the CSS rules for coverage colors.
func colors() template.CSS {
	var buf bytes.Buffer
	for i := 0; i < 11; i++ {
		fmt.Fprintf(&buf, ".cov%v { color: %v }\n", i, rgb(i))
	}
	return template.CSS(buf.String())
}

var htmlTemplate = template.Must(template.New("html").Funcs(template.FuncMap{
	"colors": colors,
}).Parse(tmplHTML))

type templateData struct {
	Files []*templateFile
	Set   bool
}

type templateFile struct {
	Name string
	Body template.HTML
}

const tmplHTML = `
<!DOCTYPE html>

		<meta http-equiv="Content-Type" content="text/html; charset=utf-8">

				<select id="files">
				{{range $i, $f := .Files}}
				<option value="file{{$i}}">{{$f.Name}}</option>
				{{end}}
				</select>

				not tracked
			{{if .Set}}
				not covered
				covered
			{{else}}
				no coverage
				low coverage
				*
				*
				*
				*
				*
				*
				*
				*
				high coverage
			{{end}}

		{{range $i, $f := .Files}}

{{$f.Body}}

		{{end}}

`
<|endoftext|>


<commit_before>/*
 * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"time"

	jsoniter "github.com/json-iterator/go"
	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/hash"
)

const (
	dataUsageObjName       = "data-usage"
	dataUsageCrawlInterval = 12 * time.Hour
)

func initDataUsageStats() {
	go runDataUsageInfoUpdateRoutine()
}

func runDataUsageInfoUpdateRoutine() {
	// Wait until the object layer is ready
	var objAPI ObjectLayer
	for {
		objAPI = newObjectLayerWithoutSafeModeFn()
		if objAPI == nil {
			time.Sleep(time.Second)
			continue
		}
		break
	}

	runDataUsageInfo(context.Background(), objAPI, GlobalServiceDoneCh)
}

// timeToNextCrawl returns the duration until next crawl should occur
// this is validated by verifying the LastUpdate time.
func timeToCrawl(ctx context.Context, objAPI ObjectLayer) time.Duration {
	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objAPI)
	if err != nil {
		// Upon an error wait for like 10
		// seconds to start the crawler.
		return 10 * time.Second
	}
	// File indeed doesn't exist when LastUpdate is zero
	// so we have never crawled, start crawl right away.
	if dataUsageInfo.LastUpdate.IsZero() {
		return 1 * time.Second
	}
	waitDuration := dataUsageInfo.LastUpdate.Sub(UTCNow())
	if waitDuration > dataUsageCrawlInterval {
		// Waited long enough start crawl in a 1 second
		return 1 * time.Second
	}
	// No crawling needed, ask the routine to wait until
	// the daily interval 12hrs - delta between last update
	// with current time.
	return dataUsageCrawlInterval - waitDuration
}

var dataUsageLockTimeout = lifecycleLockTimeout

func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer, endCh <-chan struct{}) {
	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
	for {
		err := locker.GetLock(dataUsageLockTimeout)
		if err != nil {
			time.Sleep(5 * time.Minute)
			continue
		}
		// Break without unlocking, this node will acquire
		// data usage calculator role for its lifetime.
		break
	}

	for {
		wait := timeToCrawl(ctx, objAPI)
		select {
		case <-endCh:
			locker.Unlock()
			return
		case <-time.NewTimer(wait).C:
			// Crawl only when no previous crawl has occurred,
			// or its been too long since last crawl.
			err := storeDataUsageInBackend(ctx, objAPI, objAPI.CrawlAndGetDataUsage(ctx, endCh))
			logger.LogIf(ctx, err)
		}
	}
}

func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dataUsageInfo DataUsageInfo) error {
	dataUsageJSON, err := json.Marshal(dataUsageInfo)
	if err != nil {
		return err
	}

	size := int64(len(dataUsageJSON))
	r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
	if err != nil {
		return err
	}

	_, err = objAPI.PutObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
	return err
}

func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
	var dataUsageInfoJSON bytes.Buffer

	err := objAPI.GetObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
	if err != nil {
		if isErrObjectNotFound(err) {
			return DataUsageInfo{}, nil
		}
		return DataUsageInfo{}, toObjectErr(err, minioMetaBackgroundOpsBucket, dataUsageObjName)
	}

	var dataUsageInfo DataUsageInfo
	var json = jsoniter.ConfigCompatibleWithStandardLibrary
	err = json.Unmarshal(dataUsageInfoJSON.Bytes(), &dataUsageInfo)
	if err != nil {
		return DataUsageInfo{}, err
	}

	return dataUsageInfo, nil
}

// Item represents each file while walking.
type Item struct {
	Path string
	Typ  os.FileMode
}

type getSizeFn func(item Item) (int64, error)

func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO func(), getSize getSizeFn) DataUsageInfo {
	var dataUsageInfo = DataUsageInfo{
		BucketsSizes:          make(map[string]uint64),
		ObjectsSizesHistogram: make(map[string]uint64),
	}

	fastWalk(basePath, 1, doneCh, func(path string, typ os.FileMode) error {
		// Wait for I/O to go down.
		waitForLowActiveIO()

		bucket, entry := path2BucketObjectWithBasePath(basePath, path)
		if bucket == "" {
			return nil
		}

		if isReservedOrInvalidBucket(bucket, false) {
			return filepath.SkipDir
		}

		if entry == "" && typ&os.ModeDir != 0 {
			dataUsageInfo.BucketsCount++
			dataUsageInfo.BucketsSizes[bucket] = 0
			return nil
		}

		if typ&os.ModeDir != 0 {
			return nil
		}

		t := time.Now()
		size, err := getSize(Item{path, typ})
		// Use the response time of the getSize call to guess system load.
		// Sleep equivalent time.
		if d := time.Since(t); d > 100*time.Microsecond {
			time.Sleep(d)
		}
		if err != nil {
			return errSkipFile
		}

		dataUsageInfo.ObjectsCount++
		dataUsageInfo.ObjectsTotalSize += uint64(size)
		dataUsageInfo.BucketsSizes[bucket] += uint64(size)
		dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(size))]++
		return nil
	})

	return dataUsageInfo
}
<commit_msg>Add env. variable to disable data usage crawling (#9086)<commit_after>/*
 * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"context"
	"encoding/json"
	"os"
	"path/filepath"
	"time"

	jsoniter "github.com/json-iterator/go"
	"github.com/minio/minio/cmd/config"
	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/env"
	"github.com/minio/minio/pkg/hash"
)

const (
	dataUsageObjName       = "data-usage"
	dataUsageCrawlInterval = 12 * time.Hour
	dataUsageCrawlConf     = "MINIO_DISK_USAGE_CRAWL"
)

func initDataUsageStats() {
	dataUsageEnabled, err := config.ParseBool(env.Get(dataUsageCrawlConf, config.EnableOn))
	if err == nil && !dataUsageEnabled {
		return
	}
	go runDataUsageInfoUpdateRoutine()
}

func runDataUsageInfoUpdateRoutine() {
	// Wait until the object layer is ready
	var objAPI ObjectLayer
	for {
		objAPI = newObjectLayerWithoutSafeModeFn()
		if objAPI == nil {
			time.Sleep(time.Second)
			continue
		}
		break
	}

	runDataUsageInfo(context.Background(), objAPI, GlobalServiceDoneCh)
}

// timeToNextCrawl returns the duration until next crawl should occur
// this is validated by verifying the LastUpdate time.
func timeToCrawl(ctx context.Context, objAPI ObjectLayer) time.Duration {
	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objAPI)
	if err != nil {
		// Upon an error wait for like 10
		// seconds to start the crawler.
		return 10 * time.Second
	}
	// File indeed doesn't exist when LastUpdate is zero
	// so we have never crawled, start crawl right away.
	if dataUsageInfo.LastUpdate.IsZero() {
		return 1 * time.Second
	}
	waitDuration := dataUsageInfo.LastUpdate.Sub(UTCNow())
	if waitDuration > dataUsageCrawlInterval {
		// Waited long enough start crawl in a 1 second
		return 1 * time.Second
	}
	// No crawling needed, ask the routine to wait until
	// the daily interval 12hrs - delta between last update
	// with current time.
	return dataUsageCrawlInterval - waitDuration
}

var dataUsageLockTimeout = lifecycleLockTimeout

func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer, endCh <-chan struct{}) {
	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
	for {
		err := locker.GetLock(dataUsageLockTimeout)
		if err != nil {
			time.Sleep(5 * time.Minute)
			continue
		}
		// Break without unlocking, this node will acquire
		// data usage calculator role for its lifetime.
		break
	}

	for {
		wait := timeToCrawl(ctx, objAPI)
		select {
		case <-endCh:
			locker.Unlock()
			return
		case <-time.NewTimer(wait).C:
			// Crawl only when no previous crawl has occurred,
			// or its been too long since last crawl.
			err := storeDataUsageInBackend(ctx, objAPI, objAPI.CrawlAndGetDataUsage(ctx, endCh))
			logger.LogIf(ctx, err)
		}
	}
}

func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dataUsageInfo DataUsageInfo) error {
	dataUsageJSON, err := json.Marshal(dataUsageInfo)
	if err != nil {
		return err
	}

	size := int64(len(dataUsageJSON))
	r, err := hash.NewReader(bytes.NewReader(dataUsageJSON), size, "", "", size, false)
	if err != nil {
		return err
	}

	_, err = objAPI.PutObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
	return err
}

func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
	var dataUsageInfoJSON bytes.Buffer

	err := objAPI.GetObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
	if err != nil {
		if isErrObjectNotFound(err) {
			return DataUsageInfo{}, nil
		}
		return DataUsageInfo{}, toObjectErr(err, minioMetaBackgroundOpsBucket, dataUsageObjName)
	}

	var dataUsageInfo DataUsageInfo
	var json = jsoniter.ConfigCompatibleWithStandardLibrary
	err = json.Unmarshal(dataUsageInfoJSON.Bytes(), &dataUsageInfo)
	if err != nil {
		return DataUsageInfo{}, err
	}

	return dataUsageInfo, nil
}

// Item represents each file while walking.
type Item struct {
	Path string
	Typ  os.FileMode
}

type getSizeFn func(item Item) (int64, error)

func updateUsage(basePath string, doneCh <-chan struct{}, waitForLowActiveIO func(), getSize getSizeFn) DataUsageInfo {
	var dataUsageInfo = DataUsageInfo{
		BucketsSizes:          make(map[string]uint64),
		ObjectsSizesHistogram: make(map[string]uint64),
	}

	fastWalk(basePath, 1, doneCh, func(path string, typ os.FileMode) error {
		// Wait for I/O to go down.
		waitForLowActiveIO()

		bucket, entry := path2BucketObjectWithBasePath(basePath, path)
		if bucket == "" {
			return nil
		}

		if isReservedOrInvalidBucket(bucket, false) {
			return filepath.SkipDir
		}

		if entry == "" && typ&os.ModeDir != 0 {
			dataUsageInfo.BucketsCount++
			dataUsageInfo.BucketsSizes[bucket] = 0
			return nil
		}

		if typ&os.ModeDir != 0 {
			return nil
		}

		t := time.Now()
		size, err := getSize(Item{path, typ})
		// Use the response time of the getSize call to guess system load.
		// Sleep equivalent time.
		if d := time.Since(t); d > 100*time.Microsecond {
			time.Sleep(d)
		}
		if err != nil {
			return errSkipFile
		}

		dataUsageInfo.ObjectsCount++
		dataUsageInfo.ObjectsTotalSize += uint64(size)
		dataUsageInfo.BucketsSizes[bucket] += uint64(size)
		dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(size))]++
		return nil
	})

	return dataUsageInfo
}
<|endoftext|>