<commit_before>package openjpeg

// #cgo LDFLAGS: -lopenjp2
// #include <openjpeg-2.1/openjpeg.h>
import "C"

import (
	"errors"
	"fmt"
	"image"
	"reflect"
	"runtime"
	"unsafe"
)

type JP2Image struct {
	filename                  string
	stream                    *C.opj_stream_t
	codec                     *C.opj_codec_t
	image                     *C.opj_image_t
	decodeWidth, decodeHeight int
	decodeArea                image.Rectangle
	crop, resize              bool
}

func NewJP2Image(filename string) (*JP2Image, error) {
	i := &JP2Image{filename: filename}
	runtime.SetFinalizer(i, finalizer)

	if err := i.initializeStream(); err != nil {
		return nil, err
	}

	return i, nil
}

func (i *JP2Image) SetResize(width, height int) {
	i.decodeWidth = width
	i.decodeHeight = height
	i.resize = true
}

func (i *JP2Image) SetCrop(r image.Rectangle) {
	i.decodeArea = r
	i.crop = true
}

func (i *JP2Image) RawImage() (*RawImage, error) {
	// If we want to resize, but not crop, we have to set the decode area to the
	// full image - which means reading in the image header and then
	// cleaning up all previously-initialized data
	if i.resize && !i.crop {
		if err := i.ReadHeader(); err != nil {
			goLog(3, "Error getting dimensions - aborting")
			return nil, err
		}
		i.decodeArea = i.Dimensions()
		i.CleanupResources()
	}

	// Get progression level if we're resizing (it's zero if there isn't any
	// scaling of the output)
	if i.resize {
		if err := i.initializeCodec(); err != nil {
			goLog(3, "Error initializing codec before setting decode resolution factor - aborting")
			return nil, err
		}

		level := desiredProgressionLevel(i.decodeArea, i.decodeWidth, i.decodeHeight)
		goLog(6, fmt.Sprintf("desired level: %d", level))

		if C.opj_set_decoded_resolution_factor(i.codec, C.OPJ_UINT32(level)) == C.OPJ_FALSE {
			return nil, errors.New("failed to set decode resolution factor")
		}
	}

	if err := i.ReadHeader(); err != nil {
		goLog(3, "Error reading header before decode - aborting")
		return nil, err
	}

	goLog(6, fmt.Sprintf("num comps: %d", i.image.numcomps))
	goLog(6, fmt.Sprintf("x0: %d, x1: %d, y0: %d, y1: %d", i.image.x0, i.image.x1, i.image.y0, i.image.y1))

	// Setting decode area has to happen *after* reading the header / image data
	if i.crop {
		r := i.decodeArea
		if C.opj_set_decode_area(i.codec, i.image, C.OPJ_INT32(r.Min.X), C.OPJ_INT32(r.Min.Y), C.OPJ_INT32(r.Max.X), C.OPJ_INT32(r.Max.Y)) == C.OPJ_FALSE {
			return nil, errors.New("failed to set the decoded area")
		}
	}

	// Decode the JP2 into the image stream
	if C.opj_decode(i.codec, i.stream, i.image) == C.OPJ_FALSE {
		return nil, errors.New("failed to decode image")
	}
	if C.opj_end_decompress(i.codec, i.stream) == C.OPJ_FALSE {
		return nil, errors.New("failed to close decompression")
	}

	var comps []C.opj_image_comp_t
	compsSlice := (*reflect.SliceHeader)((unsafe.Pointer(&comps)))
	compsSlice.Cap = int(i.image.numcomps)
	compsSlice.Len = int(i.image.numcomps)
	compsSlice.Data = uintptr(unsafe.Pointer(i.image.comps))

	bounds := image.Rect(0, 0, int(comps[0].w), int(comps[0].h))

	var data []int32
	dataSlice := (*reflect.SliceHeader)((unsafe.Pointer(&data)))
	dataSlice.Cap = bounds.Dx() * bounds.Dy()
	dataSlice.Len = bounds.Dx() * bounds.Dy()
	dataSlice.Data = uintptr(unsafe.Pointer(comps[0].data))

	return &RawImage{data, bounds, bounds.Dx()}, nil
}

func (i *JP2Image) ReadHeader() error {
	if i.image != nil {
		return nil
	}

	if err := i.initializeCodec(); err != nil {
		return err
	}

	if err := i.initializeStream(); err != nil {
		return err
	}

	if C.opj_read_header(i.stream, i.codec, &i.image) == C.OPJ_FALSE {
		return errors.New("failed to read the header")
	}

	return nil
}

func (i *JP2Image) Dimensions() image.Rectangle {
	return image.Rect(int(i.image.x0), int(i.image.y0), int(i.image.x1), int(i.image.y1))
}
<commit_msg>Move codec init to top of RawImage()<commit_after>package openjpeg

// #cgo LDFLAGS: -lopenjp2
// #include <openjpeg-2.1/openjpeg.h>
import "C"

import (
	"errors"
	"fmt"
	"image"
	"reflect"
	"runtime"
	"unsafe"
)

type JP2Image struct {
	filename                  string
	stream                    *C.opj_stream_t
	codec                     *C.opj_codec_t
	image                     *C.opj_image_t
	decodeWidth, decodeHeight int
	decodeArea                image.Rectangle
	crop, resize              bool
}

func NewJP2Image(filename string) (*JP2Image, error) {
	i := &JP2Image{filename: filename}
	runtime.SetFinalizer(i, finalizer)

	if err := i.initializeStream(); err != nil {
		return nil, err
	}

	return i, nil
}

func (i *JP2Image) SetResize(width, height int) {
	i.decodeWidth = width
	i.decodeHeight = height
	i.resize = true
}

func (i *JP2Image) SetCrop(r image.Rectangle) {
	i.decodeArea = r
	i.crop = true
}

func (i *JP2Image) RawImage() (*RawImage, error) {
	// We need the codec to be ready for all operations below
	if err := i.initializeCodec(); err != nil {
		goLog(3, "Error initializing codec - aborting")
		return nil, err
	}

	// If we want to resize, but not crop, we have to set the decode area to the
	// full image - which means reading in the image header and then
	// cleaning up all previously-initialized data
	if i.resize && !i.crop {
		if err := i.ReadHeader(); err != nil {
			goLog(3, "Error getting dimensions - aborting")
			return nil, err
		}
		i.decodeArea = i.Dimensions()
		i.CleanupResources()
	}

	// Get progression level if we're resizing (it's zero if there isn't any
	// scaling of the output)
	if i.resize {
		level := desiredProgressionLevel(i.decodeArea, i.decodeWidth, i.decodeHeight)
		goLog(6, fmt.Sprintf("desired level: %d", level))

		if C.opj_set_decoded_resolution_factor(i.codec, C.OPJ_UINT32(level)) == C.OPJ_FALSE {
			return nil, errors.New("failed to set decode resolution factor")
		}
	}

	if err := i.ReadHeader(); err != nil {
		goLog(3, "Error reading header before decode - aborting")
		return nil, err
	}

	goLog(6, fmt.Sprintf("num comps: %d", i.image.numcomps))
	goLog(6, fmt.Sprintf("x0: %d, x1: %d, y0: %d, y1: %d", i.image.x0, i.image.x1, i.image.y0, i.image.y1))

	// Setting decode area has to happen *after* reading the header / image data
	if i.crop {
		r := i.decodeArea
		if C.opj_set_decode_area(i.codec, i.image, C.OPJ_INT32(r.Min.X), C.OPJ_INT32(r.Min.Y), C.OPJ_INT32(r.Max.X), C.OPJ_INT32(r.Max.Y)) == C.OPJ_FALSE {
			return nil, errors.New("failed to set the decoded area")
		}
	}

	// Decode the JP2 into the image stream
	if C.opj_decode(i.codec, i.stream, i.image) == C.OPJ_FALSE {
		return nil, errors.New("failed to decode image")
	}
	if C.opj_end_decompress(i.codec, i.stream) == C.OPJ_FALSE {
		return nil, errors.New("failed to close decompression")
	}

	var comps []C.opj_image_comp_t
	compsSlice := (*reflect.SliceHeader)((unsafe.Pointer(&comps)))
	compsSlice.Cap = int(i.image.numcomps)
	compsSlice.Len = int(i.image.numcomps)
	compsSlice.Data = uintptr(unsafe.Pointer(i.image.comps))

	bounds := image.Rect(0, 0, int(comps[0].w), int(comps[0].h))

	var data []int32
	dataSlice := (*reflect.SliceHeader)((unsafe.Pointer(&data)))
	dataSlice.Cap = bounds.Dx() * bounds.Dy()
	dataSlice.Len = bounds.Dx() * bounds.Dy()
	dataSlice.Data = uintptr(unsafe.Pointer(comps[0].data))

	return &RawImage{data, bounds, bounds.Dx()}, nil
}

func (i *JP2Image) ReadHeader() error {
	if i.image != nil {
		return nil
	}

	if err := i.initializeCodec(); err != nil {
		return err
	}

	if err := i.initializeStream(); err != nil {
		return err
	}

	if C.opj_read_header(i.stream, i.codec, &i.image) == C.OPJ_FALSE {
		return errors.New("failed to read the header")
	}

	return nil
}

func (i *JP2Image) Dimensions() image.Rectangle {
	return image.Rect(int(i.image.x0), int(i.image.y0), int(i.image.x1), int(i.image.y1))
}
<|endoftext|>


<commit_before>package sqlm

import (
	"reflect"
)

var derefCount int64 = 0
var flatCount int64 = 0

// Deref
func deRef(i interface{}) interface{} {
	if i == nil {
		return i
	}
	derefCount += 1
	typeOfI := reflect.TypeOf(i)
	switch typeOfI.Kind() {
	case reflect.Ptr:
		return deRef(reflect.ValueOf(i).Elem().Interface())
	default:
		return i
	}
}

func flat(i interface{}) []interface{} {
	flatCount += 1

	result := reflect.ValueOf([]interface{}{})

	kindOfI := reflect.TypeOf(i).Kind()
	valueOfI := reflect.ValueOf(i)

	switch kindOfI {
	case reflect.Ptr:
		result = reflect.Append(result, reflect.ValueOf(i))
	case reflect.Slice, reflect.Array:
		// Iterate the slice and flat each of them
		for index := 0; index < valueOfI.Len(); index++ {
			v := valueOfI.Index(index)
			/*
			if v.CanInterface() {
				v = v.Elem()
			}
			*/
			if v.Elem().Kind() == reflect.Slice || v.Elem().Kind() == reflect.Array {
				result = reflect.AppendSlice(result,
					reflect.ValueOf(flat(v.Interface())))
			} else {
				result = reflect.Append(result, v)
			}
		}
	default:
		result = reflect.Append(result, reflect.ValueOf(i))
	}

	back := result.Interface()
	return back.([]interface{})
}

func assign(target interface{}, value interface{}) error {
	switch t := target.(type) {
	case *string:
		*t = deRef(value).(string)
	case *int:
		*t = deRef(value).(int)
	case *int8:
		*t = deRef(value).(int8)
	case *int16:
		*t = deRef(value).(int16)
	case *int32:
		*t = deRef(value).(int32)
	case *int64:
		*t = deRef(value).(int64)
	case *float32:
		*t = deRef(value).(float32)
	case *float64:
		*t = deRef(value).(float64)
	case *interface{}:
		*t = deRef(value)
	}

	return nil
}
<commit_msg>Fix miss use of reflect.CanInterface, check kind to interface directly<commit_after>package sqlm

import (
	"reflect"
)

var derefCount int64 = 0
var flatCount int64 = 0

// Deref
func deRef(i interface{}) interface{} {
	if i == nil {
		return i
	}
	derefCount += 1
	typeOfI := reflect.TypeOf(i)
	switch typeOfI.Kind() {
	case reflect.Ptr:
		return deRef(reflect.ValueOf(i).Elem().Interface())
	default:
		return i
	}
}

func flat(i interface{}) []interface{} {
	flatCount += 1

	result := reflect.ValueOf([]interface{}{})

	kindOfI := reflect.TypeOf(i).Kind()
	valueOfI := reflect.ValueOf(i)

	switch kindOfI {
	case reflect.Ptr:
		result = reflect.Append(result, reflect.ValueOf(i))
	case reflect.Slice, reflect.Array:
		// Iterate the slice and flat each of them
		for index := 0; index < valueOfI.Len(); index++ {
			v := valueOfI.Index(index)
			if v.Kind() == reflect.Interface {
				if v.Elem().Kind() == reflect.Slice || v.Elem().Kind() == reflect.Array {
					result = reflect.AppendSlice(result,
						reflect.ValueOf(flat(v.Interface())))
				} else {
					result = reflect.Append(result, v)
				}
			} else {
				result = reflect.Append(result, v)
			}
		}
	default:
		result = reflect.Append(result, reflect.ValueOf(i))
	}

	back := result.Interface()
	return back.([]interface{})
}

func assign(target interface{}, value interface{}) error {
	switch t := target.(type) {
	case *string:
		*t = deRef(value).(string)
	case *int:
		*t = deRef(value).(int)
	case *int8:
		*t = deRef(value).(int8)
	case *int16:
		*t = deRef(value).(int16)
	case *int32:
		*t = deRef(value).(int32)
	case *int64:
		*t = deRef(value).(int64)
	case *float32:
		*t = deRef(value).(float32)
	case *float64:
		*t = deRef(value).(float64)
	case *interface{}:
		*t = deRef(value)
	}

	return nil
}
<|endoftext|>


<commit_before>package operation

import (
	"log"
	"os"
	"github.com/codegangsta/cli"
	"fmt"
	"github.com/stormcat24/ecs-formation/aws"
	"github.com/stormcat24/ecs-formation/service"
	"github.com/stormcat24/ecs-formation/task"
	"strings"
	"github.com/str1ngs/ansi/color"
	"github.com/stormcat24/ecs-formation/plan"
	"github.com/stormcat24/ecs-formation/util"
	"github.com/stormcat24/ecs-formation/bluegreen"
	"github.com/stormcat24/ecs-formation/logger"
	"errors"
)

var Commands = []cli.Command{
	commandService,
	commandTask,
	commandDeploy,
}

var commandService = cli.Command{
	Name: "service",
	Usage: "Manage ECS services on cluster",
	Description: `
	Manage services on ECS cluster.
`,
	Action: doService,
}

var commandTask = cli.Command{
	Name: "task",
	Usage: "Manage ECS Task Definitions",
	Description: `
	Manage ECS Task Definitions.
`,
	Action: doTask,
}

var commandDeploy = cli.Command{
	Name: "deploy",
	Usage: "Manage bluegreen deployment on ECS",
	Description: `
	Manage bluegreen deployment on ECS.
`,
	Action: doDeploy,
}

func debug(v ...interface{}) {
	if os.Getenv("DEBUG") != "" {
		log.Println(v...)
	}
}

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func doService(c *cli.Context) {

	ecsManager, err := buildECSManager()

	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	operation, errSubCommand := createOperation(c.Args())

	if errSubCommand != nil {
		logger.Main.Error(color.Red(errSubCommand.Error()))
		os.Exit(1)
	}

	projectDir, err := os.Getwd()
	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	clusterController, err := service.NewServiceController(ecsManager, projectDir, operation.TargetResource)

	plans, err := createClusterPlans(clusterController, projectDir)

	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	if (operation.SubCommand == "apply") {
		clusterController.ApplyServicePlans(plans)
	}
}

func doTask(c *cli.Context) {

	ecsManager, err := buildECSManager()

	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	operation, errSubCommand := createOperation(c.Args())

	if errSubCommand != nil {
		logger.Main.Error(color.Red(errSubCommand.Error()))
		os.Exit(1)
	}

	projectDir, err := os.Getwd()
	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	taskController, err := task.NewTaskDefinitionController(ecsManager, projectDir, operation.TargetResource)
	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	plans := createTaskPlans(taskController, projectDir)

	if (operation.SubCommand == "apply") {
		results, errapp := taskController.ApplyTaskDefinitionPlans(plans)

		if errapp != nil {
			logger.Main.Error(color.Red(errapp.Error()))
			os.Exit(1)
		}

		for _, output := range results {
			logger.Main.Infof("Registered Task Definition '%s'", *output.TaskDefinition.Family)
			logger.Main.Info(color.Cyan(util.StringValueWithIndent(output.TaskDefinition, 1)))
		}
	}
}

func doDeploy(c *cli.Context) {

	ecsManager, err := buildECSManager()

	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	operation, errSubCommand := createOperation(c.Args())

	if errSubCommand != nil {
		logger.Main.Error(color.Red(errSubCommand.Error()))
		os.Exit(1)
	}

	projectDir, err := os.Getwd()
	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	bgController, errbgc := bluegreen.NewBlueGreenController(ecsManager, projectDir)
	if errbgc != nil {
		logger.Main.Error(color.Red(errbgc.Error()))
		os.Exit(1)
	}

	bgPlans, err := createBlueGreenPlans(bgController)

	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	// cluster check

	if (operation.SubCommand == "apply") {

		errbg := bgController.ApplyBlueGreenDeploys(bgPlans)
		if errbg != nil {
			logger.Main.Error(color.Red(errbg.Error()))
			os.Exit(1)
		}

	}
}

func createClusterPlans(controller *service.ServiceController, projectDir string) ([]*plan.ServiceUpdatePlan, error) {

	logger.Main.Infoln("Checking services on clusters...")
	plans, err := controller.CreateServiceUpdatePlans()

	if err != nil {
		return []*plan.ServiceUpdatePlan{}, err
	}

	for _, plan := range plans {

		fmt.Println(color.Yellow(fmt.Sprintf("Current status of ECS Cluster '%s':", plan.Name)))

		if len(plan.CurrentServices) > 0 {
			fmt.Println(color.Yellow("    Services as follows:"))
		} else {
			fmt.Println(color.Yellow("    No services are deployed."))
		}

		for _, cs := range plan.CurrentServices {
			fmt.Println(color.Yellow(fmt.Sprintf("        ServiceName = %s", *cs.ServiceName)))
			fmt.Println(color.Yellow(fmt.Sprintf("        ServiceARN = %s", *cs.ServiceARN)))
			fmt.Println(color.Yellow(fmt.Sprintf("        TaskDefinition = %s", *cs.TaskDefinition)))
			fmt.Println(color.Yellow(fmt.Sprintf("        DesiredCount = %d", *cs.DesiredCount)))
			fmt.Println(color.Yellow(fmt.Sprintf("        PendingCount = %d", *cs.PendingCount)))
			fmt.Println(color.Yellow(fmt.Sprintf("        RunningCount = %d", *cs.RunningCount)))
			for _, lb := range cs.LoadBalancers {
				fmt.Println(color.Yellow(fmt.Sprintf("        ELB = %s:", *lb.LoadBalancerName)))
				fmt.Println(color.Yellow(fmt.Sprintf("            ContainerName = %s", *lb.ContainerName)))
				fmt.Println(color.Yellow(fmt.Sprintf("            ContainerName = %d", *lb.ContainerPort)))
			}
			fmt.Println(color.Yellow(fmt.Sprintf("        STATUS = %s", *cs.Status)))
		}

		for _, add := range plan.NewServices {
			for _, lb := range add.LoadBalancers {
				logger.Main.Info(color.Cyan(fmt.Sprintf("            ELB:%s", lb.Name)))
			}
		}

		fmt.Println()
	}

	return plans, nil
}

func createTaskPlans(controller *task.TaskDefinitionController, projectDir string) []*plan.TaskUpdatePlan {

	taskDefs := controller.GetTaskDefinitionMap()
	plans := controller.CreateTaskUpdatePlans(taskDefs)

	for _, plan := range plans {
		logger.Main.Infof("Task Definition '%s'", plan.Name)

		for _, add := range plan.NewContainers {
			fmt.Println(color.Cyan(fmt.Sprintf("    (+) %s", add.Name)))
			fmt.Println(color.Cyan(fmt.Sprintf("      image: %s", add.Image)))
			fmt.Println(color.Cyan(fmt.Sprintf("      ports: %s", add.Ports)))
			fmt.Println(color.Cyan(fmt.Sprintf("      environment:\n%s", util.StringValueWithIndent(add.Environment, 4))))
			fmt.Println(color.Cyan(fmt.Sprintf("      links: %s", add.Links)))
			fmt.Println(color.Cyan(fmt.Sprintf("      volumes: %s", add.Volumes)))
		}

		fmt.Println()
	}

	return plans
}

func createBlueGreenPlans(controller *bluegreen.BlueGreenController) ([]*plan.BlueGreenPlan, error) {

	bgDefs := controller.GetBlueGreenDefs()
	bgPlans := []*plan.BlueGreenPlan{}

	cplans, errcp := controller.ClusterController.CreateServiceUpdatePlans()
	if errcp != nil {
		return bgPlans, errcp
	}

	for _, bg := range bgDefs {

		bgPlan, err := controller.CreateBlueGreenPlan(bg.Blue, bg.Green, cplans)
		if err != nil {
			return bgPlans, err
		}

		if bgPlan.Blue.CurrentService == nil {
			return bgPlans, errors.New(fmt.Sprintf("Service '%s' is not found. ", bg.Blue.Service))
		}

		if bgPlan.Green.CurrentService == nil {
			return bgPlans, errors.New(fmt.Sprintf("Service '%s' is not found. ", bg.Green.Service))
		}

		if bgPlan.Blue.AutoScalingGroup == nil {
			return bgPlans, errors.New(fmt.Sprintf("AutoScaling Group '%s' is not found. ", bg.Blue.AutoscalingGroup))
		}

		if bgPlan.Green.AutoScalingGroup == nil {
			return bgPlans, errors.New(fmt.Sprintf("AutoScaling Group '%s' is not found. ", bg.Green.AutoscalingGroup))
		}

		if bgPlan.Blue.ClusterUpdatePlan == nil {
			return bgPlans, errors.New(fmt.Sprintf("ECS Cluster '%s' is not found. ", bg.Blue.Cluster))
		}

		if bgPlan.Green.ClusterUpdatePlan == nil {
			return bgPlans, errors.New(fmt.Sprintf("ECS Cluster '%s' is not found. ", bg.Green.Cluster))
		}

		bgPlans = append(bgPlans, bgPlan)
	}

	return bgPlans, nil
}

func buildECSManager() (*aws.ECSManager, error) {

	accessKey := strings.Trim(os.Getenv("AWS_ACCESS_KEY"), " ")
	accessSecretKey := strings.Trim(os.Getenv("AWS_SECRET_ACCESS_KEY"), " ")
	region := strings.Trim(os.Getenv("AWS_REGION"), " ")

	if len(accessKey) == 0 {
		return nil, fmt.Errorf("'AWS_ACCESS_KEY' is not specified.")
	}

	if len(accessSecretKey) == 0 {
		return nil, fmt.Errorf("'AWS_SECRET_ACCESS_KEY' is not specified.")
	}

	if len(region) == 0 {
		return nil, fmt.Errorf("'AWS_REGION' is not specified.")
	}

	return aws.NewECSManager(accessKey, accessSecretKey, region), nil
}

func createOperation(args cli.Args) (Operation, error) {

	if len(args) == 0 {
		return Operation{}, fmt.Errorf("subcommand is not specified.")
	}

	sub := args[0]
	if sub == "plan" || sub == "apply" {

		var targetResource string
		if len(args) > 1 {
			targetResource = args[1]
		}

		return Operation{
			SubCommand: sub,
			TargetResource: targetResource,
		}, nil
	} else {
		return Operation{}, fmt.Errorf("'%s' is invalid subcommand.", sub)
	}
}

type Operation struct {
	SubCommand     string
	TargetResource string
}
<commit_msg>:speaker: output bluegreen current status<commit_after>package operation

import (
	"log"
	"os"
	"github.com/codegangsta/cli"
	"fmt"
	"github.com/stormcat24/ecs-formation/aws"
	"github.com/stormcat24/ecs-formation/service"
	"github.com/stormcat24/ecs-formation/task"
	"strings"
	"github.com/str1ngs/ansi/color"
	"github.com/stormcat24/ecs-formation/plan"
	"github.com/stormcat24/ecs-formation/util"
	"github.com/stormcat24/ecs-formation/bluegreen"
	"github.com/stormcat24/ecs-formation/logger"
	"errors"
)

var Commands = []cli.Command{
	commandService,
	commandTask,
	commandDeploy,
}

var commandService = cli.Command{
	Name: "service",
	Usage: "Manage ECS services on cluster",
	Description: `
	Manage services on ECS cluster.
`,
	Action: doService,
}

var commandTask = cli.Command{
	Name: "task",
	Usage: "Manage ECS Task Definitions",
	Description: `
	Manage ECS Task Definitions.
`,
	Action: doTask,
}

var commandDeploy = cli.Command{
	Name: "deploy",
	Usage: "Manage bluegreen deployment on ECS",
	Description: `
	Manage bluegreen deployment on ECS.
`,
	Action: doDeploy,
}

func debug(v ...interface{}) {
	if os.Getenv("DEBUG") != "" {
		log.Println(v...)
	}
}

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func doService(c *cli.Context) {

	ecsManager, err := buildECSManager()

	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	operation, errSubCommand := createOperation(c.Args())

	if errSubCommand != nil {
		logger.Main.Error(color.Red(errSubCommand.Error()))
		os.Exit(1)
	}

	projectDir, err := os.Getwd()
	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	clusterController, err := service.NewServiceController(ecsManager, projectDir, operation.TargetResource)

	plans, err := createClusterPlans(clusterController, projectDir)

	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	if (operation.SubCommand == "apply") {
		clusterController.ApplyServicePlans(plans)
	}
}

func doTask(c *cli.Context) {

	ecsManager, err := buildECSManager()

	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	operation, errSubCommand := createOperation(c.Args())

	if errSubCommand != nil {
		logger.Main.Error(color.Red(errSubCommand.Error()))
		os.Exit(1)
	}

	projectDir, err := os.Getwd()
	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	taskController, err := task.NewTaskDefinitionController(ecsManager, projectDir, operation.TargetResource)
	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	plans := createTaskPlans(taskController, projectDir)

	if (operation.SubCommand == "apply") {
		results, errapp := taskController.ApplyTaskDefinitionPlans(plans)

		if errapp != nil {
			logger.Main.Error(color.Red(errapp.Error()))
			os.Exit(1)
		}

		for _, output := range results {
			logger.Main.Infof("Registered Task Definition '%s'", *output.TaskDefinition.Family)
			logger.Main.Info(color.Cyan(util.StringValueWithIndent(output.TaskDefinition, 1)))
		}
	}
}

func doDeploy(c *cli.Context) {

	ecsManager, err := buildECSManager()

	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	operation, errSubCommand := createOperation(c.Args())

	if errSubCommand != nil {
		logger.Main.Error(color.Red(errSubCommand.Error()))
		os.Exit(1)
	}

	projectDir, err := os.Getwd()
	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	bgController, errbgc := bluegreen.NewBlueGreenController(ecsManager, projectDir)
	if errbgc != nil {
		logger.Main.Error(color.Red(errbgc.Error()))
		os.Exit(1)
	}

	bgPlans, err := createBlueGreenPlans(bgController)

	if err != nil {
		logger.Main.Error(color.Red(err.Error()))
		os.Exit(1)
	}

	// cluster check

	if (operation.SubCommand == "apply") {

		errbg := bgController.ApplyBlueGreenDeploys(bgPlans)
		if errbg != nil {
			logger.Main.Error(color.Red(errbg.Error()))
			os.Exit(1)
		}

	}
}

func createClusterPlans(controller *service.ServiceController, projectDir string) ([]*plan.ServiceUpdatePlan, error) {

	logger.Main.Infoln("Checking services on clusters...")
	plans, err := controller.CreateServiceUpdatePlans()

	if err != nil {
		return []*plan.ServiceUpdatePlan{}, err
	}

	for _, plan := range plans {

		fmt.Println(color.Yellow(fmt.Sprintf("Current status of ECS Cluster '%s':", plan.Name)))

		if len(plan.CurrentServices) > 0 {
			fmt.Println(color.Yellow("    Services as follows:"))
		} else {
			fmt.Println(color.Yellow("    No services are deployed."))
		}

		for _, cs := range plan.CurrentServices {
			fmt.Println(color.Yellow(fmt.Sprintf("        ServiceName = %s", *cs.ServiceName)))
			fmt.Println(color.Yellow(fmt.Sprintf("        ServiceARN = %s", *cs.ServiceARN)))
			fmt.Println(color.Yellow(fmt.Sprintf("        TaskDefinition = %s", *cs.TaskDefinition)))
			fmt.Println(color.Yellow(fmt.Sprintf("        DesiredCount = %d", *cs.DesiredCount)))
			fmt.Println(color.Yellow(fmt.Sprintf("        PendingCount = %d", *cs.PendingCount)))
			fmt.Println(color.Yellow(fmt.Sprintf("        RunningCount = %d", *cs.RunningCount)))
			for _, lb := range cs.LoadBalancers {
				fmt.Println(color.Yellow(fmt.Sprintf("        ELB = %s:", *lb.LoadBalancerName)))
				fmt.Println(color.Yellow(fmt.Sprintf("            ContainerName = %s", *lb.ContainerName)))
				fmt.Println(color.Yellow(fmt.Sprintf("            ContainerName = %d", *lb.ContainerPort)))
			}
			fmt.Println(color.Yellow(fmt.Sprintf("        STATUS = %s", *cs.Status)))
		}

		for _, add := range plan.NewServices {
			for _, lb := range add.LoadBalancers {
				logger.Main.Info(color.Cyan(fmt.Sprintf("            ELB:%s", lb.Name)))
			}
		}

		fmt.Println()
	}

	return plans, nil
}

func createTaskPlans(controller *task.TaskDefinitionController, projectDir string) []*plan.TaskUpdatePlan {

	taskDefs := controller.GetTaskDefinitionMap()
	plans := controller.CreateTaskUpdatePlans(taskDefs)

	for _, plan := range plans {
		logger.Main.Infof("Task Definition '%s'", plan.Name)

		for _, add := range plan.NewContainers {
			fmt.Println(color.Cyan(fmt.Sprintf("    (+) %s", add.Name)))
			fmt.Println(color.Cyan(fmt.Sprintf("      image: %s", add.Image)))
			fmt.Println(color.Cyan(fmt.Sprintf("      ports: %s", add.Ports)))
			fmt.Println(color.Cyan(fmt.Sprintf("      environment:\n%s", util.StringValueWithIndent(add.Environment, 4))))
			fmt.Println(color.Cyan(fmt.Sprintf("      links: %s", add.Links)))
			fmt.Println(color.Cyan(fmt.Sprintf("      volumes: %s", add.Volumes)))
		}

		fmt.Println()
	}

	return plans
}

func createBlueGreenPlans(controller *bluegreen.BlueGreenController) ([]*plan.BlueGreenPlan, error) {

	bgDefs := controller.GetBlueGreenDefs()
	bgPlans := []*plan.BlueGreenPlan{}

	cplans, errcp := controller.ClusterController.CreateServiceUpdatePlans()
	if errcp != nil {
		return bgPlans, errcp
	}

	for _, bg := range bgDefs {

		bgPlan, err := controller.CreateBlueGreenPlan(bg.Blue, bg.Green, cplans)
		if err != nil {
			return bgPlans, err
		}

		if bgPlan.Blue.CurrentService == nil {
			return bgPlans, errors.New(fmt.Sprintf("Service '%s' is not found. ", bg.Blue.Service))
		}

		if bgPlan.Green.CurrentService == nil {
			return bgPlans, errors.New(fmt.Sprintf("Service '%s' is not found. ", bg.Green.Service))
		}

		if bgPlan.Blue.AutoScalingGroup == nil {
			return bgPlans, errors.New(fmt.Sprintf("AutoScaling Group '%s' is not found. ", bg.Blue.AutoscalingGroup))
		}

		if bgPlan.Green.AutoScalingGroup == nil {
			return bgPlans, errors.New(fmt.Sprintf("AutoScaling Group '%s' is not found. ", bg.Green.AutoscalingGroup))
		}

		if bgPlan.Blue.ClusterUpdatePlan == nil {
			return bgPlans, errors.New(fmt.Sprintf("ECS Cluster '%s' is not found. ", bg.Blue.Cluster))
		}

		if bgPlan.Green.ClusterUpdatePlan == nil {
			return bgPlans, errors.New(fmt.Sprintf("ECS Cluster '%s' is not found. ", bg.Green.Cluster))
		}

		fmt.Println(color.Cyan("    Blue:"))
		fmt.Println(color.Cyan(fmt.Sprintf("        Cluster = %s", bg.Blue.Cluster)))
		fmt.Println(color.Cyan(fmt.Sprintf("        LoadBalancer = %s", bgPlan.Blue.LoadBalancer)))
		fmt.Println(color.Cyan(fmt.Sprintf("        AutoScalingGroupARN = %s", *bgPlan.Blue.AutoScalingGroup.AutoScalingGroupARN)))
		fmt.Println(color.Cyan("        Current services as follows:"))
		for _, bcs := range bgPlan.Blue.ClusterUpdatePlan.CurrentServices {
			fmt.Println(color.Cyan(fmt.Sprintf("            %s:", *bcs.ServiceName)))
			fmt.Println(color.Cyan(fmt.Sprintf("                ServiceARN = %s", *bcs.ServiceARN)))
			fmt.Println(color.Cyan(fmt.Sprintf("                TaskDefinition = %s", *bcs.TaskDefinition)))
			fmt.Println(color.Cyan(fmt.Sprintf("                DesiredCount = %d", *bcs.DesiredCount)))
			fmt.Println(color.Cyan(fmt.Sprintf("                PendingCount = %d", *bcs.PendingCount)))
			fmt.Println(color.Cyan(fmt.Sprintf("                RunningCount = %d", *bcs.RunningCount)))
		}

		fmt.Println(color.Green("    Green:"))
		fmt.Println(color.Green(fmt.Sprintf("        Cluster = %s", bg.Green.Cluster)))
		fmt.Println(color.Green(fmt.Sprintf("        LoadBalancer = %s", bgPlan.Green.LoadBalancer)))
		fmt.Println(color.Green(fmt.Sprintf("        AutoScalingGroupARN = %s", *bgPlan.Green.AutoScalingGroup.AutoScalingGroupARN)))
		fmt.Println(color.Green("        Current services as follows:"))
		for _, gcs := range bgPlan.Green.ClusterUpdatePlan.CurrentServices {
			fmt.Println(color.Green(fmt.Sprintf("            %s:", *gcs.ServiceName)))
			fmt.Println(color.Green(fmt.Sprintf("                ServiceARN = %s", *gcs.ServiceARN)))
			fmt.Println(color.Green(fmt.Sprintf("                TaskDefinition = %s", *gcs.TaskDefinition)))
			fmt.Println(color.Green(fmt.Sprintf("                DesiredCount = %d", *gcs.DesiredCount)))
			fmt.Println(color.Green(fmt.Sprintf("                PendingCount = %d", *gcs.PendingCount)))
			fmt.Println(color.Green(fmt.Sprintf("                RunningCount = %d", *gcs.RunningCount)))
		}

		fmt.Println()

		bgPlans = append(bgPlans, bgPlan)
	}

	return bgPlans, nil
}

func buildECSManager() (*aws.ECSManager, error) {

	accessKey := strings.Trim(os.Getenv("AWS_ACCESS_KEY"), " ")
	accessSecretKey := strings.Trim(os.Getenv("AWS_SECRET_ACCESS_KEY"), " ")
	region := strings.Trim(os.Getenv("AWS_REGION"), " ")

	if len(accessKey) == 0 {
		return nil, fmt.Errorf("'AWS_ACCESS_KEY' is not specified.")
	}

	if len(accessSecretKey) == 0 {
		return nil, fmt.Errorf("'AWS_SECRET_ACCESS_KEY' is not specified.")
	}

	if len(region) == 0 {
		return nil, fmt.Errorf("'AWS_REGION' is not specified.")
	}

	return aws.NewECSManager(accessKey, accessSecretKey, region), nil
}

func createOperation(args cli.Args) (Operation, error) {

	if len(args) == 0 {
		return Operation{}, fmt.Errorf("subcommand is not specified.")
	}

	sub := args[0]
	if sub == "plan" || sub == "apply" {

		var targetResource string
		if len(args) > 1 {
			targetResource = args[1]
		}

		return Operation{
			SubCommand: sub,
			TargetResource: targetResource,
		}, nil
	} else {
		return Operation{}, fmt.Errorf("'%s' is invalid subcommand.", sub)
	}
}

type Operation struct {
	SubCommand     string
	TargetResource string
}
<|endoftext|>


<commit_before>/* {{{ Copyright (c) Paul R. Tagliamonte <paultag@debian.org>, 2015
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE. }}} */

package archive

import (
	"fmt"
	"io"
	"os"

	"golang.org/x/crypto/openpgp"
	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
)

// Release {{{

// The file "dists/$DIST/InRelease" shall contain meta-information about the
// distribution and checksums for the indices, possibly signed with a GPG
// clearsign signature (for example created by "gpg -a -s --clearsign"). For
// older clients there can also be a "dists/$DIST/Release" file without any
// signature and the file "dists/$DIST/Release.gpg" with a detached GPG
// signature of the "Release" file, compatible with the format used by the GPG
// options "-a -b -s".
type Release struct {
	control.Paragraph

	Description string

	// Optional field indicating the origin of the repository, a single line
	// of free form text.
	Origin string

	// Optional field including some kind of label, a single line of free form
	// text.
	//
	// Typically used extensively in repositories split over multiple media
	// such as repositories stored on CDs.
	Label string

	// The Version field, if specified, shall be the version of the release.
	// This is usually a sequence of integers separated by the character
	// "." (full stop).
	//
	// Example:
	//
	//   Version: 6.0
	Version string

	// The Suite field may describe the suite. A suite is a single word. In
	// Debian, this shall be one of oldstable, stable, testing, unstable,
	// or experimental; with optional suffixes such as -updates.
	//
	// Example:
	// //   Suite: stable
	Suite string

	// The Codename field shall describe the codename of the release. A
	// codename is a single word. Debian releases are codenamed after Toy
	// Story Characters, and the unstable suite has the codename sid, the
	// experimental suite has the codename experimental.
	//
	// Example:
	//
	//   Codename: squeeze
	Codename string

	// A whitespace separated list of areas.
	//
	// Example:
	//
	//   Components: main contrib non-free
	//
	// May also include be prefixed by parts of the path following the
	// directory beneath dists, if the Release file is not in a directory
	// directly beneath dists/. As an example, security updates are specified
	// in APT as:
	//
	// deb http://security.debian.org/ stable/updates main)
	//
	// The Release file would be located at
	// http://security.debian.org/dists/stable/updates/Release and look like:
	//
	//   Suite: stable
	//   Components: updates/main updates/contrib updates/non-free
	Components []string `delim:" "`

	// Whitespace separated unique single words identifying Debian machine
	// architectures as described in Architecture specification strings,
	// Section 11.1. Clients should ignore Architectures they do not know
	// about.
	Architectures []dependency.Arch

	// The Date field shall specify the time at which the Release file was
	// created. Clients updating a local on-disk cache should ignore a Release
	// file with an earlier date than the date in the already stored Release
	// file.
	//
	// The Valid-Until field may specify at which time the Release file should
	// be considered expired by the client. Client behaviour on expired Release
	// files is unspecified.
	//
	// The format of the dates is the same as for the Date field in .changes
	// files; and as used in debian/changelog files, and documented in Policy
	// 4.4 ( Debian changelog: debian/changelog)
	Date       string
	ValidUntil string `control:"Valid-Until"`

	// note the upper-case S in MD5Sum (unlike in Packages and Sources files)
	//
	// These fields are used for two purposes:
	//
	// describe what package index files are present when release signature is
	// available it certifies that listed index files and files referenced by
	// those index files are genuine Those fields shall be multi-line fields
	// containing multiple lines of whitespace separated data. Each line shall
	// contain
	//
	// The checksum of the file in the format corresponding to the field The
	// size of the file (integer >= 0) The filename relative to the directory
	// of the Release file Each datum must be separated by one or more
	// whitespace characters.
	//
	// Server requirements:
	//
	// The checksum and sizes shall match the actual existing files. If indexes
	// are compressed, checksum data must be provided for uncompressed files as
	// well, even if not present on the server.  Client behaviour:
	//
	// Any file should be checked at least once, either in compressed or
	// uncompressed form, depending on which data is available. If a file has
	// no associated data, the client shall inform the user about this under
	// possibly dangerous situations (such as installing a package from that
	// repository). If a file does not match the data specified in the release
	// file, the client shall not use any information from that file, inform
	// the user, and might use old information (such as the previous locally
	// kept information) instead.
	MD5Sum []control.MD5FileHash    `delim:"\n" strip:" \t\n\r" multiline:"true"`
	SHA1   []control.SHA1FileHash   `delim:"\n" strip:" \t\n\r" multiline:"true"`
	SHA256 []control.SHA256FileHash `delim:"\n" strip:" \t\n\r" multiline:"true"`
	SHA512 []control.SHA512FileHash `delim:"\n" strip:" \t\n\r" multiline:"true"`

	// The NotAutomatic and ButAutomaticUpgrades fields are optional boolean
	// fields instructing the package manager. They may contain the values
	// "yes" and "no". If one the fields is not specified, this has the same
	// meaning as a value of "no".
	//
	// If a value of "yes" is specified for the NotAutomatic field, a package
	// manager should not install packages (or upgrade to newer versions) from
	// this repository without explicit user consent (APT assigns priority 1 to
	// this) If the field ButAutomaticUpgrades is specified as well and has the
	// value "yes", the package manager should automatically install package
	// upgrades from this repository, if the installed version of the package
	// is higher than the version of the package in other sources (APT assigns
	// priority 100).
	//
	// Specifying "yes" for ButAutomaticUpgrades without specifying "yes" for
	// NotAutomatic is invalid.
	NotAutomatic         string
	ButAutomaticUpgrades string

	AcquireByHash bool `control:"Acquire-By-Hash"`
}

// Given a file declared in the Release file, get the FileHash entries
// for that file (SHA256, SHA512). These can be used to ensure the
// integrety of files in the archive.
func (r *Release) Indices() map[string]control.FileHashes {
	ret := map[string]control.FileHashes{}

	// https://wiki.debian.org/DebianRepository/Format#Size.2C_MD5sum.2C_SHA1.2C_SHA256.2C_SHA512:
	// Clients may not use the MD5Sum and SHA1 fields for security purposes, and
	// must require a SHA256 or a SHA512 field.

	for _, el := range r.SHA256 {
		ret[el.Filename] = append(ret[el.Filename], el.FileHash)
	}
	for _, el := range r.SHA512 {
		ret[el.Filename] = append(ret[el.Filename], el.FileHash)
	}
	return ret
}

func (r *Release) AddHash(h control.FileHash) error {
	switch h.Algorithm {
	case "sha256":
		r.SHA256 = append(r.SHA256, control.SHA256FileHash{h})
	case "sha1":
		r.SHA1 = append(r.SHA1, control.SHA1FileHash{h})
	case "sha512":
		r.SHA512 = append(r.SHA512, control.SHA512FileHash{h})
	case "md5":
		r.MD5Sum = append(r.MD5Sum, control.MD5FileHash{h})
	default:
		return fmt.Errorf("No known hash: '%s'", h.Algorithm)
	}
	return nil
}

// }}}

// LoadInRelease {{{

// Given an InRelease io.Reader, and the OpenPGP keyring
// to validate against, return the parsed InRelease file.
func LoadInRelease(in io.Reader, keyring *openpgp.EntityList) (*Release, error) {
	ret := Release{}
	decoder, err := control.NewDecoder(in, keyring)
	if err != nil {
		return nil, err
	}
	return &ret, decoder.Decode(&ret)
}

// }}}

// LoadInReleaseFile {{{

// Given a path to the InRelease file on the filesystem, and the OpenPGP keyring
// to validate against, return the parsed InRelease file.
func LoadInReleaseFile(path string, keyring *openpgp.EntityList) (*Release, error) {
	fd, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	return LoadInRelease(fd, keyring)
}

// }}}

// vim: foldmethod=marker
<commit_msg>Update Debian Security example to "stable-security"<commit_after>/* {{{ Copyright (c) Paul R. Tagliamonte <paultag@debian.org>, 2015
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE. }}} */

package archive

import (
	"fmt"
	"io"
	"os"

	"golang.org/x/crypto/openpgp"
	"pault.ag/go/debian/control"
	"pault.ag/go/debian/dependency"
)

// Release {{{

// The file "dists/$DIST/InRelease" shall contain meta-information about the
// distribution and checksums for the indices, possibly signed with a GPG
// clearsign signature (for example created by "gpg -a -s --clearsign"). For
// older clients there can also be a "dists/$DIST/Release" file without any
// signature and the file "dists/$DIST/Release.gpg" with a detached GPG
// signature of the "Release" file, compatible with the format used by the GPG
// options "-a -b -s".
type Release struct {
	control.Paragraph

	Description string

	// Optional field indicating the origin of the repository, a single line
	// of free form text.
	Origin string

	// Optional field including some kind of label, a single line of free form
	// text.
	//
	// Typically used extensively in repositories split over multiple media
	// such as repositories stored on CDs.
	Label string

	// The Version field, if specified, shall be the version of the release.
	// This is usually a sequence of integers separated by the character
	// "." (full stop).
	//
	// Example:
	//
	//   Version: 6.0
	Version string

	// The Suite field may describe the suite. A suite is a single word. In
	// Debian, this shall be one of oldstable, stable, testing, unstable,
	// or experimental; with optional suffixes such as -updates.
	//
	// Example:
	// //   Suite: stable
	Suite string

	// The Codename field shall describe the codename of the release. A
	// codename is a single word. Debian releases are codenamed after Toy
	// Story Characters, and the unstable suite has the codename sid, the
	// experimental suite has the codename experimental.
	//
	// Example:
	//
	//   Codename: squeeze
	Codename string

	// A whitespace separated list of areas.
	//
	// Example:
	//
	//   Components: main contrib non-free
	//
	// May also include be prefixed by parts of the path following the
	// directory beneath dists, if the Release file is not in a directory
	// directly beneath dists/. As an example, security updates are specified
	// in APT as:
	//
	// deb http://security.debian.org/debian-security stable-security main)
	//
	// The Release file would be located at
	// http://security.debian.org/dists/stable-security/Release and look like:
	//
	//   Suite: stable-security
	//   Components: main contrib non-free
	Components []string `delim:" "`

	// Whitespace separated unique single words identifying Debian machine
	// architectures as described in Architecture specification strings,
	// Section 11.1. Clients should ignore Architectures they do not know
	// about.
	Architectures []dependency.Arch

	// The Date field shall specify the time at which the Release file was
	// created. Clients updating a local on-disk cache should ignore a Release
	// file with an earlier date than the date in the already stored Release
	// file.
	//
	// The Valid-Until field may specify at which time the Release file should
	// be considered expired by the client. Client behaviour on expired Release
	// files is unspecified.
	//
	// The format of the dates is the same as for the Date field in .changes
	// files; and as used in debian/changelog files, and documented in Policy
	// 4.4 ( Debian changelog: debian/changelog)
	Date       string
	ValidUntil string `control:"Valid-Until"`

	// note the upper-case S in MD5Sum (unlike in Packages and Sources files)
	//
	// These fields are used for two purposes:
	//
	// describe what package index files are present when release signature is
	// available it certifies that listed index files and files referenced by
	// those index files are genuine Those fields shall be multi-line fields
	// containing multiple lines of whitespace separated data. Each line shall
	// contain
	//
	// The checksum of the file in the format corresponding to the field The
	// size of the file (integer >= 0) The filename relative to the directory
	// of the Release file Each datum must be separated by one or more
	// whitespace characters.
	//
	// Server requirements:
	//
	// The checksum and sizes shall match the actual existing files. If indexes
	// are compressed, checksum data must be provided for uncompressed files as
	// well, even if not present on the server.  Client behaviour:
	//
	// Any file should be checked at least once, either in compressed or
	// uncompressed form, depending on which data is available. If a file has
	// no associated data, the client shall inform the user about this under
	// possibly dangerous situations (such as installing a package from that
	// repository). If a file does not match the data specified in the release
	// file, the client shall not use any information from that file, inform
	// the user, and might use old information (such as the previous locally
	// kept information) instead.
	MD5Sum []control.MD5FileHash    `delim:"\n" strip:" \t\n\r" multiline:"true"`
	SHA1   []control.SHA1FileHash   `delim:"\n" strip:" \t\n\r" multiline:"true"`
	SHA256 []control.SHA256FileHash `delim:"\n" strip:" \t\n\r" multiline:"true"`
	SHA512 []control.SHA512FileHash `delim:"\n" strip:" \t\n\r" multiline:"true"`

	// The NotAutomatic and ButAutomaticUpgrades fields are optional boolean
	// fields instructing the package manager. They may contain the values
	// "yes" and "no". If one the fields is not specified, this has the same
	// meaning as a value of "no".
	//
	// If a value of "yes" is specified for the NotAutomatic field, a package
	// manager should not install packages (or upgrade to newer versions) from
	// this repository without explicit user consent (APT assigns priority 1 to
	// this) If the field ButAutomaticUpgrades is specified as well and has the
	// value "yes", the package manager should automatically install package
	// upgrades from this repository, if the installed version of the package
	// is higher than the version of the package in other sources (APT assigns
	// priority 100).
	//
	// Specifying "yes" for ButAutomaticUpgrades without specifying "yes" for
	// NotAutomatic is invalid.
	NotAutomatic         string
	ButAutomaticUpgrades string

	AcquireByHash bool `control:"Acquire-By-Hash"`
}

// Given a file declared in the Release file, get the FileHash entries
// for that file (SHA256, SHA512). These can be used to ensure the
// integrety of files in the archive.
func (r *Release) Indices() map[string]control.FileHashes {
	ret := map[string]control.FileHashes{}

	// https://wiki.debian.org/DebianRepository/Format#Size.2C_MD5sum.2C_SHA1.2C_SHA256.2C_SHA512:
	// Clients may not use the MD5Sum and SHA1 fields for security purposes, and
	// must require a SHA256 or a SHA512 field.

	for _, el := range r.SHA256 {
		ret[el.Filename] = append(ret[el.Filename], el.FileHash)
	}
	for _, el := range r.SHA512 {
		ret[el.Filename] = append(ret[el.Filename], el.FileHash)
	}
	return ret
}

func (r *Release) AddHash(h control.FileHash) error {
	switch h.Algorithm {
	case "sha256":
		r.SHA256 = append(r.SHA256, control.SHA256FileHash{h})
	case "sha1":
		r.SHA1 = append(r.SHA1, control.SHA1FileHash{h})
	case "sha512":
		r.SHA512 = append(r.SHA512, control.SHA512FileHash{h})
	case "md5":
		r.MD5Sum = append(r.MD5Sum, control.MD5FileHash{h})
	default:
		return fmt.Errorf("No known hash: '%s'", h.Algorithm)
	}
	return nil
}

// }}}

// LoadInRelease {{{

// Given an InRelease io.Reader, and the OpenPGP keyring
// to validate against, return the parsed InRelease file.
func LoadInRelease(in io.Reader, keyring *openpgp.EntityList) (*Release, error) {
	ret := Release{}
	decoder, err := control.NewDecoder(in, keyring)
	if err != nil {
		return nil, err
	}
	return &ret, decoder.Decode(&ret)
}

// }}}

// LoadInReleaseFile {{{

// Given a path to the InRelease file on the filesystem, and the OpenPGP keyring
// to validate against, return the parsed InRelease file.
func LoadInReleaseFile(path string, keyring *openpgp.EntityList) (*Release, error) {
	fd, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	return LoadInRelease(fd, keyring)
}

// }}}

// vim: foldmethod=marker
<|endoftext|>


<commit_before>/*
 *    Copyright (C) 2017 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

package chains

import (
	"github.com/emicklei/go-restful"
	"github.com/muesli/beehive/bees"
	"github.com/muesli/smolder"
)

// ChainPostStruct holds all values of an incoming POST request
type ChainPostStruct struct {
	Chain struct {
		Name        string     `json:"name"`
		Description string     `json:"description"`
		Event       bees.Event `json:"event"`
		Filters     []string   `json:"filters"`
		Actions     []string   `json:"actions"`
	} `json:"chain"`
}

// PostAuthRequired returns true because all requests need authentication
func (r *ChainResource) PostAuthRequired() bool {
	return false
}

// PostDoc returns the description of this API endpoint
func (r *ChainResource) PostDoc() string {
	return "create a new chain"
}

// PostParams returns the parameters supported by this API endpoint
func (r *ChainResource) PostParams() []*restful.Parameter {
	return nil
}

// Post processes an incoming POST (create) request
func (r *ChainResource) Post(context smolder.APIContext, data interface{}, request *restful.Request, response *restful.Response) {
	resp := ChainResponse{}
	resp.Init(context)

	pps := data.(*ChainPostStruct)
	dupe := bees.GetChain(pps.Chain.Name)
	if dupe != nil {
		smolder.ErrorResponseHandler(request, response, nil, smolder.NewErrorResponse(
			422, // Go 1.7+: http.StatusUnprocessableEntity,
			nil,
			"ChainResource POST"))
		return
	}

	chain := bees.Chain{
		Name:        pps.Chain.Name,
		Description: pps.Chain.Description,
		Event:       &pps.Chain.Event,
		Actions:     pps.Chain.Actions,
		Filters:     pps.Chain.Filters,
	}
	chains := append(bees.GetChains(), chain)
	bees.SetChains(chains)

	resp.AddChain(chain)
	resp.Send(response)
}
<commit_msg>Fixed error response in ChainResource.Post<commit_after>/*
 *    Copyright (C) 2017 Christian Muehlhaeuser
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Christian Muehlhaeuser <muesli@gmail.com>
 */

package chains

import (
	"errors"

	"github.com/emicklei/go-restful"
	"github.com/muesli/beehive/bees"
	"github.com/muesli/smolder"
)

// ChainPostStruct holds all values of an incoming POST request
type ChainPostStruct struct {
	Chain struct {
		Name        string     `json:"name"`
		Description string     `json:"description"`
		Event       bees.Event `json:"event"`
		Filters     []string   `json:"filters"`
		Actions     []string   `json:"actions"`
	} `json:"chain"`
}

// PostAuthRequired returns true because all requests need authentication
func (r *ChainResource) PostAuthRequired() bool {
	return false
}

// PostDoc returns the description of this API endpoint
func (r *ChainResource) PostDoc() string {
	return "create a new chain"
}

// PostParams returns the parameters supported by this API endpoint
func (r *ChainResource) PostParams() []*restful.Parameter {
	return nil
}

// Post processes an incoming POST (create) request
func (r *ChainResource) Post(context smolder.APIContext, data interface{}, request *restful.Request, response *restful.Response) {
	resp := ChainResponse{}
	resp.Init(context)

	pps := data.(*ChainPostStruct)
	dupe := bees.GetChain(pps.Chain.Name)
	if dupe != nil {
		smolder.ErrorResponseHandler(request, response, nil, smolder.NewErrorResponse(
			422, // Go 1.7+: http.StatusUnprocessableEntity,
			errors.New("A Chain with that name exists already"),
			"ChainResource POST"))
		return
	}

	chain := bees.Chain{
		Name:        pps.Chain.Name,
		Description: pps.Chain.Description,
		Event:       &pps.Chain.Event,
		Actions:     pps.Chain.Actions,
		Filters:     pps.Chain.Filters,
	}
	chains := append(bees.GetChains(), chain)
	bees.SetChains(chains)

	resp.AddChain(chain)
	resp.Send(response)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package blake2b

import "encoding/binary"

// the precomputed values for BLAKE2b
// there are 12 16-byte arrays - one for each round
// the entries are calculated from the sigma constants.
var precomputed = [12][16]byte{
	{0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15},
	{14, 4, 9, 13, 10, 8, 15, 6, 1, 0, 11, 5, 12, 2, 7, 3},
	{11, 12, 5, 15, 8, 0, 2, 13, 10, 3, 7, 9, 14, 6, 1, 4},
	{7, 3, 13, 11, 9, 1, 12, 14, 2, 5, 4, 15, 6, 10, 0, 8},
	{9, 5, 2, 10, 0, 7, 4, 15, 14, 11, 6, 3, 1, 12, 8, 13},
	{2, 6, 0, 8, 12, 10, 11, 3, 4, 7, 15, 1, 13, 5, 14, 9},
	{12, 1, 14, 4, 5, 15, 13, 10, 0, 6, 9, 8, 7, 3, 2, 11},
	{13, 7, 12, 3, 11, 14, 1, 9, 5, 15, 8, 2, 0, 4, 6, 10},
	{6, 14, 11, 0, 15, 9, 3, 8, 12, 13, 1, 10, 2, 7, 4, 5},
	{10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0},
	{0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15}, // equal to the first
	{14, 4, 9, 13, 10, 8, 15, 6, 1, 0, 11, 5, 12, 2, 7, 3}, // equal to the second
}

func hashBlocksGeneric(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) {
	var m [16]uint64
	c0, c1 := c[0], c[1]

	for i := 0; i < len(blocks); {
		c0 += BlockSize
		if c0 < BlockSize {
			c1++
		}

		v0, v1, v2, v3, v4, v5, v6, v7 := h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7]
		v8, v9, v10, v11, v12, v13, v14, v15 := iv[0], iv[1], iv[2], iv[3], iv[4], iv[5], iv[6], iv[7]
		v12 ^= c0
		v13 ^= c1
		v14 ^= flag

		for j := range m {
			m[j] = binary.LittleEndian.Uint64(blocks[i:])
			i += 8
		}

		for j := range precomputed {
			s := &(precomputed[j])

			v0 += m[s[0]]
			v0 += v4
			v12 ^= v0
			v12 = v12<<(64-32) | v12>>32
			v8 += v12
			v4 ^= v8
			v4 = v4<<(64-24) | v4>>24
			v1 += m[s[1]]
			v1 += v5
			v13 ^= v1
			v13 = v13<<(64-32) | v13>>32
			v9 += v13
			v5 ^= v9
			v5 = v5<<(64-24) | v5>>24
			v2 += m[s[2]]
			v2 += v6
			v14 ^= v2
			v14 = v14<<(64-32) | v14>>32
			v10 += v14
			v6 ^= v10
			v6 = v6<<(64-24) | v6>>24
			v3 += m[s[3]]
			v3 += v7
			v15 ^= v3
			v15 = v15<<(64-32) | v15>>32
			v11 += v15
			v7 ^= v11
			v7 = v7<<(64-24) | v7>>24

			v0 += m[s[4]]
			v0 += v4
			v12 ^= v0
			v12 = v12<<(64-16) | v12>>16
			v8 += v12
			v4 ^= v8
			v4 = v4<<(64-63) | v4>>63
			v1 += m[s[5]]
			v1 += v5
			v13 ^= v1
			v13 = v13<<(64-16) | v13>>16
			v9 += v13
			v5 ^= v9
			v5 = v5<<(64-63) | v5>>63
			v2 += m[s[6]]
			v2 += v6
			v14 ^= v2
			v14 = v14<<(64-16) | v14>>16
			v10 += v14
			v6 ^= v10
			v6 = v6<<(64-63) | v6>>63
			v3 += m[s[7]]
			v3 += v7
			v15 ^= v3
			v15 = v15<<(64-16) | v15>>16
			v11 += v15
			v7 ^= v11
			v7 = v7<<(64-63) | v7>>63

			v0 += m[s[8]]
			v0 += v5
			v15 ^= v0
			v15 = v15<<(64-32) | v15>>32
			v10 += v15
			v5 ^= v10
			v5 = v5<<(64-24) | v5>>24
			v1 += m[s[9]]
			v1 += v6
			v12 ^= v1
			v12 = v12<<(64-32) | v12>>32
			v11 += v12
			v6 ^= v11
			v6 = v6<<(64-24) | v6>>24
			v2 += m[s[10]]
			v2 += v7
			v13 ^= v2
			v13 = v13<<(64-32) | v13>>32
			v8 += v13
			v7 ^= v8
			v7 = v7<<(64-24) | v7>>24
			v3 += m[s[11]]
			v3 += v4
			v14 ^= v3
			v14 = v14<<(64-32) | v14>>32
			v9 += v14
			v4 ^= v9
			v4 = v4<<(64-24) | v4>>24

			v0 += m[s[12]]
			v0 += v5
			v15 ^= v0
			v15 = v15<<(64-16) | v15>>16
			v10 += v15
			v5 ^= v10
			v5 = v5<<(64-63) | v5>>63
			v1 += m[s[13]]
			v1 += v6
			v12 ^= v1
			v12 = v12<<(64-16) | v12>>16
			v11 += v12
			v6 ^= v11
			v6 = v6<<(64-63) | v6>>63
			v2 += m[s[14]]
			v2 += v7
			v13 ^= v2
			v13 = v13<<(64-16) | v13>>16
			v8 += v13
			v7 ^= v8
			v7 = v7<<(64-63) | v7>>63
			v3 += m[s[15]]
			v3 += v4
			v14 ^= v3
			v14 = v14<<(64-16) | v14>>16
			v9 += v14
			v4 ^= v9
			v4 = v4<<(64-63) | v4>>63

		}

		h[0] ^= v0 ^ v8
		h[1] ^= v1 ^ v9
		h[2] ^= v2 ^ v10
		h[3] ^= v3 ^ v11
		h[4] ^= v4 ^ v12
		h[5] ^= v5 ^ v13
		h[6] ^= v6 ^ v14
		h[7] ^= v7 ^ v15
	}
	c[0], c[1] = c0, c1
}
<commit_msg>blake2b: use math.bits rotate functions instead of ad-hoc implementations<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package blake2b

import (
	"encoding/binary"
	"math/bits"
)

// the precomputed values for BLAKE2b
// there are 12 16-byte arrays - one for each round
// the entries are calculated from the sigma constants.
var precomputed = [12][16]byte{
	{0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15},
	{14, 4, 9, 13, 10, 8, 15, 6, 1, 0, 11, 5, 12, 2, 7, 3},
	{11, 12, 5, 15, 8, 0, 2, 13, 10, 3, 7, 9, 14, 6, 1, 4},
	{7, 3, 13, 11, 9, 1, 12, 14, 2, 5, 4, 15, 6, 10, 0, 8},
	{9, 5, 2, 10, 0, 7, 4, 15, 14, 11, 6, 3, 1, 12, 8, 13},
	{2, 6, 0, 8, 12, 10, 11, 3, 4, 7, 15, 1, 13, 5, 14, 9},
	{12, 1, 14, 4, 5, 15, 13, 10, 0, 6, 9, 8, 7, 3, 2, 11},
	{13, 7, 12, 3, 11, 14, 1, 9, 5, 15, 8, 2, 0, 4, 6, 10},
	{6, 14, 11, 0, 15, 9, 3, 8, 12, 13, 1, 10, 2, 7, 4, 5},
	{10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0},
	{0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15}, // equal to the first
	{14, 4, 9, 13, 10, 8, 15, 6, 1, 0, 11, 5, 12, 2, 7, 3}, // equal to the second
}

func hashBlocksGeneric(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) {
	var m [16]uint64
	c0, c1 := c[0], c[1]

	for i := 0; i < len(blocks); {
		c0 += BlockSize
		if c0 < BlockSize {
			c1++
		}

		v0, v1, v2, v3, v4, v5, v6, v7 := h[0], h[1], h[2], h[3], h[4], h[5], h[6], h[7]
		v8, v9, v10, v11, v12, v13, v14, v15 := iv[0], iv[1], iv[2], iv[3], iv[4], iv[5], iv[6], iv[7]
		v12 ^= c0
		v13 ^= c1
		v14 ^= flag

		for j := range m {
			m[j] = binary.LittleEndian.Uint64(blocks[i:])
			i += 8
		}

		for j := range precomputed {
			s := &(precomputed[j])

			v0 += m[s[0]]
			v0 += v4
			v12 ^= v0
			v12 = bits.RotateLeft64(v12, -32)
			v8 += v12
			v4 ^= v8
			v4 = bits.RotateLeft64(v4, -24)
			v1 += m[s[1]]
			v1 += v5
			v13 ^= v1
			v13 = bits.RotateLeft64(v13, -32)
			v9 += v13
			v5 ^= v9
			v5 = bits.RotateLeft64(v5, -24)
			v2 += m[s[2]]
			v2 += v6
			v14 ^= v2
			v14 = bits.RotateLeft64(v14, -32)
			v10 += v14
			v6 ^= v10
			v6 = bits.RotateLeft64(v6, -24)
			v3 += m[s[3]]
			v3 += v7
			v15 ^= v3
			v15 = bits.RotateLeft64(v15, -32)
			v11 += v15
			v7 ^= v11
			v7 = bits.RotateLeft64(v7, -24)

			v0 += m[s[4]]
			v0 += v4
			v12 ^= v0
			v12 = bits.RotateLeft64(v12, -16)
			v8 += v12
			v4 ^= v8
			v4 = bits.RotateLeft64(v4, -63)
			v1 += m[s[5]]
			v1 += v5
			v13 ^= v1
			v13 = bits.RotateLeft64(v13, -16)
			v9 += v13
			v5 ^= v9
			v5 = bits.RotateLeft64(v5, -63)
			v2 += m[s[6]]
			v2 += v6
			v14 ^= v2
			v14 = bits.RotateLeft64(v14, -16)
			v10 += v14
			v6 ^= v10
			v6 = bits.RotateLeft64(v6, -63)
			v3 += m[s[7]]
			v3 += v7
			v15 ^= v3
			v15 = bits.RotateLeft64(v15, -16)
			v11 += v15
			v7 ^= v11
			v7 = bits.RotateLeft64(v7, -63)

			v0 += m[s[8]]
			v0 += v5
			v15 ^= v0
			v15 = bits.RotateLeft64(v15, -32)
			v10 += v15
			v5 ^= v10
			v5 = bits.RotateLeft64(v5, -24)
			v1 += m[s[9]]
			v1 += v6
			v12 ^= v1
			v12 = bits.RotateLeft64(v12, -32)
			v11 += v12
			v6 ^= v11
			v6 = bits.RotateLeft64(v6, -24)
			v2 += m[s[10]]
			v2 += v7
			v13 ^= v2
			v13 = bits.RotateLeft64(v13, -32)
			v8 += v13
			v7 ^= v8
			v7 = bits.RotateLeft64(v7, -24)
			v3 += m[s[11]]
			v3 += v4
			v14 ^= v3
			v14 = bits.RotateLeft64(v14, -32)
			v9 += v14
			v4 ^= v9
			v4 = bits.RotateLeft64(v4, -24)

			v0 += m[s[12]]
			v0 += v5
			v15 ^= v0
			v15 = bits.RotateLeft64(v15, -16)
			v10 += v15
			v5 ^= v10
			v5 = bits.RotateLeft64(v5, -63)
			v1 += m[s[13]]
			v1 += v6
			v12 ^= v1
			v12 = bits.RotateLeft64(v12, -16)
			v11 += v12
			v6 ^= v11
			v6 = bits.RotateLeft64(v6, -63)
			v2 += m[s[14]]
			v2 += v7
			v13 ^= v2
			v13 = bits.RotateLeft64(v13, -16)
			v8 += v13
			v7 ^= v8
			v7 = bits.RotateLeft64(v7, -63)
			v3 += m[s[15]]
			v3 += v4
			v14 ^= v3
			v14 = bits.RotateLeft64(v14, -16)
			v9 += v14
			v4 ^= v9
			v4 = bits.RotateLeft64(v4, -63)

		}

		h[0] ^= v0 ^ v8
		h[1] ^= v1 ^ v9
		h[2] ^= v2 ^ v10
		h[3] ^= v3 ^ v11
		h[4] ^= v4 ^ v12
		h[5] ^= v5 ^ v13
		h[6] ^= v6 ^ v14
		h[7] ^= v7 ^ v15
	}
	c[0], c[1] = c0, c1
}
<|endoftext|>


<commit_before>package blobstore

import (
	"io"
)

type DirBlobReader interface {
	// Open blob for reading
	Open(bid, key string) error

	// Test whether there's any more entry left
	IsNextEntry() bool

	// Get the next entry from the reader
	NextEntry() (DirEntry, error)
}

type dirBlobReader struct {
	baseBlobReader             // Inherit methods of base blob reader
	Storage        BlobStorage // Blob storage
	currentReader  io.Reader   // Current reader we work on
	entriesLeft    int64       // Number of directory entries left to read
}

func NewDirBlobReader(storage BlobStorage) DirBlobReader {
	return &dirBlobReader{
		baseBlobReader: baseBlobReader{
			storage: storage}}
}

func (d *dirBlobReader) Open(bid, key string) error {

	// Get the raw blob reader
	reader, blobType, err := d.openInternal(bid, key, validationMethodHash)
	if err != nil {
		return err
	}

	// Validate the blob type
	switch blobType {

	case blobTypeSimpleStaticDir:
		d.currentReader = reader
		if d.entriesLeft, err = deserializeInt(reader); err != nil {
			return err
		}
		if d.entriesLeft < 0 || d.entriesLeft > maxSimpleDirEntries {
			return ErrMalformedDirInvalidEntriesCount
		}
		if err = d.eofTest(); err != nil {
			return err
		}
		return nil

	case blobTypeSplitStaticFile:
		panic("Split directory blobs are unimplemented")
	}

	return ErrInvalidFileBlobType
}

func (d *dirBlobReader) IsNextEntry() bool {
	return d.entriesLeft > 0
}

func (d *dirBlobReader) NextEntry() (entry DirEntry, err error) {

	// Test if there's anything else to read
	if !d.IsNextEntry() {
		err = ErrNoMoreDirEntries
		return
	}

	// Make sure the nober of entries left decreases
	// even in case of an error
	d.entriesLeft--

	// Read one entry
	if err = entry.deserialize(d.currentReader); err != nil {
		return
	}

	err = nil
	return
}

func (d *dirBlobReader) eofTest() error {
	if !d.IsNextEntry() && !d.atEOF() {
		// There must be no more data if we're at the end
		// of data stream
		return ErrMalformedDirExtraData
	}
	return nil
}

func (d *dirBlobReader) atEOF() bool {
	// TODO: We're using this for validation only, implement the proper version
	return true
}
<commit_msg>Fix blob type if in the directory reader<commit_after>package blobstore

import (
	"io"
)

type DirBlobReader interface {
	// Open blob for reading
	Open(bid, key string) error

	// Test whether there's any more entry left
	IsNextEntry() bool

	// Get the next entry from the reader
	NextEntry() (DirEntry, error)
}

type dirBlobReader struct {
	baseBlobReader             // Inherit methods of base blob reader
	Storage        BlobStorage // Blob storage
	currentReader  io.Reader   // Current reader we work on
	entriesLeft    int64       // Number of directory entries left to read
}

func NewDirBlobReader(storage BlobStorage) DirBlobReader {
	return &dirBlobReader{
		baseBlobReader: baseBlobReader{
			storage: storage}}
}

func (d *dirBlobReader) Open(bid, key string) error {

	// Get the raw blob reader
	reader, blobType, err := d.openInternal(bid, key, validationMethodHash)
	if err != nil {
		return err
	}

	// Validate the blob type
	switch blobType {

	case blobTypeSimpleStaticDir:
		d.currentReader = reader
		if d.entriesLeft, err = deserializeInt(reader); err != nil {
			return err
		}
		if d.entriesLeft < 0 || d.entriesLeft > maxSimpleDirEntries {
			return ErrMalformedDirInvalidEntriesCount
		}
		if err = d.eofTest(); err != nil {
			return err
		}
		return nil

	case blobTypeSplitStaticDir:
		panic("Split directory blobs are unimplemented")
	}

	return ErrInvalidFileBlobType
}

func (d *dirBlobReader) IsNextEntry() bool {
	return d.entriesLeft > 0
}

func (d *dirBlobReader) NextEntry() (entry DirEntry, err error) {

	// Test if there's anything else to read
	if !d.IsNextEntry() {
		err = ErrNoMoreDirEntries
		return
	}

	// Make sure the nober of entries left decreases
	// even in case of an error
	d.entriesLeft--

	// Read one entry
	if err = entry.deserialize(d.currentReader); err != nil {
		return
	}

	err = nil
	return
}

func (d *dirBlobReader) eofTest() error {
	if !d.IsNextEntry() && !d.atEOF() {
		// There must be no more data if we're at the end
		// of data stream
		return ErrMalformedDirExtraData
	}
	return nil
}

func (d *dirBlobReader) atEOF() bool {
	// TODO: We're using this for validation only, implement the proper version
	return true
}
<|endoftext|>


<commit_before>// 12 february 2014
package main

import (
	"fmt"
	"sync"
)

// A Button represents a clickable button with some text.
type Button struct {
	// This channel gets a message when the button is clicked. Unlike other channels in this package, this channel is initialized to non-nil when creating a new button, and cannot be set to nil later.
	Clicked	chan struct{}

	lock		sync.Mutex
	created	bool
	parent	Control
	pWin		*Window
	sysData	*sysData
	initText	string
}

// NewButton creates a new button with the specified text.
func NewButton(text string) (b *Button) {
	return &Button{
		sysData:	&sysData{
			cSysData:		cSysData{
				ctype:	c_button,
			},
		},
		initText:	text,
		Clicked:	make(chan struct{}),
	}
}

// SetText sets the button's text.
func (b *Button) SetText(text string) (err error) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if b.pWin != nil && b.pWin.created {
		panic("TODO")
	}
	b.initText = text
	return nil
}

func (b *Button) apply() error {
	b.lock.Lock()
	defer b.lock.Unlock()

	if b.pWin == nil {
		panic(fmt.Sprintf("button (initial text: %q) without parent window", b.initText))
	}
	if !b.pWin.created {
		b.sysData.clicked = b.Clicked
		b.sysData.parentWindow = b.pWin.sysData
		return b.sysData.make(b.initText, 300, 300)
		// TODO size to parent size
	}
	return b.sysData.show()
}

func (b *Button) setParent(c Control) {
	b.parent = c
	if w, ok := b.parent.(*Window); ok {
		b.pWin = w
	} else {
		b.pWin = c.parentWindow()
	}
}

func (b *Button) parentWindow() *Window {
	return b.pWin
}
<commit_msg>Modified Button for the new changes. Now I just need to actually create the window class and edit main()...<commit_after>// 12 february 2014
package main

import (
	"fmt"
	"sync"
)

// A Button represents a clickable button with some text.
type Button struct {
	// This channel gets a message when the button is clicked. Unlike other channels in this package, this channel is initialized to non-nil when creating a new button, and cannot be set to nil later.
	Clicked	chan struct{}

	lock		sync.Mutex
	created	bool
	parent	Control
	sysData	*sysData
	initText	string
}

// NewButton creates a new button with the specified text.
func NewButton(text string) (b *Button) {
	return &Button{
		sysData:	&sysData{
			cSysData:		cSysData{
				ctype:	c_button,
			},
		},
		initText:	text,
		Clicked:	make(chan struct{}),
	}
}

// SetText sets the button's text.
func (b *Button) SetText(text string) (err error) {
	b.lock.Lock()
	defer b.lock.Unlock()

	if b.pWin != nil && b.pWin.created {
		panic("TODO")
	}
	b.initText = text
	return nil
}

func (b *Button) apply(window *sysData) error {
	b.lock.Lock()
	defer b.lock.Unlock()

	b.sysData.clicked = b.Clicked
	return b.sysData.make(b.initText, 300, 300, window)
	// TODO size to parent size
}

func (b *Button) setParent(c Control) {
	b.parent = c
}
<|endoftext|>


<commit_before>package handler

import (
	"net/http"
	"os"
	"strconv"
	"strings"

	"ratings/parser"
	"ratings/responses"

	"github.com/go-playground/validator"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/labstack/gommon/log"
)

type RequestValidator struct {
	validator *validator.Validate
}

func (rv *RequestValidator) Validate(request interface{}) error {
	return rv.validator.Struct(request)
}

func badRequestMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if !hasAcceptHeader(context) {
			message = "Accept header is missing"

			return responses.ErrorResponse(http.StatusBadRequest, message, context)
		}

		if context.Request().Method == echo.OPTIONS && hasContentTypeHeader(context) {
			message = "OPTIONS requests must have no body"

			return responses.ErrorResponse(http.StatusBadRequest, message, context)
		}

		if context.Request().Method == echo.POST && !hasContentTypeHeader(context) {
			message = "Content-Type header is missing"

			return responses.ErrorResponse(http.StatusBadRequest, message, context)
		}

		return next(context)
	}
}

func notAcceptableMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if !isValidAcceptHeader(context) {
			message = "Not accepting JSON responses"

			return responses.ErrorResponse(http.StatusNotAcceptable, message, context)
		}

		return next(context)
	}
}

func unsupportedMediaTypeMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if context.Request().Method == echo.POST {
			if !isValidContentTypeHeader(context) || !isValidCharacterEncoding(context) {
				message = "Request body must be UTF-8 encoded JSON"

				return responses.ErrorResponse(http.StatusUnsupportedMediaType, message, context)
			}
		}

		return next(context)
	}
}

func notImplementedMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		if context.Request().Method != echo.POST && context.Request().Method != echo.OPTIONS {
			return responses.ErrorResponse(http.StatusNotImplemented, "", context)
		}

		return next(context)
	}
}

func corsMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		context.Response().Header().Set("Access-Control-Allow-Origin", "*")
		context.Response().Header().Set("Access-Control-Allow-Methods", "OPTIONS, POST")
		context.Response().Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, Authorization")

		return next(context)
	}
}

func hasAcceptHeader(context echo.Context) bool {
	if header := context.Request().Header.Get("Accept"); strings.TrimSpace(header) != "" {
		return true
	}

	return false
}

func isValidAcceptHeader(context echo.Context) bool {
	accepted := "application/json"

	if header := context.Request().Header.Get("Accept"); strings.Contains(strings.ToLower(header), accepted) || header == "*/*" {
		return true
	}

	return false
}

func hasContentTypeHeader(context echo.Context) bool {
	if header := context.Request().Header.Get("Content-Type"); strings.TrimSpace(header) != "" {
		return true
	}

	return false
}

func isValidContentTypeHeader(context echo.Context) bool {
	text := "text/plain"
	json := "application/json"

	if header := context.Request().Header.Get("Content-Type"); strings.Contains(strings.ToLower(header), text) || strings.Contains(strings.ToLower(header), json) {
		return true
	}

	return false
}

func isValidCharacterEncoding(context echo.Context) bool {
	charset := "charset=utf-8"

	if header := context.Request().Header.Get("Content-Type"); strings.HasSuffix(strings.ToLower(header), charset) {
		return true
	}

	return false
}

func Handler(port int, handlers map[string]echo.HandlerFunc) http.Handler {
	e := echo.New()
	validate := validator.New()
	env := os.Getenv("API_RATINGS_ENV")

	parser.RegisterCustomValidators(validate)

	e.Use(middleware.Secure())
	e.Use(middleware.Logger())
	e.Use(middleware.Recover())
	e.Use(middleware.BodyLimit("20K"))
	e.Use(notImplementedMiddleware)
	e.Use(notAcceptableMiddleware)
	e.Use(badRequestMiddleware)
	e.Use(unsupportedMediaTypeMiddleware)
	e.Use(corsMiddleware)

	e.OPTIONS("/", handlers["OptionsRoot"])
	e.OPTIONS("/ratings", handlers["OptionsRatings"])
	e.POST("/ratings", handlers["PostRatings"])

	if env == "DEV" {
		e.Logger.SetLevel(log.DEBUG)
		e.Debug = true
	} else {
		e.Logger.SetLevel(log.ERROR)
	}

	e.Server.Addr = ":" + strconv.Itoa(port)
	e.HTTPErrorHandler = responses.ErrorHandler
	e.Validator = &RequestValidator{validator: validate}

	return e
}
<commit_msg>Added https redirect middleware for production environment<commit_after>package handler

import (
	"net/http"
	"os"
	"strconv"
	"strings"

	"ratings/parser"
	"ratings/responses"

	"github.com/go-playground/validator"
	"github.com/labstack/echo"
	"github.com/labstack/echo/middleware"
	"github.com/labstack/gommon/log"
)

type RequestValidator struct {
	validator *validator.Validate
}

func (rv *RequestValidator) Validate(request interface{}) error {
	return rv.validator.Struct(request)
}

func badRequestMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if !hasAcceptHeader(context) {
			message = "Accept header is missing"

			return responses.ErrorResponse(http.StatusBadRequest, message, context)
		}

		if context.Request().Method == echo.OPTIONS && hasContentTypeHeader(context) {
			message = "OPTIONS requests must have no body"

			return responses.ErrorResponse(http.StatusBadRequest, message, context)
		}

		if context.Request().Method == echo.POST && !hasContentTypeHeader(context) {
			message = "Content-Type header is missing"

			return responses.ErrorResponse(http.StatusBadRequest, message, context)
		}

		return next(context)
	}
}

func notAcceptableMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if !isValidAcceptHeader(context) {
			message = "Not accepting JSON responses"

			return responses.ErrorResponse(http.StatusNotAcceptable, message, context)
		}

		return next(context)
	}
}

func unsupportedMediaTypeMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		var message string

		if context.Request().Method == echo.POST {
			if !isValidContentTypeHeader(context) || !isValidCharacterEncoding(context) {
				message = "Request body must be UTF-8 encoded JSON"

				return responses.ErrorResponse(http.StatusUnsupportedMediaType, message, context)
			}
		}

		return next(context)
	}
}

func notImplementedMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		if context.Request().Method != echo.POST && context.Request().Method != echo.OPTIONS {
			return responses.ErrorResponse(http.StatusNotImplemented, "", context)
		}

		return next(context)
	}
}

func corsMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
	return func(context echo.Context) error {
		context.Response().Header().Set("Access-Control-Allow-Origin", "*")
		context.Response().Header().Set("Access-Control-Allow-Methods", "OPTIONS, POST")
		context.Response().Header().Set("Access-Control-Allow-Headers", "Content-Type, Accept, Authorization")

		return next(context)
	}
}

func hasAcceptHeader(context echo.Context) bool {
	if header := context.Request().Header.Get("Accept"); strings.TrimSpace(header) != "" {
		return true
	}

	return false
}

func isValidAcceptHeader(context echo.Context) bool {
	accepted := "application/json"

	if header := context.Request().Header.Get("Accept"); strings.Contains(strings.ToLower(header), accepted) || header == "*/*" {
		return true
	}

	return false
}

func hasContentTypeHeader(context echo.Context) bool {
	if header := context.Request().Header.Get("Content-Type"); strings.TrimSpace(header) != "" {
		return true
	}

	return false
}

func isValidContentTypeHeader(context echo.Context) bool {
	text := "text/plain"
	json := "application/json"

	if header := context.Request().Header.Get("Content-Type"); strings.Contains(strings.ToLower(header), text) || strings.Contains(strings.ToLower(header), json) {
		return true
	}

	return false
}

func isValidCharacterEncoding(context echo.Context) bool {
	charset := "charset=utf-8"

	if header := context.Request().Header.Get("Content-Type"); strings.HasSuffix(strings.ToLower(header), charset) {
		return true
	}

	return false
}

func Handler(port int, handlers map[string]echo.HandlerFunc) http.Handler {
	e := echo.New()
	validate := validator.New()
	env := os.Getenv("API_RATINGS_ENV")

	parser.RegisterCustomValidators(validate)

	if env == "DEV" {
		e.Logger.SetLevel(log.DEBUG)
		e.Debug = true
	} else {
		e.Pre(middleware.HTTPSRedirect())
		e.Logger.SetLevel(log.ERROR)
	}

	e.Use(middleware.Secure())
	e.Use(middleware.Logger())
	e.Use(middleware.Recover())
	e.Use(middleware.BodyLimit("20K"))
	e.Use(notImplementedMiddleware)
	e.Use(notAcceptableMiddleware)
	e.Use(badRequestMiddleware)
	e.Use(unsupportedMediaTypeMiddleware)
	e.Use(corsMiddleware)

	e.OPTIONS("/", handlers["OptionsRoot"])
	e.OPTIONS("/ratings", handlers["OptionsRatings"])
	e.POST("/ratings", handlers["PostRatings"])

	e.Server.Addr = ":" + strconv.Itoa(port)
	e.HTTPErrorHandler = responses.ErrorHandler
	e.Validator = &RequestValidator{validator: validate}

	return e
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package spyglass

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/url"
	"path"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"github.com/sirupsen/logrus"
	"google.golang.org/api/iterator"

	"k8s.io/test-infra/prow/spyglass/viewers"
	"k8s.io/test-infra/testgrid/util/gcs"
)

const (
	httpScheme  = "http"
	httpsScheme = "https"
)

var (
	ErrCannotParseSource = errors.New("could not create job source from provided source")
)

// GCSArtifactFetcher contains information used for fetching artifacts from GCS
type GCSArtifactFetcher struct {
	client *storage.Client
}

// gcsJobSource is a location in GCS where Prow job-specific artifacts are stored. This implementation assumes
// Prow's native GCS upload format (treating GCS keys as a directory structure), and is not
// intended to support arbitrary GCS bucket upload formats.
type gcsJobSource struct {
	source     string
	linkPrefix string
	bucket     string
	jobPrefix  string
	jobName    string
	buildID    string
}

// NewGCSArtifactFetcher creates a new ArtifactFetcher with a real GCS Client
func NewGCSArtifactFetcher(c *storage.Client) *GCSArtifactFetcher {
	return &GCSArtifactFetcher{
		client: c,
	}
}

func fieldsForJob(src *gcsJobSource) logrus.Fields {
	return logrus.Fields{
		"jobPrefix": src.jobPath(),
	}
}

// newGCSJobSource creates a new gcsJobSource from a given bucket and jobPrefix
func newGCSJobSource(src string) (*gcsJobSource, error) {
	gcsURL, err := url.Parse(fmt.Sprintf("gs://%s", src))
	if err != nil {
		return &gcsJobSource{}, ErrCannotParseSource
	}
	gcsPath := &gcs.Path{}
	err = gcsPath.SetURL(gcsURL)
	if err != nil {
		return &gcsJobSource{}, ErrCannotParseSource
	}

	tokens := strings.FieldsFunc(gcsPath.Object(), func(c rune) bool { return c == '/' })
	if len(tokens) < 2 {
		return &gcsJobSource{}, ErrCannotParseSource
	}
	buildID := tokens[len(tokens)-1]
	name := tokens[len(tokens)-2]
	return &gcsJobSource{
		source:     src,
		linkPrefix: "gs://",
		bucket:     gcsPath.Bucket(),
		jobPrefix:  path.Clean(gcsPath.Object()) + "/",
		jobName:    name,
		buildID:    buildID,
	}, nil
}

// Artifacts lists all artifacts available for the given job source
func (af *GCSArtifactFetcher) artifacts(key string) ([]string, error) {
	src, err := newGCSJobSource(key)
	if err != nil {
		return nil, fmt.Errorf("Failed to get GCS job source from %s: %v", key, err)
	}

	listStart := time.Now()
	bucketName, prefix := extractBucketPrefixPair(src.jobPath())
	artifacts := []string{}
	bkt := af.client.Bucket(bucketName)
	q := storage.Query{
		Prefix:   prefix,
		Versions: false,
	}
	objIter := bkt.Objects(context.Background(), &q)
	for {
		oAttrs, err := objIter.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			logrus.WithFields(fieldsForJob(src)).WithError(err).Error("Error accessing GCS artifact.")
			continue
		}
		artifacts = append(artifacts, strings.TrimPrefix(oAttrs.Name, prefix))

	}
	listElapsed := time.Since(listStart)
	logrus.WithField("duration", listElapsed).Infof("Listed %d artifacts.", len(artifacts))
	return artifacts, nil
}

type gcsArtifactHandle struct {
	*storage.ObjectHandle
}

func (h *gcsArtifactHandle) NewReader(ctx context.Context) (io.ReadCloser, error) {
	return h.ObjectHandle.NewReader(ctx)
}

func (h *gcsArtifactHandle) NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
	return h.ObjectHandle.NewRangeReader(ctx, offset, length)
}

// Artifact contructs a GCS artifact from the given GCS bucket and key. Uses the golang GCS library
// to get read handles. If the artifactName is not a valid key in the bucket a handle will still be
// constructed and returned, but all read operations will fail (dictated by behavior of golang GCS lib).
func (af *GCSArtifactFetcher) artifact(key string, artifactName string, sizeLimit int64) (viewers.Artifact, error) {
	src, err := newGCSJobSource(key)
	if err != nil {
		return nil, fmt.Errorf("Failed to get GCS job source from %s: %v", key, err)
	}

	bucketName, prefix := extractBucketPrefixPair(src.jobPath())
	bkt := af.client.Bucket(bucketName)
	obj := &gcsArtifactHandle{bkt.Object(path.Join(prefix, artifactName))}
	artifactLink := &url.URL{
		Scheme: httpsScheme,
		Host:   "storage.googleapis.com",
		Path:   path.Join(src.jobPath(), artifactName),
	}
	return NewGCSArtifact(context.Background(), obj, artifactLink.String(), artifactName, sizeLimit), nil
}

func extractBucketPrefixPair(gcsPath string) (string, string) {
	split := strings.SplitN(gcsPath, "/", 2)
	return split[0], split[1]
}

// CanonicalLink gets a link to the location of job-specific artifacts in GCS
func (src *gcsJobSource) canonicalLink() string {
	return path.Join(src.linkPrefix, src.bucket, src.jobPrefix)
}

// JobPath gets the prefix to all artifacts in GCS in the job
func (src *gcsJobSource) jobPath() string {
	return fmt.Sprintf("%s/%s", src.bucket, src.jobPrefix)
}
<commit_msg>doc comment ErrCannotParseSource<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package spyglass

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net/url"
	"path"
	"strings"
	"time"

	"cloud.google.com/go/storage"
	"github.com/sirupsen/logrus"
	"google.golang.org/api/iterator"

	"k8s.io/test-infra/prow/spyglass/viewers"
	"k8s.io/test-infra/testgrid/util/gcs"
)

const (
	httpScheme  = "http"
	httpsScheme = "https"
)

var (
	// returned by newGCSJobSource when an incorrectly formatted source string is passed
	ErrCannotParseSource = errors.New("could not create job source from provided source")
)

// GCSArtifactFetcher contains information used for fetching artifacts from GCS
type GCSArtifactFetcher struct {
	client *storage.Client
}

// gcsJobSource is a location in GCS where Prow job-specific artifacts are stored. This implementation assumes
// Prow's native GCS upload format (treating GCS keys as a directory structure), and is not
// intended to support arbitrary GCS bucket upload formats.
type gcsJobSource struct {
	source     string
	linkPrefix string
	bucket     string
	jobPrefix  string
	jobName    string
	buildID    string
}

// NewGCSArtifactFetcher creates a new ArtifactFetcher with a real GCS Client
func NewGCSArtifactFetcher(c *storage.Client) *GCSArtifactFetcher {
	return &GCSArtifactFetcher{
		client: c,
	}
}

func fieldsForJob(src *gcsJobSource) logrus.Fields {
	return logrus.Fields{
		"jobPrefix": src.jobPath(),
	}
}

// newGCSJobSource creates a new gcsJobSource from a given bucket and jobPrefix
func newGCSJobSource(src string) (*gcsJobSource, error) {
	gcsURL, err := url.Parse(fmt.Sprintf("gs://%s", src))
	if err != nil {
		return &gcsJobSource{}, ErrCannotParseSource
	}
	gcsPath := &gcs.Path{}
	err = gcsPath.SetURL(gcsURL)
	if err != nil {
		return &gcsJobSource{}, ErrCannotParseSource
	}

	tokens := strings.FieldsFunc(gcsPath.Object(), func(c rune) bool { return c == '/' })
	if len(tokens) < 2 {
		return &gcsJobSource{}, ErrCannotParseSource
	}
	buildID := tokens[len(tokens)-1]
	name := tokens[len(tokens)-2]
	return &gcsJobSource{
		source:     src,
		linkPrefix: "gs://",
		bucket:     gcsPath.Bucket(),
		jobPrefix:  path.Clean(gcsPath.Object()) + "/",
		jobName:    name,
		buildID:    buildID,
	}, nil
}

// Artifacts lists all artifacts available for the given job source
func (af *GCSArtifactFetcher) artifacts(key string) ([]string, error) {
	src, err := newGCSJobSource(key)
	if err != nil {
		return nil, fmt.Errorf("Failed to get GCS job source from %s: %v", key, err)
	}

	listStart := time.Now()
	bucketName, prefix := extractBucketPrefixPair(src.jobPath())
	artifacts := []string{}
	bkt := af.client.Bucket(bucketName)
	q := storage.Query{
		Prefix:   prefix,
		Versions: false,
	}
	objIter := bkt.Objects(context.Background(), &q)
	for {
		oAttrs, err := objIter.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			logrus.WithFields(fieldsForJob(src)).WithError(err).Error("Error accessing GCS artifact.")
			continue
		}
		artifacts = append(artifacts, strings.TrimPrefix(oAttrs.Name, prefix))

	}
	listElapsed := time.Since(listStart)
	logrus.WithField("duration", listElapsed).Infof("Listed %d artifacts.", len(artifacts))
	return artifacts, nil
}

type gcsArtifactHandle struct {
	*storage.ObjectHandle
}

func (h *gcsArtifactHandle) NewReader(ctx context.Context) (io.ReadCloser, error) {
	return h.ObjectHandle.NewReader(ctx)
}

func (h *gcsArtifactHandle) NewRangeReader(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
	return h.ObjectHandle.NewRangeReader(ctx, offset, length)
}

// Artifact contructs a GCS artifact from the given GCS bucket and key. Uses the golang GCS library
// to get read handles. If the artifactName is not a valid key in the bucket a handle will still be
// constructed and returned, but all read operations will fail (dictated by behavior of golang GCS lib).
func (af *GCSArtifactFetcher) artifact(key string, artifactName string, sizeLimit int64) (viewers.Artifact, error) {
	src, err := newGCSJobSource(key)
	if err != nil {
		return nil, fmt.Errorf("Failed to get GCS job source from %s: %v", key, err)
	}

	bucketName, prefix := extractBucketPrefixPair(src.jobPath())
	bkt := af.client.Bucket(bucketName)
	obj := &gcsArtifactHandle{bkt.Object(path.Join(prefix, artifactName))}
	artifactLink := &url.URL{
		Scheme: httpsScheme,
		Host:   "storage.googleapis.com",
		Path:   path.Join(src.jobPath(), artifactName),
	}
	return NewGCSArtifact(context.Background(), obj, artifactLink.String(), artifactName, sizeLimit), nil
}

func extractBucketPrefixPair(gcsPath string) (string, string) {
	split := strings.SplitN(gcsPath, "/", 2)
	return split[0], split[1]
}

// CanonicalLink gets a link to the location of job-specific artifacts in GCS
func (src *gcsJobSource) canonicalLink() string {
	return path.Join(src.linkPrefix, src.bucket, src.jobPrefix)
}

// JobPath gets the prefix to all artifacts in GCS in the job
func (src *gcsJobSource) jobPath() string {
	return fmt.Sprintf("%s/%s", src.bucket, src.jobPrefix)
}
<|endoftext|>