<commit_before>package main

import (
  "flag"
	"fmt"
	"text/template"
	"os"
  "io/ioutil"
  "gopkg.in/yaml.v2"
  "github.com/winkapp/libclc"
)

type File struct {
  Path string
  Content string
  Owner string
  Permissions string
}

type CloudConfig struct {
  DiscoveryUrl string
  Files []*File
}

var template_dir string
var root string
var config_file string

func main() {
  // Figure out which directory we are going to prep files for
  template_def := os.ExpandEnv("$GOPATH/src/github.com/winkapp/clc/templates")
  flag.StringVar(&template_dir, "templates", template_def, "Your templates.")
  flag.StringVar(&root, "root", "./", "Directory for configs and output.")
  flag.StringVar(&config_file, "config", "clc.yaml", "Optional config file.")
  flag.Parse()
  command := flag.Arg(0)

  switch command {
  case "cc": cc(root)
  case "vagrant": vagrant(root)
  case "units": units(root)
  case "new": newCluster(root)
  }

}

func newCluster(path string) {
  cc(root)
  vagrant(root)
  units(root)
  return
}

func vagrant(path string) {
  ud(path)
  copyFile(path + "/Vagrantfile", template_dir + "/Vagrantfile.template")
  copyFile(path + "/config.rb", template_dir + "/config.rb.template")
  return
}

func units(path string) {
  unitsYaml := getFileBytes(path + "/units.yaml")
  var units libclc.UnitConfig
  err := yaml.Unmarshal(unitsYaml, &units)
  checkError(err)
  t := unitsTemplate()
  err = libclc.WriteUnits(&units, t, path + "/units")
  checkError(err)
}

func ud(path string) {
  // TODO: Open the files document for file includes
  file1 := File{
    Content: getFile("./etcd_env.sh"),
    Path: "/home/core/etcd_env.sh",
    Owner: "core:core",
    Permissions: "744",
  }

  data := CloudConfig{
    Files: []*File{&file1},
  }

  t := udTemplate(data)
  err := t.Execute(os.Stdout, data)
  checkError(err)
}

func cc(path string) {
  data := ccData(path)
  t := ccTemplate(data)
  err := t.Execute(os.Stdout, data)
	checkError(err)
}

func ccData(path string) (data CloudConfig) {
  // Open the files document for file includes
  file1 := File{
    Content: getFile("./etcd_env.sh"),
    Path: "/home/core/etcd_env.sh",
    Owner: "core:core",
    Permissions: "744",
  }

  // Open the document for the discovery url
  discovery_url := getFile(path + "/discovery_url")

  // Include files on cloud config
  data = CloudConfig{
    DiscoveryUrl: discovery_url,
    Files: []*File{&file1},
  }
  return
}

func unitsTemplate() (t *template.Template) {
  t = getTemplate("Service Template", "service.template")
  return
}

func ccTemplate(data CloudConfig) (t *template.Template) {
  t = getTemplate("Cloud Config Template", "cloud-config.template")
  return
}

func udTemplate(data CloudConfig) (t *template.Template) {
  t = getTemplate("User Data Template", "user-data.template")
  return
}

func addFile(path string, f os.FileInfo, err error) error {
  fmt.Printf("Visited: %s\n", path)
  return nil
}

func getFileBytes(path string) []byte {
  dat, err := ioutil.ReadFile(path)
  checkError(err)
  return dat
}

func getFile(path string) string {
  return string(getFileBytes(path))
}

func copyFile(dstPath string, srcPath string) {
  src, err := ioutil.ReadFile(srcPath)
  checkError(err)
  mode := int(0644)
  err = ioutil.WriteFile(dstPath, src, os.FileMode(mode))
  checkError(err)
  return
}

func getTemplate(name string, filename string) (t *template.Template) {
  templ := getFile(template_dir + "/" + filename)

  t = template.New(name)
  t, err := t.Parse(templ)
  checkError(err)
  return
}

func checkError(err error) {
	if err != nil {
		fmt.Println("Fatal error:", err.Error())
		os.Exit(1)
	}
}
<commit_msg>Started to factor out user data and cloud config generation to libclc.<commit_after>package main

import (
  "flag"
	"fmt"
	"text/template"
	"os"
  "io/ioutil"
  "gopkg.in/yaml.v2"
  "github.com/winkapp/libclc"
)

type File struct {
  Path string
  Content string
  Owner string
  Permissions string
}

type CloudConfig struct {
  DiscoveryUrl string
  Files []*File
}

var template_dir string
var root string
var config_file string

func main() {
  // Figure out which directory we are going to prep files for
  template_def := os.ExpandEnv("$GOPATH/src/github.com/winkapp/clc/templates")
  flag.StringVar(&template_dir, "templates", template_def, "Your templates.")
  flag.StringVar(&root, "root", "./", "Directory for configs and output.")
  flag.StringVar(&config_file, "config", "clc.yaml", "Optional config file.")
  flag.Parse()
  command := flag.Arg(0)

  switch command {
  case "cc": cc(root)
  case "vagrant": vagrant(root)
  case "units": units(root)
  case "new": newCluster(root)
  }

}

func newCluster(path string) {
  cc(root)
  vagrant(root)
  units(root)
  return
}

func vagrant(path string) {
  ud(path)
  copyFile(path + "/Vagrantfile", template_dir + "/Vagrantfile.template")
  copyFile(path + "/config.rb", template_dir + "/config.rb.template")
  return
}

func units(path string) {
  unitsYaml := getFileBytes(path + "/units.yaml")
  var units libclc.UnitConfig
  err := yaml.Unmarshal(unitsYaml, &units)
  checkError(err)
  t := unitsTemplate()
  err = libclc.WriteUnits(&units, t, path + "/units")
  checkError(err)
}

func ud(path string) {
  // TODO: Open the files document for file includes
  file1 := libclc.File{
    Content: getFile("./etcd_env.sh"),
    Path: "/home/core/etcd_env.sh",
    Owner: "core:core",
    Permissions: "744",
  }

  data := libclc.CloudConfig{
    Files: []*libclc.File{&file1},
  }

  t := udTemplate(data)
  err := libclc.WriteUserData(&data, t, path + "/user-data")
  checkError(err)
}

func cc(path string) {
  data := ccData(path)
  t := ccTemplate(data)
  err := t.Execute(os.Stdout, data)
	checkError(err)
}

func ccData(path string) (data libclc.CloudConfig) {
  // Open the files document for file includes
  file1 := libclc.File{
    Content: getFile("./etcd_env.sh"),
    Path: "/home/core/etcd_env.sh",
    Owner: "core:core",
    Permissions: "744",
  }

  // Open the document for the discovery url
  discovery_url := getFile(path + "/discovery_url")

  // Include files on cloud config
  data = libclc.CloudConfig{
    DiscoveryUrl: discovery_url,
    Files: []*libclc.File{&file1},
  }
  return
}

func unitsTemplate() (t *template.Template) {
  t = getTemplate("Service Template", "service.template")
  return
}

func ccTemplate(data libclc.CloudConfig) (t *template.Template) {
  t = getTemplate("Cloud Config Template", "cloud-config.template")
  return
}

func udTemplate(data libclc.CloudConfig) (t *template.Template) {
  t = getTemplate("User Data Template", "user-data.template")
  return
}

func addFile(path string, f os.FileInfo, err error) error {
  fmt.Printf("Visited: %s\n", path)
  return nil
}

func getFileBytes(path string) []byte {
  dat, err := ioutil.ReadFile(path)
  checkError(err)
  return dat
}

func getFile(path string) string {
  return string(getFileBytes(path))
}

func copyFile(dstPath string, srcPath string) {
  src, err := ioutil.ReadFile(srcPath)
  checkError(err)
  mode := int(0644)
  err = ioutil.WriteFile(dstPath, src, os.FileMode(mode))
  checkError(err)
  return
}

func getTemplate(name string, filename string) (t *template.Template) {
  templ := getFile(template_dir + "/" + filename)

  t = template.New(name)
  t, err := t.Parse(templ)
  checkError(err)
  return
}

func checkError(err error) {
	if err != nil {
		fmt.Println("Fatal error:", err.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package provider

import (
	"context"
	"fmt"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/hashicorp/terraform-plugin-go/tfprotov5"
	"github.com/hashicorp/terraform-plugin-go/tftypes"
	"github.com/hashicorp/terraform-provider-kubernetes/manifest/morph"
	"github.com/hashicorp/terraform-provider-kubernetes/manifest/payload"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

// ImportResourceState function
func (s *RawProviderServer) ImportResourceState(ctx context.Context, req *tfprotov5.ImportResourceStateRequest) (*tfprotov5.ImportResourceStateResponse, error) {
	// Terraform only gives us the schema name of the resource and an ID string, as passed by the user on the command line.
	// The ID should be a combination of a Kubernetes GVK and a namespace/name type of resource identifier.
	// Without the user supplying the GRV there is no way to fully identify the resource when making the Get API call to K8s.
	// Presumably the Kubernetes API machinery already has a standard for expressing such a group. We should look there first.
	resp := &tfprotov5.ImportResourceStateResponse{}
	gvk, name, namespace, err := parseImportID(req.ID)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to parse import ID",
			Detail:   err.Error(),
		})
	}
	s.logger.Trace("[ImportResourceState]", "[ID]", gvk, name, namespace)
	rt, err := GetResourceType(req.TypeName)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to determine resource type",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	rm, err := s.getRestMapper()
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to get RESTMapper client",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	client, err := s.getDynamicClient()
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "failed to get Dynamic client",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	ns, err := IsResourceNamespaced(gvk, rm)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to get namespacing requirement from RESTMapper",
			Detail:   err.Error(),
		})
		return resp, nil
	}

	io := unstructured.Unstructured{}
	io.SetKind(gvk.Kind)
	io.SetAPIVersion(gvk.GroupVersion().String())
	io.SetName(name)
	io.SetNamespace(namespace)

	gvr, err := GVRFromUnstructured(&io, rm)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to get GVR from GVK via RESTMapper",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	rcl := client.Resource(gvr)

	var ro *unstructured.Unstructured
	if ns {
		ro, err = rcl.Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
	} else {
		ro, err = rcl.Get(ctx, name, metav1.GetOptions{})
	}
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  fmt.Sprintf("Failed to get resource %s from API", spew.Sdump(io)),
			Detail:   err.Error(),
		})
		return resp, nil
	}
	s.logger.Trace("[ImportResourceState]", "[API Resource]", spew.Sdump(ro))

	objectType, err := s.TFTypeFromOpenAPI(ctx, gvk, false)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  fmt.Sprintf("Failed to determine resource type from GVK: %s", gvk),
			Detail:   err.Error(),
		})
		return resp, nil
	}

	fo := RemoveServerSideFields(ro.UnstructuredContent())
	nobj, err := payload.ToTFValue(fo, objectType, tftypes.NewAttributePath())
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to convert unstructured to tftypes.Value",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	nobj, err = morph.DeepUnknown(objectType, nobj, tftypes.NewAttributePath())
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to backfill unknown values during import",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	s.logger.Trace("[ImportResourceState]", "[tftypes.Value]", spew.Sdump(nobj))

	newState := make(map[string]tftypes.Value)
	wftype := rt.(tftypes.Object).AttributeTypes["wait_for"]
	newState["manifest"] = tftypes.NewValue(tftypes.Object{AttributeTypes: map[string]tftypes.Type{}}, nil)
	newState["object"] = morph.UnknownToNull(nobj)
	newState["wait_for"] = tftypes.NewValue(wftype, nil)
	nsVal := tftypes.NewValue(rt, newState)

	impState, err := tfprotov5.NewDynamicValue(nsVal.Type(), nsVal)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to construct dynamic value for imported state",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	resp.ImportedResources = append(resp.ImportedResources, &tfprotov5.ImportedResource{
		TypeName: req.TypeName,
		State:    &impState,
	})
	return resp, nil
}

// parseImportID processes the resource ID string passed by the user to the "terraform import" command
// and extracts the values for GVK, name and (optionally) namespace of the target resource as required
// during the import process.
//
// The expected format for the import resource ID is:
//
// "<apiGroup/><apiVersion>#<Kind>#<namespace>#<name>"
//
// where 'namespace' is only required for resources that expect a namespace.
//
// Note the '#' separator between the elements of the ID string.
//
// Example: "v1#Secret#default#default-token-qgm6s"
//
func parseImportID(id string) (gvk schema.GroupVersionKind, name string, namespace string, err error) {
	parts := strings.Split(id, "#")
	if len(parts) < 3 || len(parts) > 4 {
		err = fmt.Errorf("invalid format for import ID [%s]", id)
		return
	}
	gvk = schema.FromAPIVersionAndKind(parts[0], parts[1])
	if len(parts) == 4 {
		namespace = parts[2]
		name = parts[3]
	} else {
		name = parts[2]
	}
	return
}
<commit_msg>Remove usage of go-spew<commit_after>package provider

import (
	"context"
	"fmt"
	"strings"

	"github.com/hashicorp/terraform-plugin-go/tfprotov5"
	"github.com/hashicorp/terraform-plugin-go/tftypes"
	"github.com/hashicorp/terraform-provider-kubernetes/manifest/morph"
	"github.com/hashicorp/terraform-provider-kubernetes/manifest/payload"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

// ImportResourceState function
func (s *RawProviderServer) ImportResourceState(ctx context.Context, req *tfprotov5.ImportResourceStateRequest) (*tfprotov5.ImportResourceStateResponse, error) {
	// Terraform only gives us the schema name of the resource and an ID string, as passed by the user on the command line.
	// The ID should be a combination of a Kubernetes GVK and a namespace/name type of resource identifier.
	// Without the user supplying the GRV there is no way to fully identify the resource when making the Get API call to K8s.
	// Presumably the Kubernetes API machinery already has a standard for expressing such a group. We should look there first.
	resp := &tfprotov5.ImportResourceStateResponse{}
	gvk, name, namespace, err := parseImportID(req.ID)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to parse import ID",
			Detail:   err.Error(),
		})
	}
	s.logger.Trace("[ImportResourceState]", "[ID]", gvk, name, namespace)
	rt, err := GetResourceType(req.TypeName)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to determine resource type",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	rm, err := s.getRestMapper()
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to get RESTMapper client",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	client, err := s.getDynamicClient()
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "failed to get Dynamic client",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	ns, err := IsResourceNamespaced(gvk, rm)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to get namespacing requirement from RESTMapper",
			Detail:   err.Error(),
		})
		return resp, nil
	}

	io := unstructured.Unstructured{}
	io.SetKind(gvk.Kind)
	io.SetAPIVersion(gvk.GroupVersion().String())
	io.SetName(name)
	io.SetNamespace(namespace)

	gvr, err := GVRFromUnstructured(&io, rm)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to get GVR from GVK via RESTMapper",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	rcl := client.Resource(gvr)

	var ro *unstructured.Unstructured
	if ns {
		ro, err = rcl.Namespace(namespace).Get(ctx, name, metav1.GetOptions{})
	} else {
		ro, err = rcl.Get(ctx, name, metav1.GetOptions{})
	}
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  fmt.Sprintf("Failed to get resource %+v from API", io),
			Detail:   err.Error(),
		})
		return resp, nil
	}
	s.logger.Trace("[ImportResourceState]", "[API Resource]", ro)

	objectType, err := s.TFTypeFromOpenAPI(ctx, gvk, false)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  fmt.Sprintf("Failed to determine resource type from GVK: %s", gvk),
			Detail:   err.Error(),
		})
		return resp, nil
	}

	fo := RemoveServerSideFields(ro.UnstructuredContent())
	nobj, err := payload.ToTFValue(fo, objectType, tftypes.NewAttributePath())
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to convert unstructured to tftypes.Value",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	nobj, err = morph.DeepUnknown(objectType, nobj, tftypes.NewAttributePath())
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to backfill unknown values during import",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	s.logger.Trace("[ImportResourceState]", "[tftypes.Value]", nobj)

	newState := make(map[string]tftypes.Value)
	wftype := rt.(tftypes.Object).AttributeTypes["wait_for"]
	newState["manifest"] = tftypes.NewValue(tftypes.Object{AttributeTypes: map[string]tftypes.Type{}}, nil)
	newState["object"] = morph.UnknownToNull(nobj)
	newState["wait_for"] = tftypes.NewValue(wftype, nil)
	nsVal := tftypes.NewValue(rt, newState)

	impState, err := tfprotov5.NewDynamicValue(nsVal.Type(), nsVal)
	if err != nil {
		resp.Diagnostics = append(resp.Diagnostics, &tfprotov5.Diagnostic{
			Severity: tfprotov5.DiagnosticSeverityError,
			Summary:  "Failed to construct dynamic value for imported state",
			Detail:   err.Error(),
		})
		return resp, nil
	}
	resp.ImportedResources = append(resp.ImportedResources, &tfprotov5.ImportedResource{
		TypeName: req.TypeName,
		State:    &impState,
	})
	return resp, nil
}

// parseImportID processes the resource ID string passed by the user to the "terraform import" command
// and extracts the values for GVK, name and (optionally) namespace of the target resource as required
// during the import process.
//
// The expected format for the import resource ID is:
//
// "<apiGroup/><apiVersion>#<Kind>#<namespace>#<name>"
//
// where 'namespace' is only required for resources that expect a namespace.
//
// Note the '#' separator between the elements of the ID string.
//
// Example: "v1#Secret#default#default-token-qgm6s"
//
func parseImportID(id string) (gvk schema.GroupVersionKind, name string, namespace string, err error) {
	parts := strings.Split(id, "#")
	if len(parts) < 3 || len(parts) > 4 {
		err = fmt.Errorf("invalid format for import ID [%s]", id)
		return
	}
	gvk = schema.FromAPIVersionAndKind(parts[0], parts[1])
	if len(parts) == 4 {
		namespace = parts[2]
		name = parts[3]
	} else {
		name = parts[2]
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"reflect"
	"testing"
	"time"
)

// TODO refactor tests to use testify/assert library like `bitrot_test.go`
func TestManifestComparison(t *testing.T) {
	newCreatedAt := time.Now()
	newModTime := time.Now()
	oldCreatedAt := newCreatedAt.Add(-24 * time.Hour)
	oldModTime := newModTime.Add(-24 * time.Hour)
	oldManifest := Manifest{
		Path:      "/old/stuff",
		CreatedAt: oldCreatedAt,
		Entries: map[string]ChecksumRecord{
			"silently_corrupted": {Checksum: "asdf", ModTime: oldModTime},
			"not_changed":        {Checksum: "zxcv", ModTime: oldModTime},
			"modified":           {Checksum: "qwer", ModTime: oldModTime},
			"touched":            {Checksum: "olkm", ModTime: oldModTime},
			"deleted":            {Checksum: "jklh", ModTime: oldModTime},
			"renamedOld":         {Checksum: "xxxx", ModTime: oldModTime},
		},
	}

	newManifest := Manifest{
		Path:      "/new/thing",
		CreatedAt: newCreatedAt,
		Entries: map[string]ChecksumRecord{
			"silently_corrupted": {Checksum: "zzzz", ModTime: oldModTime},
			"not_changed":        {Checksum: "zxcv", ModTime: oldModTime},
			"modified":           {Checksum: "tyui", ModTime: newModTime},
			"touched":            {Checksum: "olkm", ModTime: newModTime},
			"added":              {Checksum: "bnmv", ModTime: newModTime},
			"renamedNew":         {Checksum: "xxxx", ModTime: oldModTime},
		},
	}

	comparison := CompareManifests(&oldManifest, &newManifest)

	expectedUnchangedPaths := []string{"not_changed", "touched"}
	if !reflect.DeepEqual(comparison.UnchangedPaths, expectedUnchangedPaths) {
		t.Fatalf("expected UnchangedPaths %v; got %v", expectedUnchangedPaths, comparison.UnchangedPaths)
	}

	expectedDeletedPaths := []string{"deleted"}
	if !reflect.DeepEqual(comparison.DeletedPaths, expectedDeletedPaths) {
		t.Fatalf("expected DeletedPaths %v; got %v", expectedDeletedPaths, comparison.DeletedPaths)
	}

	expectedModifiedPaths := []string{"modified"}
	if !reflect.DeepEqual(comparison.ModifiedPaths, expectedModifiedPaths) {
		t.Fatalf("expected ModifiedPaths %v; got %v", expectedModifiedPaths, comparison.ModifiedPaths)
	}

	expectedFlaggedPaths := []string{"silently_corrupted"}
	if !reflect.DeepEqual(comparison.FlaggedPaths, expectedFlaggedPaths) {
		t.Fatalf("expected FlaggedPaths %v; got %v", expectedFlaggedPaths, comparison.FlaggedPaths)
	}

	expectedAddedPaths := []string{"added"}
	if !reflect.DeepEqual(comparison.AddedPaths, expectedAddedPaths) {
		t.Fatalf("expected AddedPaths %v; got %v", expectedAddedPaths, comparison.AddedPaths)
	}

	expectedRenamedPaths := []RenamedPath{{OldPath: "renamedOld", NewPath: "renamedNew"}}
	if !reflect.DeepEqual(comparison.RenamedPaths, expectedRenamedPaths) {
		t.Fatalf("expected RenamedPaths %v; got %v", expectedRenamedPaths, comparison.RenamedPaths)
	}
}
<commit_msg>Refactor ManifestComparison test to be less flaky<commit_after>package main

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestManifestComparison(t *testing.T) {
	newCreatedAt := time.Now()
	newModTime := time.Now()
	oldCreatedAt := newCreatedAt.Add(-24 * time.Hour)
	oldModTime := newModTime.Add(-24 * time.Hour)
	oldManifest := Manifest{
		Path:      "/old/stuff",
		CreatedAt: oldCreatedAt,
		Entries: map[string]ChecksumRecord{
			"silently_corrupted": {Checksum: "asdf", ModTime: oldModTime},
			"not_changed":        {Checksum: "zxcv", ModTime: oldModTime},
			"modified":           {Checksum: "qwer", ModTime: oldModTime},
			"touched":            {Checksum: "olkm", ModTime: oldModTime},
			"deleted":            {Checksum: "jklh", ModTime: oldModTime},
			"renamedOld":         {Checksum: "xxxx", ModTime: oldModTime},
		},
	}

	newManifest := Manifest{
		Path:      "/new/thing",
		CreatedAt: newCreatedAt,
		Entries: map[string]ChecksumRecord{
			"silently_corrupted": {Checksum: "zzzz", ModTime: oldModTime},
			"not_changed":        {Checksum: "zxcv", ModTime: oldModTime},
			"modified":           {Checksum: "tyui", ModTime: newModTime},
			"touched":            {Checksum: "olkm", ModTime: newModTime},
			"added":              {Checksum: "bnmv", ModTime: newModTime},
			"renamedNew":         {Checksum: "xxxx", ModTime: oldModTime},
		},
	}

	comparison := CompareManifests(&oldManifest, &newManifest)

	assert.ElementsMatch(t, comparison.UnchangedPaths, []string{"not_changed", "touched"})
	assert.ElementsMatch(t, comparison.DeletedPaths, []string{"deleted"})
	assert.ElementsMatch(t, comparison.ModifiedPaths, []string{"modified"})
	assert.ElementsMatch(t, comparison.FlaggedPaths, []string{"silently_corrupted"})
	assert.ElementsMatch(t, comparison.AddedPaths, []string{"added"})
	assert.ElementsMatch(t, comparison.RenamedPaths, []RenamedPath{{OldPath: "renamedOld", NewPath: "renamedNew"}})
}
<|endoftext|>


<commit_before>package worker

import (
	"github.com/mitchellh/multistep"
	"github.com/travis-ci/worker/context"
)

type stepCheckCancellation struct{}

func (s *stepCheckCancellation) Run(state multistep.StateBag) multistep.StepAction {
	ctx := state.Get("ctx").(gocontext.Context)
	buildJob := state.Get("buildJob").(Job)
	logWriter := state.Get("logWriter").(LogWriter)
	cancelChan := state.Get("cancelChan").(<-chan struct{})

	select {
	case <-cancelChan:
		s.writeLogAndFinishWithState(ctx, logWriter, buildJob, FinishStateCancelled, "\n\nDone: Job Cancelled\n\n")
	default:
	}

	return multistep.ActionHalt
}

func (s *stepCheckCancellation) Cleanup(state multistep.StateBag) {
}

func (s *stepCheckCancellation) writeLogAndFinishWithState(ctx gocontext.Context, logWriter LogWriter, buildJob Job, state FinishState, logMessage string) {
	_, err := logWriter.WriteAndClose([]byte(logMessage))
	if err != nil {
		context.LoggerFromContext(ctx).WithField("err", err).Error("couldn't write final log message")
	}

	err = buildJob.Finish(state)
	if err != nil {
		context.LoggerFromContext(ctx).WithField("err", err).WithField("state", state).Error("couldn't update job state")
	}
}
<commit_msg>step_check_cancellation: add missing context package import<commit_after>package worker

import (
	gocontext "context"
	"github.com/mitchellh/multistep"
	"github.com/travis-ci/worker/context"
)

type stepCheckCancellation struct{}

func (s *stepCheckCancellation) Run(state multistep.StateBag) multistep.StepAction {
	ctx := state.Get("ctx").(gocontext.Context)
	buildJob := state.Get("buildJob").(Job)
	logWriter := state.Get("logWriter").(LogWriter)
	cancelChan := state.Get("cancelChan").(<-chan struct{})

	select {
	case <-cancelChan:
		s.writeLogAndFinishWithState(ctx, logWriter, buildJob, FinishStateCancelled, "\n\nDone: Job Cancelled\n\n")
	default:
	}

	return multistep.ActionHalt
}

func (s *stepCheckCancellation) Cleanup(state multistep.StateBag) {
}

func (s *stepCheckCancellation) writeLogAndFinishWithState(ctx gocontext.Context, logWriter LogWriter, buildJob Job, state FinishState, logMessage string) {
	_, err := logWriter.WriteAndClose([]byte(logMessage))
	if err != nil {
		context.LoggerFromContext(ctx).WithField("err", err).Error("couldn't write final log message")
	}

	err = buildJob.Finish(state)
	if err != nil {
		context.LoggerFromContext(ctx).WithField("err", err).WithField("state", state).Error("couldn't update job state")
	}
}
<|endoftext|>


<commit_before>package parser

/*
#cgo LDFLAGS: -lavformat -lavutil -lavcodec
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libavcodec/avcodec.h>
#include <string.h>
#include <stdlib.h>

#define TIMEBASE_Q (AVRational){1, 90000}

AVStream* get_stream(AVStream **streams, int pos)
{
  return streams[pos];
}

int64_t rescale_to_generic_timebase(int64_t val, AVRational timebase)
{
  return av_rescale_q(val, timebase, TIMEBASE_Q);
}

int64_t rescale_to_timebase(int64_t val, int in_tb, int out_tb)
{
  AVRational in_r;
  AVRational out_r;

  in_r.num = 1;
  in_r.den = in_tb;

  out_r.num = 1;
  out_r.den = out_tb;

  return av_rescale_q(val, in_r, out_r);
}

char *convert_byte_slice(void *buffer, int size)
{
  char *res = malloc(size);
  memcpy(res, (char *)buffer, size);
  return res;
}
*/
import "C"
import "fmt"
import "errors"
import "unsafe"
import "runtime"

/* Structure used to reference FFMPEG C AVFormatContext structure */
type FFMPEGDemuxer struct {
	context   *C.AVFormatContext
	pkt       C.AVPacket
}

/* Structure used to store a Sample for chunk generation */
type Sample struct {
	pts      int64
	dts      int64
	duration int64
	keyFrame bool
	data	 unsafe.Pointer
	size     C.int
	encrypt  *SampleEncryption
}

/* Called when starting the program, initialise FFMPEG demuxers */
func FFMPEGInitialise() error {
	_, err := C.av_register_all()
	return err
}

func CInt(val int) C.int {
	return C.int(val)
}

func CArray(buffer []byte) unsafe.Pointer {
	if len(buffer) > 0 {
		return unsafe.Pointer(C.convert_byte_slice(unsafe.Pointer(&buffer[0]), C.int(len(buffer))))
	}
	return nil
}

func CFree(ptr unsafe.Pointer) {
	if ptr != nil {
		C.free(ptr)
	}
}

func TimebaseRescale(val int, tbIn int, tbOut int) int {
	return int(C.rescale_to_timebase(C.int64_t(val), C.int(tbIn), C.int(tbOut)))
}

/* Return byte data from a sample */
func (s *Sample) GetData() []byte {
	return C.GoBytes(s.data, s.size)
}

/* Find a track using its index */
func findTrack(tracks []*Track, index int) *Track {
	for i := 0; i < len(tracks); i++ {
		if tracks[i].index == index {
			return tracks[i]
		}
	}
	return nil
}

/* Open FFMPEG specific demuxer */
func (d *FFMPEGDemuxer) Open(path string) error {
	res, err := C.avformat_open_input(&(d.context), C.CString(path), nil, nil)
	if err != nil {
		return err
	} else if res < 0 {
		return errors.New("Could not open source file " + path)
	} else {
		C.av_opt_set_int(unsafe.Pointer(d.context), C.CString("max_analyze_duration"), C.int64_t(0), C.int(0))
		return nil
	}
}

/* Find the first Video track for use as chunk size reference */
func (d *FFMPEGDemuxer) findMainIndex() int {
	var stream *C.AVStream
	for i := 0; i < int(d.context.nb_streams); i++ {
		stream = C.get_stream(d.context.streams, C.int(i))
		if (stream.codec.codec_type == C.AVMEDIA_TYPE_VIDEO) {
			return int(stream.index)
		}
	}
	return 0
}

/* Called by GC to free sample data memory */
func packetFinalizer(s *Sample) {
	C.av_free(unsafe.Pointer(s.data))
}

/* Append a sample to a track */
func (d *FFMPEGDemuxer) AppendSample(track *Track, stream *C.AVStream) {
	sample := new(Sample)
	/* Copy packet metadata in sample */
	sample.pts = int64(C.rescale_to_generic_timebase(d.pkt.pts, stream.time_base))
	sample.dts = int64(C.rescale_to_generic_timebase(d.pkt.dts, stream.time_base))
	sample.duration = int64(C.rescale_to_generic_timebase(C.int64_t(d.pkt.duration), stream.time_base))
	sample.keyFrame = (d.pkt.flags) & 0x1 > 0
	/* Copy packet data in sample */
	sample.size = d.pkt.size
	sample.data = unsafe.Pointer(C.av_malloc(C.size_t(d.pkt.size)))
	C.memcpy(sample.data, unsafe.Pointer(d.pkt.data), C.size_t(d.pkt.size))
	/* Set finalizer to free memory when GC is called */
	runtime.SetFinalizer(sample, packetFinalizer)
	/* Append sample to track */
	track.appendSample(sample)
}

/*
Extract one chunk for each track from input, size of the chunk depends on the first
 video track found.
 */
func (d *FFMPEGDemuxer) ExtractChunk(tracks *[]*Track, isLive bool) bool {
	var track *Track
	var stream *C.AVStream
	/* Find first video track to use as reference for chunk size */
	mainIndex := d.findMainIndex()
	/* Append last extracted chunk */
	d.AppendSample(findTrack(*tracks, int(d.pkt.stream_index)), C.get_stream(d.context.streams, C.int(d.pkt.stream_index)))
	C.av_free_packet(&d.pkt)
	/* Read frames until reference track sample uis a key frame */
	res := C.av_read_frame(d.context, &d.pkt)
	for ; res >= 0; res = C.av_read_frame(d.context, &d.pkt) {
		/* Retrieve track corresponding to packet, if we have one*/
		track = findTrack(*tracks, int(d.pkt.stream_index))
		stream = C.get_stream(d.context.streams, C.int(d.pkt.stream_index))
		if track != nil {
			/* quit if pkt is from reference track and is a key frame*/
			if (track.index == mainIndex && ((d.pkt.flags) & 0x1 > 0) && len(track.samples) > 0) {
				break
			}
			/* Otherwise append sample to chunk */
			d.AppendSample(track, stream)
			C.av_free_packet(&d.pkt)
		}
	}
	/* Return if we have reached EOF */
	return res >= 0
}

/* Retrieve tracks from previously opened file using FFMPEG */
func (d *FFMPEGDemuxer) GetTracks(tracks *[]*Track) error {
	var track *Track
	var stream *C.AVStream
	/* Iterate over streams found by ffmpeg */
	for i := 0; i < int(d.context.nb_streams); i++ {
		/* Little hack to retrieve the stream due to pointer arithmetic */
		stream = C.get_stream(d.context.streams, C.int(i))
		if stream.codec.codec_type == C.AVMEDIA_TYPE_VIDEO {
			/* Test if video is H264 */
			if stream.codec.codec_id != C.AV_CODEC_ID_H264 {
				return fmt.Errorf("Video track is not encoded in H264 (codec_id=%d)", stream.codec.codec_id)
			}
			/* Set video specific info in track structure */
			track = new(Track)
			track.width = int(stream.codec.width)
			track.height = int(stream.codec.height)
			track.bitsPerSample = int(stream.codec.bits_per_coded_sample)
			track.colorTableId = int(stream.codec.color_table_id)
			track.isAudio = false
		} else if stream.codec.codec_type == C.AVMEDIA_TYPE_AUDIO {
			/* Test if audio is AAC */
			if stream.codec.codec_id != C.AV_CODEC_ID_AAC  {
				return fmt.Errorf("Audio track is not encoded in AAC (codec_id=%d)", stream.codec.codec_id)
			}
			/* Set audio specific info in track structure */
			track = new(Track)
			track.sampleRate = int(stream.codec.sample_rate)
			track.isAudio = true
		} else {
			continue
		}
		/* Set common properties in track structure */
		track.SetTimeFields()
		track.duration = int(C.rescale_to_generic_timebase(stream.duration, stream.time_base))
		track.globalTimescale = 90000
		track.timescale = 90000
		track.extradata = C.GoBytes(unsafe.Pointer(stream.codec.extradata), stream.codec.extradata_size)
		track.index = int(stream.index)
		/* Append track to slice */
		*tracks = append(*tracks, track)
	}
	C.av_read_frame(d.context, &d.pkt)
	return nil
}

/* Close demuxer and free FFMPEG specific data */
func (d *FFMPEGDemuxer) Close() {
	C.avformat_close_input(&d.context);
}
<commit_msg>disable deprecated color_table_id<commit_after>package parser

/*
#cgo LDFLAGS: -lavformat -lavutil -lavcodec
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libavcodec/avcodec.h>
#include <string.h>
#include <stdlib.h>

#define TIMEBASE_Q (AVRational){1, 90000}

AVStream* get_stream(AVStream **streams, int pos)
{
  return streams[pos];
}

int64_t rescale_to_generic_timebase(int64_t val, AVRational timebase)
{
  return av_rescale_q(val, timebase, TIMEBASE_Q);
}

int64_t rescale_to_timebase(int64_t val, int in_tb, int out_tb)
{
  AVRational in_r;
  AVRational out_r;

  in_r.num = 1;
  in_r.den = in_tb;

  out_r.num = 1;
  out_r.den = out_tb;

  return av_rescale_q(val, in_r, out_r);
}

char *convert_byte_slice(void *buffer, int size)
{
  char *res = malloc(size);
  memcpy(res, (char *)buffer, size);
  return res;
}
*/
import "C"
import "fmt"
import "errors"
import "unsafe"
import "runtime"

/* Structure used to reference FFMPEG C AVFormatContext structure */
type FFMPEGDemuxer struct {
	context   *C.AVFormatContext
	pkt       C.AVPacket
}

/* Structure used to store a Sample for chunk generation */
type Sample struct {
	pts      int64
	dts      int64
	duration int64
	keyFrame bool
	data	 unsafe.Pointer
	size     C.int
	encrypt  *SampleEncryption
}

/* Called when starting the program, initialise FFMPEG demuxers */
func FFMPEGInitialise() error {
	_, err := C.av_register_all()
	return err
}

func CInt(val int) C.int {
	return C.int(val)
}

func CArray(buffer []byte) unsafe.Pointer {
	if len(buffer) > 0 {
		return unsafe.Pointer(C.convert_byte_slice(unsafe.Pointer(&buffer[0]), C.int(len(buffer))))
	}
	return nil
}

func CFree(ptr unsafe.Pointer) {
	if ptr != nil {
		C.free(ptr)
	}
}

func TimebaseRescale(val int, tbIn int, tbOut int) int {
	return int(C.rescale_to_timebase(C.int64_t(val), C.int(tbIn), C.int(tbOut)))
}

/* Return byte data from a sample */
func (s *Sample) GetData() []byte {
	return C.GoBytes(s.data, s.size)
}

/* Find a track using its index */
func findTrack(tracks []*Track, index int) *Track {
	for i := 0; i < len(tracks); i++ {
		if tracks[i].index == index {
			return tracks[i]
		}
	}
	return nil
}

/* Open FFMPEG specific demuxer */
func (d *FFMPEGDemuxer) Open(path string) error {
	res, err := C.avformat_open_input(&(d.context), C.CString(path), nil, nil)
	if err != nil {
		return err
	} else if res < 0 {
		return errors.New("Could not open source file " + path)
	} else {
		C.av_opt_set_int(unsafe.Pointer(d.context), C.CString("max_analyze_duration"), C.int64_t(0), C.int(0))
		return nil
	}
}

/* Find the first Video track for use as chunk size reference */
func (d *FFMPEGDemuxer) findMainIndex() int {
	var stream *C.AVStream
	for i := 0; i < int(d.context.nb_streams); i++ {
		stream = C.get_stream(d.context.streams, C.int(i))
		if (stream.codec.codec_type == C.AVMEDIA_TYPE_VIDEO) {
			return int(stream.index)
		}
	}
	return 0
}

/* Called by GC to free sample data memory */
func packetFinalizer(s *Sample) {
	C.av_free(unsafe.Pointer(s.data))
}

/* Append a sample to a track */
func (d *FFMPEGDemuxer) AppendSample(track *Track, stream *C.AVStream) {
	sample := new(Sample)
	/* Copy packet metadata in sample */
	sample.pts = int64(C.rescale_to_generic_timebase(d.pkt.pts, stream.time_base))
	sample.dts = int64(C.rescale_to_generic_timebase(d.pkt.dts, stream.time_base))
	sample.duration = int64(C.rescale_to_generic_timebase(C.int64_t(d.pkt.duration), stream.time_base))
	sample.keyFrame = (d.pkt.flags) & 0x1 > 0
	/* Copy packet data in sample */
	sample.size = d.pkt.size
	sample.data = unsafe.Pointer(C.av_malloc(C.size_t(d.pkt.size)))
	C.memcpy(sample.data, unsafe.Pointer(d.pkt.data), C.size_t(d.pkt.size))
	/* Set finalizer to free memory when GC is called */
	runtime.SetFinalizer(sample, packetFinalizer)
	/* Append sample to track */
	track.appendSample(sample)
}

/*
Extract one chunk for each track from input, size of the chunk depends on the first
 video track found.
 */
func (d *FFMPEGDemuxer) ExtractChunk(tracks *[]*Track, isLive bool) bool {
	var track *Track
	var stream *C.AVStream
	/* Find first video track to use as reference for chunk size */
	mainIndex := d.findMainIndex()
	/* Append last extracted chunk */
	d.AppendSample(findTrack(*tracks, int(d.pkt.stream_index)), C.get_stream(d.context.streams, C.int(d.pkt.stream_index)))
	C.av_free_packet(&d.pkt)
	/* Read frames until reference track sample uis a key frame */
	res := C.av_read_frame(d.context, &d.pkt)
	for ; res >= 0; res = C.av_read_frame(d.context, &d.pkt) {
		/* Retrieve track corresponding to packet, if we have one*/
		track = findTrack(*tracks, int(d.pkt.stream_index))
		stream = C.get_stream(d.context.streams, C.int(d.pkt.stream_index))
		if track != nil {
			/* quit if pkt is from reference track and is a key frame*/
			if (track.index == mainIndex && ((d.pkt.flags) & 0x1 > 0) && len(track.samples) > 0) {
				break
			}
			/* Otherwise append sample to chunk */
			d.AppendSample(track, stream)
			C.av_free_packet(&d.pkt)
		}
	}
	/* Return if we have reached EOF */
	return res >= 0
}

/* Retrieve tracks from previously opened file using FFMPEG */
func (d *FFMPEGDemuxer) GetTracks(tracks *[]*Track) error {
	var track *Track
	var stream *C.AVStream
	/* Iterate over streams found by ffmpeg */
	for i := 0; i < int(d.context.nb_streams); i++ {
		/* Little hack to retrieve the stream due to pointer arithmetic */
		stream = C.get_stream(d.context.streams, C.int(i))
		if stream.codec.codec_type == C.AVMEDIA_TYPE_VIDEO {
			/* Test if video is H264 */
			if stream.codec.codec_id != C.AV_CODEC_ID_H264 {
				return fmt.Errorf("Video track is not encoded in H264 (codec_id=%d)", stream.codec.codec_id)
			}
			/* Set video specific info in track structure */
			track = new(Track)
			track.width = int(stream.codec.width)
			track.height = int(stream.codec.height)
			track.bitsPerSample = int(stream.codec.bits_per_coded_sample)
			// track.colorTableId = int(stream.codec.color_table_id)
			track.isAudio = false
		} else if stream.codec.codec_type == C.AVMEDIA_TYPE_AUDIO {
			/* Test if audio is AAC */
			if stream.codec.codec_id != C.AV_CODEC_ID_AAC  {
				return fmt.Errorf("Audio track is not encoded in AAC (codec_id=%d)", stream.codec.codec_id)
			}
			/* Set audio specific info in track structure */
			track = new(Track)
			track.sampleRate = int(stream.codec.sample_rate)
			track.isAudio = true
		} else {
			continue
		}
		/* Set common properties in track structure */
		track.SetTimeFields()
		track.duration = int(C.rescale_to_generic_timebase(stream.duration, stream.time_base))
		track.globalTimescale = 90000
		track.timescale = 90000
		track.extradata = C.GoBytes(unsafe.Pointer(stream.codec.extradata), stream.codec.extradata_size)
		track.index = int(stream.index)
		/* Append track to slice */
		*tracks = append(*tracks, track)
	}
	C.av_read_frame(d.context, &d.pkt)
	return nil
}

/* Close demuxer and free FFMPEG specific data */
func (d *FFMPEGDemuxer) Close() {
	C.avformat_close_input(&d.context);
}
<|endoftext|>


<commit_before>package main

//go:generate echo mm

import (
	"log"

	"github.com/codegangsta/cli"
	"github.com/mozilla-services/go-bouncer/bouncer"
	"github.com/mozilla-services/go-bouncer/go-sentry/sentry"
)

func main() {
	app := cli.NewApp()
	app.Name = "sentry"
	app.Action = Main
	app.Version = bouncer.Version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "db-dsn",
			Value:  "user:password@tcp(localhost:3306)/bouncer",
			Usage:  "database DSN (https://github.com/go-sql-driver/mysql#dsn-data-source-name)",
			EnvVar: "SENTRY_DB_DSN",
		},
		cli.BoolFlag{
			Name:  "checknow",
			Usage: "Checks mirrors marked with checknow",
		},
		cli.StringFlag{
			Name:  "mirror",
			Usage: "If set, checks a specific mirror",
		},
		cli.IntFlag{
			Name:  "mirror-routines",
			Usage: "How many mirrors can be checked at once.",
			Value: 5,
		},
		cli.IntFlag{
			Name:  "location-routines",
			Usage: "How many locations can be checked at once.",
			Value: 15,
		},
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "Output run log",
		},
	}
	app.RunAndExitOnError()
}

func Main(c *cli.Context) {
	db, err := bouncer.NewDB(c.String("db-dsn"))
	if err != nil {
		log.Fatalf("Could not open DB: %v", err)
	}
	defer db.Close()

	sentry, err := sentry.New(db, c.Bool("checknow"), c.String("mirror"), c.Int("mirror-routines"), c.Int("location-routines"))
	if err != nil {
		log.Fatal(err)
	}
	sentry.Verbose = c.Bool("verbose")

	err = sentry.Run()
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>sentry: log in mozlog format<commit_after>package main

//go:generate echo mm

import (
	"log"

	"github.com/codegangsta/cli"
	"github.com/mozilla-services/go-bouncer/bouncer"
	"github.com/mozilla-services/go-bouncer/go-sentry/sentry"

	_ "github.com/mozilla-services/go-bouncer/mozlog"
)

func main() {
	app := cli.NewApp()
	app.Name = "sentry"
	app.Action = Main
	app.Version = bouncer.Version
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "db-dsn",
			Value:  "user:password@tcp(localhost:3306)/bouncer",
			Usage:  "database DSN (https://github.com/go-sql-driver/mysql#dsn-data-source-name)",
			EnvVar: "SENTRY_DB_DSN",
		},
		cli.BoolFlag{
			Name:  "checknow",
			Usage: "Checks mirrors marked with checknow",
		},
		cli.StringFlag{
			Name:  "mirror",
			Usage: "If set, checks a specific mirror",
		},
		cli.IntFlag{
			Name:  "mirror-routines",
			Usage: "How many mirrors can be checked at once.",
			Value: 5,
		},
		cli.IntFlag{
			Name:  "location-routines",
			Usage: "How many locations can be checked at once.",
			Value: 15,
		},
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "Output run log",
		},
	}
	app.RunAndExitOnError()
}

func Main(c *cli.Context) {
	db, err := bouncer.NewDB(c.String("db-dsn"))
	if err != nil {
		log.Fatalf("Could not open DB: %v", err)
	}
	defer db.Close()

	sentry, err := sentry.New(db, c.Bool("checknow"), c.String("mirror"), c.Int("mirror-routines"), c.Int("location-routines"))
	if err != nil {
		log.Fatal(err)
	}
	sentry.Verbose = c.Bool("verbose")

	err = sentry.Run()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"log"
	"github.com/qadium/plumb/shell"
	// "gopkg.in/yaml.v2"
	// "io/ioutil"
)

func Bootstrap(commit string) error {
	log.Printf("==> Bootstraping plumb.")
	defer log.Printf("<== Bootstrap complete.")

	if err := shell.RunAndLog("docker",
		"run",
		"--rm",
		"--entrypoint=\"/bin/bash\"",
		"-v",
		"/var/run/docker.sock:/var/run/docker.sock",
		"centurylink/golang-builder",
		"-c",
		fmt.Sprintf(`git clone https://github.com/qadium/plumb /plumb \
&& cd /plumb \
&& git checkout -b build %s \
&& cp -r /plumb/manager/* /src \
&& cd /src \
&& /build.sh`, commit)); err != nil {
		return err
	}

	// use docker to compile the manager and copy the binary into
	// another docker container
	//
	// tag this new container

	return nil
}
<commit_msg>commits the manager image to plumb/manager<commit_after>package cli

import (
	"fmt"
	"log"
	"github.com/qadium/plumb/shell"
	// "gopkg.in/yaml.v2"
	// "io/ioutil"
)

func Bootstrap(commit string) error {
	log.Printf("==> Bootstraping plumb.")
	defer log.Printf("<== Bootstrap complete.")

	if err := shell.RunAndLog("docker",
		"run",
		"--rm",
		"--entrypoint=\"/bin/bash\"",
		"-v",
		"/var/run/docker.sock:/var/run/docker.sock",
		"centurylink/golang-builder",
		"-c",
		fmt.Sprintf(`git clone https://github.com/qadium/plumb /plumb \
&& cd /plumb \
&& git checkout -b build %s \
&& cp -r /plumb/manager/* /src \
&& cd /src \
&& /build.sh plumb/manager`, commit)); err != nil {
		return err
	}

	// use docker to compile the manager and copy the binary into
	// another docker container
	//
	// tag this new container

	return nil
}
<|endoftext|>


<commit_before>// basictypes tests the debuggers' ability
// to interpret basic types.
package main

func Stack() {
	var i int
	// BREAKPOINT
	// (gdb) info locals
	// b = false
	// i = 0
	i = 5
	// BREAKPOINT
	// (gdb) printf "%d", i
	// 5
	// (gdb) print i
	// \$[0-9]+ = 5
	_ = i
	var b bool
	// BREAKPOINT
	// (gdb) print b
	// \$[0-9]+ = false
	_ = b
}

func Heap() (*int, *bool) {
	i := 5
	b := false
	/* BROKEN, SKIPPED:
	// BREAKPOINT
	// (gdb) print i
	// \$[0-9]+ = 5
	// (gdb) print b
	// \$[0-9]+ = false
	*/
	return &i, &b
}

func main() {
	Stack()
	Heap()
}
<commit_msg>Don't rely on info locals scoping<commit_after>// basictypes tests the debuggers' ability
// to interpret basic types.
package main

func Stack() {
	var i int
	var b bool
	// BREAKPOINT
	// (gdb) info locals
	// b = false
	// i = 0
	i = 5
	// BREAKPOINT
	// (gdb) printf "%d", i
	// 5
	// (gdb) print i
	// \$[0-9]+ = 5
	_ = i
	// BREAKPOINT
	// (gdb) print b
	// \$[0-9]+ = false
	_ = b
}

func Heap() (*int, *bool) {
	i := 5
	b := false
	/* BROKEN, SKIPPED:
	// BREAKPOINT
	// (gdb) print i
	// \$[0-9]+ = 5
	// (gdb) print b
	// \$[0-9]+ = false
	*/
	return &i, &b
}

func main() {
	Stack()
	Heap()
}
<|endoftext|>


<commit_before>package integer_partition_generation

type partition []int

func Partitions(n int) []partition {
	partitions := []partition{}

	// The first partition of n in lexicographically decreasing order is {n} itself.
	p := partition{n}

	for p != nil {
		partitions = append(partitions, p)
		p = next(p)
	}

	return partitions
}

func next(p partition) partition {
	g := indexLastGreaterThan1(p)
	if g < 0 {
		// Root case: p is made entirely of 1s. There is no next partition.
		return nil
	}

	// To generate a new partition that is after p in lexicographically decreasing order, we remove
	// 1 from the last element greater than 1, then append it back at the end of the partition.
	// Example: {3, 1, 1} -> {2, 1, 1, 1}.
	c := clone(p)
	c[g]--
	c = append(c, 1)

	// This new partition is after p in lexicographically decreasing order, however it is not
	// necessarily *immediately* after p. To fix that, we must "collect" all the 1s that follow g.
	// Example: {2, 1, 1, 1} -> {2, 2, 1}.
	// Note that in the above example, we must be careful not to "overcollect" the 1s into {2, 3},
	// because that's the same partition as {3, 2}, which has already been generated since it's
	// before p in lexicographically decreasing order. The "collection" must stop when it reaches
	// the value of p[g].
	return collect(c, g)
}

func indexLastGreaterThan1(p partition) int {
	for i := len(p) - 1; i >= 0; i-- {
		if v := p[i]; v > 1 {
			return i
		}
	}
	return -1
}

func clone(p partition) partition {
	c := make(partition, len(p))
	copy(c, p)
	return c
}

func collect(p partition, g int) partition {
	max := p[g]
	for i := g + 1; i < len(p); i++ {
		// Collect the last element of p into p[i] until there's nothing left to collect or p[i]
		// reaches the max.
		for len(p)-1 > i && p[i] < max {
			p[i]++
			p = p[:len(p)-1]
		}
	}
	return p
}
<commit_msg>[integer_partition_generation/go] Condense loop.<commit_after>package integer_partition_generation

type partition []int

func Partitions(n int) []partition {
	partitions := []partition{}

	// The first partition of n in lexicographically decreasing order is {n} itself.
	for p := (partition{n}); p != nil; p = next(p) {
		partitions = append(partitions, p)
	}

	return partitions
}

func next(p partition) partition {
	g := indexLastGreaterThan1(p)
	if g < 0 {
		// Root case: p is made entirely of 1s. There is no next partition.
		return nil
	}

	// To generate a new partition that is after p in lexicographically decreasing order, we remove
	// 1 from the last element greater than 1, then append it back at the end of the partition.
	// Example: {3, 1, 1} -> {2, 1, 1, 1}.
	c := clone(p)
	c[g]--
	c = append(c, 1)

	// This new partition is after p in lexicographically decreasing order, however it is not
	// necessarily *immediately* after p. To fix that, we must "collect" all the 1s that follow g.
	// Example: {2, 1, 1, 1} -> {2, 2, 1}.
	// Note that in the above example, we must be careful not to "overcollect" the 1s into {2, 3},
	// because that's the same partition as {3, 2}, which has already been generated since it's
	// before p in lexicographically decreasing order. The "collection" must stop when it reaches
	// the value of p[g].
	return collect(c, g)
}

func indexLastGreaterThan1(p partition) int {
	for i := len(p) - 1; i >= 0; i-- {
		if v := p[i]; v > 1 {
			return i
		}
	}
	return -1
}

func clone(p partition) partition {
	c := make(partition, len(p))
	copy(c, p)
	return c
}

func collect(p partition, g int) partition {
	max := p[g]
	for i := g + 1; i < len(p); i++ {
		// Collect the last element of p into p[i] until there's nothing left to collect or p[i]
		// reaches the max.
		for len(p)-1 > i && p[i] < max {
			p[i]++
			p = p[:len(p)-1]
		}
	}
	return p
}
<|endoftext|>


<commit_before>package glock

import (
	"bufio"
	"errors"
	"fmt"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/iron-io/golog"
	"github.com/stathat/consistent"
)

type connectionError struct {
	error
}

type internalError struct {
	error
}

type Client struct {
	endpoints       []string
	consistent      *consistent.Consistent
	poolsLock       sync.RWMutex
	connectionPools map[string]chan *connection
	poolSize        int
}

type connection struct {
	endpoint string
	conn     net.Conn
	reader   *bufio.Reader
}

// func (c *Client) ClosePool() error {
// 	size := len(c.connectionPool)
// 	for x := 0; x < size; x++ {
// 		connection := <-c.connectionPool
// 		err := connection.Close()
// 		if err != nil {
// 			return err
// 		}
// 	}
// 	return nil
// }

func (c *Client) Size() int {
	var size int
	for _, pool := range c.connectionPools {
		size += len(pool)
	}
	return size
}

func NewClient(endpoints []string, size int) (*Client, error) {
	client := &Client{consistent: consistent.New(), connectionPools: make(map[string]chan *connection), endpoints: endpoints,
		poolSize: size}
	err := client.initPool()
	if err != nil {
		golog.Errorln("GlockClient - ", "Initing pool ", err)
		return nil, err
	}
	client.CheckServerStatus()

	golog.Debugf("Init with connection pool of %d to Glock server", size)
	return client, nil
}

func (c *Client) initPool() error {
	c.addEndpoints(c.endpoints)
	return nil
}

func (c *Client) addEndpoints(endpoints []string) {
	for _, endpoint := range endpoints {
		golog.Infoln("GlockClient -", "Attempting to add endpoint:", endpoint)
		conn, err := net.Dial("tcp", endpoint)
		if err == nil {
			c.connectionPools[endpoint] = make(chan *connection, c.poolSize)
			c.connectionPools[endpoint] <- &connection{conn: conn, reader: bufio.NewReader(conn), endpoint: endpoint}
			c.consistent.Add(endpoint)
			golog.Infoln("GlockClient -", "Added endpoint:", endpoint)
		} else {
			golog.Errorln("GlockClient -", "Error adding endoint, could not connect, not added. endpoint:", endpoint, "error:", err)
		}
	}
}

func (c *Client) getConnection(key string) (*connection, error) {
	server, err := c.consistent.Get(key)
	if err != nil {
		golog.Errorln("GlockClient -", "Consistent hashing error, could not get server for key:", key, "error:", err)
		return nil, err
	}
	golog.Debugln("GlockClient -", "in getConn, got server", server, "for key", key)
	select {
	case conn := <-c.connectionPools[server]:
		return conn, nil
	default:
		golog.Infoln("GlockClient - Creating new connection... server:", server)
		conn, err := net.Dial("tcp", server)
		if err != nil {
			golog.Errorln("GlockClient - getConnection - could not connect to:", server, "error:", err)
			c.removeEndpoint(server)
			return nil, err
		}
		return &connection{conn: conn, reader: bufio.NewReader(conn), endpoint: server}, nil
	}
}

func (c *Client) releaseConnection(connection *connection) {
	c.poolsLock.RLock()
	connectionPool, ok := c.connectionPools[connection.endpoint]
	c.poolsLock.RUnlock()
	if !ok {
		connection.Close()
		return
	}

	select {
	case connectionPool <- connection:
	default:
		connection.Close()
	}
}

func (c *Client) Lock(key string, duration time.Duration) (id int64, err error) {
	// its important that we get the server before we do getConnection (instead of inside getConnection) because if that error drops we need to put the connection back to the original mapping.

	connection, err := c.getConnection(key)
	if err != nil {
		return id, err
	}
	defer c.releaseConnection(connection)

	id, err = connection.lock(key, duration)
	if err != nil {
		if err, ok := err.(*connectionError); ok {
			golog.Errorln("GlockClient -", "Connection error, couldn't get lock. Removing endpoint from hash table, server: ", connection.endpoint, " error: ", err)
			c.removeEndpoint(connection.endpoint)
			// todo for evan/treeder, if it is a connection error remove the failed server and then lock again recursively
			return c.Lock(key, duration)
		}
		golog.Errorln("GlockClient -", "Error trying to get lock. endpoint: ", connection.endpoint, " error: ", err)
		return id, err
	}
	return id, nil
}

func (c *connection) lock(key string, duration time.Duration) (id int64, err error) {
	err = c.fprintf("LOCK %s %d\n", key, int(duration/time.Millisecond))
	if err != nil {
		golog.Errorln("GlockClient -", "lock error: ", err)
		return id, err
	}

	splits, err := c.readResponse()
	if err != nil {
		golog.Errorln("GlockClient - ", "Lock readResponse error: ", err)
		return id, err
	}

	id, err = strconv.ParseInt(splits[1], 10, 64)
	if err != nil {
		return id, &internalError{err}
	}

	return id, nil
}

func (c *Client) removeEndpoint(endpoint string) {
	golog.Errorln("GlockClient -", "Removing endpoint: ", endpoint)
	// remove from hash first
	c.consistent.Remove(endpoint)
	// then we should get rid of all the connections

	c.poolsLock.RLock()
	_, ok := c.connectionPools[endpoint]
	c.poolsLock.RUnlock()
	if !ok {
		return
	}

	c.poolsLock.Lock()
	defer c.poolsLock.Unlock()
	if _, ok := c.connectionPools[endpoint]; ok {
		delete(c.connectionPools, endpoint)
	}
}

func (c *Client) Unlock(key string, id int64) (err error) {

	connection, err := c.getConnection(key)
	if err != nil {
		return err
	}
	defer c.releaseConnection(connection)

	err = connection.fprintf("UNLOCK %s %d\n", key, id)
	if err != nil {
		golog.Errorln("GlockClient - ", "unlock error: ", err)
		return err
	}

	splits, err := connection.readResponse()
	if err != nil {
		golog.Errorln("GlockClient -", "unlock readResponse error: ", err)
		return err
	}

	cmd := splits[0]
	switch cmd {
	case "NOT_UNLOCKED":
		return errors.New("NOT_UNLOCKED")
	case "UNLOCKED":
		return nil
	}
	return errors.New("Unknown reponse format")
}

func (c *connection) fprintf(format string, a ...interface{}) error {
	for i := 0; i < 3; i++ {
		_, err := fmt.Fprintf(c.conn, format, a...)
		if err != nil {
			err = c.redial()
			if err != nil {
				return &internalError{err}
			}
		} else {
			break
		}
	}
	return nil
}

func (c *connection) readResponse() (splits []string, err error) {
	response, err := c.reader.ReadString('\n')
	golog.Debugln("GlockClient -", "glockResponse: ", response)
	if err != nil {
		return nil, &connectionError{err}
	}

	trimmedResponse := strings.TrimRight(response, "\n")
	splits = strings.Split(trimmedResponse, " ")
	if splits[0] == "ERROR" {
		return nil, &internalError{errors.New(trimmedResponse)}
	}

	return splits, nil
}

func (c *connection) redial() error {
	c.conn.Close()
	conn, err := net.Dial("tcp", c.endpoint)
	if err != nil {
		return err
	}
	c.conn = conn
	c.reader = bufio.NewReader(conn)

	return nil
}

func (c *connection) Close() error {
	c.reader = nil
	return c.conn.Close()
}
<commit_msg>Thread-safety improvements around reading the connectionPools map<commit_after>package glock

import (
	"bufio"
	"errors"
	"fmt"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/iron-io/golog"
	"github.com/stathat/consistent"
)

type connectionError struct {
	error
}

type internalError struct {
	error
}

type Client struct {
	endpoints       []string
	consistent      *consistent.Consistent
	poolsLock       sync.RWMutex
	connectionPools map[string]chan *connection
	poolSize        int
}

type connection struct {
	endpoint string
	conn     net.Conn
	reader   *bufio.Reader
}

// func (c *Client) ClosePool() error {
// 	size := len(c.connectionPool)
// 	for x := 0; x < size; x++ {
// 		connection := <-c.connectionPool
// 		err := connection.Close()
// 		if err != nil {
// 			return err
// 		}
// 	}
// 	return nil
// }

func (c *Client) Size() int {
	var size int
	for _, pool := range c.connectionPools {
		size += len(pool)
	}
	return size
}

func NewClient(endpoints []string, size int) (*Client, error) {
	client := &Client{consistent: consistent.New(), connectionPools: make(map[string]chan *connection), endpoints: endpoints,
		poolSize: size}
	err := client.initPool()
	if err != nil {
		golog.Errorln("GlockClient - ", "Initing pool ", err)
		return nil, err
	}
	client.CheckServerStatus()

	golog.Debugf("Init with connection pool of %d to Glock server", size)
	return client, nil
}

func (c *Client) initPool() error {
	c.addEndpoints(c.endpoints)
	return nil
}

func (c *Client) addEndpoints(endpoints []string) {
	for _, endpoint := range endpoints {
		golog.Infoln("GlockClient -", "Attempting to add endpoint:", endpoint)
		conn, err := net.Dial("tcp", endpoint)
		if err == nil {
			pool := make(chan *connection, c.poolSize)
			pool <- &connection{conn: conn, reader: bufio.NewReader(conn), endpoint: endpoint}

			c.poolsLock.Lock()
			c.connectionPools[endpoint] = pool
			c.poolsLock.Unlock()

			c.consistent.Add(endpoint)
			golog.Infoln("GlockClient -", "Added endpoint:", endpoint)
		} else {
			golog.Errorln("GlockClient -", "Error adding endoint, could not connect, not added. endpoint:", endpoint, "error:", err)
		}
	}
}

func (c *Client) getConnection(key string) (*connection, error) {
	server, err := c.consistent.Get(key)
	if err != nil {
		golog.Errorln("GlockClient -", "Consistent hashing error, could not get server for key:", key, "error:", err)
		return nil, err
	}
	golog.Debugln("GlockClient -", "in getConn, got server", server, "for key", key)

	c.poolsLock.RLock()
	connectionPool, ok := c.connectionPools[server]
	c.poolsLock.RUnlock()
	if !ok {
		return nil, errors.New("connectionPool removed")
	}

	select {
	case conn := <-connectionPool:
		return conn, nil
	default:
		golog.Infoln("GlockClient - Creating new connection... server:", server)
		conn, err := net.Dial("tcp", server)
		if err != nil {
			golog.Errorln("GlockClient - getConnection - could not connect to:", server, "error:", err)
			c.removeEndpoint(server)
			return nil, err
		}
		return &connection{conn: conn, reader: bufio.NewReader(conn), endpoint: server}, nil
	}
}

func (c *Client) releaseConnection(connection *connection) {
	c.poolsLock.RLock()
	connectionPool, ok := c.connectionPools[connection.endpoint]
	c.poolsLock.RUnlock()
	if !ok {
		connection.Close()
		return
	}

	select {
	case connectionPool <- connection:
	default:
		connection.Close()
	}
}

func (c *Client) Lock(key string, duration time.Duration) (id int64, err error) {
	// its important that we get the server before we do getConnection (instead of inside getConnection) because if that error drops we need to put the connection back to the original mapping.

	connection, err := c.getConnection(key)
	if err != nil {
		return id, err
	}
	defer c.releaseConnection(connection)

	id, err = connection.lock(key, duration)
	if err != nil {
		if err, ok := err.(*connectionError); ok {
			golog.Errorln("GlockClient -", "Connection error, couldn't get lock. Removing endpoint from hash table, server: ", connection.endpoint, " error: ", err)
			c.removeEndpoint(connection.endpoint)
			// todo for evan/treeder, if it is a connection error remove the failed server and then lock again recursively
			return c.Lock(key, duration)
		}
		golog.Errorln("GlockClient -", "Error trying to get lock. endpoint: ", connection.endpoint, " error: ", err)
		return id, err
	}
	return id, nil
}

func (c *connection) lock(key string, duration time.Duration) (id int64, err error) {
	err = c.fprintf("LOCK %s %d\n", key, int(duration/time.Millisecond))
	if err != nil {
		golog.Errorln("GlockClient -", "lock error: ", err)
		return id, err
	}

	splits, err := c.readResponse()
	if err != nil {
		golog.Errorln("GlockClient - ", "Lock readResponse error: ", err)
		return id, err
	}

	id, err = strconv.ParseInt(splits[1], 10, 64)
	if err != nil {
		return id, &internalError{err}
	}

	return id, nil
}

func (c *Client) removeEndpoint(endpoint string) {
	golog.Errorln("GlockClient -", "Removing endpoint: ", endpoint)
	// remove from hash first
	c.consistent.Remove(endpoint)
	// then we should get rid of all the connections

	c.poolsLock.RLock()
	_, ok := c.connectionPools[endpoint]
	c.poolsLock.RUnlock()
	if !ok {
		return
	}

	c.poolsLock.Lock()
	defer c.poolsLock.Unlock()
	if _, ok := c.connectionPools[endpoint]; ok {
		delete(c.connectionPools, endpoint)
	}
}

func (c *Client) Unlock(key string, id int64) (err error) {

	connection, err := c.getConnection(key)
	if err != nil {
		return err
	}
	defer c.releaseConnection(connection)

	err = connection.fprintf("UNLOCK %s %d\n", key, id)
	if err != nil {
		golog.Errorln("GlockClient - ", "unlock error: ", err)
		return err
	}

	splits, err := connection.readResponse()
	if err != nil {
		golog.Errorln("GlockClient -", "unlock readResponse error: ", err)
		return err
	}

	cmd := splits[0]
	switch cmd {
	case "NOT_UNLOCKED":
		return errors.New("NOT_UNLOCKED")
	case "UNLOCKED":
		return nil
	}
	return errors.New("Unknown reponse format")
}

func (c *connection) fprintf(format string, a ...interface{}) error {
	for i := 0; i < 3; i++ {
		_, err := fmt.Fprintf(c.conn, format, a...)
		if err != nil {
			err = c.redial()
			if err != nil {
				return &internalError{err}
			}
		} else {
			break
		}
	}
	return nil
}

func (c *connection) readResponse() (splits []string, err error) {
	response, err := c.reader.ReadString('\n')
	golog.Debugln("GlockClient -", "glockResponse: ", response)
	if err != nil {
		return nil, &connectionError{err}
	}

	trimmedResponse := strings.TrimRight(response, "\n")
	splits = strings.Split(trimmedResponse, " ")
	if splits[0] == "ERROR" {
		return nil, &internalError{errors.New(trimmedResponse)}
	}

	return splits, nil
}

func (c *connection) redial() error {
	c.conn.Close()
	conn, err := net.Dial("tcp", c.endpoint)
	if err != nil {
		return err
	}
	c.conn = conn
	c.reader = bufio.NewReader(conn)

	return nil
}

func (c *connection) Close() error {
	c.reader = nil
	return c.conn.Close()
}
<|endoftext|>


<commit_before>package controller

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"time"

	ct "github.com/flynn/flynn-controller/types"
	"github.com/flynn/rpcplus"
)

func New(uri string) (*Client, error) {
	u, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}
	return &Client{url: uri, addr: u.Host}, nil
}

type Client struct {
	url  string
	addr string
}

var ErrNotFound = errors.New("controller: not found")

func (c *Client) send(method, path string, in, out interface{}) error {
	data, err := json.Marshal(in)
	if err != nil {
		return err
	}
	req, err := http.NewRequest(method, c.url+path, bytes.NewBuffer(data))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return fmt.Errorf("controller: unexpected status %d", res.StatusCode)
	}
	if out != nil {
		return json.NewDecoder(res.Body).Decode(out)
	}
	return nil
}

func (c *Client) put(path string, in, out interface{}) error {
	return c.send("PUT", path, in, out)
}

func (c *Client) post(path string, in, out interface{}) error {
	return c.send("POST", path, in, out)
}

func (c *Client) get(path string, out interface{}) error {
	res, err := http.Get(c.url + path)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 404 {
			return ErrNotFound
		}
		return fmt.Errorf("controller: unexpected status %d", res.StatusCode)
	}
	return json.NewDecoder(res.Body).Decode(out)
}

func (c *Client) StreamFormations(since *time.Time) (<-chan *ct.ExpandedFormation, *error) {
	if since == nil {
		*since = time.Unix(0, 0)
	}
	// TODO: handle TLS
	client, err := rpcplus.DialHTTP("tcp", c.addr)
	if err != nil {
		return nil, &err
	}
	ch := make(chan *ct.ExpandedFormation)
	return ch, &client.StreamGo("Controller.StreamFormations", since, ch).Error
}

func (c *Client) CreateArtifact(artifact *ct.Artifact) error {
	return c.post("/artifacts", artifact, artifact)
}

func (c *Client) CreateRelease(release *ct.Release) error {
	return c.post("/releases", release, release)
}

func (c *Client) CreateApp(app *ct.App) error {
	return c.post("/apps", app, app)
}

func (c *Client) CreateProvider(provider *ct.App) error {
	return c.post("/providers", provider, provider)
}

func (c *Client) PutResource(resource *ct.Resource) error {
	if resource.ID == "" || resource.ProviderID == "" {
		return errors.New("controller: missing id and/or provider id")
	}
	return c.put(fmt.Sprintf("/providers/%s/resources/%s", resource.ProviderID, resource.ID), resource, resource)
}

func (c *Client) PutFormation(formation *ct.Formation) error {
	if formation.AppID == "" || formation.ReleaseID == "" {
		return errors.New("controller missing app id and/or release id")
	}
	return c.put(fmt.Sprintf("/apps/%s/formations/%s", formation.AppID, formation.ReleaseID), formation, formation)
}

func (c *Client) SetAppRelease(appID, releaseID string) error {
	return c.put("/apps/"+appID+"/release", &ct.Release{ID: releaseID}, nil)
}

func (c *Client) GetAppRelease(appID string) (*ct.Release, error) {
	release := &ct.Release{}
	return release, c.get("/apps/"+appID+"/release", release)
}
<commit_msg>Fix typo in client error<commit_after>package controller

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"time"

	ct "github.com/flynn/flynn-controller/types"
	"github.com/flynn/rpcplus"
)

func New(uri string) (*Client, error) {
	u, err := url.Parse(uri)
	if err != nil {
		return nil, err
	}
	return &Client{url: uri, addr: u.Host}, nil
}

type Client struct {
	url  string
	addr string
}

var ErrNotFound = errors.New("controller: not found")

func (c *Client) send(method, path string, in, out interface{}) error {
	data, err := json.Marshal(in)
	if err != nil {
		return err
	}
	req, err := http.NewRequest(method, c.url+path, bytes.NewBuffer(data))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		return fmt.Errorf("controller: unexpected status %d", res.StatusCode)
	}
	if out != nil {
		return json.NewDecoder(res.Body).Decode(out)
	}
	return nil
}

func (c *Client) put(path string, in, out interface{}) error {
	return c.send("PUT", path, in, out)
}

func (c *Client) post(path string, in, out interface{}) error {
	return c.send("POST", path, in, out)
}

func (c *Client) get(path string, out interface{}) error {
	res, err := http.Get(c.url + path)
	if err != nil {
		return err
	}
	defer res.Body.Close()
	if res.StatusCode != 200 {
		if res.StatusCode == 404 {
			return ErrNotFound
		}
		return fmt.Errorf("controller: unexpected status %d", res.StatusCode)
	}
	return json.NewDecoder(res.Body).Decode(out)
}

func (c *Client) StreamFormations(since *time.Time) (<-chan *ct.ExpandedFormation, *error) {
	if since == nil {
		*since = time.Unix(0, 0)
	}
	// TODO: handle TLS
	client, err := rpcplus.DialHTTP("tcp", c.addr)
	if err != nil {
		return nil, &err
	}
	ch := make(chan *ct.ExpandedFormation)
	return ch, &client.StreamGo("Controller.StreamFormations", since, ch).Error
}

func (c *Client) CreateArtifact(artifact *ct.Artifact) error {
	return c.post("/artifacts", artifact, artifact)
}

func (c *Client) CreateRelease(release *ct.Release) error {
	return c.post("/releases", release, release)
}

func (c *Client) CreateApp(app *ct.App) error {
	return c.post("/apps", app, app)
}

func (c *Client) CreateProvider(provider *ct.App) error {
	return c.post("/providers", provider, provider)
}

func (c *Client) PutResource(resource *ct.Resource) error {
	if resource.ID == "" || resource.ProviderID == "" {
		return errors.New("controller: missing id and/or provider id")
	}
	return c.put(fmt.Sprintf("/providers/%s/resources/%s", resource.ProviderID, resource.ID), resource, resource)
}

func (c *Client) PutFormation(formation *ct.Formation) error {
	if formation.AppID == "" || formation.ReleaseID == "" {
		return errors.New("controller: missing app id and/or release id")
	}
	return c.put(fmt.Sprintf("/apps/%s/formations/%s", formation.AppID, formation.ReleaseID), formation, formation)
}

func (c *Client) SetAppRelease(appID, releaseID string) error {
	return c.put("/apps/"+appID+"/release", &ct.Release{ID: releaseID}, nil)
}

func (c *Client) GetAppRelease(appID string) (*ct.Release, error) {
	release := &ct.Release{}
	return release, c.get("/apps/"+appID+"/release", release)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package client is a Go client for the GopherPit API.

For more information about the Engine API, see the documentation:
https://gopherpit.com/docs/api

Authorization is performed by Personal Access Token which can
be obtained on Settings page of the GopherPit site.

Usage

Communication with the API is performed by creating a Client
object and calling methods on it.

Example:

package main

    import (
        "fmt"
        "os"

        "gopherpit.com/gopherpit/client"
    )

    func main() {
        c := client.NewClient(os.Getenv("GOPHERPIT_TOKEN"))

        domains, err := c.Domains("", 0)
        if err != nil {
            fmt.Fprintln(os.Stderr, "get domains", err)
            os.Exit(1)
        }

        for _, domain := range domains.Domains {
            fmt.Printf("%s %s\n", domain.ID, domain.FQDN)
        }
    }

To use GopherPit installation on-premises:

    c := client.NewClientWithEndpoint("https://go.example.com/api/v1", "TOKEN")

*/
package client // import "gopherpit.com/gopherpit/client"

import (
	apiClient "resenje.org/httputils/client/api"

	"gopherpit.com/gopherpit/api"
)

var (
	version           = "0.1"
	userAgent         = "GopherPitClient/" + version
	gopherpitEndpoint = "https://gopherpit.com/api/v1"
)

// Client is the API client that performs all operations
// against GopherPit server.
type Client struct {
	apiClient.Client
}

// NewClient creates a new Client object with
// gopherpit.com endpoint. It is intended for connecting to
// publicly available GopherPit service.
func NewClient(key string) *Client {
	return NewClientWithEndpoint(gopherpitEndpoint, key)
}

// NewClientWithEndpoint creates a new Client object with
// HTTP endpoint and Personal Access Token as a key. It is intended
// for connecting to on-premises GopherPit installations.
// Endpoint URL must include schema, host and path components.
// For example: https://go.example.com/api/v1
func NewClientWithEndpoint(endpoint, key string) *Client {
	return newClientWithAPIClient(&apiClient.Client{
		Endpoint:  endpoint,
		Key:       key,
		UserAgent: userAgent,
	})
}

// newClientWithAPIClient creates a new Client and injects
// api.ErrorRegistry in the API Client.
func newClientWithAPIClient(c *apiClient.Client) *Client {
	if c == nil {
		c = &apiClient.Client{}
	}
	c.ErrorRegistry = api.ErrorRegistry
	return &Client{Client: *c}
}
<commit_msg>A minor simplification in API client<commit_after>// Copyright (c) 2017, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package client is a Go client for the GopherPit API.

For more information about the Engine API, see the documentation:
https://gopherpit.com/docs/api

Authorization is performed by Personal Access Token which can
be obtained on Settings page of the GopherPit site.

Usage

Communication with the API is performed by creating a Client
object and calling methods on it.

Example:

package main

    import (
        "fmt"
        "os"

        "gopherpit.com/gopherpit/client"
    )

    func main() {
        c := client.NewClient(os.Getenv("GOPHERPIT_TOKEN"))

        domains, err := c.Domains("", 0)
        if err != nil {
            fmt.Fprintln(os.Stderr, "get domains", err)
            os.Exit(1)
        }

        for _, domain := range domains.Domains {
            fmt.Printf("%s %s\n", domain.ID, domain.FQDN)
        }
    }

To use GopherPit installation on-premises:

    c := client.NewClientWithEndpoint("https://go.example.com/api/v1", "TOKEN")

*/
package client // import "gopherpit.com/gopherpit/client"

import (
	apiClient "resenje.org/httputils/client/api"

	"gopherpit.com/gopherpit/api"
)

var (
	version           = "0.1"
	userAgent         = "GopherPitClient/" + version
	gopherpitEndpoint = "https://gopherpit.com/api/v1"
)

// Client is the API client that performs all operations
// against GopherPit server.
type Client struct {
	apiClient.Client
}

// NewClient creates a new Client object with
// gopherpit.com endpoint. It is intended for connecting to
// publicly available GopherPit service.
func NewClient(key string) *Client {
	return NewClientWithEndpoint(gopherpitEndpoint, key)
}

// NewClientWithEndpoint creates a new Client object with
// HTTP endpoint and Personal Access Token as a key. It is intended
// for connecting to on-premises GopherPit installations.
// Endpoint URL must include schema, host and path components.
// For example: https://go.example.com/api/v1
func NewClientWithEndpoint(endpoint, key string) *Client {
	return newClientWithAPIClient(apiClient.Client{
		Endpoint:  endpoint,
		Key:       key,
		UserAgent: userAgent,
	})
}

// newClientWithAPIClient creates a new Client and injects
// api.ErrorRegistry in the API Client.
func newClientWithAPIClient(c apiClient.Client) *Client {
	c.ErrorRegistry = api.ErrorRegistry
	return &Client{Client: c}
}
<|endoftext|>


<commit_before>package sawyer

import (
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"strings"
)

var httpClient = &http.Client{}

type Client struct {
	HttpClient *http.Client
	Endpoint   *url.URL
	Decoders   map[string]DecoderFunc
}

type DecoderFunc func(r io.Reader) Decoder

type Decoder interface {
	Decode(v interface{}) error
}

func New(endpoint *url.URL, client *http.Client) *Client {
	if client == nil {
		client = httpClient
	}

	if len(endpoint.Path) > 0 && !strings.HasSuffix(endpoint.Path, "/") {
		endpoint.Path = endpoint.Path + "/"
	}

	decoders := map[string]DecoderFunc{
		"json": func(r io.Reader) Decoder {
			return json.NewDecoder(r)
		},
	}
	return &Client{client, endpoint, decoders}
}

func NewFromString(endpoint string, client *http.Client) (*Client, error) {
	e, err := url.Parse(endpoint)
	if err != nil {
		return nil, err
	}

	return New(e, client), nil
}

func (c *Client) Do(resource interface{}, apierr interface{}, req *http.Request) (*http.Response, error) {
	res, err := c.HttpClient.Do(req)
	if err != nil {
		return res, err
	}
	defer res.Body.Close()

	return res, c.decode(resource, apierr, res)
}

func (c *Client) Get(resource interface{}, apierr interface{}, rawurl string) (*http.Response, error) {
	req, err := c.NewRequest("GET", rawurl, nil)
	if err != nil {
		return nil, err
	}

	return c.Do(resource, apierr, req)
}

func (c *Client) NewRequest(method string, rawurl string, body io.Reader) (*http.Request, error) {
	u, err := c.resolveReferenceString(rawurl)
	if err != nil {
		return nil, err
	}

	return http.NewRequest("GET", u, nil)
}

func (c *Client) ResolveReference(u *url.URL) *url.URL {
	return c.Endpoint.ResolveReference(u)
}

func (c *Client) decode(resource interface{}, apierr interface{}, res *http.Response) error {
	// TODO: content type negotiation to find the right decoder
	dec := c.Decoders["json"](res.Body)

	if UseApiError(res.StatusCode) {
		return dec.Decode(apierr)
	}

	return dec.Decode(resource)
}

func (c *Client) resolveReferenceString(rawurl string) (string, error) {
	u, err := url.Parse(rawurl)
	if err != nil {
		return "", err
	}
	return c.ResolveReference(u).String(), nil
}

func UseApiError(status int) bool {
	switch {
	case status > 199 && status < 300:
		return false
	case status == 304:
		return false
	case status == 0:
		return false
	}
	return true
}
<commit_msg>use content negotiation to pick a decoder<commit_after>package sawyer

import (
	"encoding/json"
	"github.com/lostisland/go-sawyer/mediatype"
	"io"
	"net/http"
	"net/url"
	"strings"
)

var httpClient = &http.Client{}

type Client struct {
	HttpClient *http.Client
	Endpoint   *url.URL
	Decoders   map[string]DecoderFunc
}

type DecoderFunc func(r io.Reader) Decoder

type Decoder interface {
	Decode(v interface{}) error
}

func New(endpoint *url.URL, client *http.Client) *Client {
	if client == nil {
		client = httpClient
	}

	if len(endpoint.Path) > 0 && !strings.HasSuffix(endpoint.Path, "/") {
		endpoint.Path = endpoint.Path + "/"
	}

	decoders := map[string]DecoderFunc{
		"json": func(r io.Reader) Decoder {
			return json.NewDecoder(r)
		},
	}
	return &Client{client, endpoint, decoders}
}

func NewFromString(endpoint string, client *http.Client) (*Client, error) {
	e, err := url.Parse(endpoint)
	if err != nil {
		return nil, err
	}

	return New(e, client), nil
}

func (c *Client) Do(resource interface{}, apierr interface{}, req *http.Request) (*http.Response, error) {
	res, err := c.HttpClient.Do(req)
	if err != nil {
		return res, err
	}
	defer res.Body.Close()

	return res, c.decode(resource, apierr, res)
}

func (c *Client) Get(resource interface{}, apierr interface{}, rawurl string) (*http.Response, error) {
	req, err := c.NewRequest("GET", rawurl, nil)
	if err != nil {
		return nil, err
	}

	return c.Do(resource, apierr, req)
}

func (c *Client) NewRequest(method string, rawurl string, body io.Reader) (*http.Request, error) {
	u, err := c.resolveReferenceString(rawurl)
	if err != nil {
		return nil, err
	}

	return http.NewRequest("GET", u, nil)
}

func (c *Client) ResolveReference(u *url.URL) *url.URL {
	return c.Endpoint.ResolveReference(u)
}

func (c *Client) decode(resource interface{}, apierr interface{}, res *http.Response) error {
	if UseApiError(res.StatusCode) {
		return c.decodeResource(apierr, res)
	}
	return c.decodeResource(resource, res)
}

func (c *Client) decodeResource(resource interface{}, res *http.Response) error {
	if resource == nil {
		return nil
	}

	dec, err := c.decoder(res)
	if err != nil {
		return err
	} else if dec != nil {
		return dec.Decode(resource)
	}

	return nil
}

func (c *Client) decoder(res *http.Response) (Decoder, error) {
	mt, err := c.mediaType(res)
	if err != nil {
		return nil, err
	}

	if decfunc, ok := c.Decoders[mt.Format]; ok {
		return decfunc(res.Body), nil
	}
	return nil, nil
}

func (c *Client) mediaType(res *http.Response) (*mediatype.MediaType, error) {
	if ctype := res.Header.Get("Content-Type"); len(ctype) > 0 {
		return mediatype.Parse(ctype)
	}
	return nil, nil
}

func (c *Client) resolveReferenceString(rawurl string) (string, error) {
	u, err := url.Parse(rawurl)
	if err != nil {
		return "", err
	}
	return c.ResolveReference(u).String(), nil
}

func UseApiError(status int) bool {
	switch {
	case status > 199 && status < 300:
		return false
	case status == 304:
		return false
	case status == 0:
		return false
	}
	return true
}
<|endoftext|>