<commit_before>package scheduling

import (
	"bytes"
	"encoding/base64"
	"encoding/gob"
	"fmt"
	"path"
	"strings"
	"time"

	swarming_api "github.com/luci/luci-go/common/api/swarming/swarming/v1"
	"go.skia.org/infra/go/isolate"
	"go.skia.org/infra/go/swarming"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/task_scheduler/go/db"
	"go.skia.org/infra/task_scheduler/go/specs"
)

// taskCandidate is a struct used for determining which tasks to schedule.
type taskCandidate struct {
	Commits        []string
	IsolatedInput  string
	IsolatedHashes []string
	JobCreated     time.Time
	ParentTaskIds  []string
	RetryOf        string
	Score          float64
	StealingFromId string
	db.TaskKey
	TaskSpec *specs.TaskSpec
}

// Copy returns a copy of the taskCandidate.
func (c *taskCandidate) Copy() *taskCandidate {
	return &taskCandidate{
		Commits:        util.CopyStringSlice(c.Commits),
		IsolatedInput:  c.IsolatedInput,
		IsolatedHashes: util.CopyStringSlice(c.IsolatedHashes),
		JobCreated:     c.JobCreated,
		ParentTaskIds:  util.CopyStringSlice(c.ParentTaskIds),
		RetryOf:        c.RetryOf,
		Score:          c.Score,
		StealingFromId: c.StealingFromId,
		TaskKey:        c.TaskKey.Copy(),
		TaskSpec:       c.TaskSpec.Copy(),
	}
}

// MakeId generates a string ID for the taskCandidate.
func (c *taskCandidate) MakeId() string {
	var buf bytes.Buffer
	if err := gob.NewEncoder(&buf).Encode(&c.TaskKey); err != nil {
		panic(fmt.Sprintf("Failed to GOB encode TaskKey: %s", err))
	}
	b64 := base64.StdEncoding.EncodeToString(buf.Bytes())
	return fmt.Sprintf("taskCandidate|%s", b64)
}

// parseId generates taskCandidate information from the ID.
func parseId(id string) (db.TaskKey, error) {
	var rv db.TaskKey
	split := strings.Split(id, "|")
	if len(split) != 2 {
		return rv, fmt.Errorf("Invalid ID, not enough parts: %q", id)
	}
	if split[0] != "taskCandidate" {
		return rv, fmt.Errorf("Invalid ID, no 'taskCandidate' prefix: %q", id)
	}
	b, err := base64.StdEncoding.DecodeString(split[1])
	if err != nil {
		return rv, fmt.Errorf("Failed to base64 decode ID: %s", err)
	}
	if err := gob.NewDecoder(bytes.NewBuffer(b)).Decode(&rv); err != nil {
		return rv, fmt.Errorf("Failed to GOB decode ID: %s", err)
	}
	return rv, nil
}

// MakeTask instantiates a db.Task from the taskCandidate.
func (c *taskCandidate) MakeTask() *db.Task {
	commits := make([]string, len(c.Commits))
	copy(commits, c.Commits)
	parentTaskIds := make([]string, len(c.ParentTaskIds))
	copy(parentTaskIds, c.ParentTaskIds)
	return &db.Task{
		Commits:       commits,
		Id:            "", // Filled in when the task is inserted into the DB.
		ParentTaskIds: parentTaskIds,
		RetryOf:       c.RetryOf,
		TaskKey:       c.TaskKey.Copy(),
	}
}

// MakeIsolateTask creates an isolate.Task from this taskCandidate.
func (c *taskCandidate) MakeIsolateTask(infraBotsDir, baseDir string) *isolate.Task {
	return &isolate.Task{
		BaseDir:     baseDir,
		Blacklist:   isolate.DEFAULT_BLACKLIST,
		Deps:        c.IsolatedHashes,
		IsolateFile: path.Join(infraBotsDir, c.TaskSpec.Isolate),
		OsType:      "linux", // TODO(borenet)
	}
}

// getPatchStorage returns "gerrit" or "rietveld" based on the Server URL.
func getPatchStorage(server string) string {
	if server == "" {
		return ""
	}
	if strings.Contains(server, "codereview.chromium") {
		return "rietveld"
	}
	return "gerrit"
}

// replaceVars replaces variable names with their values in a given string.
func replaceVars(c *taskCandidate, s string) string {
	issueShort := ""
	if len(c.Issue) < specs.ISSUE_SHORT_LENGTH {
		issueShort = c.Issue
	} else {
		issueShort = c.Issue[len(c.Issue)-specs.ISSUE_SHORT_LENGTH:]
	}
	replacements := map[string]string{
		specs.VARIABLE_CODEREVIEW_SERVER: c.Server,
		specs.VARIABLE_ISSUE:             c.Issue,
		specs.VARIABLE_ISSUE_SHORT:       issueShort,
		specs.VARIABLE_PATCH_STORAGE:     getPatchStorage(c.Server),
		specs.VARIABLE_PATCHSET:          c.Patchset,
		specs.VARIABLE_REPO:              c.Repo,
		specs.VARIABLE_REVISION:          c.Revision,
		specs.VARIABLE_TASK_NAME:         c.Name,
	}
	for k, v := range replacements {
		s = strings.Replace(s, fmt.Sprintf(specs.VARIABLE_SYNTAX, k), v, -1)
	}
	return s
}

// MakeTaskRequest creates a SwarmingRpcsNewTaskRequest object from the taskCandidate.
func (c *taskCandidate) MakeTaskRequest(id string) *swarming_api.SwarmingRpcsNewTaskRequest {
	var cipdInput *swarming_api.SwarmingRpcsCipdInput
	if len(c.TaskSpec.CipdPackages) > 0 {
		cipdInput = &swarming_api.SwarmingRpcsCipdInput{
			Packages: make([]*swarming_api.SwarmingRpcsCipdPackage, 0, len(c.TaskSpec.CipdPackages)),
		}
		for _, p := range c.TaskSpec.CipdPackages {
			cipdInput.Packages = append(cipdInput.Packages, &swarming_api.SwarmingRpcsCipdPackage{
				PackageName: p.Name,
				Path:        p.Path,
				Version:     p.Version,
			})
		}
	}

	dims := make([]*swarming_api.SwarmingRpcsStringPair, 0, len(c.TaskSpec.Dimensions))
	dimsMap := make(map[string]string, len(c.TaskSpec.Dimensions))
	for _, d := range c.TaskSpec.Dimensions {
		split := strings.SplitN(d, ":", 2)
		key := split[0]
		val := split[1]
		dims = append(dims, &swarming_api.SwarmingRpcsStringPair{
			Key:   key,
			Value: val,
		})
		dimsMap[key] = val
	}

	var env []*swarming_api.SwarmingRpcsStringPair
	if len(c.TaskSpec.Environment) > 0 {
		env = make([]*swarming_api.SwarmingRpcsStringPair, 0, len(c.TaskSpec.Environment))
		for k, v := range c.TaskSpec.Environment {
			env = append(env, &swarming_api.SwarmingRpcsStringPair{
				Key:   k,
				Value: v,
			})
		}
	}

	extraArgs := make([]string, 0, len(c.TaskSpec.ExtraArgs))
	for _, arg := range c.TaskSpec.ExtraArgs {
		extraArgs = append(extraArgs, replaceVars(c, arg))
	}

	expirationSecs := int64(c.TaskSpec.Expiration.Seconds())
	if expirationSecs == int64(0) {
		expirationSecs = int64(swarming.RECOMMENDED_EXPIRATION)
	}
	executionTimeoutSecs := int64(c.TaskSpec.ExecutionTimeout.Seconds())
	if executionTimeoutSecs == int64(0) {
		executionTimeoutSecs = int64(swarming.RECOMMENDED_HARD_TIMEOUT)
	}
	ioTimeoutSecs := int64(c.TaskSpec.IoTimeout.Seconds())
	if ioTimeoutSecs == int64(0) {
		ioTimeoutSecs = int64(swarming.RECOMMENDED_IO_TIMEOUT)
	}
	return &swarming_api.SwarmingRpcsNewTaskRequest{
		ExpirationSecs: expirationSecs,
		Name:           c.Name,
		Priority:       int64(100.0 * c.TaskSpec.Priority),
		Properties: &swarming_api.SwarmingRpcsTaskProperties{
			CipdInput:            cipdInput,
			Dimensions:           dims,
			Env:                  env,
			ExecutionTimeoutSecs: executionTimeoutSecs,
			ExtraArgs:            extraArgs,
			InputsRef: &swarming_api.SwarmingRpcsFilesRef{
				Isolated:       c.IsolatedInput,
				Isolatedserver: isolate.ISOLATE_SERVER_URL,
				Namespace:      isolate.DEFAULT_NAMESPACE,
			},
			IoTimeoutSecs: ioTimeoutSecs,
		},
		Tags: db.TagsForTask(c.Name, id, c.TaskSpec.Priority, c.RepoState, c.RetryOf, dimsMap, c.ForcedJobId, c.ParentTaskIds),
		User: "skia-task-scheduler",
	}
}

// allDepsMet determines whether all dependencies for the given task candidate
// have been satisfied, and if so, returns a map of whose keys are task IDs and
// values are their isolated outputs.
func (c *taskCandidate) allDepsMet(cache db.TaskCache) (bool, map[string]string, error) {
	rv := make(map[string]string, len(c.TaskSpec.Dependencies))
	for _, depName := range c.TaskSpec.Dependencies {
		key := c.TaskKey.Copy()
		key.Name = depName
		byKey, err := cache.GetTasksByKey(&key)
		if err != nil {
			return false, nil, err
		}
		ok := false
		for _, t := range byKey {
			if t.Done() && t.Success() && t.IsolatedOutput != "" {
				rv[t.Id] = t.IsolatedOutput
				ok = true
				break
			}
		}
		if !ok {
			return false, nil, nil
		}
	}
	return true, rv, nil
}

// taskCandidateSlice is an alias used for sorting a slice of taskCandidates.
type taskCandidateSlice []*taskCandidate

func (s taskCandidateSlice) Len() int { return len(s) }
func (s taskCandidateSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s taskCandidateSlice) Less(i, j int) bool {
	return s[i].Score > s[j].Score // candidates sort in decreasing order.
}
<commit_msg>[task scheduler] Fix timeouts<commit_after>package scheduling

import (
	"bytes"
	"encoding/base64"
	"encoding/gob"
	"fmt"
	"path"
	"strings"
	"time"

	swarming_api "github.com/luci/luci-go/common/api/swarming/swarming/v1"
	"go.skia.org/infra/go/isolate"
	"go.skia.org/infra/go/swarming"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/task_scheduler/go/db"
	"go.skia.org/infra/task_scheduler/go/specs"
)

// taskCandidate is a struct used for determining which tasks to schedule.
type taskCandidate struct {
	Commits        []string
	IsolatedInput  string
	IsolatedHashes []string
	JobCreated     time.Time
	ParentTaskIds  []string
	RetryOf        string
	Score          float64
	StealingFromId string
	db.TaskKey
	TaskSpec *specs.TaskSpec
}

// Copy returns a copy of the taskCandidate.
func (c *taskCandidate) Copy() *taskCandidate {
	return &taskCandidate{
		Commits:        util.CopyStringSlice(c.Commits),
		IsolatedInput:  c.IsolatedInput,
		IsolatedHashes: util.CopyStringSlice(c.IsolatedHashes),
		JobCreated:     c.JobCreated,
		ParentTaskIds:  util.CopyStringSlice(c.ParentTaskIds),
		RetryOf:        c.RetryOf,
		Score:          c.Score,
		StealingFromId: c.StealingFromId,
		TaskKey:        c.TaskKey.Copy(),
		TaskSpec:       c.TaskSpec.Copy(),
	}
}

// MakeId generates a string ID for the taskCandidate.
func (c *taskCandidate) MakeId() string {
	var buf bytes.Buffer
	if err := gob.NewEncoder(&buf).Encode(&c.TaskKey); err != nil {
		panic(fmt.Sprintf("Failed to GOB encode TaskKey: %s", err))
	}
	b64 := base64.StdEncoding.EncodeToString(buf.Bytes())
	return fmt.Sprintf("taskCandidate|%s", b64)
}

// parseId generates taskCandidate information from the ID.
func parseId(id string) (db.TaskKey, error) {
	var rv db.TaskKey
	split := strings.Split(id, "|")
	if len(split) != 2 {
		return rv, fmt.Errorf("Invalid ID, not enough parts: %q", id)
	}
	if split[0] != "taskCandidate" {
		return rv, fmt.Errorf("Invalid ID, no 'taskCandidate' prefix: %q", id)
	}
	b, err := base64.StdEncoding.DecodeString(split[1])
	if err != nil {
		return rv, fmt.Errorf("Failed to base64 decode ID: %s", err)
	}
	if err := gob.NewDecoder(bytes.NewBuffer(b)).Decode(&rv); err != nil {
		return rv, fmt.Errorf("Failed to GOB decode ID: %s", err)
	}
	return rv, nil
}

// MakeTask instantiates a db.Task from the taskCandidate.
func (c *taskCandidate) MakeTask() *db.Task {
	commits := make([]string, len(c.Commits))
	copy(commits, c.Commits)
	parentTaskIds := make([]string, len(c.ParentTaskIds))
	copy(parentTaskIds, c.ParentTaskIds)
	return &db.Task{
		Commits:       commits,
		Id:            "", // Filled in when the task is inserted into the DB.
		ParentTaskIds: parentTaskIds,
		RetryOf:       c.RetryOf,
		TaskKey:       c.TaskKey.Copy(),
	}
}

// MakeIsolateTask creates an isolate.Task from this taskCandidate.
func (c *taskCandidate) MakeIsolateTask(infraBotsDir, baseDir string) *isolate.Task {
	return &isolate.Task{
		BaseDir:     baseDir,
		Blacklist:   isolate.DEFAULT_BLACKLIST,
		Deps:        c.IsolatedHashes,
		IsolateFile: path.Join(infraBotsDir, c.TaskSpec.Isolate),
		OsType:      "linux", // TODO(borenet)
	}
}

// getPatchStorage returns "gerrit" or "rietveld" based on the Server URL.
func getPatchStorage(server string) string {
	if server == "" {
		return ""
	}
	if strings.Contains(server, "codereview.chromium") {
		return "rietveld"
	}
	return "gerrit"
}

// replaceVars replaces variable names with their values in a given string.
func replaceVars(c *taskCandidate, s string) string {
	issueShort := ""
	if len(c.Issue) < specs.ISSUE_SHORT_LENGTH {
		issueShort = c.Issue
	} else {
		issueShort = c.Issue[len(c.Issue)-specs.ISSUE_SHORT_LENGTH:]
	}
	replacements := map[string]string{
		specs.VARIABLE_CODEREVIEW_SERVER: c.Server,
		specs.VARIABLE_ISSUE:             c.Issue,
		specs.VARIABLE_ISSUE_SHORT:       issueShort,
		specs.VARIABLE_PATCH_STORAGE:     getPatchStorage(c.Server),
		specs.VARIABLE_PATCHSET:          c.Patchset,
		specs.VARIABLE_REPO:              c.Repo,
		specs.VARIABLE_REVISION:          c.Revision,
		specs.VARIABLE_TASK_NAME:         c.Name,
	}
	for k, v := range replacements {
		s = strings.Replace(s, fmt.Sprintf(specs.VARIABLE_SYNTAX, k), v, -1)
	}
	return s
}

// MakeTaskRequest creates a SwarmingRpcsNewTaskRequest object from the taskCandidate.
func (c *taskCandidate) MakeTaskRequest(id string) *swarming_api.SwarmingRpcsNewTaskRequest {
	var cipdInput *swarming_api.SwarmingRpcsCipdInput
	if len(c.TaskSpec.CipdPackages) > 0 {
		cipdInput = &swarming_api.SwarmingRpcsCipdInput{
			Packages: make([]*swarming_api.SwarmingRpcsCipdPackage, 0, len(c.TaskSpec.CipdPackages)),
		}
		for _, p := range c.TaskSpec.CipdPackages {
			cipdInput.Packages = append(cipdInput.Packages, &swarming_api.SwarmingRpcsCipdPackage{
				PackageName: p.Name,
				Path:        p.Path,
				Version:     p.Version,
			})
		}
	}

	dims := make([]*swarming_api.SwarmingRpcsStringPair, 0, len(c.TaskSpec.Dimensions))
	dimsMap := make(map[string]string, len(c.TaskSpec.Dimensions))
	for _, d := range c.TaskSpec.Dimensions {
		split := strings.SplitN(d, ":", 2)
		key := split[0]
		val := split[1]
		dims = append(dims, &swarming_api.SwarmingRpcsStringPair{
			Key:   key,
			Value: val,
		})
		dimsMap[key] = val
	}

	var env []*swarming_api.SwarmingRpcsStringPair
	if len(c.TaskSpec.Environment) > 0 {
		env = make([]*swarming_api.SwarmingRpcsStringPair, 0, len(c.TaskSpec.Environment))
		for k, v := range c.TaskSpec.Environment {
			env = append(env, &swarming_api.SwarmingRpcsStringPair{
				Key:   k,
				Value: v,
			})
		}
	}

	extraArgs := make([]string, 0, len(c.TaskSpec.ExtraArgs))
	for _, arg := range c.TaskSpec.ExtraArgs {
		extraArgs = append(extraArgs, replaceVars(c, arg))
	}

	expirationSecs := int64(c.TaskSpec.Expiration.Seconds())
	if expirationSecs == int64(0) {
		expirationSecs = int64(swarming.RECOMMENDED_EXPIRATION.Seconds())
	}
	executionTimeoutSecs := int64(c.TaskSpec.ExecutionTimeout.Seconds())
	if executionTimeoutSecs == int64(0) {
		executionTimeoutSecs = int64(swarming.RECOMMENDED_HARD_TIMEOUT.Seconds())
	}
	ioTimeoutSecs := int64(c.TaskSpec.IoTimeout.Seconds())
	if ioTimeoutSecs == int64(0) {
		ioTimeoutSecs = int64(swarming.RECOMMENDED_IO_TIMEOUT.Seconds())
	}
	return &swarming_api.SwarmingRpcsNewTaskRequest{
		ExpirationSecs: expirationSecs,
		Name:           c.Name,
		Priority:       int64(100.0 * c.TaskSpec.Priority),
		Properties: &swarming_api.SwarmingRpcsTaskProperties{
			CipdInput:            cipdInput,
			Dimensions:           dims,
			Env:                  env,
			ExecutionTimeoutSecs: executionTimeoutSecs,
			ExtraArgs:            extraArgs,
			InputsRef: &swarming_api.SwarmingRpcsFilesRef{
				Isolated:       c.IsolatedInput,
				Isolatedserver: isolate.ISOLATE_SERVER_URL,
				Namespace:      isolate.DEFAULT_NAMESPACE,
			},
			IoTimeoutSecs: ioTimeoutSecs,
		},
		Tags: db.TagsForTask(c.Name, id, c.TaskSpec.Priority, c.RepoState, c.RetryOf, dimsMap, c.ForcedJobId, c.ParentTaskIds),
		User: "skia-task-scheduler",
	}
}

// allDepsMet determines whether all dependencies for the given task candidate
// have been satisfied, and if so, returns a map of whose keys are task IDs and
// values are their isolated outputs.
func (c *taskCandidate) allDepsMet(cache db.TaskCache) (bool, map[string]string, error) {
	rv := make(map[string]string, len(c.TaskSpec.Dependencies))
	for _, depName := range c.TaskSpec.Dependencies {
		key := c.TaskKey.Copy()
		key.Name = depName
		byKey, err := cache.GetTasksByKey(&key)
		if err != nil {
			return false, nil, err
		}
		ok := false
		for _, t := range byKey {
			if t.Done() && t.Success() && t.IsolatedOutput != "" {
				rv[t.Id] = t.IsolatedOutput
				ok = true
				break
			}
		}
		if !ok {
			return false, nil, nil
		}
	}
	return true, rv, nil
}

// taskCandidateSlice is an alias used for sorting a slice of taskCandidates.
type taskCandidateSlice []*taskCandidate

func (s taskCandidateSlice) Len() int { return len(s) }
func (s taskCandidateSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s taskCandidateSlice) Less(i, j int) bool {
	return s[i].Score > s[j].Score // candidates sort in decreasing order.
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package constants

import (
	"path/filepath"
	"testing"

	"k8s.io/apimachinery/pkg/util/version"
	apimachineryversion "k8s.io/apimachinery/pkg/version"
)

func TestGetStaticPodDirectory(t *testing.T) {
	expected := "/etc/kubernetes/manifests"
	actual := GetStaticPodDirectory()

	if actual != expected {
		t.Errorf(
			"failed GetStaticPodDirectory:\n\texpected: %s\n\t  actual: %s",
			expected,
			actual,
		)
	}
}

func TestGetAdminKubeConfigPath(t *testing.T) {
	expected := filepath.Join(KubernetesDir, AdminKubeConfigFileName)
	actual := GetAdminKubeConfigPath()

	if actual != expected {
		t.Errorf(
			"failed GetAdminKubeConfigPath:\n\texpected: %s\n\t  actual: %s",
			expected,
			actual,
		)
	}
}

func TestGetBootstrapKubeletKubeConfigPath(t *testing.T) {
	expected := "/etc/kubernetes/bootstrap-kubelet.conf"
	actual := GetBootstrapKubeletKubeConfigPath()

	if actual != expected {
		t.Errorf(
			"failed GetBootstrapKubeletKubeConfigPath:\n\texpected: %s\n\t  actual: %s",
			expected,
			actual,
		)
	}
}

func TestGetKubeletKubeConfigPath(t *testing.T) {
	expected := "/etc/kubernetes/kubelet.conf"
	actual := GetKubeletKubeConfigPath()

	if actual != expected {
		t.Errorf(
			"failed GetKubeletKubeConfigPath:\n\texpected: %s\n\t  actual: %s",
			expected,
			actual,
		)
	}
}

func TestGetStaticPodFilepath(t *testing.T) {
	var tests = []struct {
		componentName, manifestsDir, expected string
	}{
		{
			componentName: "kube-apiserver",
			manifestsDir:  "/etc/kubernetes/manifests",
			expected:      "/etc/kubernetes/manifests/kube-apiserver.yaml",
		},
		{
			componentName: "kube-controller-manager",
			manifestsDir:  "/etc/kubernetes/manifests/",
			expected:      "/etc/kubernetes/manifests/kube-controller-manager.yaml",
		},
		{
			componentName: "foo",
			manifestsDir:  "/etc/bar/",
			expected:      "/etc/bar/foo.yaml",
		},
	}
	for _, rt := range tests {
		t.Run(rt.componentName, func(t *testing.T) {
			actual := GetStaticPodFilepath(rt.componentName, rt.manifestsDir)
			if actual != rt.expected {
				t.Errorf(
					"failed GetStaticPodFilepath:\n\texpected: %s\n\t  actual: %s",
					rt.expected,
					actual,
				)
			}
		})
	}
}

func TestEtcdSupportedVersion(t *testing.T) {
	var supportedEtcdVersion = map[uint8]string{
		13: "3.2.24",
		14: "3.3.10",
		15: "3.3.10",
		16: "3.3.17-0",
		17: "3.4.3-0",
		18: "3.4.3-0",
	}
	var tests = []struct {
		kubernetesVersion string
		expectedVersion   *version.Version
		expectedWarning   bool
		expectedError     bool
	}{
		{
			kubernetesVersion: "1.x.1",
			expectedVersion:   nil,
			expectedWarning:   false,
			expectedError:     true,
		},
		{
			kubernetesVersion: "1.10.1",
			expectedVersion:   version.MustParseSemantic("3.2.24"),
			expectedWarning:   true,
			expectedError:     false,
		},
		{
			kubernetesVersion: "1.99.0",
			expectedVersion:   version.MustParseSemantic("3.4.3-0"),
			expectedWarning:   true,
			expectedError:     false,
		},
		{
			kubernetesVersion: "v1.16.0",
			expectedVersion:   version.MustParseSemantic("3.3.17-0"),
			expectedWarning:   false,
			expectedError:     false,
		},
		{
			kubernetesVersion: "1.17.2",
			expectedVersion:   version.MustParseSemantic("3.4.3-0"),
			expectedWarning:   false,
			expectedError:     false,
		},
	}
	for _, rt := range tests {
		t.Run(rt.kubernetesVersion, func(t *testing.T) {
			actualVersion, actualWarning, actualError := EtcdSupportedVersion(supportedEtcdVersion, rt.kubernetesVersion)
			if (actualError != nil) != rt.expectedError {
				t.Fatalf("expected error %v, got %v", rt.expectedError, actualError != nil)
			}
			if (actualWarning != nil) != rt.expectedWarning {
				t.Fatalf("expected warning %v, got %v", rt.expectedWarning, actualWarning != nil)
			}
			if actualError == nil && actualVersion.String() != rt.expectedVersion.String() {
				t.Errorf("expected version %s, got %s", rt.expectedVersion.String(), actualVersion.String())
			}
		})
	}
}

func TestGetKubernetesServiceCIDR(t *testing.T) {
	var tests = []struct {
		svcSubnetList string
		isDualStack   bool
		expected      string
		expectedError bool
		name          string
	}{
		{
			svcSubnetList: "192.168.10.0/24",
			expected:      "192.168.10.0/24",
			expectedError: false,
			name:          "valid: valid IPv4 range from single-stack",
		},
		{
			svcSubnetList: "fd03::/112",
			expected:      "fd03::/112",
			expectedError: false,
			name:          "valid: valid IPv6 range from single-stack",
		},
		{
			svcSubnetList: "192.168.10.0/24,fd03::/112",
			expected:      "192.168.10.0/24",
			expectedError: false,
			name:          "valid: valid <IPv4,IPv6> ranges from dual-stack",
		},
		{
			svcSubnetList: "fd03::/112,192.168.10.0/24",
			expected:      "fd03::/112",
			expectedError: false,
			name:          "valid: valid <IPv6,IPv4> ranges from dual-stack",
		},
		{
			svcSubnetList: "192.168.10.0/24,fd03:x::/112",
			expected:      "",
			expectedError: true,
			name:          "invalid: failed to parse subnet range for dual-stack",
		},
	}

	for _, rt := range tests {
		t.Run(rt.name, func(t *testing.T) {
			actual, actualError := GetKubernetesServiceCIDR(rt.svcSubnetList)
			if rt.expectedError {
				if actualError == nil {
					t.Errorf("failed GetKubernetesServiceCIDR:\n\texpected error, but got no error")
				}
			} else if !rt.expectedError && actualError != nil {
				t.Errorf("failed GetKubernetesServiceCIDR:\n\texpected no error, but got: %v", actualError)
			} else {
				if actual.String() != rt.expected {
					t.Errorf(
						"failed GetKubernetesServiceCIDR:\n\texpected: %s\n\t  actual: %s",
						rt.expected,
						actual.String(),
					)
				}
			}
		})
	}
}

func TestGetSkewedKubernetesVersionImpl(t *testing.T) {
	tests := []struct {
		name           string
		versionInfo    *apimachineryversion.Info
		n              int
		expectedResult *version.Version
	}{
		{
			name:           "invalid versionInfo; placeholder version is returned",
			versionInfo:    &apimachineryversion.Info{},
			expectedResult: defaultKubernetesPlaceholderVersion,
		},
		{
			name:           "valid skew of -1",
			versionInfo:    &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"},
			n:              -1,
			expectedResult: version.MustParseSemantic("v1.22.0"),
		},
		{
			name:           "valid skew of 0",
			versionInfo:    &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"},
			n:              0,
			expectedResult: version.MustParseSemantic("v1.23.0"),
		},
		{
			name:           "valid skew of +1",
			versionInfo:    &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"},
			n:              1,
			expectedResult: version.MustParseSemantic("v1.24.0"),
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			result := getSkewedKubernetesVersionImpl(tc.versionInfo, tc.n)
			if cmp, _ := result.Compare(tc.expectedResult.String()); cmp != 0 {
				t.Errorf("expected result: %v, got %v", tc.expectedResult, result)
			}
		})
	}
}
<commit_msg>add test for GetAPIServerVirtualIP<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package constants

import (
	"path/filepath"
	"testing"

	"k8s.io/apimachinery/pkg/util/version"
	apimachineryversion "k8s.io/apimachinery/pkg/version"
)

func TestGetStaticPodDirectory(t *testing.T) {
	expected := "/etc/kubernetes/manifests"
	actual := GetStaticPodDirectory()

	if actual != expected {
		t.Errorf(
			"failed GetStaticPodDirectory:\n\texpected: %s\n\t  actual: %s",
			expected,
			actual,
		)
	}
}

func TestGetAdminKubeConfigPath(t *testing.T) {
	expected := filepath.Join(KubernetesDir, AdminKubeConfigFileName)
	actual := GetAdminKubeConfigPath()

	if actual != expected {
		t.Errorf(
			"failed GetAdminKubeConfigPath:\n\texpected: %s\n\t  actual: %s",
			expected,
			actual,
		)
	}
}

func TestGetBootstrapKubeletKubeConfigPath(t *testing.T) {
	expected := "/etc/kubernetes/bootstrap-kubelet.conf"
	actual := GetBootstrapKubeletKubeConfigPath()

	if actual != expected {
		t.Errorf(
			"failed GetBootstrapKubeletKubeConfigPath:\n\texpected: %s\n\t  actual: %s",
			expected,
			actual,
		)
	}
}

func TestGetKubeletKubeConfigPath(t *testing.T) {
	expected := "/etc/kubernetes/kubelet.conf"
	actual := GetKubeletKubeConfigPath()

	if actual != expected {
		t.Errorf(
			"failed GetKubeletKubeConfigPath:\n\texpected: %s\n\t  actual: %s",
			expected,
			actual,
		)
	}
}

func TestGetStaticPodFilepath(t *testing.T) {
	var tests = []struct {
		componentName, manifestsDir, expected string
	}{
		{
			componentName: "kube-apiserver",
			manifestsDir:  "/etc/kubernetes/manifests",
			expected:      "/etc/kubernetes/manifests/kube-apiserver.yaml",
		},
		{
			componentName: "kube-controller-manager",
			manifestsDir:  "/etc/kubernetes/manifests/",
			expected:      "/etc/kubernetes/manifests/kube-controller-manager.yaml",
		},
		{
			componentName: "foo",
			manifestsDir:  "/etc/bar/",
			expected:      "/etc/bar/foo.yaml",
		},
	}
	for _, rt := range tests {
		t.Run(rt.componentName, func(t *testing.T) {
			actual := GetStaticPodFilepath(rt.componentName, rt.manifestsDir)
			if actual != rt.expected {
				t.Errorf(
					"failed GetStaticPodFilepath:\n\texpected: %s\n\t  actual: %s",
					rt.expected,
					actual,
				)
			}
		})
	}
}

func TestEtcdSupportedVersion(t *testing.T) {
	var supportedEtcdVersion = map[uint8]string{
		13: "3.2.24",
		14: "3.3.10",
		15: "3.3.10",
		16: "3.3.17-0",
		17: "3.4.3-0",
		18: "3.4.3-0",
	}
	var tests = []struct {
		kubernetesVersion string
		expectedVersion   *version.Version
		expectedWarning   bool
		expectedError     bool
	}{
		{
			kubernetesVersion: "1.x.1",
			expectedVersion:   nil,
			expectedWarning:   false,
			expectedError:     true,
		},
		{
			kubernetesVersion: "1.10.1",
			expectedVersion:   version.MustParseSemantic("3.2.24"),
			expectedWarning:   true,
			expectedError:     false,
		},
		{
			kubernetesVersion: "1.99.0",
			expectedVersion:   version.MustParseSemantic("3.4.3-0"),
			expectedWarning:   true,
			expectedError:     false,
		},
		{
			kubernetesVersion: "v1.16.0",
			expectedVersion:   version.MustParseSemantic("3.3.17-0"),
			expectedWarning:   false,
			expectedError:     false,
		},
		{
			kubernetesVersion: "1.17.2",
			expectedVersion:   version.MustParseSemantic("3.4.3-0"),
			expectedWarning:   false,
			expectedError:     false,
		},
	}
	for _, rt := range tests {
		t.Run(rt.kubernetesVersion, func(t *testing.T) {
			actualVersion, actualWarning, actualError := EtcdSupportedVersion(supportedEtcdVersion, rt.kubernetesVersion)
			if (actualError != nil) != rt.expectedError {
				t.Fatalf("expected error %v, got %v", rt.expectedError, actualError != nil)
			}
			if (actualWarning != nil) != rt.expectedWarning {
				t.Fatalf("expected warning %v, got %v", rt.expectedWarning, actualWarning != nil)
			}
			if actualError == nil && actualVersion.String() != rt.expectedVersion.String() {
				t.Errorf("expected version %s, got %s", rt.expectedVersion.String(), actualVersion.String())
			}
		})
	}
}

func TestGetKubernetesServiceCIDR(t *testing.T) {
	var tests = []struct {
		svcSubnetList string
		isDualStack   bool
		expected      string
		expectedError bool
		name          string
	}{
		{
			svcSubnetList: "192.168.10.0/24",
			expected:      "192.168.10.0/24",
			expectedError: false,
			name:          "valid: valid IPv4 range from single-stack",
		},
		{
			svcSubnetList: "fd03::/112",
			expected:      "fd03::/112",
			expectedError: false,
			name:          "valid: valid IPv6 range from single-stack",
		},
		{
			svcSubnetList: "192.168.10.0/24,fd03::/112",
			expected:      "192.168.10.0/24",
			expectedError: false,
			name:          "valid: valid <IPv4,IPv6> ranges from dual-stack",
		},
		{
			svcSubnetList: "fd03::/112,192.168.10.0/24",
			expected:      "fd03::/112",
			expectedError: false,
			name:          "valid: valid <IPv6,IPv4> ranges from dual-stack",
		},
		{
			svcSubnetList: "192.168.10.0/24,fd03:x::/112",
			expected:      "",
			expectedError: true,
			name:          "invalid: failed to parse subnet range for dual-stack",
		},
	}

	for _, rt := range tests {
		t.Run(rt.name, func(t *testing.T) {
			actual, actualError := GetKubernetesServiceCIDR(rt.svcSubnetList)
			if rt.expectedError {
				if actualError == nil {
					t.Errorf("failed GetKubernetesServiceCIDR:\n\texpected error, but got no error")
				}
			} else if !rt.expectedError && actualError != nil {
				t.Errorf("failed GetKubernetesServiceCIDR:\n\texpected no error, but got: %v", actualError)
			} else {
				if actual.String() != rt.expected {
					t.Errorf(
						"failed GetKubernetesServiceCIDR:\n\texpected: %s\n\t  actual: %s",
						rt.expected,
						actual.String(),
					)
				}
			}
		})
	}
}

func TestGetSkewedKubernetesVersionImpl(t *testing.T) {
	tests := []struct {
		name           string
		versionInfo    *apimachineryversion.Info
		n              int
		expectedResult *version.Version
	}{
		{
			name:           "invalid versionInfo; placeholder version is returned",
			versionInfo:    &apimachineryversion.Info{},
			expectedResult: defaultKubernetesPlaceholderVersion,
		},
		{
			name:           "valid skew of -1",
			versionInfo:    &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"},
			n:              -1,
			expectedResult: version.MustParseSemantic("v1.22.0"),
		},
		{
			name:           "valid skew of 0",
			versionInfo:    &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"},
			n:              0,
			expectedResult: version.MustParseSemantic("v1.23.0"),
		},
		{
			name:           "valid skew of +1",
			versionInfo:    &apimachineryversion.Info{Major: "1", GitVersion: "v1.23.0"},
			n:              1,
			expectedResult: version.MustParseSemantic("v1.24.0"),
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			result := getSkewedKubernetesVersionImpl(tc.versionInfo, tc.n)
			if cmp, _ := result.Compare(tc.expectedResult.String()); cmp != 0 {
				t.Errorf("expected result: %v, got %v", tc.expectedResult, result)
			}
		})
	}
}

func TestGetAPIServerVirtualIP(t *testing.T) {
	var tests = []struct {
		name, svcSubnet, expectedIP string
		expectedErr                 bool
	}{
		{
			name:        "subnet mask 24",
			svcSubnet:   "10.96.0.12/24",
			expectedIP:  "10.96.0.1",
			expectedErr: false,
		},
		{
			name:        "subnet mask 12",
			svcSubnet:   "10.96.0.0/12",
			expectedIP:  "10.96.0.1",
			expectedErr: false,
		},
		{
			name:        "subnet mask 26",
			svcSubnet:   "10.87.116.64/26",
			expectedIP:  "10.87.116.65",
			expectedErr: false,
		},
		{
			name:        "dual-stack ipv4 primary, subnet mask 26",
			svcSubnet:   "10.87.116.64/26,fd03::/112",
			expectedIP:  "10.87.116.65",
			expectedErr: false,
		},
		{
			name:        "dual-stack, subnet mask 26 , missing first ip segment",
			svcSubnet:   ",10.87.116.64/26",
			expectedErr: true,
		},
		{
			name:        "dual-stack ipv4 primary, subnet mask 26, missing second ip segment",
			svcSubnet:   "10.87.116.64/26,",
			expectedErr: true,
		},
		{
			name:        "dual-stack ipv6 primary, subnet mask 112",
			svcSubnet:   "fd03::/112,10.87.116.64/26",
			expectedIP:  "fd03::1",
			expectedErr: false,
		},
		{
			name:        "dual-stack, subnet mask 26, missing first ip segment",
			svcSubnet:   ",fd03::/112",
			expectedErr: true,
		},
		{
			name:        "dual-stack, subnet mask 26, missing second ip segment",
			svcSubnet:   "fd03::/112,",
			expectedErr: true,
		},
	}
	for _, rt := range tests {
		t.Run(rt.name, func(t *testing.T) {
			virtualIP, err := GetAPIServerVirtualIP(rt.svcSubnet)
			if (err != nil) != rt.expectedErr {
				t.Errorf("failed APIServerVirtualIP:\n\texpectedErr: %v, got: %v", rt.expectedErr, err)
			} else if !rt.expectedErr {
				if virtualIP.String() != rt.expectedIP {
					t.Errorf(
						"failed APIServerVirtualIP:\n\texpected: %s\n\t  actual: %s",
						rt.expectedIP,
						virtualIP.String(),
					)
				}
			}
		})
	}
}
<|endoftext|>


<commit_before>package http

import (
	"database/sql"
	"net/http"

	http_cors "github.com/rs/cors"
	auth_proto "github.com/vardius/go-api-boilerplate/cmd/auth/proto"
	user_security "github.com/vardius/go-api-boilerplate/cmd/user/internal/application/security"
	"github.com/vardius/go-api-boilerplate/cmd/user/internal/domain/user"
	user_persistence "github.com/vardius/go-api-boilerplate/cmd/user/internal/infrastructure/persistence"
	handlers "github.com/vardius/go-api-boilerplate/cmd/user/internal/interfaces/http/handlers"
	commandbus "github.com/vardius/go-api-boilerplate/internal/commandbus"
	http_middleware "github.com/vardius/go-api-boilerplate/internal/http/middleware"
	http_authenticator "github.com/vardius/go-api-boilerplate/internal/http/middleware/authenticator"
	"github.com/vardius/go-api-boilerplate/internal/http/middleware/firewall"
	http_metadata_middleware "github.com/vardius/go-api-boilerplate/internal/http/middleware/metadata"
	log "github.com/vardius/go-api-boilerplate/internal/log"
	gorouter "github.com/vardius/gorouter/v4"
	"golang.org/x/oauth2"
	"google.golang.org/grpc"
)

const googleAPIURL = "https://www.googleapis.com/oauth2/v2/userinfo"
const facebookAPIURL = "https://graph.facebook.com/me"

// NewRouter provides new router
func NewRouter(logger *log.Logger, repository user_persistence.UserRepository, commandBus commandbus.CommandBus, mysqlConnection *sql.DB, grpAuthClient auth_proto.AuthenticationServiceClient, grpcConnectionMap map[string]*grpc.ClientConn, oauth2Config oauth2.Config, secretKey string) gorouter.Router {
	auth := http_authenticator.NewToken(user_security.TokenAuthHandler(grpAuthClient, repository))

	// Global middleware
	router := gorouter.New(
		http_middleware.Recover(logger),
		http_metadata_middleware.WithMetadata(),
		http_middleware.Logger(logger),
		http_cors.Default().Handler,
		http_middleware.XSS(),
		http_middleware.HSTS(),
		http_middleware.Metrics(),
		http_middleware.LimitRequestBody(int64(10<<20)), // 10 MB is a lot of text.
		auth.FromHeader("USER"),
		auth.FromQuery("authToken"),
	)

	// Liveness probes are to indicate that your application is running
	router.GET("/v1/health", handlers.BuildLivenessHandler())
	// Readiness is meant to check if your application is ready to serve traffic
	router.GET("/v1/readiness", handlers.BuildReadinessHandler(mysqlConnection, grpcConnectionMap))

	// Auth routes
	router.POST("/v1/google/callback", handlers.BuildSocialAuthHandler(googleAPIURL, commandBus, user.RegisterUserWithGoogle, secretKey, oauth2Config))
	router.POST("/v1/facebook/callback", handlers.BuildSocialAuthHandler(facebookAPIURL, commandBus, user.RegisterUserWithFacebook, secretKey, oauth2Config))

	// Protected User routes
	router.POST("/v1/dispatch/"+user.ChangeUserEmailAddress, handlers.BuildCommandDispatchHandler(commandBus))
	router.USE(http.MethodPost, "/v1/dispatch/"+user.ChangeUserEmailAddress, firewall.GrantAccessFor("USER"))

	// Public User routes
	router.POST("/v1/dispatch/{command}", handlers.BuildCommandDispatchHandler(commandBus))

	router.GET("/v1/me", handlers.BuildMeHandler(repository))
	router.USE(http.MethodGet, "/v1/me", firewall.GrantAccessFor("USER"))

	router.GET("/v1/", handlers.BuildListUserHandler(repository))
	router.GET("/v1/{id}", handlers.BuildGetUserHandler(repository))

	return router
}
<commit_msg>Use same handler for both routes(upstream)<commit_after>package http

import (
	"database/sql"
	"net/http"

	http_cors "github.com/rs/cors"
	auth_proto "github.com/vardius/go-api-boilerplate/cmd/auth/proto"
	user_security "github.com/vardius/go-api-boilerplate/cmd/user/internal/application/security"
	"github.com/vardius/go-api-boilerplate/cmd/user/internal/domain/user"
	user_persistence "github.com/vardius/go-api-boilerplate/cmd/user/internal/infrastructure/persistence"
	handlers "github.com/vardius/go-api-boilerplate/cmd/user/internal/interfaces/http/handlers"
	commandbus "github.com/vardius/go-api-boilerplate/internal/commandbus"
	http_middleware "github.com/vardius/go-api-boilerplate/internal/http/middleware"
	http_authenticator "github.com/vardius/go-api-boilerplate/internal/http/middleware/authenticator"
	"github.com/vardius/go-api-boilerplate/internal/http/middleware/firewall"
	http_metadata_middleware "github.com/vardius/go-api-boilerplate/internal/http/middleware/metadata"
	log "github.com/vardius/go-api-boilerplate/internal/log"
	gorouter "github.com/vardius/gorouter/v4"
	"golang.org/x/oauth2"
	"google.golang.org/grpc"
)

const googleAPIURL = "https://www.googleapis.com/oauth2/v2/userinfo"
const facebookAPIURL = "https://graph.facebook.com/me"

// NewRouter provides new router
func NewRouter(logger *log.Logger, repository user_persistence.UserRepository, commandBus commandbus.CommandBus, mysqlConnection *sql.DB, grpAuthClient auth_proto.AuthenticationServiceClient, grpcConnectionMap map[string]*grpc.ClientConn, oauth2Config oauth2.Config, secretKey string) gorouter.Router {
	auth := http_authenticator.NewToken(user_security.TokenAuthHandler(grpAuthClient, repository))

	// Global middleware
	router := gorouter.New(
		http_middleware.Recover(logger),
		http_metadata_middleware.WithMetadata(),
		http_middleware.Logger(logger),
		http_cors.Default().Handler,
		http_middleware.XSS(),
		http_middleware.HSTS(),
		http_middleware.Metrics(),
		http_middleware.LimitRequestBody(int64(10<<20)), // 10 MB is a lot of text.
		auth.FromHeader("USER"),
		auth.FromQuery("authToken"),
	)

	// Liveness probes are to indicate that your application is running
	router.GET("/v1/health", handlers.BuildLivenessHandler())
	// Readiness is meant to check if your application is ready to serve traffic
	router.GET("/v1/readiness", handlers.BuildReadinessHandler(mysqlConnection, grpcConnectionMap))

	// Auth routes
	router.POST("/v1/google/callback", handlers.BuildSocialAuthHandler(googleAPIURL, commandBus, user.RegisterUserWithGoogle, secretKey, oauth2Config))
	router.POST("/v1/facebook/callback", handlers.BuildSocialAuthHandler(facebookAPIURL, commandBus, user.RegisterUserWithFacebook, secretKey, oauth2Config))

	commandDispatchHandler := handlers.BuildCommandDispatchHandler(commandBus)

	// Protected User routes
	router.POST("/v1/dispatch/"+user.ChangeUserEmailAddress, commandDispatchHandler)
	router.USE(http.MethodPost, "/v1/dispatch/"+user.ChangeUserEmailAddress, firewall.GrantAccessFor("USER"))

	// Public User routes
	router.POST("/v1/dispatch/{command}", commandDispatchHandler)

	router.GET("/v1/me", handlers.BuildMeHandler(repository))
	router.USE(http.MethodGet, "/v1/me", firewall.GrantAccessFor("USER"))

	router.GET("/v1/", handlers.BuildListUserHandler(repository))
	router.GET("/v1/{id}", handlers.BuildGetUserHandler(repository))

	return router
}
<|endoftext|>


<commit_before>/*
 * Manifesto - generates a Greybus Module Manifest from git config syntax input
 *
 * Copyright 2014 Google Inc.
 * Copyright 2014 Linaro Ltd.
 *
 * Provided under the three clause BSD license found in the LICENSE file.
 *
 * Note: quick and dirty, needs a bunch of error checking on I/O and data read
 * 	 from the gcfg-style manifest source file
 */

package main

import (
	"bufio"
	"code.google.com/p/gcfg"
	"encoding/binary"
	"path/filepath"
	"log"
	"os"
	"strings"
)

const (
	INVALID_TYPE		uint8	= iota
	INTERFACE_TYPE		uint8	= iota
	STRING_TYPE		uint8	= iota
	BUNDLE_TYPE		uint8	= iota
	CPORT_TYPE		uint8	= iota
)

const (
	MANIFEST_HEADER_SIZE	uint16	= 0x04
	INTERFACE_SIZE		uint16	= 0x05
	STRING_SIZE		uint16	= 0x05
	BUNDLE_SIZE		uint16	= 0x05
	CPORT_SIZE		uint16	= 0x07
)

// Greybus version 0.1 manifest format
type Manifest struct {
	Manifest_header struct {
		Size uint16
		Version_major uint8
		Version_minor uint8
	}
	Interface_descriptor struct {
		Size uint16
		Type uint8
		Vendor_string_id uint8
		Product_string_id uint8
	}
	Bundle_descriptor map[string] *struct {
		Size uint16
		Type uint8
		Id uint8
		Class uint8
	}
	String_descriptor map[string] *struct {
		Size uint16
		Type uint8
		Length uint8
		Id uint8
		String string
	}
	Cport_descriptor map[string] *struct {
		Size uint16
		Type uint8
		Bundle uint8
		Id uint16
		Protocol uint8
	}
}

func right_pad(s string, padStr string, pLen int) string {
	return s + strings.Repeat(padStr, pLen);
}

func populate_manifest(mnf Manifest) Manifest {
	var mnf_size uint16
	mnf_size = 0

	mnf.Interface_descriptor.Type = INTERFACE_TYPE
	mnf.Interface_descriptor.Size = INTERFACE_SIZE
	mnf_size = mnf_size +
		(uint16)(mnf.Interface_descriptor.Size)

	for k := range mnf.String_descriptor {
		var size uint16

		/* Raw string length for the length field */
		length := len(mnf.String_descriptor[k].String)
		mnf.String_descriptor[k].Length = (uint8)(length)
		length = (int)(STRING_SIZE) + length

		/* Pad string descriptor to 4 byte boundaries */
		if length % 4 != 0 {
			mnf.String_descriptor[k].String =
				right_pad(mnf.String_descriptor[k].String,
						"\x00", 4 - length % 4)
		}

		/* Total string descriptor size includes string pad */
		size = STRING_SIZE +
			(uint16)(len(mnf.String_descriptor[k].String))
		mnf.String_descriptor[k].Type = STRING_TYPE
		mnf.String_descriptor[k].Size = size
		mnf_size = mnf_size + (uint16)(size)
	}

	for k := range mnf.Bundle_descriptor {
		mnf.Bundle_descriptor[k].Type = BUNDLE_TYPE
		mnf.Bundle_descriptor[k].Size = BUNDLE_SIZE
		mnf_size = mnf_size +
			(uint16)(mnf.Bundle_descriptor[k].Size)
	}

	for k := range mnf.Cport_descriptor {
		mnf.Cport_descriptor[k].Type = CPORT_TYPE
		mnf.Cport_descriptor[k].Size = CPORT_SIZE
		mnf_size = mnf_size +
			(uint16)(mnf.Cport_descriptor[k].Size)
	}

	/* Total size of all descriptors plus our header */
	mnf.Manifest_header.Size = MANIFEST_HEADER_SIZE + mnf_size

	return mnf
}

func write_manifest(m *os.File, mnf Manifest) {
	mwriter := bufio.NewWriter(m)

	/* Manifest header */
	binary.Write(mwriter, binary.LittleEndian, mnf.Manifest_header)

	/* Interface descriptor */
	binary.Write(mwriter, binary.LittleEndian, mnf.Interface_descriptor)

	/* Cport descriptors */
	for k := range mnf.Cport_descriptor {
		binary.Write(mwriter, binary.LittleEndian,
			     mnf.Cport_descriptor[k])
	}

	/* Bundle descriptors */
	for k := range mnf.Bundle_descriptor {
		binary.Write(mwriter, binary.LittleEndian,
			     mnf.Bundle_descriptor[k])
	}

	/* String descriptors */
	for k := range mnf.String_descriptor {
		strdesc := mnf.String_descriptor[k]

		/* Binary writer doesn't work on strings, so do it manually */
		binary.Write(mwriter, binary.LittleEndian, strdesc.Size)
		binary.Write(mwriter, binary.LittleEndian, strdesc.Type)
		binary.Write(mwriter, binary.LittleEndian, strdesc.Length)
		binary.Write(mwriter, binary.LittleEndian, strdesc.Id)
		mwriter.WriteString(strdesc.String)
	}

	mwriter.Flush()
}

func main() {
	var mnf Manifest

	// Manifest source file is our only argument
	mnfs := os.Args[1];

	// TODO: error checking on args

	// Open a binary manifest output file
	basename := mnfs[0:len(mnfs)-len(filepath.Ext(mnfs))]
	mnfb := basename + ".mnfb"
	m, err := os.Create(mnfb)
	if err != nil {
		panic(err)
	}
	defer m.Close()

	// Read in static manifest fields
	err = gcfg.ReadFileInto(&mnf, mnfs)
	if err != nil {
		log.Fatalf("Failed to parse manifest source: %s", err)
	}

	// Populate calculated manifest fields
	mnf = populate_manifest(mnf)

	// Write out manifest
	write_manifest(m, mnf);
}

<commit_msg>manifesto: Move id-field before bundle-field in CPort Descriptor.<commit_after>/*
 * Manifesto - generates a Greybus Module Manifest from git config syntax input
 *
 * Copyright 2014 Google Inc.
 * Copyright 2014 Linaro Ltd.
 *
 * Provided under the three clause BSD license found in the LICENSE file.
 *
 * Note: quick and dirty, needs a bunch of error checking on I/O and data read
 * 	 from the gcfg-style manifest source file
 */

package main

import (
	"bufio"
	"code.google.com/p/gcfg"
	"encoding/binary"
	"path/filepath"
	"log"
	"os"
	"strings"
)

const (
	INVALID_TYPE		uint8	= iota
	INTERFACE_TYPE		uint8	= iota
	STRING_TYPE		uint8	= iota
	BUNDLE_TYPE		uint8	= iota
	CPORT_TYPE		uint8	= iota
)

const (
	MANIFEST_HEADER_SIZE	uint16	= 0x04
	INTERFACE_SIZE		uint16	= 0x05
	STRING_SIZE		uint16	= 0x05
	BUNDLE_SIZE		uint16	= 0x05
	CPORT_SIZE		uint16	= 0x07
)

// Greybus version 0.1 manifest format
type Manifest struct {
	Manifest_header struct {
		Size uint16
		Version_major uint8
		Version_minor uint8
	}
	Interface_descriptor struct {
		Size uint16
		Type uint8
		Vendor_string_id uint8
		Product_string_id uint8
	}
	Bundle_descriptor map[string] *struct {
		Size uint16
		Type uint8
		Id uint8
		Class uint8
	}
	String_descriptor map[string] *struct {
		Size uint16
		Type uint8
		Length uint8
		Id uint8
		String string
	}
	Cport_descriptor map[string] *struct {
		Size uint16
		Type uint8
		Id uint16
		Bundle uint8
		Protocol uint8
	}
}

func right_pad(s string, padStr string, pLen int) string {
	return s + strings.Repeat(padStr, pLen);
}

func populate_manifest(mnf Manifest) Manifest {
	var mnf_size uint16
	mnf_size = 0

	mnf.Interface_descriptor.Type = INTERFACE_TYPE
	mnf.Interface_descriptor.Size = INTERFACE_SIZE
	mnf_size = mnf_size +
		(uint16)(mnf.Interface_descriptor.Size)

	for k := range mnf.String_descriptor {
		var size uint16

		/* Raw string length for the length field */
		length := len(mnf.String_descriptor[k].String)
		mnf.String_descriptor[k].Length = (uint8)(length)
		length = (int)(STRING_SIZE) + length

		/* Pad string descriptor to 4 byte boundaries */
		if length % 4 != 0 {
			mnf.String_descriptor[k].String =
				right_pad(mnf.String_descriptor[k].String,
						"\x00", 4 - length % 4)
		}

		/* Total string descriptor size includes string pad */
		size = STRING_SIZE +
			(uint16)(len(mnf.String_descriptor[k].String))
		mnf.String_descriptor[k].Type = STRING_TYPE
		mnf.String_descriptor[k].Size = size
		mnf_size = mnf_size + (uint16)(size)
	}

	for k := range mnf.Bundle_descriptor {
		mnf.Bundle_descriptor[k].Type = BUNDLE_TYPE
		mnf.Bundle_descriptor[k].Size = BUNDLE_SIZE
		mnf_size = mnf_size +
			(uint16)(mnf.Bundle_descriptor[k].Size)
	}

	for k := range mnf.Cport_descriptor {
		mnf.Cport_descriptor[k].Type = CPORT_TYPE
		mnf.Cport_descriptor[k].Size = CPORT_SIZE
		mnf_size = mnf_size +
			(uint16)(mnf.Cport_descriptor[k].Size)
	}

	/* Total size of all descriptors plus our header */
	mnf.Manifest_header.Size = MANIFEST_HEADER_SIZE + mnf_size

	return mnf
}

func write_manifest(m *os.File, mnf Manifest) {
	mwriter := bufio.NewWriter(m)

	/* Manifest header */
	binary.Write(mwriter, binary.LittleEndian, mnf.Manifest_header)

	/* Interface descriptor */
	binary.Write(mwriter, binary.LittleEndian, mnf.Interface_descriptor)

	/* Cport descriptors */
	for k := range mnf.Cport_descriptor {
		binary.Write(mwriter, binary.LittleEndian,
			     mnf.Cport_descriptor[k])
	}

	/* Bundle descriptors */
	for k := range mnf.Bundle_descriptor {
		binary.Write(mwriter, binary.LittleEndian,
			     mnf.Bundle_descriptor[k])
	}

	/* String descriptors */
	for k := range mnf.String_descriptor {
		strdesc := mnf.String_descriptor[k]

		/* Binary writer doesn't work on strings, so do it manually */
		binary.Write(mwriter, binary.LittleEndian, strdesc.Size)
		binary.Write(mwriter, binary.LittleEndian, strdesc.Type)
		binary.Write(mwriter, binary.LittleEndian, strdesc.Length)
		binary.Write(mwriter, binary.LittleEndian, strdesc.Id)
		mwriter.WriteString(strdesc.String)
	}

	mwriter.Flush()
}

func main() {
	var mnf Manifest

	// Manifest source file is our only argument
	mnfs := os.Args[1];

	// TODO: error checking on args

	// Open a binary manifest output file
	basename := mnfs[0:len(mnfs)-len(filepath.Ext(mnfs))]
	mnfb := basename + ".mnfb"
	m, err := os.Create(mnfb)
	if err != nil {
		panic(err)
	}
	defer m.Close()

	// Read in static manifest fields
	err = gcfg.ReadFileInto(&mnf, mnfs)
	if err != nil {
		log.Fatalf("Failed to parse manifest source: %s", err)
	}

	// Populate calculated manifest fields
	mnf = populate_manifest(mnf)

	// Write out manifest
	write_manifest(m, mnf);
}

<|endoftext|>


<commit_before>package nyb

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"strings"
	"time"

	"github.com/hako/durafmt"
	kitty "github.com/ugjka/kittybot"
	"gopkg.in/ugjka/go-tz.v2/tz"
)

const helpMsg = "COMMANDS: '%shny <location>', '%stime <location>', '%snext', '%sprevious', '%sremaining', '%shelp', '%ssource'"

func (bot *Settings) addTriggers() {
	irc := bot.IRC
	//Log Notices
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "NOTICE"
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("[NOTICE] " + m.Content)
		},
	})

	//Trigger for !source
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%ssource", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Reply(m, "https://github.com/ugjka/newyearsbot")
		},
	})

	//Trigger for !help
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%shelp", bot.Prefix)) ||
				normalize(m.Content) == fmt.Sprintf("%shny", bot.Prefix)
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying help...")
			b.Reply(m, fmt.Sprintf(helpMsg, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix))
		},
	})
	//Trigger for !next
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%snext", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying next...")
			dur := time.Minute * time.Duration(bot.next.Offset*60)
			if timeNow().UTC().Add(dur).After(target) {
				b.Reply(m, fmt.Sprintf("No more next, %d is here AoE", target.Year()))
				return
			}
			humandur := durafmt.Parse(target.Sub(timeNow().UTC().Add(dur)))
			b.Reply(m, fmt.Sprintf("Next New Year in %s in %s",
				roundDuration(humandur), bot.next))
		},
	})
	//Trigger for !previous
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%sprevious", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying previous...")
			dur := time.Minute * time.Duration(bot.previous.Offset*60)
			humandur := durafmt.Parse(timeNow().UTC().Add(dur).Sub(target))
			if bot.previous.Offset == -12 {
				humandur = durafmt.Parse(timeNow().UTC().Add(dur).Sub(target.AddDate(-1, 0, 0)))
			}
			b.Reply(m, fmt.Sprintf("Previous New Year %s ago in %s",
				roundDuration(humandur), bot.previous))
		},
	})
	//Trigger for !remaining
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%sremaining", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying remaining...")
			ss := "s"
			if bot.remaining == 1 {
				ss = ""
			}
			b.Reply(m, fmt.Sprintf("%s: %d timezone%s remaining", m.Name, bot.remaining, ss))
		},
	})
	//Trigger for time in location
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%stime ", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying time...")
			res, err := bot.getTime(normalize(m.Content)[len(bot.Prefix)+len("time")+1:])
			if err == errNoZone || err == errNoPlace {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, fmt.Sprintf("%s: %s", m.Name, err))
				return
			}
			if err != nil {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, fmt.Sprintf("%s: Some error occurred!", m.Name))
				return
			}
			b.Reply(m, res)
		},
	})
	//UTC
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				normalize(m.Content) == fmt.Sprintf("%stime", bot.Prefix)
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying time...")
			res := fmt.Sprintf("Time is %s", time.Now().UTC().Format("Mon Jan 2 15:04:05 -0700 MST 2006"))
			b.Reply(m, res)
		},
	})

	//Trigger for location queries
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%shny ", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			tz, err := bot.getNewYear(normalize(m.Content)[len(bot.Prefix)+len("hny")+1:])
			if err == errNoZone || err == errNoPlace {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, fmt.Sprintf("%s: %s", m.Name, err))
				return
			}
			if err != nil {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, fmt.Sprintf("%s: Some error occurred!", m.Name))
				return
			}
			b.Reply(m, fmt.Sprintf("%s: %s", m.Name, tz))
		},
	})
}

var (
	errNoZone  = errors.New("couldn't get the timezone for that location")
	errNoPlace = errors.New("Couldn't find that place")
)

func (bot *Settings) getNominatimReqURL(location *string) string {
	maps := url.Values{}
	maps.Add("q", *location)
	maps.Add("format", "json")
	maps.Add("accept-language", "en")
	maps.Add("limit", "1")
	maps.Add("email", bot.Email)
	return bot.Nominatim + NominatimEndpoint + maps.Encode()
}

func (bot *Settings) getTime(location string) (string, error) {
	bot.IRC.Info("Querying location: " + location)
	data, err := NominatimGetter(bot.getNominatimReqURL(&location))
	if err != nil {
		bot.IRC.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	var res NominatimResults
	if err = json.Unmarshal(data, &res); err != nil {
		bot.IRC.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	if len(res) == 0 {
		return "", errNoPlace
	}
	p := tz.Point{
		Lat: res[0].Lat,
		Lon: res[0].Lon,
	}
	tzid, err := tz.GetZone(p)
	if err != nil {
		return "", errNoZone
	}
	zone, err := time.LoadLocation(tzid[0])
	if err != nil {
		return "", errNoZone
	}
	address := res[0].DisplayName
	return fmt.Sprintf("Time in %s is %s", address, time.Now().In(zone).Format("Mon Jan 2 15:04:05 -0700 MST 2006")), nil
}

func (bot *Settings) getNewYear(location string) (string, error) {
	bot.IRC.Info("Querying location: " + location)
	data, err := NominatimGetter(bot.getNominatimReqURL(&location))
	if err != nil {
		bot.IRC.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	var res NominatimResults
	if err = json.Unmarshal(data, &res); err != nil {
		bot.IRC.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	if len(res) == 0 {
		return "", errNoPlace
	}
	p := tz.Point{
		Lat: res[0].Lat,
		Lon: res[0].Lon,
	}
	tzid, err := tz.GetZone(p)
	if err != nil {
		return "", errNoZone
	}
	zone, err := time.LoadLocation(tzid[0])
	if err != nil {
		return "", errNoZone
	}
	offset := time.Second * time.Duration(getOffset(target, zone))
	address := res[0].DisplayName

	if timeNow().UTC().Add(offset).Before(target) {
		humandur := durafmt.Parse(target.Sub(timeNow().UTC().Add(offset)))
		const newYearFutureMsg = "New Year in %s will happen in %s"
		return fmt.Sprintf(newYearFutureMsg, address, roundDuration(humandur)), nil
	}
	humandur := durafmt.Parse(timeNow().UTC().Add(offset).Sub(target))
	const newYearPastMsg = "New Year in %s happened %s ago"
	return fmt.Sprintf(newYearPastMsg, address, roundDuration(humandur)), nil
}

func normalize(s string) string {
	s = strings.TrimSpace(s)
	s = strings.ToLower(s)
	split := strings.Split(s, " ")
	s = ""
	for _, w := range split {
		if w == "" {
			continue
		}
		s += w + " "
	}
	return s[:len(s)-1]
}
<commit_msg>Simplify responses<commit_after>package nyb

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"strings"
	"time"

	"github.com/hako/durafmt"
	kitty "github.com/ugjka/kittybot"
	"gopkg.in/ugjka/go-tz.v2/tz"
)

const helpMsg = "COMMANDS: '%shny <location>', '%stime <location>', '%snext', '%sprevious', '%sremaining', '%shelp', '%ssource'"

func (bot *Settings) addTriggers() {
	irc := bot.IRC
	//Log Notices
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "NOTICE"
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("[NOTICE] " + m.Content)
		},
	})

	//Trigger for !source
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%ssource", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Reply(m, "https://github.com/ugjka/newyearsbot")
		},
	})

	//Trigger for !help
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%shelp", bot.Prefix)) ||
				normalize(m.Content) == fmt.Sprintf("%shny", bot.Prefix)
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying help...")
			b.Reply(m, fmt.Sprintf(helpMsg, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix, bot.Prefix))
		},
	})
	//Trigger for !next
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%snext", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying next...")
			dur := time.Minute * time.Duration(bot.next.Offset*60)
			if timeNow().UTC().Add(dur).After(target) {
				b.Reply(m, fmt.Sprintf("No more next, %d is here AoE", target.Year()))
				return
			}
			humandur := durafmt.Parse(target.Sub(timeNow().UTC().Add(dur)))
			b.Reply(m, fmt.Sprintf("Next New Year in %s in %s",
				roundDuration(humandur), bot.next))
		},
	})
	//Trigger for !previous
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%sprevious", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying previous...")
			dur := time.Minute * time.Duration(bot.previous.Offset*60)
			humandur := durafmt.Parse(timeNow().UTC().Add(dur).Sub(target))
			if bot.previous.Offset == -12 {
				humandur = durafmt.Parse(timeNow().UTC().Add(dur).Sub(target.AddDate(-1, 0, 0)))
			}
			b.Reply(m, fmt.Sprintf("Previous New Year %s ago in %s",
				roundDuration(humandur), bot.previous))
		},
	})
	//Trigger for !remaining
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%sremaining", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying remaining...")
			plural := "s"
			if bot.remaining == 1 {
				plural = ""
			}
			b.Reply(m, fmt.Sprintf("%d timezone%s remaining", bot.remaining, plural))
		},
	})
	//Trigger for time in location
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%stime ", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying time...")
			result, err := bot.getTime(normalize(m.Content)[len(bot.Prefix)+len("time")+1:])
			if err == errNoZone || err == errNoPlace {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, err.Error())
				return
			}
			if err != nil {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, "Some error occurred!")
				return
			}
			b.Reply(m, result)
		},
	})
	//UTC
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				normalize(m.Content) == fmt.Sprintf("%stime", bot.Prefix)
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			b.Info("Querying time...")
			result := fmt.Sprintf("Time is %s", time.Now().UTC().Format("Mon Jan 2 15:04:05 -0700 MST 2006"))
			b.Reply(m, result)
		},
	})

	//Trigger for location queries
	irc.AddTrigger(kitty.Trigger{
		Condition: func(b *kitty.Bot, m *kitty.Message) bool {
			return m.Command == "PRIVMSG" &&
				strings.HasPrefix(normalize(m.Content), fmt.Sprintf("%shny ", bot.Prefix))
		},
		Action: func(b *kitty.Bot, m *kitty.Message) {
			result, err := bot.getNewYear(normalize(m.Content)[len(bot.Prefix)+len("hny")+1:])
			if err == errNoZone || err == errNoPlace {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, err.Error())
				return
			}
			if err != nil {
				b.Warn("Query error: " + err.Error())
				b.Reply(m, "Some error occurred!")
				return
			}
			b.Reply(m, result)
		},
	})
}

var (
	errNoZone  = errors.New("couldn't get the timezone for that location")
	errNoPlace = errors.New("Couldn't find that place")
)

func (bot *Settings) getNominatimReqURL(location *string) string {
	maps := url.Values{}
	maps.Add("q", *location)
	maps.Add("format", "json")
	maps.Add("accept-language", "en")
	maps.Add("limit", "1")
	maps.Add("email", bot.Email)
	return bot.Nominatim + NominatimEndpoint + maps.Encode()
}

func (bot *Settings) getTime(location string) (string, error) {
	bot.IRC.Info("Querying location: " + location)
	data, err := NominatimGetter(bot.getNominatimReqURL(&location))
	if err != nil {
		bot.IRC.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	var res NominatimResults
	if err = json.Unmarshal(data, &res); err != nil {
		bot.IRC.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	if len(res) == 0 {
		return "", errNoPlace
	}
	p := tz.Point{
		Lat: res[0].Lat,
		Lon: res[0].Lon,
	}
	tzid, err := tz.GetZone(p)
	if err != nil {
		return "", errNoZone
	}
	zone, err := time.LoadLocation(tzid[0])
	if err != nil {
		return "", errNoZone
	}
	address := res[0].DisplayName
	return fmt.Sprintf("Time in %s is %s", address, time.Now().In(zone).Format("Mon Jan 2 15:04:05 -0700 MST 2006")), nil
}

func (bot *Settings) getNewYear(location string) (string, error) {
	bot.IRC.Info("Querying location: " + location)
	data, err := NominatimGetter(bot.getNominatimReqURL(&location))
	if err != nil {
		bot.IRC.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	var res NominatimResults
	if err = json.Unmarshal(data, &res); err != nil {
		bot.IRC.Warn("Nominatim error: " + err.Error())
		return "", err
	}
	if len(res) == 0 {
		return "", errNoPlace
	}
	p := tz.Point{
		Lat: res[0].Lat,
		Lon: res[0].Lon,
	}
	tzid, err := tz.GetZone(p)
	if err != nil {
		return "", errNoZone
	}
	zone, err := time.LoadLocation(tzid[0])
	if err != nil {
		return "", errNoZone
	}
	offset := time.Second * time.Duration(getOffset(target, zone))
	address := res[0].DisplayName

	if timeNow().UTC().Add(offset).Before(target) {
		humandur := durafmt.Parse(target.Sub(timeNow().UTC().Add(offset)))
		const newYearFutureMsg = "New Year in %s will happen in %s"
		return fmt.Sprintf(newYearFutureMsg, address, roundDuration(humandur)), nil
	}
	humandur := durafmt.Parse(timeNow().UTC().Add(offset).Sub(target))
	const newYearPastMsg = "New Year in %s happened %s ago"
	return fmt.Sprintf(newYearPastMsg, address, roundDuration(humandur)), nil
}

func normalize(s string) string {
	s = strings.TrimSpace(s)
	s = strings.ToLower(s)
	split := strings.Split(s, " ")
	s = ""
	for _, w := range split {
		if w == "" {
			continue
		}
		s += w + " "
	}
	return s[:len(s)-1]
}
<|endoftext|>


<commit_before>package oauth

import (
	"fmt"
	"net/http"
	"os"
	"reflect"
	"strings"

	"github.com/go-chi/chi"
	"github.com/gocontrib/auth"
	"github.com/gocontrib/log"
	"github.com/gocontrib/request"
	"github.com/markbates/goth"
	"github.com/markbates/goth/gothic"
)

type providerFactory = func(clientKey, secret, callbackURL string, scopes ...string) goth.Provider

var providerNames []string

func WithProviders(config *auth.Config, providers ...interface{}) {
	providerNames = []string{}
	var gothProviders []goth.Provider
	for i := 0; i+1 < len(providers); i++ {
		name := providers[i].(string)
		factory := makeProviderFactory(providers[i+1])
		provider := makeProvider(config, name, factory)
		if provider != nil {
			providerNames = append(providerNames, name)
			gothProviders = append(gothProviders, provider)
		}
	}
	if len(gothProviders) > 0 {
		goth.UseProviders(gothProviders...)
	} else {
		log.Warning("no oauth providers")
	}
}

func makeProviderFactory(v interface{}) providerFactory {
	return func(clientKey, secret, callbackURL string, scopes ...string) goth.Provider {
		f := reflect.ValueOf(v)
		// f := reflect.FuncOf(v)
		args := []reflect.Value{
			reflect.ValueOf(clientKey),
			reflect.ValueOf(secret),
			reflect.ValueOf(callbackURL),
		}
		for _, scope := range scopes {
			args = append(args, reflect.ValueOf(scope))
		}
		result := f.Call(args)
		return result[0].Interface().(goth.Provider)
	}
}

func makeProvider(config *auth.Config, provider string, factory providerFactory) goth.Provider {
	p := strings.ToUpper(provider)
	key := os.Getenv(p + "_KEY")
	secret := os.Getenv(p + "_SECRET")
	if len(key) == 0 || len(secret) == 0 {
		log.Warning("%s has no keys\n", provider)
		return nil
	}

	baseURL := strings.TrimRight(getBaseURL(config), "/")
	callbackURL := baseURL + "/api/oauth/callback/" + provider
	log.Debug("%s callback: %s\n", provider, callbackURL)

	// TODO support scopes via env
	return factory(key, secret, callbackURL)
}

func getBaseURL(config *auth.Config) string {
	if len(config.ServerURL) > 0 {
		return config.ServerURL
	}
	if config.ServerPort != 0 {
		return fmt.Sprintf("http://localhost:%d", config.ServerPort)
	}
	hostname, err := os.Hostname()
	if err == nil {
		return fmt.Sprintf("http://%s", hostname)
	}
	return "http://localhost:4200"
}

type Router interface {
	Get(pattern string, h http.HandlerFunc)
}

func RegisterAPI(mux Router, config *auth.Config) {
	config = config.SetDefaults()

	userStore := config.UserStoreEx
	if userStore == nil {
		return
	}

	defaultGetProviderName := gothic.GetProviderName
	gothic.GetProviderName = func(r *http.Request) (string, error) {
		providerName := chi.URLParam(r, "provider")
		if len(providerName) > 0 {
			return providerName, nil
		}
		return defaultGetProviderName(r)
	}

	mux.Get("/api/oauth/providers", func(w http.ResponseWriter, r *http.Request) {
		auth.SendJSON(w, providerNames)
	})

	mux.Get("/api/oauth/login/{provider}", func(w http.ResponseWriter, r *http.Request) {
		// try to get the user without re-authenticating
		if user, err := gothic.CompleteUserAuth(w, r); err == nil {
			completeOAuthFlow(w, r, config, user)
		} else {
			gothic.BeginAuthHandler(w, r)
		}
	})

	mux.Get("/api/oauth/logout/{provider}", func(w http.ResponseWriter, r *http.Request) {
		gothic.Logout(w, r)
		w.Header().Set("Location", "/")
		w.WriteHeader(http.StatusTemporaryRedirect)
	})

	mux.Get("/api/oauth/callback/{provider}", func(w http.ResponseWriter, r *http.Request) {
		user, err := gothic.CompleteUserAuth(w, r)
		if err != nil {
			fmt.Fprintln(w, err)
			return
		}
		completeOAuthFlow(w, r, config, user)
	})
}

func completeOAuthFlow(w http.ResponseWriter, r *http.Request, config *auth.Config, account goth.User) {
	ctx := r.Context()
	userStore := config.UserStoreEx
	if userStore == nil {
		return
	}

	user, err := userStore.FindUserByEmail(ctx, account.Email)
	if err != nil {
		// create user and link with external account
		data := auth.UserData{
			RawData:           account.RawData,
			Provider:          account.Provider,
			Email:             account.Email,
			Name:              account.Name,
			FirstName:         account.FirstName,
			LastName:          account.LastName,
			NickName:          account.NickName,
			Description:       account.Description,
			UserID:            account.UserID,
			AvatarURL:         account.AvatarURL,
			Location:          account.Location,
			AccessToken:       account.AccessToken,
			AccessTokenSecret: account.AccessTokenSecret,
			RefreshToken:      account.RefreshToken,
			ExpiresAt:         account.ExpiresAt,
		}
		user, err = userStore.CreateUser(ctx, data)
		if err != nil {
			fmt.Fprintln(w, err)
			return
		}
	}

	token := auth.MakeToken(r, config, user)
	tokenString, err3 := token.Encode(config)
	if err3 != nil {
		auth.SendError(w, err3)
		return
	}

	request.SetCookie(w, r, config.TokenCookie, tokenString)

	// TODO support return_url, absolute url if needed
	http.Redirect(w, r, "/", http.StatusFound)
}
<commit_msg>include token to redirect url<commit_after>package oauth

import (
	"fmt"
	"net/http"
	"os"
	"reflect"
	"strings"

	"github.com/go-chi/chi"
	"github.com/gocontrib/auth"
	"github.com/gocontrib/log"
	"github.com/gocontrib/request"
	"github.com/markbates/goth"
	"github.com/markbates/goth/gothic"
)

type providerFactory = func(clientKey, secret, callbackURL string, scopes ...string) goth.Provider

var providerNames []string

func WithProviders(config *auth.Config, providers ...interface{}) {
	providerNames = []string{}
	var gothProviders []goth.Provider
	for i := 0; i+1 < len(providers); i++ {
		name := providers[i].(string)
		factory := makeProviderFactory(providers[i+1])
		provider := makeProvider(config, name, factory)
		if provider != nil {
			providerNames = append(providerNames, name)
			gothProviders = append(gothProviders, provider)
		}
	}
	if len(gothProviders) > 0 {
		goth.UseProviders(gothProviders...)
	} else {
		log.Warning("no oauth providers")
	}
}

func makeProviderFactory(v interface{}) providerFactory {
	return func(clientKey, secret, callbackURL string, scopes ...string) goth.Provider {
		f := reflect.ValueOf(v)
		// f := reflect.FuncOf(v)
		args := []reflect.Value{
			reflect.ValueOf(clientKey),
			reflect.ValueOf(secret),
			reflect.ValueOf(callbackURL),
		}
		for _, scope := range scopes {
			args = append(args, reflect.ValueOf(scope))
		}
		result := f.Call(args)
		return result[0].Interface().(goth.Provider)
	}
}

func makeProvider(config *auth.Config, provider string, factory providerFactory) goth.Provider {
	p := strings.ToUpper(provider)
	key := os.Getenv(p + "_KEY")
	secret := os.Getenv(p + "_SECRET")
	if len(key) == 0 || len(secret) == 0 {
		log.Warning("%s has no keys\n", provider)
		return nil
	}

	baseURL := strings.TrimRight(getBaseURL(config), "/")
	callbackURL := baseURL + "/api/oauth/callback/" + provider
	log.Debug("%s callback: %s\n", provider, callbackURL)

	// TODO support scopes via env
	return factory(key, secret, callbackURL)
}

func getBaseURL(config *auth.Config) string {
	if len(config.ServerURL) > 0 {
		return config.ServerURL
	}
	if config.ServerPort != 0 {
		return fmt.Sprintf("http://localhost:%d", config.ServerPort)
	}
	hostname, err := os.Hostname()
	if err == nil {
		return fmt.Sprintf("http://%s", hostname)
	}
	return "http://localhost:4200"
}

type Router interface {
	Get(pattern string, h http.HandlerFunc)
}

func RegisterAPI(mux Router, config *auth.Config) {
	config = config.SetDefaults()

	userStore := config.UserStoreEx
	if userStore == nil {
		return
	}

	defaultGetProviderName := gothic.GetProviderName
	gothic.GetProviderName = func(r *http.Request) (string, error) {
		providerName := chi.URLParam(r, "provider")
		if len(providerName) > 0 {
			return providerName, nil
		}
		return defaultGetProviderName(r)
	}

	mux.Get("/api/oauth/providers", func(w http.ResponseWriter, r *http.Request) {
		auth.SendJSON(w, providerNames)
	})

	mux.Get("/api/oauth/login/{provider}", func(w http.ResponseWriter, r *http.Request) {
		// try to get the user without re-authenticating
		if user, err := gothic.CompleteUserAuth(w, r); err == nil {
			completeOAuthFlow(w, r, config, user)
		} else {
			gothic.BeginAuthHandler(w, r)
		}
	})

	mux.Get("/api/oauth/logout/{provider}", func(w http.ResponseWriter, r *http.Request) {
		gothic.Logout(w, r)
		w.Header().Set("Location", "/")
		w.WriteHeader(http.StatusTemporaryRedirect)
	})

	mux.Get("/api/oauth/callback/{provider}", func(w http.ResponseWriter, r *http.Request) {
		user, err := gothic.CompleteUserAuth(w, r)
		if err != nil {
			fmt.Fprintln(w, err)
			return
		}
		completeOAuthFlow(w, r, config, user)
	})
}

func completeOAuthFlow(w http.ResponseWriter, r *http.Request, config *auth.Config, account goth.User) {
	ctx := r.Context()
	userStore := config.UserStoreEx
	if userStore == nil {
		return
	}

	user, err := userStore.FindUserByEmail(ctx, account.Email)
	if err != nil {
		// create user and link with external account
		data := auth.UserData{
			RawData:           account.RawData,
			Provider:          account.Provider,
			Email:             account.Email,
			Name:              account.Name,
			FirstName:         account.FirstName,
			LastName:          account.LastName,
			NickName:          account.NickName,
			Description:       account.Description,
			UserID:            account.UserID,
			AvatarURL:         account.AvatarURL,
			Location:          account.Location,
			AccessToken:       account.AccessToken,
			AccessTokenSecret: account.AccessTokenSecret,
			RefreshToken:      account.RefreshToken,
			ExpiresAt:         account.ExpiresAt,
		}
		user, err = userStore.CreateUser(ctx, data)
		if err != nil {
			fmt.Fprintln(w, err)
			return
		}
	}

	token := auth.MakeToken(r, config, user)
	tokenString, err3 := token.Encode(config)
	if err3 != nil {
		auth.SendError(w, err3)
		return
	}

	request.SetCookie(w, r, config.TokenCookie, tokenString)

	// TODO support return_url, absolute url if needed
	http.Redirect(w, r, "/token="+tokenString, http.StatusFound)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"
)

type Crawler interface {
	Crawl()
}

func NewCrawler(travis *Travis, db *DB) []Crawler {
	return []Crawler{
		&NewBuildCrawler{travis, db, log.New(os.Stderr, "[NewBuildCrawler] ", log.LstdFlags)},
		&FinishedBuildCrawler{travis, db, log.New(os.Stderr, "[FinishedBuildCrawler] ", log.LstdFlags)},
	}
}

type NewBuildCrawler struct {
	Travis *Travis
	DB     *DB
	Logger *log.Logger
}

func (c *NewBuildCrawler) Crawl() {
	ch := time.Tick(10 * time.Second)
	for _ = range ch {
		c.Logger.Println("crawling for new builds...")
		c.crawlNewBuilds()
	}
}

func (c *NewBuildCrawler) crawlNewBuilds() {
	repos, err := c.Travis.Repos()
	if err != nil {
		c.Logger.Println(err)
		return
	}

	newBuilds := []string{}
	for _, repo := range repos {
		updated, err := c.DB.Upsert("new_builds", Query{"lastbuildid": repo.LastBuildId}, repo)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		if updated {
			newBuilds = append(newBuilds, repo.Slug)
		}
	}

	c.Logger.Printf("harvested %d builds with %d new builds: %s\n", len(repos), len(newBuilds), strings.Join(newBuilds, ", "))
}

type FinishedBuildCrawler struct {
	Travis *Travis
	DB     *DB
	Logger *log.Logger
}

func (c *FinishedBuildCrawler) Crawl() {
	ch := time.Tick(1 * time.Minute)
	for _ = range ch {
		c.Logger.Println("crawling for finsihed builds...")
		c.crawlFinishedBuilds()
	}
}

func (c *FinishedBuildCrawler) crawlFinishedBuilds() {
	colNames, finishedBuilds, skippedBuilds := c.doCrawlFinishedBuilds()
	c.Logger.Printf("fetched %d builds with %d finsihed and %d skipped. Finsihed builds: %s\n", len(finishedBuilds)+len(skippedBuilds), len(finishedBuilds), len(skippedBuilds), strings.Join(finishedBuilds, ", "))

	err := c.ensureColIndexes(colNames)
	if err != nil {
		c.Logger.Println(err)
	}
}

func (c *FinishedBuildCrawler) doCrawlFinishedBuilds() (colNames map[string]string, finishedBuilds []string, skippedBuilds []string) {
	colNames = make(map[string]string)

	var (
		repo  *Repo
		query Query
	)
	//query = Query{"lastbuildstartedat": Query{"$gte": oneMinuteAgo()}}
	iter := c.DB.C("new_builds").Find(query).Sort("-lastbuildstartedat").Iter()
	for iter.Next(&repo) {
		build, err := c.crawlFinsihedBuild(repo)
		if err != nil {
			c.Logger.Println(err)
			skippedBuilds = append(skippedBuilds, repo.Slug)
			continue
		}

		colName, updated, err := c.upsertBuild(build)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		colNames[colName] = colName

		if updated {
			finishedBuilds = append(finishedBuilds, repo.Slug)
		}
	}

	return
}

func (c *FinishedBuildCrawler) crawlFinsihedBuild(repo *Repo) (build *Build, err error) {
	build, err = c.Travis.Build(repo.LastBuildId)
	if err != nil {
		return
	}

	isFinished := !(build.FinishedAt == nil || build.StartedAt == nil)
	if !isFinished {
		err = fmt.Errorf("skipping build: %s - %d\n", repo.Slug, repo.LastBuildId)
		return
	}

	build.Repository = repo

	return
}

func (c *FinishedBuildCrawler) upsertBuild(build *Build) (colName string, updated bool, err error) {
	colName = buildColName(build.StartedAt)

	updated, err = c.DB.Upsert(colName, Query{"id": build.Id}, build)
	if err != nil {
		return
	}

	err = c.DB.C("new_builds").Remove(Query{"lastbuildid": build.Repository.LastBuildId})
	if err != nil {
		return
	}

	return
}

func (c *FinishedBuildCrawler) ensureColIndexes(colNames map[string]string) error {
	for _, colName := range colNames {
		c.Logger.Printf("ensuring index for collection %s\n", colName)
		err := c.DB.EnsureUniqueIndexKey(colName, "id")
		if err != nil {
			return err
		}
	}

	return nil
}

func buildColName(date *time.Time) string {
	buildDate := date.UTC().Format("2006_01_02")
	return fmt.Sprintf("builds_%s", buildDate)
}

func oneMinuteAgo() time.Time {
	now := time.Now()
	return now.Add(-1 * time.Minute).UTC()
}
<commit_msg>Increase time interval for finished builds<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"time"
)

var (
	newBuildCrawlerInterval      = 10 * time.Second
	finishedBuildCrawlerInterval = 2 * time.Minute
)

type Crawler interface {
	Crawl()
}

func NewCrawler(travis *Travis, db *DB) []Crawler {
	return []Crawler{
		&NewBuildCrawler{travis, db, log.New(os.Stderr, "[NewBuildCrawler] ", log.LstdFlags)},
		&FinishedBuildCrawler{travis, db, log.New(os.Stderr, "[FinishedBuildCrawler] ", log.LstdFlags)},
	}
}

type NewBuildCrawler struct {
	Travis *Travis
	DB     *DB
	Logger *log.Logger
}

func (c *NewBuildCrawler) Crawl() {
	ch := time.Tick(newBuildCrawlerInterval)
	for _ = range ch {
		c.Logger.Println("crawling for new builds...")
		c.crawlNewBuilds()
	}
}

func (c *NewBuildCrawler) crawlNewBuilds() {
	repos, err := c.Travis.Repos()
	if err != nil {
		c.Logger.Println(err)
		return
	}

	newBuilds := []string{}
	for _, repo := range repos {
		updated, err := c.DB.Upsert("new_builds", Query{"lastbuildid": repo.LastBuildId}, repo)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		if updated {
			newBuilds = append(newBuilds, repo.Slug)
		}
	}

	c.Logger.Printf("harvested %d builds with %d new builds: %s\n", len(repos), len(newBuilds), strings.Join(newBuilds, ", "))
}

type FinishedBuildCrawler struct {
	Travis *Travis
	DB     *DB
	Logger *log.Logger
}

func (c *FinishedBuildCrawler) Crawl() {
	ch := time.Tick(finishedBuildCrawlerInterval)
	for _ = range ch {
		c.Logger.Println("crawling for finsihed builds...")
		c.crawlFinishedBuilds()
	}
}

func (c *FinishedBuildCrawler) crawlFinishedBuilds() {
	colNames, finishedBuilds, skippedBuilds := c.doCrawlFinishedBuilds()
	c.Logger.Printf("fetched %d builds with %d finsihed and %d skipped. Finsihed builds: %s\n", len(finishedBuilds)+len(skippedBuilds), len(finishedBuilds), len(skippedBuilds), strings.Join(finishedBuilds, ", "))

	err := c.ensureColIndexes(colNames)
	if err != nil {
		c.Logger.Println(err)
	}
}

func (c *FinishedBuildCrawler) doCrawlFinishedBuilds() (colNames map[string]string, finishedBuilds []string, skippedBuilds []string) {
	colNames = make(map[string]string)

	var (
		repo  *Repo
		query Query
	)
	//query = Query{"lastbuildstartedat": Query{"$gte": oneMinuteAgo()}}
	iter := c.DB.C("new_builds").Find(query).Sort("-lastbuildstartedat").Iter()
	for iter.Next(&repo) {
		build, err := c.crawlFinsihedBuild(repo)
		if err != nil {
			c.Logger.Println(err)
			skippedBuilds = append(skippedBuilds, repo.Slug)
			continue
		}

		colName, updated, err := c.upsertBuild(build)
		if err != nil {
			c.Logger.Println(err)
			continue
		}

		colNames[colName] = colName

		if updated {
			finishedBuilds = append(finishedBuilds, repo.Slug)
		}
	}

	return
}

func (c *FinishedBuildCrawler) crawlFinsihedBuild(repo *Repo) (build *Build, err error) {
	build, err = c.Travis.Build(repo.LastBuildId)
	if err != nil {
		return
	}

	isFinished := !(build.FinishedAt == nil || build.StartedAt == nil)
	if !isFinished {
		err = fmt.Errorf("skipping build: %s - %d\n", repo.Slug, repo.LastBuildId)
		return
	}

	build.Repository = repo

	return
}

func (c *FinishedBuildCrawler) upsertBuild(build *Build) (colName string, updated bool, err error) {
	colName = buildColName(build.StartedAt)

	updated, err = c.DB.Upsert(colName, Query{"id": build.Id}, build)
	if err != nil {
		return
	}

	err = c.DB.C("new_builds").Remove(Query{"lastbuildid": build.Repository.LastBuildId})
	if err != nil {
		return
	}

	return
}

func (c *FinishedBuildCrawler) ensureColIndexes(colNames map[string]string) error {
	for _, colName := range colNames {
		c.Logger.Printf("ensuring index for collection %s\n", colName)
		err := c.DB.EnsureUniqueIndexKey(colName, "id")
		if err != nil {
			return err
		}
	}

	return nil
}

func buildColName(date *time.Time) string {
	buildDate := date.UTC().Format("2006_01_02")
	return fmt.Sprintf("builds_%s", buildDate)
}

func oneMinuteAgo() time.Time {
	now := time.Now()
	return now.Add(-1 * time.Minute).UTC()
}
<|endoftext|>