<commit_before>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build unit

package metrics

import (
	"errors"
	"reflect"
	"testing"
	"time"

	"github.com/control-center/serviced/utils"
)

func TestCache(t *testing.T) {

	clock := utils.NewTestClock()

	cache := MemoryUsageCache{
		Usages: make(map[string]*MemoryUsageItem),
		TTL:    time.Minute,
		Clock:  clock,
	}

	memusage1 := []MemoryUsageStats{
		MemoryUsageStats{ServiceID: "memusage1"},
	}
	memusage2 := []MemoryUsageStats{
		MemoryUsageStats{ServiceID: "memusage2"},
	}

	getter1 := func() ([]MemoryUsageStats, error) {
		return memusage1, nil
	}

	getter2 := func() ([]MemoryUsageStats, error) {
		return memusage2, nil
	}

	errgetter := func() ([]MemoryUsageStats, error) {
		return nil, errors.New("")
	}

	// Cache is empty, test error propagation in getter
	x, err := cache.Get("first", errgetter)
	if err == nil {
		t.Errorf("Cache did not propagate error in getter properly")
	}

	// Cache still empty, test getter
	x, err = cache.Get("first", getter1)
	if !reflect.DeepEqual(x, memusage1) {
		t.Errorf("Empty cache did not return a new item when a key did not yet exist")
	}

	// Cache has value for key, try error getter again
	x, err = cache.Get("first", errgetter)
	if err != nil {
		t.Errorf("Cache returned an error when it should have returned a cached value")
	}

	// Cache has value for key, try a new key with a different getter
	x, err = cache.Get("second", getter2)
	if !reflect.DeepEqual(x, memusage2) {
		t.Errorf("Non-empty cache did not return a new item when a key did not yet exist")
	}

	// Cache has a value for key, try different getter
	x, err = cache.Get("first", getter2)
	if !reflect.DeepEqual(x, memusage1) {
		t.Errorf("Cache did not return the correct item")
	}

	// Force expiration
	done := make(chan struct{})
	go func() {
		close(done)
		clock.Fire()
	}()
	<-done

	// Cache should no longer have a value for key, try with different getter
	x, err = cache.Get("first", getter2)
	if !reflect.DeepEqual(x, memusage2) {
		t.Errorf("Cache returned a value that should have expired")
	}
}
<commit_msg>elaborated on a test process<commit_after>// Copyright 2015 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build unit

package metrics

import (
	"errors"
	"reflect"
	"testing"
	"time"

	"github.com/control-center/serviced/utils"
)

func TestCache(t *testing.T) {

	clock := utils.NewTestClock()

	cache := MemoryUsageCache{
		Usages: make(map[string]*MemoryUsageItem),
		TTL:    time.Minute,
		Clock:  clock,
	}

	memusage1 := []MemoryUsageStats{
		MemoryUsageStats{ServiceID: "memusage1"},
	}
	memusage2 := []MemoryUsageStats{
		MemoryUsageStats{ServiceID: "memusage2"},
	}

	getter1 := func() ([]MemoryUsageStats, error) {
		return memusage1, nil
	}

	getter2 := func() ([]MemoryUsageStats, error) {
		return memusage2, nil
	}

	errgetter := func() ([]MemoryUsageStats, error) {
		return nil, errors.New("")
	}

	// Cache is empty, test error propagation in getter
	x, err := cache.Get("first", errgetter)
	if err == nil {
		t.Errorf("Cache did not propagate error in getter properly")
	}

	// Cache still empty, test getter
	x, err = cache.Get("first", getter1)
	if !reflect.DeepEqual(x, memusage1) {
		t.Errorf("Empty cache did not return a new item when a key did not yet exist")
	}

	// Cache has value for key, try error getter again
	x, err = cache.Get("first", errgetter)
	if err != nil {
		t.Errorf("Cache returned an error when it should have returned a cached value")
	}

	// Cache has value for key, try a new key with a different getter
	x, err = cache.Get("second", getter2)
	if !reflect.DeepEqual(x, memusage2) {
		t.Errorf("Non-empty cache did not return a new item when a key did not yet exist")
	}

	// Cache has a value for key, try different getter
	x, err = cache.Get("first", getter2)
	if !reflect.DeepEqual(x, memusage1) {
		t.Errorf("Cache did not return the correct item")
	}

	// Force expiration
	// I know this seems crazy, but go clock.Fire() may not trigger before
	// cache.Get is called.
	done := make(chan struct{})
	go func() {
		close(done)
		clock.Fire()
	}()
	<-done

	// Cache should no longer have a value for key, try with different getter
	x, err = cache.Get("first", getter2)
	if !reflect.DeepEqual(x, memusage2) {
		t.Errorf("Cache returned a value that should have expired")
	}
}
<|endoftext|>


<commit_before>package metrics

import (
	"database/sql"
	"errors"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

type SlowQueryMetrics struct {
	mutex        sync.Mutex
	metrics      map[string]prometheus.Gauge
	milliseconds int64
}

func NewSlowQueryMetrics(durationToConsiderSlow time.Duration) *SlowQueryMetrics {
	return &SlowQueryMetrics{
		milliseconds: durationToConsiderSlow.Nanoseconds() / 1000000,
		metrics: map[string]prometheus.Gauge{
			"slow_queries": prometheus.NewGauge(prometheus.GaugeOpts{
				Namespace: namespace,
				Name:      "slow_queries_total",
				Help:      "Number of slow queries",
			}),
			"slow_select_queries": prometheus.NewGauge(prometheus.GaugeOpts{
				Namespace: namespace,
				Name:      "slow_select_queries_total",
				Help:      "Number of slow SELECT queries",
			}),
			"slow_dml_queries": prometheus.NewGauge(prometheus.GaugeOpts{
				Namespace: namespace,
				Name:      "slow_dml_queries_total",
				Help:      "Number of slow data manipulation queries (INSERT, UPDATE, DELETE)",
			}),
		},
	}
}

func (s *SlowQueryMetrics) Scrape(db *sql.DB) error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	count := new(float64)
	err := db.QueryRow(
		`SELECT count(*)
				FROM pg_stat_activity
				WHERE state = 'active' AND
				NOW() - query_start > ($1 || ' milliseconds')::interval`,
		s.milliseconds).Scan(count)
	if err != nil {
		return errors.New("error counting slow queries: " + err.Error())
	}

	s.metrics["slow_queries"].Set(*count)

	err = db.QueryRow(
		`SELECT count(*)
				FROM pg_stat_activity
				WHERE state = 'active' AND
				NOW() - query_start > ($1 || ' milliseconds' )::interval AND
				query ilike 'select%'`,
		s.milliseconds).Scan(count)
	if err != nil {
		return errors.New("error counting slow select queries: " + err.Error())
	}

	s.metrics["slow_select_queries"].Set(*count)

	err = db.QueryRow(
		`SELECT count(*)
				FROM pg_stat_activity
				WHERE state = 'active' AND
				NOW() - query_start > ($1 || ' milliseconds')::interval AND
				(query ilike 'insert%' OR query ilike 'update%' OR query ilike 'delete%')`,
		s.milliseconds).Scan(count)
	if err != nil {
		return errors.New("error counting slow dml queries: " + err.Error())
	}

	s.metrics["slow_dml_queries"].Set(*count)

	return nil
}

func (s *SlowQueryMetrics) Describe(ch chan<- *prometheus.Desc) {
	for _, m := range s.metrics {
		m.Describe(ch)
	}
}

func (s *SlowQueryMetrics) Collect(ch chan<- prometheus.Metric) {
	for _, m := range s.metrics {
		m.Collect(ch)
	}
}

// check interface
var _ Collection = new(SlowQueryMetrics)
<commit_msg>MFD-7 use named constant<commit_after>package metrics

import (
	"database/sql"
	"errors"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

type SlowQueryMetrics struct {
	mutex        sync.Mutex
	metrics      map[string]prometheus.Gauge
	milliseconds int64
}

func NewSlowQueryMetrics(durationToConsiderSlow time.Duration) *SlowQueryMetrics {
	return &SlowQueryMetrics{
		milliseconds: durationToConsiderSlow.Nanoseconds() / int64(time.Millisecond),
		metrics: map[string]prometheus.Gauge{
			"slow_queries": prometheus.NewGauge(prometheus.GaugeOpts{
				Namespace: namespace,
				Name:      "slow_queries_total",
				Help:      "Number of slow queries",
			}),
			"slow_select_queries": prometheus.NewGauge(prometheus.GaugeOpts{
				Namespace: namespace,
				Name:      "slow_select_queries_total",
				Help:      "Number of slow SELECT queries",
			}),
			"slow_dml_queries": prometheus.NewGauge(prometheus.GaugeOpts{
				Namespace: namespace,
				Name:      "slow_dml_queries_total",
				Help:      "Number of slow data manipulation queries (INSERT, UPDATE, DELETE)",
			}),
		},
	}
}

func (s *SlowQueryMetrics) Scrape(db *sql.DB) error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	count := new(float64)
	err := db.QueryRow(
		`SELECT count(*)
				FROM pg_stat_activity
				WHERE state = 'active' AND
				NOW() - query_start > ($1 || ' milliseconds')::interval`,
		s.milliseconds).Scan(count)
	if err != nil {
		return errors.New("error counting slow queries: " + err.Error())
	}

	s.metrics["slow_queries"].Set(*count)

	err = db.QueryRow(
		`SELECT count(*)
				FROM pg_stat_activity
				WHERE state = 'active' AND
				NOW() - query_start > ($1 || ' milliseconds' )::interval AND
				query ilike 'select%'`,
		s.milliseconds).Scan(count)
	if err != nil {
		return errors.New("error counting slow select queries: " + err.Error())
	}

	s.metrics["slow_select_queries"].Set(*count)

	err = db.QueryRow(
		`SELECT count(*)
				FROM pg_stat_activity
				WHERE state = 'active' AND
				NOW() - query_start > ($1 || ' milliseconds')::interval AND
				(query ilike 'insert%' OR query ilike 'update%' OR query ilike 'delete%')`,
		s.milliseconds).Scan(count)
	if err != nil {
		return errors.New("error counting slow dml queries: " + err.Error())
	}

	s.metrics["slow_dml_queries"].Set(*count)

	return nil
}

func (s *SlowQueryMetrics) Describe(ch chan<- *prometheus.Desc) {
	for _, m := range s.metrics {
		m.Describe(ch)
	}
}

func (s *SlowQueryMetrics) Collect(ch chan<- prometheus.Metric) {
	for _, m := range s.metrics {
		m.Collect(ch)
	}
}

// check interface
var _ Collection = new(SlowQueryMetrics)
<|endoftext|>


<commit_before>package monitor

import (
	"errors"
	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/cloud-tools/mm"
	"github.com/percona/cloud-tools/mm/mysql"
	"github.com/percona/cloud-tools/pct"
)

type Factory struct {
	logChan chan *proto.LogEntry
}

func NewFactory(logChan chan *proto.LogEntry) *Factory {
	f := &Factory{
		logChan: logChan,
	}
	return f
}

func (f *Factory) Make(mtype, name string) (mm.Monitor, error) {
	var monitor mm.Monitor
	switch mtype {
	case "mysql":
		monitor = mysql.NewMonitor(pct.NewLogger(f.logChan, name))
	default:
		return nil, errors.New("Unknown monitor type: " + mtype)
	}
	return monitor, nil
}
<commit_msg>Add system monitor type to mm factory.<commit_after>package monitor

import (
	"errors"
	"github.com/percona/cloud-protocol/proto"
	"github.com/percona/cloud-tools/mm"
	"github.com/percona/cloud-tools/mm/mysql"
	"github.com/percona/cloud-tools/mm/system"
	"github.com/percona/cloud-tools/pct"
)

type Factory struct {
	logChan chan *proto.LogEntry
}

func NewFactory(logChan chan *proto.LogEntry) *Factory {
	f := &Factory{
		logChan: logChan,
	}
	return f
}

func (f *Factory) Make(mtype, name string) (mm.Monitor, error) {
	var monitor mm.Monitor
	switch mtype {
	case "mysql":
		monitor = mysql.NewMonitor(pct.NewLogger(f.logChan, name))
	case "system":
		monitor = system.NewMonitor(pct.NewLogger(f.logChan, name))
	default:
		return nil, errors.New("Unknown monitor type: " + mtype)
	}
	return monitor, nil
}
<|endoftext|>


<commit_before>package tests

import (
	"fmt"
	"testing"
	"time"

	"github.com/deis/deis/tests/dockercliutils"
	"github.com/deis/deis/tests/etcdutils"
	"github.com/deis/deis/tests/mockserviceutils"
	"github.com/deis/deis/tests/utils"
)

func runDeisControllerTest(t *testing.T, testSessionUID string, etcdPort string, servicePort string) {
	cli, stdout, stdoutPipe := dockercliutils.GetNewClient()
	done := make(chan bool, 1)
	dockercliutils.BuildDockerfile(t, "../", "deis/controller:"+testSessionUID)
	//docker run --name deis-controller -p 8000:8000 -e PUBLISH=8000
	// -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from=deis-logger deis/controller
	IPAddress := utils.GetHostIPAddress()
	done <- true
	go func() {
		<-done
		dockercliutils.RunContainer(t, cli, "--name",
			"deis-controller-"+testSessionUID,
			"-p", servicePort+":8000",
			"-e", "PUBLISH="+servicePort,
			"-e", "HOST="+IPAddress,
			"-e", "ETCD_PORT="+etcdPort,
			"deis/controller:"+testSessionUID)
	}()
	time.Sleep(5000 * time.Millisecond)
	dockercliutils.PrintToStdout(t, stdout, stdoutPipe, "Booting")

}

func TestBuild(t *testing.T) {
	setkeys := []string{"/deis/registry/protocol",
		"deis/registry/host",
		"/deis/registry/port",
		"/deis/cache/host",
		"/deis/cache/port"}
	setdir := []string{"/deis/controller",
		"/deis/cache",
		"/deis/database",
		"/deis/registry",
		"/deis/domains"}
	var testSessionUID = utils.NewUuid()
	fmt.Println("UUID for the session Controller Test :" + testSessionUID)
	etcdPort := utils.GetRandomPort()
	servicePort := utils.GetRandomPort()
	dbPort := utils.GetRandomPort()
	dockercliutils.RunEtcdTest(t, testSessionUID, etcdPort)
	fmt.Println("starting controller test:")
	Controllerhandler := etcdutils.InitetcdValues(setdir, setkeys, etcdPort)
	etcdutils.Publishvalues(t, Controllerhandler)
	mockserviceutils.RunMockDatabase(t, testSessionUID, etcdPort, dbPort)
	fmt.Println("starting Controller component test")
	runDeisControllerTest(t, testSessionUID, etcdPort, servicePort)
	dockercliutils.DeisServiceTest(
		t, "deis-controller-"+testSessionUID, servicePort, "http")
	dockercliutils.ClearTestSession(t, testSessionUID)
}
<commit_msg>fix(tests): use `docker run --rm` instead of explicitly removing containers<commit_after>package tests

import (
	"fmt"
	"testing"
	"time"

	"github.com/deis/deis/tests/dockercliutils"
	"github.com/deis/deis/tests/etcdutils"
	"github.com/deis/deis/tests/mockserviceutils"
	"github.com/deis/deis/tests/utils"
)

func runDeisControllerTest(t *testing.T, testSessionUID string, etcdPort string, servicePort string) {
	cli, stdout, stdoutPipe := dockercliutils.GetNewClient()
	done := make(chan bool, 1)
	dockercliutils.BuildDockerfile(t, "../", "deis/controller:"+testSessionUID)
	//docker run --name deis-controller -p 8000:8000 -e PUBLISH=8000
	// -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from=deis-logger deis/controller
	IPAddress := utils.GetHostIPAddress()
	done <- true
	go func() {
		<-done
		dockercliutils.RunContainer(t, cli,
			"--name", "deis-controller-"+testSessionUID,
			"--rm",
			"-p", servicePort+":8000",
			"-e", "PUBLISH="+servicePort,
			"-e", "HOST="+IPAddress,
			"-e", "ETCD_PORT="+etcdPort,
			"deis/controller:"+testSessionUID)
	}()
	time.Sleep(5000 * time.Millisecond)
	dockercliutils.PrintToStdout(t, stdout, stdoutPipe, "Booting")

}

func TestBuild(t *testing.T) {
	setkeys := []string{"/deis/registry/protocol",
		"deis/registry/host",
		"/deis/registry/port",
		"/deis/cache/host",
		"/deis/cache/port"}
	setdir := []string{"/deis/controller",
		"/deis/cache",
		"/deis/database",
		"/deis/registry",
		"/deis/domains"}
	var testSessionUID = utils.NewUuid()
	fmt.Println("UUID for the session Controller Test :" + testSessionUID)
	etcdPort := utils.GetRandomPort()
	servicePort := utils.GetRandomPort()
	dbPort := utils.GetRandomPort()
	dockercliutils.RunEtcdTest(t, testSessionUID, etcdPort)
	fmt.Println("starting controller test:")
	Controllerhandler := etcdutils.InitetcdValues(setdir, setkeys, etcdPort)
	etcdutils.Publishvalues(t, Controllerhandler)
	mockserviceutils.RunMockDatabase(t, testSessionUID, etcdPort, dbPort)
	fmt.Println("starting Controller component test")
	runDeisControllerTest(t, testSessionUID, etcdPort, servicePort)
	dockercliutils.DeisServiceTest(
		t, "deis-controller-"+testSessionUID, servicePort, "http")
	dockercliutils.ClearTestSession(t, testSessionUID)
}
<|endoftext|>


<commit_before>// Copyright 2012 The KidStuff Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package model

import (
	"errors"
)

var (
	ErrDuplicateName = errors.New("auth: duplicate Group Name")
)

type Group struct {
	Id        *string    `bson:"Id,omitempty"`
	Name      *string    `bson:"Name,omitempty" json:",omitempty"`
	Info      *GroupInfo `bson:"Info,omitempty" json:",omitempty"`
	Privilege []string   `bson:"Privilege,omitempty" json:",omitempty"`
}

type GroupInfo struct {
	Description *string `bson:"Description" json:",omitempty"`
}

type GroupManager interface {
	// AddDetail adds a group with full detail to database.
	AddDetail(*Group) (*Group, error)
	// UpdateDetail updates group detail specific by id.
	UpdateDetail(*Group) error
	// Find find the group specific by id.
	Find(id string) (*Group, error)
	// FindByName find the group specific by name.
	FindByName(name string) (*Group, error)
	// FindSome find and return a slice of group specific by thier id.
	FindSome(id ...string) ([]*Group, error)
	// FindAll finds and return a slice of group.
	// If limit < 0 the mean using the default upper limit.
	// If limit == 0 return empty result with error indicate no result found.
	// If limit can't be greater than the default upper limit.
	// Specific fields name for porjection select.
	FindAll(limit int, offsetId string, fields []string) ([]*Group, error)
	// Delete deletes a group from database base on the given id;
	// It returns an error describes the first issue encountered, if any.
	Delete(id string) error
}
<commit_msg>change interface of group manager according to user nanaer<commit_after>// Copyright 2012 The KidStuff Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package model

import (
	"errors"
)

var (
	ErrDuplicateName = errors.New("auth: duplicate Group Name")
)

type Group struct {
	Id        *string    `bson:"Id,omitempty"`
	Name      *string    `bson:"Name,omitempty" json:",omitempty"`
	Info      *GroupInfo `bson:"Info,omitempty" json:",omitempty"`
	Privilege []string   `bson:"Privilege,omitempty" json:",omitempty"`
}

type GroupInfo struct {
	Description *string `bson:"Description" json:",omitempty"`
}

type GroupManager interface {
	// AddDetail adds a group with full detail to database.
	AddDetail(name string, pri []string, info *GroupInfo) (*Group, error)
	// UpdateDetail updates group detail specific by id.
	UpdateDetail(id string, pri []string, info *GroupInfo) error
	// Find find the group specific by id.
	Find(id string) (*Group, error)
	// FindByName find the group specific by name.
	FindByName(name string) (*Group, error)
	// FindSome find and return a slice of group specific by thier id.
	FindSome(id ...string) ([]*Group, error)
	// FindAll finds and return a slice of group.
	// If limit < 0 the mean using the default upper limit.
	// If limit == 0 return empty result with error indicate no result found.
	// If limit can't be greater than the default upper limit.
	// Specific fields name for porjection select.
	FindAll(limit int, offsetId string, fields []string) ([]*Group, error)
	// Delete deletes a group from database base on the given id;
	// It returns an error describes the first issue encountered, if any.
	Delete(id string) error
}
<|endoftext|>


<commit_before>package models

import (
	"bufio"
	"crypto/md5"
	"crypto/sha256"
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/platform"
	"github.com/APTrust/exchange/util"
	"github.com/APTrust/exchange/util/fileutil"
	"github.com/satori/go.uuid"
	"hash"
	"io"
	"os"
	"path"
	"regexp"
	"strings"
	"time"
)

// VirtualBag creates an IntellectualObject from a bag on disk.
// The IntellectualObject can then be validated by workers.BagValidator.
type VirtualBag struct {
	pathToBag       string
	calculateMd5    bool
	calculateSha256 bool
	tagFilesToParse []string
	obj             *IntellectualObject
	summary         *WorkSummary
	readIterator    fileutil.ReadIterator
}

// NewVirtualBag creates a new virtual bag. Param pathToBag should
// be an absolute path to either a tar file or a directory containing
// an untarred bag. It pathToBag points to a tar file, the Read()
// function will read the bag without untarring it. Param tagFilesToParse
// should be a list of relative paths, pointing to tag files within the
// bag that should be parsed. For example, "aptrust-info.txt" or
// "dpn_tags/dpn-info.txt" Params calculateMd5 and calculateSha256
// indicate whether we should calculate md5 and/or sha256 checksums
// on the files in the bag.
func NewVirtualBag(pathToBag string, tagFilesToParse []string, calculateMd5, calculateSha256 bool) *VirtualBag {
	if tagFilesToParse == nil {
		tagFilesToParse = make([]string, 0)
	}
	return &VirtualBag{
		calculateMd5:    calculateMd5,
		calculateSha256: calculateSha256,
		pathToBag:       pathToBag,
		tagFilesToParse: tagFilesToParse,
	}
}

// Read() reads the bag and returns an IntellectualObject and a WorkSummary.
// The WorkSummary will include a list of errors, if there were any.
// The list of files contained in IntellectualObject.GenericFiles will include
// ALL files found in the bag, even some we may not want to save, such as
// those beginning with dots and dashes. If you don't want to preserve those
// files you can delete them from the IntellectualObject manually later.
func (vbag *VirtualBag) Read() (*IntellectualObject, *WorkSummary) {
	vbag.summary = NewWorkSummary()
	vbag.summary.Start()
	vbag.obj = NewIntellectualObject()
	vbag.obj.Identifier = util.CleanBagName(path.Base(vbag.pathToBag))
	if strings.HasSuffix(vbag.pathToBag, ".tar") {
		vbag.obj.IngestTarFilePath = vbag.pathToBag
	} else {
		vbag.obj.IngestUntarredPath = vbag.pathToBag
	}

	// Compile a list of the bag's contents (GenericFiles),
	// and calculate checksums for everything in the bag.
	var err error
	if vbag.obj.IngestTarFilePath != "" {
		vbag.readIterator, err = fileutil.NewTarFileIterator(vbag.obj.IngestTarFilePath)
	} else {
		vbag.readIterator, err = fileutil.NewFileSystemIterator(vbag.obj.IngestUntarredPath)
	}
	if err != nil {
		vbag.summary.AddError("Could not read bag: %v", err)
		vbag.summary.Finish()
		return vbag.obj, vbag.summary
	} else {
		vbag.addGenericFiles()
	}
	vbag.obj.IngestTopLevelDirNames = vbag.readIterator.GetTopLevelDirNames()

	// Golang's tar file reader is forward-only, so we need to
	// open a new iterator to read through a handful of tag files,
	// manifests and tag manifests.
	vbag.readIterator = nil
	if vbag.obj.IngestTarFilePath != "" {
		vbag.readIterator, err = fileutil.NewTarFileIterator(vbag.obj.IngestTarFilePath)
	} else {
		vbag.readIterator, err = fileutil.NewFileSystemIterator(vbag.obj.IngestUntarredPath)
	}
	if err != nil {
		vbag.summary.AddError("Could not read bag: %v", err)
	} else {
		vbag.parseManifestsTagFilesAndMimeTypes()
	}
	vbag.summary.Finish()
	return vbag.obj, vbag.summary
}

// Add every file in the bag to the list of generic files.
func (vbag *VirtualBag) addGenericFiles() {
	for {
		err := vbag.addGenericFile()
		if err != nil && (err == io.EOF || err.Error() == "EOF") {
			break
		} else if err != nil {
			vbag.summary.AddError(err.Error())
		}
	}
}

// Adds a single generic file to the bag.
func (vbag *VirtualBag) addGenericFile() error {
	reader, fileSummary, err := vbag.readIterator.Next()
	if err != nil {
		return err
	}
	if !fileSummary.IsRegularFile {
		return nil
	}
	_uuid := uuid.NewV4()
	gf := NewGenericFile()
	gf.Identifier = fmt.Sprintf("%s/%s", vbag.obj.Identifier, fileSummary.RelPath)
	gf.IntellectualObjectIdentifier = vbag.obj.Identifier
	gf.Size = fileSummary.Size
	gf.FileModified = fileSummary.ModTime
	gf.IngestLocalPath = fileSummary.AbsPath // will be empty if bag is tarred
	gf.IngestUUID = _uuid.String()
	gf.IngestUUIDGeneratedAt = time.Now().UTC()
	gf.IngestFileUid = fileSummary.Uid
	gf.IngestFileGid = fileSummary.Gid
	vbag.obj.GenericFiles = append(vbag.obj.GenericFiles, gf)
	vbag.setIngestFileType(gf, fileSummary)
	return vbag.calculateChecksums(reader, gf)
}

// Figure out what type of file this is.
func (vbag *VirtualBag) setIngestFileType(gf *GenericFile, fileSummary *fileutil.FileSummary) {
	if strings.HasPrefix(fileSummary.RelPath, "tagmanifest-") {
		gf.IngestFileType = constants.TAG_MANIFEST
		gf.FileFormat = "text/plain"
		vbag.obj.IngestTagManifests = append(vbag.obj.IngestTagManifests, fileSummary.RelPath)
	} else if strings.HasPrefix(fileSummary.RelPath, "manifest-") {
		gf.IngestFileType = constants.PAYLOAD_MANIFEST
		gf.FileFormat = "text/plain"
		vbag.obj.IngestManifests = append(vbag.obj.IngestManifests, fileSummary.RelPath)
	} else if strings.HasPrefix(fileSummary.RelPath, "data/") {
		gf.IngestFileType = constants.PAYLOAD_FILE
	} else {
		gf.IngestFileType = constants.TAG_FILE
	}
}

// Calculate the md5 and/or sha256 checksums on a file.
func (vbag *VirtualBag) calculateChecksums(reader io.Reader, gf *GenericFile) error {
	hashes := make([]io.Writer, 0)
	var md5Hash hash.Hash
	var sha256Hash hash.Hash
	if vbag.calculateMd5 {
		md5Hash = md5.New()
		hashes = append(hashes, md5Hash)
	}
	if vbag.calculateSha256 {
		sha256Hash = sha256.New()
		hashes = append(hashes, sha256Hash)
	}
	if len(hashes) > 0 {
		multiWriter := io.MultiWriter(hashes...)
		io.Copy(multiWriter, reader)
		utcNow := time.Now().UTC()
		if md5Hash != nil {
			gf.IngestMd5 = fmt.Sprintf("%x", md5Hash.Sum(nil))
			gf.IngestMd5GeneratedAt = utcNow
		}
		if sha256Hash != nil {
			gf.IngestSha256 = fmt.Sprintf("%x", sha256Hash.Sum(nil))
			gf.IngestSha256GeneratedAt = utcNow
		}
	}
	return nil
}

func (vbag *VirtualBag) setMimeType(reader io.Reader, gf *GenericFile) {
	// on err, defaults to application/binary
	bufLen := 128
	if gf.Size < int64(bufLen) {
		bufLen = int(gf.Size - 1)
		if bufLen < 1 {
			// We actually do permit zero-length files, and we can
			// save them in S3. These files can be necessary in
			// certain cases, such as __init__.py files for Python,
			// PHP templates whose presence is required, ".keep" files, etc.
			gf.FileFormat = "text/empty"
			return
		}
	}
	buf := make([]byte, bufLen)
	_, err := reader.Read(buf)
	if err != nil {
		vbag.summary.AddError(err.Error())
	}
	gf.FileFormat, err = platform.GuessMimeTypeByBuffer(buf)
	if err != nil {
		vbag.summary.AddError(err.Error())
	}
}

func (vbag *VirtualBag) parseManifestsTagFilesAndMimeTypes() {
	for {
		reader, fileSummary, err := vbag.readIterator.Next()
		if reader != nil {
			defer reader.Close()
		}
		if err == io.EOF {
			return
		}
		if err != nil {
			vbag.summary.AddError(err.Error())
			continue
		}
		// genericFile will sometimes be nil because the iterator
		// returns directory names as well as file names
		genericFile := vbag.obj.FindGenericFile(fileSummary.RelPath)
		if util.StringListContains(vbag.tagFilesToParse, fileSummary.RelPath) {
			vbag.parseTags(reader, fileSummary.RelPath)
			if genericFile != nil {
				// Our vbag library can only parse text files, so this
				// should be a plain text file.
				if strings.HasSuffix(genericFile.Identifier, ".txt") {
					genericFile.FileFormat = "text/plain"
				} else {
					genericFile.FileFormat = "application/binary"
				}
			}
		} else if util.StringListContains(vbag.obj.IngestManifests, fileSummary.RelPath) ||
			util.StringListContains(vbag.obj.IngestTagManifests, fileSummary.RelPath) {
			vbag.parseManifest(reader, fileSummary.RelPath)
		} else {
			if genericFile != nil {
				vbag.setMimeType(reader, genericFile)
			}
		}
	}
}

// Parse the checksums in a manifest.
func (vbag *VirtualBag) parseManifest(reader io.Reader, relFilePath string) {
	alg := ""
	if strings.Contains(relFilePath, constants.AlgSha256) {
		alg = constants.AlgSha256
	} else if strings.Contains(relFilePath, constants.AlgMd5) {
		alg = constants.AlgMd5
	} else {
		fmt.Fprintln(os.Stderr, "Not verifying checksums in", relFilePath,
			"- unsupported algorithm. Will still verify any md5 or sha256 checksums.")
		return
	}
	re := regexp.MustCompile(`^(\S*)\s*(.*)`)
	scanner := bufio.NewScanner(reader)
	lineNum := 1
	for scanner.Scan() {
		line := scanner.Text()
		if strings.TrimSpace(line) == "" {
			continue
		}
		if re.MatchString(line) {
			data := re.FindStringSubmatch(line)
			digest := data[1]
			filePath := data[2]
			genericFile := vbag.obj.FindGenericFile(filePath)
			if genericFile == nil {
				vbag.summary.AddError(
					"File '%s' in manifest '%s' is missing from bag",
					filePath, relFilePath)
				vbag.obj.IngestMissingFiles = append(vbag.obj.IngestMissingFiles,
					NewMissingFile(relFilePath, lineNum, filePath, digest))
				continue
			}
			if alg == constants.AlgMd5 {
				genericFile.IngestManifestMd5 = digest
			} else if alg == constants.AlgSha256 {
				genericFile.IngestManifestSha256 = digest
			}
		} else {
			vbag.summary.AddError(fmt.Sprintf(
				"Unable to parse data from line %d of manifest %s: %s",
				lineNum, relFilePath, line))
		}
		lineNum += 1
	}
}

// Parse the tag fields in a file.
func (vbag *VirtualBag) parseTags(reader io.Reader, relFilePath string) {
	re := regexp.MustCompile(`^(\S*\:)?(\s.*)?$`)
	scanner := bufio.NewScanner(reader)
	var tag *Tag
	for scanner.Scan() {
		line := scanner.Text()
		if strings.TrimSpace(line) == "" {
			continue
		}
		if re.MatchString(line) {
			data := re.FindStringSubmatch(line)
			data[1] = strings.Replace(data[1], ":", "", 1)
			if data[1] != "" {
				if tag != nil && tag.Label != "" {
					vbag.obj.IngestTags = append(vbag.obj.IngestTags, tag)
				}
				tag = NewTag(relFilePath, data[1], strings.Trim(data[2], " "))
				vbag.setIntelObjTagValue(tag)
				continue
			}
			value := strings.Trim(data[2], " ")
			tag.Value = strings.Join([]string{tag.Value, value}, " ")
			vbag.setIntelObjTagValue(tag)
		} else {
			vbag.summary.AddError("Unable to parse tag data from line: '%s'", line)
		}
	}
	if tag.Label != "" {
		vbag.obj.IngestTags = append(vbag.obj.IngestTags, tag)
	}
	if scanner.Err() != nil {
		vbag.summary.AddError("Error reading tag file '%s': %v",
			relFilePath, scanner.Err().Error())
	}
}

// Copy certain values from the aptrust-info.txt file into
// properties of the IntellectualObject.
func (vbag *VirtualBag) setIntelObjTagValue(tag *Tag) {
	if tag.SourceFile == "aptrust-info.txt" {
		label := strings.ToLower(tag.Label)
		switch label {
		case "title":
			vbag.obj.Title = tag.Value
		case "access":
			vbag.obj.Access = tag.Value
		}
	} else if tag.SourceFile == "bag-info.txt" {
		label := strings.ToLower(tag.Label)
		switch label {
		case "source-organization":
			vbag.obj.Institution = tag.Value
		case "internal-sender-description":
			vbag.obj.Description = tag.Value
		case "internal-sender-identifier":
			vbag.obj.AltIdentifier = tag.Value
		}
	}

}
<commit_msg>PT #144107747: Handle nil tag<commit_after>package models

import (
	"bufio"
	"crypto/md5"
	"crypto/sha256"
	"fmt"
	"github.com/APTrust/exchange/constants"
	"github.com/APTrust/exchange/platform"
	"github.com/APTrust/exchange/util"
	"github.com/APTrust/exchange/util/fileutil"
	"github.com/satori/go.uuid"
	"hash"
	"io"
	"os"
	"path"
	"regexp"
	"strings"
	"time"
)

// VirtualBag creates an IntellectualObject from a bag on disk.
// The IntellectualObject can then be validated by workers.BagValidator.
type VirtualBag struct {
	pathToBag       string
	calculateMd5    bool
	calculateSha256 bool
	tagFilesToParse []string
	obj             *IntellectualObject
	summary         *WorkSummary
	readIterator    fileutil.ReadIterator
}

// NewVirtualBag creates a new virtual bag. Param pathToBag should
// be an absolute path to either a tar file or a directory containing
// an untarred bag. It pathToBag points to a tar file, the Read()
// function will read the bag without untarring it. Param tagFilesToParse
// should be a list of relative paths, pointing to tag files within the
// bag that should be parsed. For example, "aptrust-info.txt" or
// "dpn_tags/dpn-info.txt" Params calculateMd5 and calculateSha256
// indicate whether we should calculate md5 and/or sha256 checksums
// on the files in the bag.
func NewVirtualBag(pathToBag string, tagFilesToParse []string, calculateMd5, calculateSha256 bool) *VirtualBag {
	if tagFilesToParse == nil {
		tagFilesToParse = make([]string, 0)
	}
	return &VirtualBag{
		calculateMd5:    calculateMd5,
		calculateSha256: calculateSha256,
		pathToBag:       pathToBag,
		tagFilesToParse: tagFilesToParse,
	}
}

// Read() reads the bag and returns an IntellectualObject and a WorkSummary.
// The WorkSummary will include a list of errors, if there were any.
// The list of files contained in IntellectualObject.GenericFiles will include
// ALL files found in the bag, even some we may not want to save, such as
// those beginning with dots and dashes. If you don't want to preserve those
// files you can delete them from the IntellectualObject manually later.
func (vbag *VirtualBag) Read() (*IntellectualObject, *WorkSummary) {
	vbag.summary = NewWorkSummary()
	vbag.summary.Start()
	vbag.obj = NewIntellectualObject()
	vbag.obj.Identifier = util.CleanBagName(path.Base(vbag.pathToBag))
	if strings.HasSuffix(vbag.pathToBag, ".tar") {
		vbag.obj.IngestTarFilePath = vbag.pathToBag
	} else {
		vbag.obj.IngestUntarredPath = vbag.pathToBag
	}

	// Compile a list of the bag's contents (GenericFiles),
	// and calculate checksums for everything in the bag.
	var err error
	if vbag.obj.IngestTarFilePath != "" {
		vbag.readIterator, err = fileutil.NewTarFileIterator(vbag.obj.IngestTarFilePath)
	} else {
		vbag.readIterator, err = fileutil.NewFileSystemIterator(vbag.obj.IngestUntarredPath)
	}
	if err != nil {
		vbag.summary.AddError("Could not read bag: %v", err)
		vbag.summary.Finish()
		return vbag.obj, vbag.summary
	} else {
		vbag.addGenericFiles()
	}
	vbag.obj.IngestTopLevelDirNames = vbag.readIterator.GetTopLevelDirNames()

	// Golang's tar file reader is forward-only, so we need to
	// open a new iterator to read through a handful of tag files,
	// manifests and tag manifests.
	vbag.readIterator = nil
	if vbag.obj.IngestTarFilePath != "" {
		vbag.readIterator, err = fileutil.NewTarFileIterator(vbag.obj.IngestTarFilePath)
	} else {
		vbag.readIterator, err = fileutil.NewFileSystemIterator(vbag.obj.IngestUntarredPath)
	}
	if err != nil {
		vbag.summary.AddError("Could not read bag: %v", err)
	} else {
		vbag.parseManifestsTagFilesAndMimeTypes()
	}
	vbag.summary.Finish()
	return vbag.obj, vbag.summary
}

// Add every file in the bag to the list of generic files.
func (vbag *VirtualBag) addGenericFiles() {
	for {
		err := vbag.addGenericFile()
		if err != nil && (err == io.EOF || err.Error() == "EOF") {
			break
		} else if err != nil {
			vbag.summary.AddError(err.Error())
		}
	}
}

// Adds a single generic file to the bag.
func (vbag *VirtualBag) addGenericFile() error {
	reader, fileSummary, err := vbag.readIterator.Next()
	if err != nil {
		return err
	}
	if !fileSummary.IsRegularFile {
		return nil
	}
	_uuid := uuid.NewV4()
	gf := NewGenericFile()
	gf.Identifier = fmt.Sprintf("%s/%s", vbag.obj.Identifier, fileSummary.RelPath)
	gf.IntellectualObjectIdentifier = vbag.obj.Identifier
	gf.Size = fileSummary.Size
	gf.FileModified = fileSummary.ModTime
	gf.IngestLocalPath = fileSummary.AbsPath // will be empty if bag is tarred
	gf.IngestUUID = _uuid.String()
	gf.IngestUUIDGeneratedAt = time.Now().UTC()
	gf.IngestFileUid = fileSummary.Uid
	gf.IngestFileGid = fileSummary.Gid
	vbag.obj.GenericFiles = append(vbag.obj.GenericFiles, gf)
	vbag.setIngestFileType(gf, fileSummary)
	return vbag.calculateChecksums(reader, gf)
}

// Figure out what type of file this is.
func (vbag *VirtualBag) setIngestFileType(gf *GenericFile, fileSummary *fileutil.FileSummary) {
	if strings.HasPrefix(fileSummary.RelPath, "tagmanifest-") {
		gf.IngestFileType = constants.TAG_MANIFEST
		gf.FileFormat = "text/plain"
		vbag.obj.IngestTagManifests = append(vbag.obj.IngestTagManifests, fileSummary.RelPath)
	} else if strings.HasPrefix(fileSummary.RelPath, "manifest-") {
		gf.IngestFileType = constants.PAYLOAD_MANIFEST
		gf.FileFormat = "text/plain"
		vbag.obj.IngestManifests = append(vbag.obj.IngestManifests, fileSummary.RelPath)
	} else if strings.HasPrefix(fileSummary.RelPath, "data/") {
		gf.IngestFileType = constants.PAYLOAD_FILE
	} else {
		gf.IngestFileType = constants.TAG_FILE
	}
}

// Calculate the md5 and/or sha256 checksums on a file.
func (vbag *VirtualBag) calculateChecksums(reader io.Reader, gf *GenericFile) error {
	hashes := make([]io.Writer, 0)
	var md5Hash hash.Hash
	var sha256Hash hash.Hash
	if vbag.calculateMd5 {
		md5Hash = md5.New()
		hashes = append(hashes, md5Hash)
	}
	if vbag.calculateSha256 {
		sha256Hash = sha256.New()
		hashes = append(hashes, sha256Hash)
	}
	if len(hashes) > 0 {
		multiWriter := io.MultiWriter(hashes...)
		io.Copy(multiWriter, reader)
		utcNow := time.Now().UTC()
		if md5Hash != nil {
			gf.IngestMd5 = fmt.Sprintf("%x", md5Hash.Sum(nil))
			gf.IngestMd5GeneratedAt = utcNow
		}
		if sha256Hash != nil {
			gf.IngestSha256 = fmt.Sprintf("%x", sha256Hash.Sum(nil))
			gf.IngestSha256GeneratedAt = utcNow
		}
	}
	return nil
}

func (vbag *VirtualBag) setMimeType(reader io.Reader, gf *GenericFile) {
	// on err, defaults to application/binary
	bufLen := 128
	if gf.Size < int64(bufLen) {
		bufLen = int(gf.Size - 1)
		if bufLen < 1 {
			// We actually do permit zero-length files, and we can
			// save them in S3. These files can be necessary in
			// certain cases, such as __init__.py files for Python,
			// PHP templates whose presence is required, ".keep" files, etc.
			gf.FileFormat = "text/empty"
			return
		}
	}
	buf := make([]byte, bufLen)
	_, err := reader.Read(buf)
	if err != nil {
		vbag.summary.AddError(err.Error())
	}
	gf.FileFormat, err = platform.GuessMimeTypeByBuffer(buf)
	if err != nil {
		vbag.summary.AddError(err.Error())
	}
}

func (vbag *VirtualBag) parseManifestsTagFilesAndMimeTypes() {
	for {
		reader, fileSummary, err := vbag.readIterator.Next()
		if reader != nil {
			defer reader.Close()
		}
		if err == io.EOF {
			return
		}
		if err != nil {
			vbag.summary.AddError(err.Error())
			continue
		}
		// genericFile will sometimes be nil because the iterator
		// returns directory names as well as file names
		genericFile := vbag.obj.FindGenericFile(fileSummary.RelPath)
		if util.StringListContains(vbag.tagFilesToParse, fileSummary.RelPath) {
			vbag.parseTags(reader, fileSummary.RelPath)
			if genericFile != nil {
				// Our vbag library can only parse text files, so this
				// should be a plain text file.
				if strings.HasSuffix(genericFile.Identifier, ".txt") {
					genericFile.FileFormat = "text/plain"
				} else {
					genericFile.FileFormat = "application/binary"
				}
			}
		} else if util.StringListContains(vbag.obj.IngestManifests, fileSummary.RelPath) ||
			util.StringListContains(vbag.obj.IngestTagManifests, fileSummary.RelPath) {
			vbag.parseManifest(reader, fileSummary.RelPath)
		} else {
			if genericFile != nil {
				vbag.setMimeType(reader, genericFile)
			}
		}
	}
}

// Parse the checksums in a manifest.
func (vbag *VirtualBag) parseManifest(reader io.Reader, relFilePath string) {
	alg := ""
	if strings.Contains(relFilePath, constants.AlgSha256) {
		alg = constants.AlgSha256
	} else if strings.Contains(relFilePath, constants.AlgMd5) {
		alg = constants.AlgMd5
	} else {
		fmt.Fprintln(os.Stderr, "Not verifying checksums in", relFilePath,
			"- unsupported algorithm. Will still verify any md5 or sha256 checksums.")
		return
	}
	re := regexp.MustCompile(`^(\S*)\s*(.*)`)
	scanner := bufio.NewScanner(reader)
	lineNum := 1
	for scanner.Scan() {
		line := scanner.Text()
		if strings.TrimSpace(line) == "" {
			continue
		}
		if re.MatchString(line) {
			data := re.FindStringSubmatch(line)
			digest := data[1]
			filePath := data[2]
			genericFile := vbag.obj.FindGenericFile(filePath)
			if genericFile == nil {
				vbag.summary.AddError(
					"File '%s' in manifest '%s' is missing from bag",
					filePath, relFilePath)
				vbag.obj.IngestMissingFiles = append(vbag.obj.IngestMissingFiles,
					NewMissingFile(relFilePath, lineNum, filePath, digest))
				continue
			}
			if alg == constants.AlgMd5 {
				genericFile.IngestManifestMd5 = digest
			} else if alg == constants.AlgSha256 {
				genericFile.IngestManifestSha256 = digest
			}
		} else {
			vbag.summary.AddError(fmt.Sprintf(
				"Unable to parse data from line %d of manifest %s: %s",
				lineNum, relFilePath, line))
		}
		lineNum += 1
	}
}

// Parse the tag fields in a file.
func (vbag *VirtualBag) parseTags(reader io.Reader, relFilePath string) {
	re := regexp.MustCompile(`^(\S*\:)?(\s.*)?$`)
	scanner := bufio.NewScanner(reader)
	var tag *Tag
	for scanner.Scan() {
		line := scanner.Text()
		if strings.TrimSpace(line) == "" {
			continue
		}
		if re.MatchString(line) {
			data := re.FindStringSubmatch(line)
			data[1] = strings.Replace(data[1], ":", "", 1)
			if data[1] != "" {
				if tag != nil && tag.Label != "" {
					vbag.obj.IngestTags = append(vbag.obj.IngestTags, tag)
				}
				tag = NewTag(relFilePath, data[1], strings.Trim(data[2], " "))
				vbag.setIntelObjTagValue(tag)
				continue
			}
			value := strings.Trim(data[2], " ")
			tag.Value = strings.Join([]string{tag.Value, value}, " ")
			vbag.setIntelObjTagValue(tag)
		} else {
			vbag.summary.AddError("Unable to parse tag data from line: '%s'", line)
		}
	}
	if tag != nil && tag.Label != "" {
		vbag.obj.IngestTags = append(vbag.obj.IngestTags, tag)
	}
	if scanner.Err() != nil {
		vbag.summary.AddError("Error reading tag file '%s': %v",
			relFilePath, scanner.Err().Error())
	}
}

// Copy certain values from the aptrust-info.txt file into
// properties of the IntellectualObject.
func (vbag *VirtualBag) setIntelObjTagValue(tag *Tag) {
	if tag.SourceFile == "aptrust-info.txt" {
		label := strings.ToLower(tag.Label)
		switch label {
		case "title":
			vbag.obj.Title = tag.Value
		case "access":
			vbag.obj.Access = tag.Value
		}
	} else if tag.SourceFile == "bag-info.txt" {
		label := strings.ToLower(tag.Label)
		switch label {
		case "source-organization":
			vbag.obj.Institution = tag.Value
		case "internal-sender-description":
			vbag.obj.Description = tag.Value
		case "internal-sender-identifier":
			vbag.obj.AltIdentifier = tag.Value
		}
	}

}
<|endoftext|>


<commit_before>package models

type ItemBoard interface {
}

type ItemBoards interface {
	GetBoard(string) (ItemBoard, bool)
}

type Board struct {
	ID string `json:"id"`
}

type Boards map[string]Board

func (bs *Boards) GetBoard(id string) (ItemBoard, bool) {
	board, ok := (*bs)[id]
	return board, ok
}

func NewItemBoard() ItemBoard {
	return &Board{}
}

func NewItemBoards() ItemBoards {
	return &Boards{}
}
<commit_msg>Add dummy board.<commit_after>package models

type ItemBoard interface {
}

type ItemBoards interface {
	GetBoard(string) (ItemBoard, bool)
}

type Board struct {
	ID string `json:"id"`
}

type Boards map[string]ItemBoard

func (bs Boards) GetBoard(id string) (ItemBoard, bool) {
	board, ok := bs[id]
	return board, ok
}

func NewItemBoard(id string) ItemBoard {
	return &Board{
		ID: id,
	}
}

func NewItemBoards() ItemBoards {
	board := Boards{}
	board["123"] = NewItemBoard("123")
	return board
}
<|endoftext|>


<commit_before>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package url

import (
	"crypto/tls"
	"errors"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"golang.org/x/net/html"
	"mime"
	"net/http"
	"regexp"
	"strings"
)

type Module struct {
	client   *http.Client
	regex    *regexp.Regexp
	RegexStr string `json:"regex"`
}

func Init(moduleSet *modules.ModuleSet) {
	moduleSet.Register(new(Module))
}

func (m *Module) Name() string {
	return "url"
}

func (m *Module) Help() string {
	return "Displays information about posted URLs."
}

func (m *Module) Defaults() {
	m.RegexStr = `(?i)\b((http|https)\://(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[\]{};:'".,<>?«»“”‘’]))`
}

func (m *Module) Load(client *irc.Client) error {
	regex, err := regexp.Compile(m.RegexStr)
	if err != nil {
		return err
	}

	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	m.client = &http.Client{Transport: tr}

	m.regex = regex
	client.CmdHook("privmsg", m.urlCmd)

	return nil
}

func (m *Module) urlCmd(client *irc.Client, msg irc.Message) error {
	url := m.regex.FindString(msg.Data)
	if len(url) <= 0 {
		return nil
	}

	resp, err := m.client.Head(url)
	if err != nil {
		return err
	}
	resp.Body.Close() // HEAD response doesn't have a body

	info := m.infoString(resp)
	if len(info) <= 0 {
		return nil
	}

	return client.Write("NOTICE %s :%s", msg.Receiver, info)
}

func (m *Module) infoString(resp *http.Response) string {
	var mtype string
	var infos []string

	ctype := resp.Header.Get("Content-Type")
	if len(ctype) > 0 {
		m, _, err := mime.ParseMediaType(ctype)
		if err == nil {
			mtype = m
			infos = append(infos, fmt.Sprintf("Type: %s", mtype))
		}
	}

	csize := resp.ContentLength
	if csize >= 0 {
		infos = append(infos, fmt.Sprintf("Size: %s", m.humanize(csize)))
	}

	if mtype == "text/html" {
		title, err := m.extractTitle(resp.Request.URL.String())
		if err == nil {
			infos = append(infos, fmt.Sprintf("Title: %s", title))
		}
	}

	info := strings.Join(infos, " | ")
	if len(info) > 0 {
		info = fmt.Sprintf("%s -- %s", strings.ToUpper(m.Name()), info)
	}

	return info
}

func (m *Module) extractTitle(url string) (title string, err error) {
	resp, err := m.client.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	doc, err := html.Parse(resp.Body)
	if err != nil {
		return
	}

	var parseFunc func(n *html.Node)
	parseFunc = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "title" {
			child := n.FirstChild
			if child != nil {
				title = child.Data
			} else {
				return
			}
		}

		for c := n.FirstChild; c != nil; c = c.NextSibling {
			parseFunc(c)
		}
	}

	parseFunc(doc)
	if len(title) <= 0 {
		err = errors.New("couldn't extract title")
		return
	}

	return
}

func (m *Module) humanize(count int64) string {
	switch {
	case count > (1 << 40):
		return fmt.Sprintf("%v TiB", count/(1<<40))
	case count > (1 << 30):
		return fmt.Sprintf("%v GiB", count/(1<<30))
	case count > (1 << 20):
		return fmt.Sprintf("%v MiB", count/(1<<20))
	case count > (1 << 10):
		return fmt.Sprintf("%v KiB", count/(1<<10))
	default:
		return fmt.Sprintf("%v B", count)
	}
}
<commit_msg>Revert "url: don't verify TLS certificates"<commit_after>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package url

import (
	"errors"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"golang.org/x/net/html"
	"mime"
	"net/http"
	"regexp"
	"strings"
)

type Module struct {
	regex    *regexp.Regexp
	RegexStr string `json:"regex"`
}

func Init(moduleSet *modules.ModuleSet) {
	moduleSet.Register(new(Module))
}

func (m *Module) Name() string {
	return "url"
}

func (m *Module) Help() string {
	return "Displays information about posted URLs."
}

func (m *Module) Defaults() {
	m.RegexStr = `(?i)\b((http|https)\://(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[\]{};:'".,<>?«»“”‘’]))`
}

func (m *Module) Load(client *irc.Client) error {
	regex, err := regexp.Compile(m.RegexStr)
	if err != nil {
		return err
	}

	m.regex = regex
	client.CmdHook("privmsg", m.urlCmd)

	return nil
}

func (m *Module) urlCmd(client *irc.Client, msg irc.Message) error {
	url := m.regex.FindString(msg.Data)
	if len(url) <= 0 {
		return nil
	}

	resp, err := http.Head(url)
	if err != nil {
		return err
	}
	resp.Body.Close() // HEAD response doesn't have a body

	info := m.infoString(resp)
	if len(info) <= 0 {
		return nil
	}

	return client.Write("NOTICE %s :%s", msg.Receiver, info)
}

func (m *Module) infoString(resp *http.Response) string {
	var mtype string
	var infos []string

	ctype := resp.Header.Get("Content-Type")
	if len(ctype) > 0 {
		m, _, err := mime.ParseMediaType(ctype)
		if err == nil {
			mtype = m
			infos = append(infos, fmt.Sprintf("Type: %s", mtype))
		}
	}

	csize := resp.ContentLength
	if csize >= 0 {
		infos = append(infos, fmt.Sprintf("Size: %s", m.humanize(csize)))
	}

	if mtype == "text/html" {
		title, err := m.extractTitle(resp.Request.URL.String())
		if err == nil {
			infos = append(infos, fmt.Sprintf("Title: %s", title))
		}
	}

	info := strings.Join(infos, " | ")
	if len(info) > 0 {
		info = fmt.Sprintf("%s -- %s", strings.ToUpper(m.Name()), info)
	}

	return info
}

func (m *Module) extractTitle(url string) (title string, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	doc, err := html.Parse(resp.Body)
	if err != nil {
		return
	}

	var parseFunc func(n *html.Node)
	parseFunc = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "title" {
			child := n.FirstChild
			if child != nil {
				title = child.Data
			} else {
				return
			}
		}

		for c := n.FirstChild; c != nil; c = c.NextSibling {
			parseFunc(c)
		}
	}

	parseFunc(doc)
	if len(title) <= 0 {
		err = errors.New("couldn't extract title")
		return
	}

	return
}

func (m *Module) humanize(count int64) string {
	switch {
	case count > (1 << 40):
		return fmt.Sprintf("%v TiB", count/(1<<40))
	case count > (1 << 30):
		return fmt.Sprintf("%v GiB", count/(1<<30))
	case count > (1 << 20):
		return fmt.Sprintf("%v MiB", count/(1<<20))
	case count > (1 << 10):
		return fmt.Sprintf("%v KiB", count/(1<<10))
	default:
		return fmt.Sprintf("%v B", count)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"

	"github.com/talbor49/HoneyBee/grammar"
)

const (
	success        = "OK"
	error          = "ERR"
	errNoBucket    = "You are not connected to any bucket. use the 'SET {BUCKET}'"
	errNotLoggedIn = "You are not logged in. Use 'Auth <username> <password>'"
)

// HandleQuery recieves a plain text string query, and hanles it.
// In most cases it adds it to the requests queue.
// Whilst in AUTH requests it validates the credentials and returns an answer.
func HandleQuery(query string, conn *DatabaseConnection) (returnCode string) {
	// TODO: write query in plain text to log
	requestType, tokens, err := grammar.ParseQuery(query)

	if err != nil {
		return err.Error()
	}

	switch requestType {
	case "AUTH":
		// AUTH {username} {password} {database_name}
		fmt.Println("Client wants to authenticate.")
		username := tokens[0]
		password := tokens[1]
		// bucketname := tokens[2]
		if credentialsValid(username, password) {
			conn.Username = username
		}
		fmt.Println("User logged in as: ", username)
		return success
	case "SET":
		// SET {key} {value} [ttl] [nooverride]
		fmt.Println("Client wants to set key")
		if conn.Bucket == "" {
			return errNoBucket
		}
		if conn.Username == "" {
			return errNotLoggedIn
		}
		key := tokens[0]
		value := tokens[1]
		fmt.Println("Setting " + key + ":" + value)
		setRequest := SetRequest{Key: key, Value: value, Conn: conn}
		return handleSetRequest(setRequest)

	case "GET":
		// GET {key}
		fmt.Println("Client wants to get key")
		if conn.Bucket == "" {
			return errNoBucket
		}
		if conn.Username == "" {
			return errNotLoggedIn
		}
		key := tokens[0]
		fmt.Println("Returning value of key: " + key)
		getRequest := GetRequest{Key: key, Conn: conn}
		return handleGetRequest(getRequest)

	case "DELETE":
		// DELETE {key}
		fmt.Println("Client wants to delete a bucket/key")
		if conn.Bucket == "" {
			return errNoBucket
		}
		if conn.Username == "" {
			return errNotLoggedIn
		}
		return success
	case "CREATE":
		fmt.Println("Client wants to create a bucket")
		if conn.Bucket == "" {
			return errNoBucket
		}
		if conn.Username == "" {
			return errNotLoggedIn
		}
		return success
	case "USE":
		fmt.Println("Client wants to use a specific bucket")
		if conn.Username == "" {
			return errNotLoggedIn
		}

		bucketname := tokens[0]

		useRequest := UseRequest{BucketName: bucketname, Conn: conn}

		return handleUseRequest(useRequest)
	default:
		return error
	}

}
<commit_msg>Changed DB error messages<commit_after>package server

import (
	"fmt"

	"github.com/talbor49/HoneyBee/grammar"
)

const (
	success        = "OK"
	error          = "ERR"
	errNoBucket    = "You are not connected to any bucket. use the 'USE <BUCKET>'"
	errNotLoggedIn = "You are not logged in. Use 'AUTH <username> <password>'"
)

// HandleQuery recieves a plain text string query, and hanles it.
// In most cases it adds it to the requests queue.
// Whilst in AUTH requests it validates the credentials and returns an answer.
func HandleQuery(query string, conn *DatabaseConnection) (returnCode string) {
	// TODO: write query in plain text to log
	requestType, tokens, err := grammar.ParseQuery(query)

	if err != nil {
		return err.Error()
	}

	switch requestType {
	case "AUTH":
		// AUTH {username} {password} {database_name}
		fmt.Println("Client wants to authenticate.")
		username := tokens[0]
		password := tokens[1]
		// bucketname := tokens[2]
		if credentialsValid(username, password) {
			conn.Username = username
		}
		fmt.Println("User logged in as: ", username)
		return success
	case "SET":
		// SET {key} {value} [ttl] [nooverride]
		fmt.Println("Client wants to set key")
		if conn.Bucket == "" {
			return errNoBucket
		}
		if conn.Username == "" {
			return errNotLoggedIn
		}
		key := tokens[0]
		value := tokens[1]
		fmt.Println("Setting " + key + ":" + value)
		setRequest := SetRequest{Key: key, Value: value, Conn: conn}
		return handleSetRequest(setRequest)

	case "GET":
		// GET {key}
		fmt.Println("Client wants to get key")
		if conn.Bucket == "" {
			return errNoBucket
		}
		if conn.Username == "" {
			return errNotLoggedIn
		}
		key := tokens[0]
		fmt.Println("Returning value of key: " + key)
		getRequest := GetRequest{Key: key, Conn: conn}
		return handleGetRequest(getRequest)

	case "DELETE":
		// DELETE {key}
		fmt.Println("Client wants to delete a bucket/key")
		if conn.Bucket == "" {
			return errNoBucket
		}
		if conn.Username == "" {
			return errNotLoggedIn
		}
		return success
	case "CREATE":
		fmt.Println("Client wants to create a bucket")
		if conn.Bucket == "" {
			return errNoBucket
		}
		if conn.Username == "" {
			return errNotLoggedIn
		}
		return success
	case "USE":
		fmt.Println("Client wants to use a specific bucket")
		if conn.Username == "" {
			return errNotLoggedIn
		}

		bucketname := tokens[0]

		useRequest := UseRequest{BucketName: bucketname, Conn: conn}

		return handleUseRequest(useRequest)
	default:
		return error
	}

}
<|endoftext|>


<commit_before>package http

import (
	"fmt"
	"github.com/vanng822/imgscale/imagick"
	"github.com/vanng822/imgscale/imgscale"
	"io/ioutil"
	"net/http"
	"strings"
)

type imageProviderHTTP struct {
	baseUrl string
}

func (imageProvider imageProviderHTTP) Fetch(filename string) (*imagick.MagickWand, error) {
	img := imagick.NewMagickWand()
	// %s%s will make it possible for using on arbitrary remote image
	// like http://127.0.0.1:8081/img/100x100/http://127.0.0.1:8080/img/original/kth.jpg
	resp, err := http.Get(fmt.Sprintf("%s%s", imageProvider.baseUrl, filename))
	if err != nil {
		return img, err
	}
	defer resp.Body.Close()

	imgData, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return img, err
	}
	err = img.ReadImageBlob(imgData)
	return img, err
}

/*
	NewImageProviderHTTP returns an instance of imageProviderHTTP
	where baseUrl is absolute url, together with filename it should become
	a valid url to image resource.

	One can leave baseUrl empty and specify filename as a remote image like
	http://127.0.0.1:8081/img/100x100/http://127.0.0.1:8080/img/original/kth.jpg
	Be aware what you do and also that it may not work for some handler/frameworks
*/
func New(baseUrl string) imgscale.ImageProvider {
	provider := imageProviderHTTP{}
	if baseUrl != "" {
		// assume we have a valid base url
		provider.baseUrl = fmt.Sprintf("%s/", strings.TrimSuffix(baseUrl, "/"))
	}
	return provider
}
<commit_msg>Pointer reciever<commit_after>package http

import (
	"fmt"
	"github.com/vanng822/imgscale/imagick"
	"github.com/vanng822/imgscale/imgscale"
	"io/ioutil"
	"net/http"
	"strings"
)

type imageProviderHTTP struct {
	baseUrl string
}

func (imageProvider *imageProviderHTTP) Fetch(filename string) (*imagick.MagickWand, error) {
	img := imagick.NewMagickWand()
	// %s%s will make it possible for using on arbitrary remote image
	// like http://127.0.0.1:8081/img/100x100/http://127.0.0.1:8080/img/original/kth.jpg
	resp, err := http.Get(fmt.Sprintf("%s%s", imageProvider.baseUrl, filename))
	if err != nil {
		return img, err
	}
	defer resp.Body.Close()

	imgData, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return img, err
	}
	err = img.ReadImageBlob(imgData)
	return img, err
}

/*
	NewImageProviderHTTP returns an instance of imageProviderHTTP
	where baseUrl is absolute url, together with filename it should become
	a valid url to image resource.

	One can leave baseUrl empty and specify filename as a remote image like
	http://127.0.0.1:8081/img/100x100/http://127.0.0.1:8080/img/original/kth.jpg
	Be aware what you do and also that it may not work for some handler/frameworks
*/
func New(baseUrl string) imgscale.ImageProvider {
	provider := &imageProviderHTTP{}
	if baseUrl != "" {
		// assume we have a valid base url
		provider.baseUrl = fmt.Sprintf("%s/", strings.TrimSuffix(baseUrl, "/"))
	}
	return provider
}
<|endoftext|>


<commit_before>// Copyright 2012 Jesse van den Kieboom. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package flags provides an extensive command line option parser.
// The flags package is similar in functionality to the go builtin flag package
// but provides more options and uses reflection to provide a convenient and
// succinct way of specifying command line options.
//
// Supported features:
//     Options with short names (-v)
//     Options with long names (--verbose)
//     Options with and without arguments (bool v.s. other type)
//     Options with optional arguments and default values
//     Multiple option groups each containing a set of options
//     Generate and print well-formatted help message
//     Passing remaining command line arguments after -- (optional)
//     Ignoring unknown command line options (optional)
//     Supports -I/usr/include -I=/usr/include -I /usr/include option argument specification
//     Supports multiple short options -aux
//     Supports all primitive go types (string, int{8..64}, uint{8..64}, float)
//     Supports same option multiple times (can store in slice or last option counts)
//     Supports maps
//     Supports function callbacks
//
// Additional features specific to Windows:
//     Options with short names (/v)
//     Options with long names (/verbose)
//     Windows-style options with arguments use a colon as the delimiter
//     Modify generated help message with Windows-style / options
//
// The flags package uses structs, reflection and struct field tags
// to allow users to specify command line options. This results in very simple
// and consise specification of your application options. For example:
//
//     type Options struct {
//         Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"`
//     }
//
// This specifies one option with a short name -v and a long name --verbose.
// When either -v or --verbose is found on the command line, a 'true' value
// will be appended to the Verbose field. e.g. when specifying -vvv, the
// resulting value of Verbose will be {[true, true, true]}.
//
// Slice options work exactly the same as primitive type options, except that
// whenever the option is encountered, a value is appended to the slice.
//
// Map options from string to primitive type are also supported. On the command
// line, you specify the value for such an option as key:value. For example
//
//     type Options struct {
//         AuthorInfo string[string] `short:"a"`
//     }
//
// Then, the AuthorInfo map can be filled with something like
// -a name:Jesse -a "surname:van den Kieboom".
//
// Finally, for full control over the conversion between command line argument
// values and options, user defined types can choose to implement the Marshaler
// and Unmarshaler interfaces.
//
// Available field tags:
//     short:          the short name of the option (single character)
//     long:           the long name of the option
//     description:    the description of the option (optional)
//     optional:       whether an argument of the option is optional (optional)
//     optional-value: the value of an optional option when the option occurs
//                     without an argument. This tag can be specified multiple
//                     times in the case of maps or slices (optional)
//     default:        the default value of an option. This tag can be specified
//                     multiple times in the case of slices or maps (optional).
//     default-mask:   when specified, this value will be displayed in the help
//                     instead of the actual default value. This is useful
//                     mostly for hiding otherwise sensitive information from
//                     showing up in the help. If default-mask takes the special
//                     value "-", then no default value will be shown at all
//                     (optional)
//     required:       whether an option is required to appear on the command
//                     line. If a required option is not present, the parser
//                     will return ErrRequired.
//     base:           a base (radix) used to convert strings to integer values,
//                     the default base is 10 (i.e. decimal) (optional)
//     value-name:     the name of the argument value (to be shown in the help,
//                     (optional)
//     group:          when specified on a struct field, makes the struct field
//                     a separate group with the given name (optional).
//     command:        when specified on a struct field, makes the struct field
//                     a (sub)command with the given name (optional).
//     name:           the display name of the command. This name is the name
//                     shown in the builtin generated help and can be used
//                     to provide a more informative title of a command. If not
//                     specified this defaults to the name given in the command
//                     command tag (optional).
//
// Either short: or long: must be specified to make the field eligible as an
// option.
//
//
// Option groups:
//
// Option groups are a simple way to semantically separate your options. The
// only real difference is in how your options will appear in the builtin
// generated help. All options in a particular group are shown together in the
// help under the name of the group.
//
// There are currently three ways to specify option groups.
//
//     1. Use NewNamedParser specifying the various option groups.
//     2. Use AddGroup to add a group to an existing parser.
//     3. Add a struct field to the toplevel options annotated with the
//        group:"group-name" tag.
//
//
//
// Commands:
//
// The flags package also has basic support for commands. Commands are often
// used in monolithic applications that support various commands or actions.
// Take git for example, all of the add, commit, checkout, etc. are called
// commands. Using commands you can easily separate multiple functions of your
// application.
//
// There are currently two ways to specifiy a command.
//
//     1. Use AddCommand on an existing parser.
//     2. Add a struct field to your options struct annotated with the
//        command:"command-name" tag.
//
// The most common, idiomatic way to implement commands is to define a global
// parser instance and implement each command in a separate file. These
// command files should define a go init function which calls AddCommand on
// the global parser.
//
// When parsing ends and there is an active command and that command implements
// the Commander interface, then its Execute method will be run with the
// remaining command line arguments.
//
// Command structs can have options which become valid to parse after the
// command has been specified on the command line. It is currently not valid
// to specify options from the parent level of the command after the command
// name has occurred. Thus, given a toplevel option "-v" and a command "add":
//
//     Valid:   ./app -v add
//     Invalid: ./app add -v
//
package flags
<commit_msg>Remove docs for unsupported name tag<commit_after>// Copyright 2012 Jesse van den Kieboom. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package flags provides an extensive command line option parser.
// The flags package is similar in functionality to the go builtin flag package
// but provides more options and uses reflection to provide a convenient and
// succinct way of specifying command line options.
//
// Supported features:
//     Options with short names (-v)
//     Options with long names (--verbose)
//     Options with and without arguments (bool v.s. other type)
//     Options with optional arguments and default values
//     Multiple option groups each containing a set of options
//     Generate and print well-formatted help message
//     Passing remaining command line arguments after -- (optional)
//     Ignoring unknown command line options (optional)
//     Supports -I/usr/include -I=/usr/include -I /usr/include option argument specification
//     Supports multiple short options -aux
//     Supports all primitive go types (string, int{8..64}, uint{8..64}, float)
//     Supports same option multiple times (can store in slice or last option counts)
//     Supports maps
//     Supports function callbacks
//
// Additional features specific to Windows:
//     Options with short names (/v)
//     Options with long names (/verbose)
//     Windows-style options with arguments use a colon as the delimiter
//     Modify generated help message with Windows-style / options
//
// The flags package uses structs, reflection and struct field tags
// to allow users to specify command line options. This results in very simple
// and consise specification of your application options. For example:
//
//     type Options struct {
//         Verbose []bool `short:"v" long:"verbose" description:"Show verbose debug information"`
//     }
//
// This specifies one option with a short name -v and a long name --verbose.
// When either -v or --verbose is found on the command line, a 'true' value
// will be appended to the Verbose field. e.g. when specifying -vvv, the
// resulting value of Verbose will be {[true, true, true]}.
//
// Slice options work exactly the same as primitive type options, except that
// whenever the option is encountered, a value is appended to the slice.
//
// Map options from string to primitive type are also supported. On the command
// line, you specify the value for such an option as key:value. For example
//
//     type Options struct {
//         AuthorInfo string[string] `short:"a"`
//     }
//
// Then, the AuthorInfo map can be filled with something like
// -a name:Jesse -a "surname:van den Kieboom".
//
// Finally, for full control over the conversion between command line argument
// values and options, user defined types can choose to implement the Marshaler
// and Unmarshaler interfaces.
//
// Available field tags:
//     short:          the short name of the option (single character)
//     long:           the long name of the option
//     description:    the description of the option (optional)
//     optional:       whether an argument of the option is optional (optional)
//     optional-value: the value of an optional option when the option occurs
//                     without an argument. This tag can be specified multiple
//                     times in the case of maps or slices (optional)
//     default:        the default value of an option. This tag can be specified
//                     multiple times in the case of slices or maps (optional).
//     default-mask:   when specified, this value will be displayed in the help
//                     instead of the actual default value. This is useful
//                     mostly for hiding otherwise sensitive information from
//                     showing up in the help. If default-mask takes the special
//                     value "-", then no default value will be shown at all
//                     (optional)
//     required:       whether an option is required to appear on the command
//                     line. If a required option is not present, the parser
//                     will return ErrRequired.
//     base:           a base (radix) used to convert strings to integer values,
//                     the default base is 10 (i.e. decimal) (optional)
//     value-name:     the name of the argument value (to be shown in the help,
//                     (optional)
//     group:          when specified on a struct field, makes the struct field
//                     a separate group with the given name (optional).
//     command:        when specified on a struct field, makes the struct field
//                     a (sub)command with the given name (optional).
//
// Either short: or long: must be specified to make the field eligible as an
// option.
//
//
// Option groups:
//
// Option groups are a simple way to semantically separate your options. The
// only real difference is in how your options will appear in the builtin
// generated help. All options in a particular group are shown together in the
// help under the name of the group.
//
// There are currently three ways to specify option groups.
//
//     1. Use NewNamedParser specifying the various option groups.
//     2. Use AddGroup to add a group to an existing parser.
//     3. Add a struct field to the toplevel options annotated with the
//        group:"group-name" tag.
//
//
//
// Commands:
//
// The flags package also has basic support for commands. Commands are often
// used in monolithic applications that support various commands or actions.
// Take git for example, all of the add, commit, checkout, etc. are called
// commands. Using commands you can easily separate multiple functions of your
// application.
//
// There are currently two ways to specifiy a command.
//
//     1. Use AddCommand on an existing parser.
//     2. Add a struct field to your options struct annotated with the
//        command:"command-name" tag.
//
// The most common, idiomatic way to implement commands is to define a global
// parser instance and implement each command in a separate file. These
// command files should define a go init function which calls AddCommand on
// the global parser.
//
// When parsing ends and there is an active command and that command implements
// the Commander interface, then its Execute method will be run with the
// remaining command line arguments.
//
// Command structs can have options which become valid to parse after the
// command has been specified on the command line. It is currently not valid
// to specify options from the parent level of the command after the command
// name has occurred. Thus, given a toplevel option "-v" and a command "add":
//
//     Valid:   ./app -v add
//     Invalid: ./app add -v
//
package flags
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package clonerefs

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/sirupsen/logrus"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/pod-utils/clone"
)

// Run clones the configured refs
func (o Options) Run() error {
	var env []string
	if len(o.KeyFiles) > 0 {
		var err error
		env, err = addSSHKeys(o.KeyFiles)
		if err != nil {
			return fmt.Errorf("failed to add SSH keys: %v", err)
		}
	}

	var numWorkers int
	if o.MaxParallelWorkers != 0 {
		numWorkers = o.MaxParallelWorkers
	} else {
		numWorkers = len(o.GitRefs)
	}

	wg := &sync.WaitGroup{}
	wg.Add(numWorkers)

	input := make(chan *kube.Refs)
	output := make(chan clone.Record, len(o.GitRefs))
	for i := 0; i < numWorkers; i++ {
		go func() {
			defer wg.Done()
			for ref := range input {
				output <- clone.Run(ref, o.SrcRoot, o.GitUserName, o.GitUserEmail, env)
			}
		}()
	}

	for _, ref := range o.GitRefs {
		input <- ref
	}

	close(input)
	wg.Wait()
	close(output)

	var results []clone.Record
	for record := range output {
		results = append(results, record)
	}

	logData, err := json.Marshal(results)
	if err != nil {
		return fmt.Errorf("failed to marshal clone records: %v", err)
	}

	if err := ioutil.WriteFile(o.Log, logData, 0755); err != nil {
		return fmt.Errorf("failed to write clone records: %v", err)
	}

	return nil
}

// addSSHKeys will start the ssh-agent and add all the specified
// keys, returning the ssh-agent environment variables for reuse
func addSSHKeys(paths []string) ([]string, error) {
	vars, err := exec.Command("ssh-agent").CombinedOutput()
	if err != nil {
		return []string{}, fmt.Errorf("failed to start ssh-agent: %v", err)
	}
	logrus.Info("Started SSH agent")
	// ssh-agent will output three lines of text, in the form:
	// SSH_AUTH_SOCK=xxx; export SSH_AUTH_SOCK;
	// SSH_AGENT_PID=xxx; export SSH_AGENT_PID;
	// echo Agent pid xxx;
	// We need to parse out the environment variables from that.
	parts := strings.Split(string(vars), ";")
	env := []string{strings.TrimSpace(parts[0]), strings.TrimSpace(parts[2])}
	for _, keyPath := range paths {
		// we can be given literal paths to keys or paths to dirs
		// that are mounted from a secret, so we need to check which
		// we have
		if err := filepath.Walk(keyPath, func(path string, info os.FileInfo, err error) error {
			if strings.HasPrefix(info.Name(), "..") {
				// kubernetes volumes also include files we
				// should not look be looking into for keys
				if info.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}
			if info.IsDir() {
				return nil
			}

			cmd := exec.Command("ssh-add", path)
			cmd.Env = append(cmd.Env, env...)
			if output, err := cmd.CombinedOutput(); err != nil {
				return fmt.Errorf("failed to add ssh key at %s: %v: %s", path, err, output)
			}
			logrus.Infof("Added SSH key at %s", path)
			return nil
		}); err != nil {
			return env, fmt.Errorf("error walking path %q: %v", keyPath, err)
		}
	}
	return env, nil
}
<commit_msg>Make clonerefs write errors to log on ssh error.<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package clonerefs

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"

	"github.com/sirupsen/logrus"
	"k8s.io/test-infra/prow/kube"
	"k8s.io/test-infra/prow/pod-utils/clone"
)

// Run clones the configured refs
func (o Options) Run() error {
	var env []string
	if len(o.KeyFiles) > 0 {
		var err error
		env, err = addSSHKeys(o.KeyFiles)
		if err != nil {
			logrus.WithError(err).Error("Failed to add SSH keys.")
			// Continue on error. Clones will fail with an appropriate error message
			// that initupload can consume whereas quiting without writing the clone
			// record log is silent and results in an errored prow job instead of a
			// failed one.
		}
	}

	var numWorkers int
	if o.MaxParallelWorkers != 0 {
		numWorkers = o.MaxParallelWorkers
	} else {
		numWorkers = len(o.GitRefs)
	}

	wg := &sync.WaitGroup{}
	wg.Add(numWorkers)

	input := make(chan *kube.Refs)
	output := make(chan clone.Record, len(o.GitRefs))
	for i := 0; i < numWorkers; i++ {
		go func() {
			defer wg.Done()
			for ref := range input {
				output <- clone.Run(ref, o.SrcRoot, o.GitUserName, o.GitUserEmail, env)
			}
		}()
	}

	for _, ref := range o.GitRefs {
		input <- ref
	}

	close(input)
	wg.Wait()
	close(output)

	var results []clone.Record
	for record := range output {
		results = append(results, record)
	}

	logData, err := json.Marshal(results)
	if err != nil {
		return fmt.Errorf("failed to marshal clone records: %v", err)
	}

	if err := ioutil.WriteFile(o.Log, logData, 0755); err != nil {
		return fmt.Errorf("failed to write clone records: %v", err)
	}

	return nil
}

// addSSHKeys will start the ssh-agent and add all the specified
// keys, returning the ssh-agent environment variables for reuse
func addSSHKeys(paths []string) ([]string, error) {
	vars, err := exec.Command("ssh-agent").CombinedOutput()
	if err != nil {
		return []string{}, fmt.Errorf("failed to start ssh-agent: %v", err)
	}
	logrus.Info("Started SSH agent")
	// ssh-agent will output three lines of text, in the form:
	// SSH_AUTH_SOCK=xxx; export SSH_AUTH_SOCK;
	// SSH_AGENT_PID=xxx; export SSH_AGENT_PID;
	// echo Agent pid xxx;
	// We need to parse out the environment variables from that.
	parts := strings.Split(string(vars), ";")
	env := []string{strings.TrimSpace(parts[0]), strings.TrimSpace(parts[2])}
	for _, keyPath := range paths {
		// we can be given literal paths to keys or paths to dirs
		// that are mounted from a secret, so we need to check which
		// we have
		if err := filepath.Walk(keyPath, func(path string, info os.FileInfo, err error) error {
			if strings.HasPrefix(info.Name(), "..") {
				// kubernetes volumes also include files we
				// should not look be looking into for keys
				if info.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}
			if info.IsDir() {
				return nil
			}

			cmd := exec.Command("ssh-add", path)
			cmd.Env = append(cmd.Env, env...)
			if output, err := cmd.CombinedOutput(); err != nil {
				return fmt.Errorf("failed to add ssh key at %s: %v: %s", path, err, output)
			}
			logrus.Infof("Added SSH key at %s", path)
			return nil
		}); err != nil {
			return env, fmt.Errorf("error walking path %q: %v", keyPath, err)
		}
	}
	return env, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package config knows how to read and parse config.yaml.
package config

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"text/template"
	"time"

	"github.com/ghodss/yaml"

	"k8s.io/test-infra/prow/kube"
)

// Config is a read-only snapshot of the config.
type Config struct {
	// Full repo name (such as "kubernetes/kubernetes") -> list of jobs.
	Presubmits  map[string][]Presubmit  `json:"presubmits,omitempty"`
	Postsubmits map[string][]Postsubmit `json:"postsubmits,omitempty"`

	// Periodics are not associated with any repo.
	Periodics []Periodic `json:"periodics,omitempty"`

	Tide     Tide      `json:"tide,omitempty"`
	Plank    Plank     `json:"plank,omitempty"`
	Sinker   Sinker    `json:"sinker,omitempty"`
	Triggers []Trigger `json:"triggers,omitempty"`
	Heart    Heart     `json:"heart,omitempty"`

	// ProwJobNamespace is the namespace in the cluster that prow
	// components will use for looking up ProwJobs. The namespace
	// needs to exist and will not be created by prow.
	ProwJobNamespace string `json:"prowjob_namespace,omitempty"`
	// PodNamespace is the namespace in the cluster that prow
	// components will use for looking up Pods owned by ProwJobs.
	// The namespace needs to exist and will not be created by prow.
	PodNamespace string       `json:"pod_namespace,omitempty"`
	SlackEvents  []SlackEvent `json:"slackevents,omitempty"`
}

// Tide is config for the tide pool.
type Tide struct {
	// These must be valid GitHub search queries. They should not overlap,
	// which is to say two queries should never return the same PR.
	Queries []string `json:"queries,omitempty"`
}

// Plank is config for the plank controller.
type Plank struct {
	// JobURLTemplateString compiles into JobURLTemplate at load time.
	JobURLTemplateString string `json:"job_url_template,omitempty"`
	// JobURLTemplate is compiled at load time from JobURLTemplateString. It
	// will be passed a kube.ProwJob and is used to set the URL for the
	// "details" link on GitHub as well as the link from deck.
	JobURLTemplate *template.Template `json:"-"`

	// ReportTemplateString compiles into ReportTemplate at load time.
	ReportTemplateString string `json:"report_template,omitempty"`
	// ReportTemplate is compiled at load time from ReportTemplateString. It
	// will be passed a kube.ProwJob and can provide an optional blurb below
	// the test failures comment.
	ReportTemplate *template.Template `json:"-"`
}

// Sinker is config for the sinker controller.
type Sinker struct {
	// ResyncPeriodString compiles into ResyncPeriod at load time.
	ResyncPeriodString string `json:"resync_period,omitempty"`
	// ResyncPeriod is how often the controller will perform a garbage
	// collection.
	ResyncPeriod time.Duration `json:"-"`
	// MaxProwJobAgeString compiles into MaxProwJobAge at load time.
	MaxProwJobAgeString string `json:"max_prowjob_age,omitempty"`
	// MaxProwJobAge is how old a ProwJob can be before it is garbage-collected.
	MaxProwJobAge time.Duration `json:"-"`
	// MaxPodAgeString compiles into MaxPodAge at load time.
	MaxPodAgeString string `json:"max_pod_age,omitempty"`
	// MaxPodAge is how old a Pod can be before it is garbage-collected.
	MaxPodAge time.Duration `json:"-"`
}

// Trigger is config for the trigger plugin.
type Trigger struct {
	// Repos is either of the form org/repos or just org.
	Repos []string `json:"repos,omitempty"`
	// TrustedOrg is the org whose members' PRs will be automatically built
	// for PRs to the above repos.
	TrustedOrg string `json:"trusted_org,omitempty"`
}

// Heart is config for the heart plugin
type Heart struct {
	// Adorees is a list of GitHub logins for members
	// for whom we will add emojis to comments
	Adorees []string `json:"adorees,omitempty"`
}

// SlackEvent is config for the slackevents plugin.
// If a PR is pushed to any of the repos listed in the config
// then sent message to the all the  slack channels listed if pusher is NOT in the whitelist.
type SlackEvent struct {
	// Repos is either of the form org/repos or just org.
	Repos []string `json:"repos,omitempty"`
	// List of channels on which a event is published.
	Channels []string `json:"channels,omitempty"`
	// A slack event is published if the user is not part of the WhiteList.
	WhiteList []string `json:"whitelist,omitempty"`
}

// Load loads and parses the config at path.
func Load(path string) (*Config, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("error reading %s: %v", path, err)
	}
	nc := &Config{}
	if err := yaml.Unmarshal(b, nc); err != nil {
		return nil, fmt.Errorf("error unmarshaling %s: %v", path, err)
	}
	if err := parseConfig(nc); err != nil {
		return nil, err
	}
	return nc, nil
}

func parseConfig(c *Config) error {
	// Ensure that presubmit regexes are valid.
	for _, vs := range c.Presubmits {
		if err := setRegexes(vs); err != nil {
			return fmt.Errorf("could not set regex: %v", err)
		}
		for v := range vs {
			name := vs[v].Name
			agent := vs[v].Agent
			if agent == string(kube.KubernetesAgent) && vs[v].Spec == nil {
				return fmt.Errorf("job %s has no spec", name)
			}
			if agent != string(kube.KubernetesAgent) && agent != string(kube.JenkinsAgent) {
				return fmt.Errorf("job %s has invalid agent (%s), it needs to be one of the following: %s %s",
					name, agent, kube.KubernetesAgent, kube.JenkinsAgent)
			}
		}
	}

	// Ensure that postsubmits have a pod spec.
	for _, js := range c.Postsubmits {
		for j := range js {
			name := js[j].Name
			agent := js[j].Agent
			if agent == string(kube.KubernetesAgent) && js[j].Spec == nil {
				return fmt.Errorf("job %s has no spec", name)
			}
			if agent != string(kube.KubernetesAgent) && agent != string(kube.JenkinsAgent) {
				return fmt.Errorf("job %s has invalid agent (%s), it needs to be one of the following: %s %s",
					name, agent, kube.KubernetesAgent, kube.JenkinsAgent)
			}
		}
	}

	// Ensure that the periodic durations are valid and specs exist.
	for j := range c.Periodics {
		name := c.Periodics[j].Name
		agent := c.Periodics[j].Agent
		if agent == string(kube.KubernetesAgent) && c.Periodics[j].Spec == nil {
			return fmt.Errorf("job %s has no spec", name)
		}
		if agent != string(kube.KubernetesAgent) && agent != string(kube.JenkinsAgent) {
			return fmt.Errorf("job %s has invalid agent (%s), it needs to be one of the following: %s %s",
				name, agent, kube.KubernetesAgent, kube.JenkinsAgent)
		}
		d, err := time.ParseDuration(c.Periodics[j].Interval)
		if err != nil {
			return fmt.Errorf("cannot parse duration for %s: %v", c.Periodics[j].Name, err)
		}
		c.Periodics[j].interval = d
	}

	urlTmpl, err := template.New("JobURL").Parse(c.Plank.JobURLTemplateString)
	if err != nil {
		return fmt.Errorf("parsing template: %v", err)
	}
	c.Plank.JobURLTemplate = urlTmpl

	reportTmpl, err := template.New("Report").Parse(c.Plank.ReportTemplateString)
	if err != nil {
		return fmt.Errorf("parsing template: %v", err)
	}
	c.Plank.ReportTemplate = reportTmpl

	resyncPeriod, err := time.ParseDuration(c.Sinker.ResyncPeriodString)
	if err != nil {
		return fmt.Errorf("cannot parse duration for resync_period: %v", err)
	}
	c.Sinker.ResyncPeriod = resyncPeriod
	maxProwJobAge, err := time.ParseDuration(c.Sinker.MaxProwJobAgeString)
	if err != nil {
		return fmt.Errorf("cannot parse duration for max_prowjob_age: %v", err)
	}
	c.Sinker.MaxProwJobAge = maxProwJobAge
	maxPodAge, err := time.ParseDuration(c.Sinker.MaxPodAgeString)
	if err != nil {
		return fmt.Errorf("cannot parse duration for max_pod_age: %v", err)
	}
	c.Sinker.MaxPodAge = maxPodAge
	return nil
}

func setRegexes(js []Presubmit) error {
	for i, j := range js {
		if re, err := regexp.Compile(j.Trigger); err == nil {
			js[i].re = re
		} else {
			return fmt.Errorf("could not compile trigger regex for %s: %v", j.Name, err)
		}
		if err := setRegexes(j.RunAfterSuccess); err != nil {
			return err
		}
		if j.RunIfChanged != "" {
			re, err := regexp.Compile(j.RunIfChanged)
			if err != nil {
				return fmt.Errorf("could not compile changes regex for %s: %v", j.Name, err)
			}
			js[i].reChanges = re
		}
	}
	return nil
}
<commit_msg>Fix the test for nil pod specs in child jobs.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package config knows how to read and parse config.yaml.
package config

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"text/template"
	"time"

	"github.com/ghodss/yaml"

	"k8s.io/test-infra/prow/kube"
)

// Config is a read-only snapshot of the config.
type Config struct {
	// Full repo name (such as "kubernetes/kubernetes") -> list of jobs.
	Presubmits  map[string][]Presubmit  `json:"presubmits,omitempty"`
	Postsubmits map[string][]Postsubmit `json:"postsubmits,omitempty"`

	// Periodics are not associated with any repo.
	Periodics []Periodic `json:"periodics,omitempty"`

	Tide     Tide      `json:"tide,omitempty"`
	Plank    Plank     `json:"plank,omitempty"`
	Sinker   Sinker    `json:"sinker,omitempty"`
	Triggers []Trigger `json:"triggers,omitempty"`
	Heart    Heart     `json:"heart,omitempty"`

	// ProwJobNamespace is the namespace in the cluster that prow
	// components will use for looking up ProwJobs. The namespace
	// needs to exist and will not be created by prow.
	ProwJobNamespace string `json:"prowjob_namespace,omitempty"`
	// PodNamespace is the namespace in the cluster that prow
	// components will use for looking up Pods owned by ProwJobs.
	// The namespace needs to exist and will not be created by prow.
	PodNamespace string       `json:"pod_namespace,omitempty"`
	SlackEvents  []SlackEvent `json:"slackevents,omitempty"`
}

// Tide is config for the tide pool.
type Tide struct {
	// These must be valid GitHub search queries. They should not overlap,
	// which is to say two queries should never return the same PR.
	Queries []string `json:"queries,omitempty"`
}

// Plank is config for the plank controller.
type Plank struct {
	// JobURLTemplateString compiles into JobURLTemplate at load time.
	JobURLTemplateString string `json:"job_url_template,omitempty"`
	// JobURLTemplate is compiled at load time from JobURLTemplateString. It
	// will be passed a kube.ProwJob and is used to set the URL for the
	// "details" link on GitHub as well as the link from deck.
	JobURLTemplate *template.Template `json:"-"`

	// ReportTemplateString compiles into ReportTemplate at load time.
	ReportTemplateString string `json:"report_template,omitempty"`
	// ReportTemplate is compiled at load time from ReportTemplateString. It
	// will be passed a kube.ProwJob and can provide an optional blurb below
	// the test failures comment.
	ReportTemplate *template.Template `json:"-"`
}

// Sinker is config for the sinker controller.
type Sinker struct {
	// ResyncPeriodString compiles into ResyncPeriod at load time.
	ResyncPeriodString string `json:"resync_period,omitempty"`
	// ResyncPeriod is how often the controller will perform a garbage
	// collection.
	ResyncPeriod time.Duration `json:"-"`
	// MaxProwJobAgeString compiles into MaxProwJobAge at load time.
	MaxProwJobAgeString string `json:"max_prowjob_age,omitempty"`
	// MaxProwJobAge is how old a ProwJob can be before it is garbage-collected.
	MaxProwJobAge time.Duration `json:"-"`
	// MaxPodAgeString compiles into MaxPodAge at load time.
	MaxPodAgeString string `json:"max_pod_age,omitempty"`
	// MaxPodAge is how old a Pod can be before it is garbage-collected.
	MaxPodAge time.Duration `json:"-"`
}

// Trigger is config for the trigger plugin.
type Trigger struct {
	// Repos is either of the form org/repos or just org.
	Repos []string `json:"repos,omitempty"`
	// TrustedOrg is the org whose members' PRs will be automatically built
	// for PRs to the above repos.
	TrustedOrg string `json:"trusted_org,omitempty"`
}

// Heart is config for the heart plugin
type Heart struct {
	// Adorees is a list of GitHub logins for members
	// for whom we will add emojis to comments
	Adorees []string `json:"adorees,omitempty"`
}

// SlackEvent is config for the slackevents plugin.
// If a PR is pushed to any of the repos listed in the config
// then sent message to the all the  slack channels listed if pusher is NOT in the whitelist.
type SlackEvent struct {
	// Repos is either of the form org/repos or just org.
	Repos []string `json:"repos,omitempty"`
	// List of channels on which a event is published.
	Channels []string `json:"channels,omitempty"`
	// A slack event is published if the user is not part of the WhiteList.
	WhiteList []string `json:"whitelist,omitempty"`
}

// Load loads and parses the config at path.
func Load(path string) (*Config, error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("error reading %s: %v", path, err)
	}
	nc := &Config{}
	if err := yaml.Unmarshal(b, nc); err != nil {
		return nil, fmt.Errorf("error unmarshaling %s: %v", path, err)
	}
	if err := parseConfig(nc); err != nil {
		return nil, err
	}
	return nc, nil
}

func parseConfig(c *Config) error {
	// Ensure that presubmit regexes are valid.
	for _, vs := range c.Presubmits {
		if err := setRegexes(vs); err != nil {
			return fmt.Errorf("could not set regex: %v", err)
		}
	}

	// Ensure that presubmits have a pod spec.
	for _, v := range c.AllPresubmits(nil) {
		name := v.Name
		agent := v.Agent
		if agent == string(kube.KubernetesAgent) && v.Spec == nil {
			return fmt.Errorf("job %s has no spec", name)
		}
		if agent != string(kube.KubernetesAgent) && agent != string(kube.JenkinsAgent) {
			return fmt.Errorf("job %s has invalid agent (%s), it needs to be one of the following: %s %s",
				name, agent, kube.KubernetesAgent, kube.JenkinsAgent)
		}
	}

	// Ensure that postsubmits have a pod spec.
	for _, j := range c.AllPostsubmits(nil) {
		name := j.Name
		agent := j.Agent
		if agent == string(kube.KubernetesAgent) && j.Spec == nil {
			return fmt.Errorf("job %s has no spec", name)
		}
		if agent != string(kube.KubernetesAgent) && agent != string(kube.JenkinsAgent) {
			return fmt.Errorf("job %s has invalid agent (%s), it needs to be one of the following: %s %s",
				name, agent, kube.KubernetesAgent, kube.JenkinsAgent)
		}
	}

	// Ensure that the periodic durations are valid and specs exist.
	for _, p := range c.AllPeriodics() {
		name := p.Name
		agent := p.Agent
		if agent == string(kube.KubernetesAgent) && p.Spec == nil {
			return fmt.Errorf("job %s has no spec", name)
		}
		if agent != string(kube.KubernetesAgent) && agent != string(kube.JenkinsAgent) {
			return fmt.Errorf("job %s has invalid agent (%s), it needs to be one of the following: %s %s",
				name, agent, kube.KubernetesAgent, kube.JenkinsAgent)
		}
	}
	// Set the interval on the periodic jobs. It doesn't make sense to do this
	// for child jobs.
	for j := range c.Periodics {
		d, err := time.ParseDuration(c.Periodics[j].Interval)
		if err != nil {
			return fmt.Errorf("cannot parse duration for %s: %v", c.Periodics[j].Name, err)
		}
		c.Periodics[j].interval = d
	}

	urlTmpl, err := template.New("JobURL").Parse(c.Plank.JobURLTemplateString)
	if err != nil {
		return fmt.Errorf("parsing template: %v", err)
	}
	c.Plank.JobURLTemplate = urlTmpl

	reportTmpl, err := template.New("Report").Parse(c.Plank.ReportTemplateString)
	if err != nil {
		return fmt.Errorf("parsing template: %v", err)
	}
	c.Plank.ReportTemplate = reportTmpl

	resyncPeriod, err := time.ParseDuration(c.Sinker.ResyncPeriodString)
	if err != nil {
		return fmt.Errorf("cannot parse duration for resync_period: %v", err)
	}
	c.Sinker.ResyncPeriod = resyncPeriod
	maxProwJobAge, err := time.ParseDuration(c.Sinker.MaxProwJobAgeString)
	if err != nil {
		return fmt.Errorf("cannot parse duration for max_prowjob_age: %v", err)
	}
	c.Sinker.MaxProwJobAge = maxProwJobAge
	maxPodAge, err := time.ParseDuration(c.Sinker.MaxPodAgeString)
	if err != nil {
		return fmt.Errorf("cannot parse duration for max_pod_age: %v", err)
	}
	c.Sinker.MaxPodAge = maxPodAge
	return nil
}

func setRegexes(js []Presubmit) error {
	for i, j := range js {
		if re, err := regexp.Compile(j.Trigger); err == nil {
			js[i].re = re
		} else {
			return fmt.Errorf("could not compile trigger regex for %s: %v", j.Name, err)
		}
		if err := setRegexes(j.RunAfterSuccess); err != nil {
			return err
		}
		if j.RunIfChanged != "" {
			re, err := regexp.Compile(j.RunIfChanged)
			if err != nil {
				return fmt.Errorf("could not compile changes regex for %s: %v", j.Name, err)
			}
			js[i].reChanges = re
		}
	}
	return nil
}
<|endoftext|>