<commit_before>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"unicode"

	"github.com/coreos/go-semver/semver"

	"gitlab-odx.oracle.com/odx/functions/fn/langs"
)

const (
	functionsDockerImage     = "funcy/functions"
	minRequiredDockerVersion = "17.5.0"
)

func verbwriter(verbose bool) io.Writer {
	// this is too limiting, removes all logs which isn't what we want
	// verbwriter := ioutil.Discard
	// if verbose {
	verbwriter := os.Stderr
	// }
	return verbwriter
}

func buildfunc(verbwriter io.Writer, fn string) (*funcfile, error) {
	funcfile, err := parsefuncfile(fn)
	if err != nil {
		return nil, err
	}

	if funcfile.Version == "" {
		funcfile, err = bumpversion(*funcfile)
		if err != nil {
			return nil, err
		}
		if err := storefuncfile(fn, funcfile); err != nil {
			return nil, err
		}
		funcfile, err = parsefuncfile(fn)
		if err != nil {
			return nil, err
		}
	}

	if err := localbuild(verbwriter, fn, funcfile.Build); err != nil {
		return nil, err
	}

	if err := dockerbuild(verbwriter, fn, funcfile); err != nil {
		return nil, err
	}

	return funcfile, nil
}

func localbuild(verbwriter io.Writer, path string, steps []string) error {
	for _, cmd := range steps {
		exe := exec.Command("/bin/sh", "-c", cmd)
		exe.Dir = filepath.Dir(path)
		exe.Stderr = verbwriter
		exe.Stdout = verbwriter
		if err := exe.Run(); err != nil {
			return fmt.Errorf("error running command %v (%v)", cmd, err)
		}
	}

	return nil
}

func dockerbuild(verbwriter io.Writer, path string, ff *funcfile) error {
	err := dockerVersionCheck()
	if err != nil {
		return err
	}

	dir := filepath.Dir(path)

	var helper langs.LangHelper
	dockerfile := filepath.Join(dir, "Dockerfile")
	if !exists(dockerfile) {
		helper = langs.GetLangHelper(*ff.Runtime)
		if helper == nil {
			return fmt.Errorf("Cannot build, no language helper found for %v", *ff.Runtime)
		}
		err := writeTmpDockerfile(helper, dir, ff)
		defer os.Remove(filepath.Join(dir, "Dockerfile"))
		if err != nil {
			return err
		}
		if helper.HasPreBuild() {
			err := helper.PreBuild()
			if err != nil {
				return err
			}
		}
	}

	fmt.Printf("Building image %v\n", ff.FullName())
	cmd := exec.Command("docker", "build",
		"-t", ff.FullName(),
		// "--no-cache",
		"--build-arg", "HTTP_PROXY",
		"--build-arg", "HTTPS_PROXY",
		".")
	cmd.Dir = dir
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("error running docker build: %v", err)
	}
	if helper != nil {
		err := helper.AfterBuild()
		if err != nil {
			return err
		}
	}
	return nil
}

func dockerVersionCheck() error {
	out, err := exec.Command("docker", "version", "--format", "{{.Server.Version}}").Output()
	if err != nil {
		return fmt.Errorf("could not check Docker version: %v", err)
	}
	// dev / test builds append '-ce', trim this
	trimmed := strings.TrimRightFunc(string(out), func(r rune) bool { return r != '.' && !unicode.IsDigit(r) })

	v, err := semver.NewVersion(trimmed)
	if err != nil {
		return fmt.Errorf("could not check Docker version: %v", err)
	}
	vMin, err := semver.NewVersion(minRequiredDockerVersion)
	if err != nil {
		return fmt.Errorf("our bad, sorry... please make an issue.", err)
	}
	if v.LessThan(*vMin) {
		return fmt.Errorf("please upgrade your version of Docker to %s or greater", minRequiredDockerVersion)
	}
	return nil
}

func exists(name string) bool {
	if _, err := os.Stat(name); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}

func writeTmpDockerfile(helper langs.LangHelper, dir string, ff *funcfile) error {
	if ff.Entrypoint == "" && ff.Cmd == "" {
		return errors.New("entrypoint and cmd are missing, you must provide one or the other")
	}

	fd, err := os.Create(filepath.Join(dir, "Dockerfile"))
	if err != nil {
		return err
	}
	defer fd.Close()

	// multi-stage build: https://medium.com/travis-on-docker/multi-stage-docker-builds-for-creating-tiny-go-images-e0e1867efe5a
	dfLines := []string{}
	if helper.IsMultiStage() {
		// build stage
		dfLines = append(dfLines, fmt.Sprintf("FROM %s as build-stage", helper.BuildFromImage()))
	} else {
		dfLines = append(dfLines, fmt.Sprintf("FROM %s", helper.BuildFromImage()))
	}
	dfLines = append(dfLines, "WORKDIR /function")
	dfLines = append(dfLines, helper.DockerfileBuildCmds()...)
	if helper.IsMultiStage() {
		// final stage
		dfLines = append(dfLines, fmt.Sprintf("FROM %s", helper.RunFromImage()))
		dfLines = append(dfLines, "WORKDIR /function")
		dfLines = append(dfLines, helper.DockerfileCopyCmds()...)
	}
	if ff.Entrypoint != "" {
		dfLines = append(dfLines, fmt.Sprintf("ENTRYPOINT [%s]", stringToSlice(ff.Entrypoint)))
	}
	if ff.Cmd != "" {
		dfLines = append(dfLines, fmt.Sprintf("CMD [%s]", stringToSlice(ff.Cmd)))
	}
	err = writeLines(fd, dfLines)
	if err != nil {
		return err
	}
	return err
}

func writeLines(w io.Writer, lines []string) error {
	writer := bufio.NewWriter(w)
	for _, l := range lines {
		_, err := writer.WriteString(l + "\n")
		if err != nil {
			return err
		}
	}
	writer.Flush()
	return nil
}

func stringToSlice(in string) string {
	epvals := strings.Fields(in)
	var buffer bytes.Buffer
	for i, s := range epvals {
		if i > 0 {
			buffer.WriteString(", ")
		}
		buffer.WriteString("\"")
		buffer.WriteString(s)
		buffer.WriteString("\"")
	}
	return buffer.String()
}

func extractEnvConfig(configs []string) map[string]string {
	c := make(map[string]string)
	for _, v := range configs {
		kv := strings.SplitN(v, "=", 2)
		if len(kv) == 2 {
			c[kv[0]] = os.ExpandEnv(kv[1])
		}
	}
	return c
}

func dockerpush(ff *funcfile) error {
	fmt.Println("Pushing to docker registry...")
	cmd := exec.Command("docker", "push", ff.FullName())
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("error running docker push: %v", err)
	}
	return nil
}

func appNamePath(img string) (string, string) {
	sep := strings.Index(img, "/")
	if sep < 0 {
		return "", ""
	}
	tag := strings.Index(img[sep:], ":")
	if tag < 0 {
		tag = len(img[sep:])
	}
	return img[:sep], img[sep : sep+tag]
}
<commit_msg>Clean up dockerfile orphans<commit_after>package main

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"unicode"

	"github.com/coreos/go-semver/semver"

	"gitlab-odx.oracle.com/odx/functions/fn/langs"
)

const (
	functionsDockerImage     = "funcy/functions"
	minRequiredDockerVersion = "17.5.0"
)

func verbwriter(verbose bool) io.Writer {
	// this is too limiting, removes all logs which isn't what we want
	// verbwriter := ioutil.Discard
	// if verbose {
	verbwriter := os.Stderr
	// }
	return verbwriter
}

func buildfunc(verbwriter io.Writer, fn string) (*funcfile, error) {
	funcfile, err := parsefuncfile(fn)
	if err != nil {
		return nil, err
	}

	if funcfile.Version == "" {
		funcfile, err = bumpversion(*funcfile)
		if err != nil {
			return nil, err
		}
		if err := storefuncfile(fn, funcfile); err != nil {
			return nil, err
		}
		funcfile, err = parsefuncfile(fn)
		if err != nil {
			return nil, err
		}
	}

	if err := localbuild(verbwriter, fn, funcfile.Build); err != nil {
		return nil, err
	}

	if err := dockerbuild(verbwriter, fn, funcfile); err != nil {
		return nil, err
	}

	return funcfile, nil
}

func localbuild(verbwriter io.Writer, path string, steps []string) error {
	for _, cmd := range steps {
		exe := exec.Command("/bin/sh", "-c", cmd)
		exe.Dir = filepath.Dir(path)
		exe.Stderr = verbwriter
		exe.Stdout = verbwriter
		if err := exe.Run(); err != nil {
			return fmt.Errorf("error running command %v (%v)", cmd, err)
		}
	}

	return nil
}

func dockerbuild(verbwriter io.Writer, path string, ff *funcfile) error {
	err := dockerVersionCheck()
	if err != nil {
		return err
	}

	dir := filepath.Dir(path)

	var helper langs.LangHelper
	dockerfile := filepath.Join(dir, "Dockerfile")
	if !exists(dockerfile) {
		helper = langs.GetLangHelper(*ff.Runtime)
		if helper == nil {
			return fmt.Errorf("Cannot build, no language helper found for %v", *ff.Runtime)
		}
		dockerfile, err = writeTmpDockerfile(helper, dir, ff)
		if err != nil {
			return err
		}
		defer os.Remove(dockerfile)
		if helper.HasPreBuild() {
			err := helper.PreBuild()
			if err != nil {
				return err
			}
		}
	}

	fmt.Printf("Building image %v\n", ff.FullName())

	cancel := make(chan os.Signal, 3)
	signal.Notify(cancel, os.Interrupt) // and others perhaps
	defer signal.Stop(cancel)

	result := make(chan error, 1)

	go func(done chan<- error) {
		cmd := exec.Command("docker", "build",
			"-t", ff.FullName(),
			"-f", dockerfile,
			// "--no-cache",
			"--build-arg", "HTTP_PROXY",
			"--build-arg", "HTTPS_PROXY",
			".")
		cmd.Dir = dir
		cmd.Stderr = os.Stderr
		cmd.Stdout = os.Stdout
		done <- cmd.Run()
	}(result)

	select {
	case err := <-result:
		if err != nil {
			return fmt.Errorf("error running docker build: %v", err)
		}
	case signal := <-cancel:
		return fmt.Errorf("build cancelled on signal %v", signal)
	}

	if helper != nil {
		err := helper.AfterBuild()
		if err != nil {
			return err
		}
	}
	return nil
}

func dockerVersionCheck() error {
	out, err := exec.Command("docker", "version", "--format", "{{.Server.Version}}").Output()
	if err != nil {
		return fmt.Errorf("could not check Docker version: %v", err)
	}
	// dev / test builds append '-ce', trim this
	trimmed := strings.TrimRightFunc(string(out), func(r rune) bool { return r != '.' && !unicode.IsDigit(r) })

	v, err := semver.NewVersion(trimmed)
	if err != nil {
		return fmt.Errorf("could not check Docker version: %v", err)
	}
	vMin, err := semver.NewVersion(minRequiredDockerVersion)
	if err != nil {
		return fmt.Errorf("our bad, sorry... please make an issue.", err)
	}
	if v.LessThan(*vMin) {
		return fmt.Errorf("please upgrade your version of Docker to %s or greater", minRequiredDockerVersion)
	}
	return nil
}

func exists(name string) bool {
	if _, err := os.Stat(name); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}

func writeTmpDockerfile(helper langs.LangHelper, dir string, ff *funcfile) (string, error) {
	if ff.Entrypoint == "" && ff.Cmd == "" {
		return "", errors.New("entrypoint and cmd are missing, you must provide one or the other")
	}

	fd, err := ioutil.TempFile(dir, "Dockerfile")
	if err != nil {
		return "", err
	}
	defer fd.Close()

	// multi-stage build: https://medium.com/travis-on-docker/multi-stage-docker-builds-for-creating-tiny-go-images-e0e1867efe5a
	dfLines := []string{}
	if helper.IsMultiStage() {
		// build stage
		dfLines = append(dfLines, fmt.Sprintf("FROM %s as build-stage", helper.BuildFromImage()))
	} else {
		dfLines = append(dfLines, fmt.Sprintf("FROM %s", helper.BuildFromImage()))
	}
	dfLines = append(dfLines, "WORKDIR /function")
	dfLines = append(dfLines, helper.DockerfileBuildCmds()...)
	if helper.IsMultiStage() {
		// final stage
		dfLines = append(dfLines, fmt.Sprintf("FROM %s", helper.RunFromImage()))
		dfLines = append(dfLines, "WORKDIR /function")
		dfLines = append(dfLines, helper.DockerfileCopyCmds()...)
	}
	if ff.Entrypoint != "" {
		dfLines = append(dfLines, fmt.Sprintf("ENTRYPOINT [%s]", stringToSlice(ff.Entrypoint)))
	}
	if ff.Cmd != "" {
		dfLines = append(dfLines, fmt.Sprintf("CMD [%s]", stringToSlice(ff.Cmd)))
	}
	err = writeLines(fd, dfLines)
	if err != nil {
		return "", err
	}
	return fd.Name(), err
}

func writeLines(w io.Writer, lines []string) error {
	writer := bufio.NewWriter(w)
	for _, l := range lines {
		_, err := writer.WriteString(l + "\n")
		if err != nil {
			return err
		}
	}
	writer.Flush()
	return nil
}

func stringToSlice(in string) string {
	epvals := strings.Fields(in)
	var buffer bytes.Buffer
	for i, s := range epvals {
		if i > 0 {
			buffer.WriteString(", ")
		}
		buffer.WriteString("\"")
		buffer.WriteString(s)
		buffer.WriteString("\"")
	}
	return buffer.String()
}

func extractEnvConfig(configs []string) map[string]string {
	c := make(map[string]string)
	for _, v := range configs {
		kv := strings.SplitN(v, "=", 2)
		if len(kv) == 2 {
			c[kv[0]] = os.ExpandEnv(kv[1])
		}
	}
	return c
}

func dockerpush(ff *funcfile) error {
	fmt.Println("Pushing to docker registry...")
	cmd := exec.Command("docker", "push", ff.FullName())
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("error running docker push: %v", err)
	}
	return nil
}

func appNamePath(img string) (string, string) {
	sep := strings.Index(img, "/")
	if sep < 0 {
		return "", ""
	}
	tag := strings.Index(img[sep:], ":")
	if tag < 0 {
		tag = len(img[sep:])
	}
	return img[:sep], img[sep : sep+tag]
}
<|endoftext|>


<commit_before><commit_msg>added for-while.go<commit_after><|endoftext|>


<commit_before>package deduplicate

import (
	"encoding/json"
	"github.com/henrylee2cn/pholcus/common/mgo"
	"github.com/henrylee2cn/pholcus/common/mysql"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/config"
	"github.com/henrylee2cn/pholcus/logs"
	"gopkg.in/mgo.v2/bson"
	"io/ioutil"
	"os"
	"path"
	"sync"
)

type Deduplicate interface {
	// 采集非重复样本并返回对比结果，重复为true
	Compare(obj interface{}) bool
	// 提交去重记录至指定输出方式(数据库或文件)
	Submit(provider ...string)
	// 从指定输出方式(数据库或文件)读取更新去重记录
	Update(provider ...string)
	// 取消指定去重样本
	Remove(obj interface{})
	// 清空样本缓存
	CleanCache()
}

type Deduplication struct {
	sampling struct {
		new map[string]bool
		old map[string]bool
	}
	provider string
	sync.Mutex
}

func New() Deduplicate {
	return &Deduplication{
		sampling: struct {
			new map[string]bool
			old map[string]bool
		}{
			new: make(map[string]bool),
			old: make(map[string]bool),
		},
	}
}

// 对比是否已存在，不存在则采样
func (self *Deduplication) Compare(obj interface{}) (duplicate bool) {
	self.Mutex.Lock()
	defer self.Unlock()

	s := util.MakeUnique(obj)
	if self.sampling.old[s] {
		return true
	}
	if self.sampling.new[s] {
		return true
	}
	self.sampling.new[s] = true
	return false
}

// 取消指定去重样本
func (self *Deduplication) Remove(obj interface{}) {
	self.Mutex.Lock()
	defer self.Unlock()

	s := util.MakeUnique(obj)
	if self.sampling.new[s] {
		delete(self.sampling.new, s)
	}
}

func (self *Deduplication) Submit(provider ...string) {
	if len(provider) > 0 && provider[0] != "" {
		self.provider = provider[0]
	}
	if len(self.sampling.new) == 0 {
		return
	}

	self.Mutex.Lock()
	defer self.Unlock()

	switch self.provider {
	case "mgo":
		var docs = make([]map[string]interface{}, len(self.sampling.new))
		var i int
		for key := range self.sampling.new {
			docs[i] = map[string]interface{}{"_id": key}
			self.sampling.old[key] = true
			i++
		}
		mgo.Mgo(nil, "insert", map[string]interface{}{
			"Database":   config.MGO.DB,
			"Collection": config.DEDUPLICATION.FILE_NAME,
			"Docs":       docs,
		})

	case "mysql":
		db, ok := mysql.MysqlPool.GetOne().(*mysql.MysqlSrc)
		if !ok || db == nil {
			logs.Log.Error("链接Mysql数据库超时，无法保存去重记录！")
			return
		}
		defer mysql.MysqlPool.Free(db)
		table := mysql.New(db.DB).
			SetTableName(config.DEDUPLICATION.FILE_NAME).
			CustomPrimaryKey(`id VARCHAR(255) not null primary key`).
			Create()
		for key := range self.sampling.new {
			table.AddRow(key).Update()
			self.sampling.old[key] = true
		}

	default:
		p, _ := path.Split(config.COMM_PATH.CACHE + "/" + config.DEDUPLICATION.FILE_NAME)
		// 创建/打开目录
		d, err := os.Stat(p)
		if err != nil || !d.IsDir() {
			if err := os.MkdirAll(p, 0777); err != nil {
				logs.Log.Error("Error: %v\n", err)
			}
		}

		f, _ := os.OpenFile(config.COMM_PATH.CACHE+"/"+config.DEDUPLICATION.FILE_NAME, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0660)

		b, _ := json.Marshal(self.sampling.new)
		b[0] = ','
		f.Write(b[:len(b)-1])
		f.Close()

		for key := range self.sampling.new {
			self.sampling.old[key] = true
		}
	}
	logs.Log.Informational(" *     新增 %v 条去重样本\n", len(self.sampling.new))
	self.sampling.new = make(map[string]bool)
}

func (self *Deduplication) Update(provider ...string) {
	if len(provider) == 0 || provider[0] == self.provider {
		return
	}
	self.CleanCache()
	self.provider = provider[0]

	switch self.provider {
	case "mgo":
		var docs = map[string]interface{}{}
		err := mgo.Mgo(&docs, "find", map[string]interface{}{
			"Database":   config.MGO.DB,
			"Collection": config.DEDUPLICATION.FILE_NAME,
		})
		if err != nil {
			logs.Log.Error("去重读取mgo: %v", err)
			return
		}
		for _, v := range docs["Docs"].([]interface{}) {
			self.sampling.old[v.(bson.M)["_id"].(string)] = true
		}

	case "mysql":
		db, ok := mysql.MysqlPool.GetOne().(*mysql.MysqlSrc)
		if !ok || db == nil {
			logs.Log.Error("链接Mysql数据库超时，无法读取去重记录！")
			return
		}
		defer mysql.MysqlPool.Free(db)
		rows, err := mysql.New(db.DB).
			SetTableName("`" + config.DEDUPLICATION.FILE_NAME + "`").
			SelectAll()
		if err != nil {
			// logs.Log.Error("读取Mysql数据库中去重记录失败：%v", err)
			return
		}

		for rows.Next() {
			var id string
			err = rows.Scan(&id)
			self.sampling.old[id] = true
		}

	default:
		f, err := os.Open(config.COMM_PATH.CACHE + "/" + config.DEDUPLICATION.FILE_NAME)
		if err != nil {
			return
		}
		defer f.Close()
		b, _ := ioutil.ReadAll(f)
		b[0] = '{'
		json.Unmarshal(
			append(b, '}'),
			&self.sampling.old,
		)
	}
	logs.Log.Informational(" *     读出 %v 条去重样本\n", len(self.sampling.old))
}

func (self *Deduplication) CleanCache() {
	self.Mutex.Lock()
	defer self.Unlock()
	self.sampling.old = make(map[string]bool)
	self.sampling.new = make(map[string]bool)
}
<commit_msg>每次开始任务，若继承历史去重记录，则均重新读取样本<commit_after>package deduplicate

import (
	"encoding/json"
	"github.com/henrylee2cn/pholcus/common/mgo"
	"github.com/henrylee2cn/pholcus/common/mysql"
	"github.com/henrylee2cn/pholcus/common/util"
	"github.com/henrylee2cn/pholcus/config"
	"github.com/henrylee2cn/pholcus/logs"
	"gopkg.in/mgo.v2/bson"
	"io/ioutil"
	"os"
	"path"
	"sync"
)

type Deduplicate interface {
	// 采集非重复样本并返回对比结果，重复为true
	Compare(obj interface{}) bool
	// 提交去重记录至指定输出方式(数据库或文件)
	Submit(provider ...string)
	// 从指定输出方式(数据库或文件)读取更新去重记录
	Update(provider ...string)
	// 取消指定去重样本
	Remove(obj interface{})
	// 清空样本缓存
	CleanCache()
}

type Deduplication struct {
	sampling struct {
		new map[string]bool
		old map[string]bool
	}
	provider string
	sync.Mutex
}

func New() Deduplicate {
	return &Deduplication{
		sampling: struct {
			new map[string]bool
			old map[string]bool
		}{
			new: make(map[string]bool),
			old: make(map[string]bool),
		},
	}
}

// 对比是否已存在，不存在则采样
func (self *Deduplication) Compare(obj interface{}) (duplicate bool) {
	self.Mutex.Lock()
	defer self.Unlock()

	s := util.MakeUnique(obj)
	if self.sampling.old[s] {
		return true
	}
	if self.sampling.new[s] {
		return true
	}
	self.sampling.new[s] = true
	return false
}

// 取消指定去重样本
func (self *Deduplication) Remove(obj interface{}) {
	self.Mutex.Lock()
	defer self.Unlock()

	s := util.MakeUnique(obj)
	if self.sampling.new[s] {
		delete(self.sampling.new, s)
	}
}

func (self *Deduplication) Submit(provider ...string) {
	if len(provider) > 0 && provider[0] != "" {
		self.provider = provider[0]
	}
	if len(self.sampling.new) == 0 {
		return
	}

	self.Mutex.Lock()
	defer self.Unlock()

	switch self.provider {
	case "mgo":
		var docs = make([]map[string]interface{}, len(self.sampling.new))
		var i int
		for key := range self.sampling.new {
			docs[i] = map[string]interface{}{"_id": key}
			self.sampling.old[key] = true
			i++
		}
		mgo.Mgo(nil, "insert", map[string]interface{}{
			"Database":   config.MGO.DB,
			"Collection": config.DEDUPLICATION.FILE_NAME,
			"Docs":       docs,
		})

	case "mysql":
		db, ok := mysql.MysqlPool.GetOne().(*mysql.MysqlSrc)
		if !ok || db == nil {
			logs.Log.Error("链接Mysql数据库超时，无法保存去重记录！")
			return
		}
		defer mysql.MysqlPool.Free(db)
		table := mysql.New(db.DB).
			SetTableName(config.DEDUPLICATION.FILE_NAME).
			CustomPrimaryKey(`id VARCHAR(255) not null primary key`).
			Create()
		for key := range self.sampling.new {
			table.AddRow(key).Update()
			self.sampling.old[key] = true
		}

	default:
		p, _ := path.Split(config.COMM_PATH.CACHE + "/" + config.DEDUPLICATION.FILE_NAME)
		// 创建/打开目录
		d, err := os.Stat(p)
		if err != nil || !d.IsDir() {
			if err := os.MkdirAll(p, 0777); err != nil {
				logs.Log.Error("Error: %v\n", err)
			}
		}

		f, _ := os.OpenFile(config.COMM_PATH.CACHE+"/"+config.DEDUPLICATION.FILE_NAME, os.O_CREATE|os.O_APPEND|os.O_RDWR, 0660)

		b, _ := json.Marshal(self.sampling.new)
		b[0] = ','
		f.Write(b[:len(b)-1])
		f.Close()

		for key := range self.sampling.new {
			self.sampling.old[key] = true
		}
	}
	logs.Log.Informational(" *     新增 %v 条去重样本\n", len(self.sampling.new))
	self.sampling.new = make(map[string]bool)
}

func (self *Deduplication) Update(provider ...string) {
	if len(provider) == 0 {
		return
	}
	if provider[0] != self.provider {
		self.CleanCache()
		self.provider = provider[0]
	}

	switch self.provider {
	case "mgo":
		var docs = map[string]interface{}{}
		err := mgo.Mgo(&docs, "find", map[string]interface{}{
			"Database":   config.MGO.DB,
			"Collection": config.DEDUPLICATION.FILE_NAME,
		})
		if err != nil {
			logs.Log.Error("去重读取mgo: %v", err)
			return
		}
		for _, v := range docs["Docs"].([]interface{}) {
			self.sampling.old[v.(bson.M)["_id"].(string)] = true
		}

	case "mysql":
		db, ok := mysql.MysqlPool.GetOne().(*mysql.MysqlSrc)
		if !ok || db == nil {
			logs.Log.Error("链接Mysql数据库超时，无法读取去重记录！")
			return
		}
		defer mysql.MysqlPool.Free(db)
		rows, err := mysql.New(db.DB).
			SetTableName("`" + config.DEDUPLICATION.FILE_NAME + "`").
			SelectAll()
		if err != nil {
			// logs.Log.Error("读取Mysql数据库中去重记录失败：%v", err)
			return
		}

		for rows.Next() {
			var id string
			err = rows.Scan(&id)
			self.sampling.old[id] = true
		}

	default:
		f, err := os.Open(config.COMM_PATH.CACHE + "/" + config.DEDUPLICATION.FILE_NAME)
		if err != nil {
			return
		}
		defer f.Close()
		b, _ := ioutil.ReadAll(f)
		b[0] = '{'
		json.Unmarshal(
			append(b, '}'),
			&self.sampling.old,
		)
	}
	logs.Log.Informational(" *     读出 %v 条去重样本\n", len(self.sampling.old))
}

func (self *Deduplication) CleanCache() {
	self.Mutex.Lock()
	defer self.Unlock()
	self.sampling.old = make(map[string]bool)
	self.sampling.new = make(map[string]bool)
}
<|endoftext|>


<commit_before>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec2

import (
	"context"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"

	"go.chromium.org/luci/common/system/environ"
)

func build(src, tmpdir string) (string, error) {
	binary := filepath.Join(tmpdir, "exe.exe")
	cmd := exec.Command("go", "build", "-o", binary, src)
	if err := cmd.Run(); err != nil {
		return "", err
	}
	return binary, nil
}

func TestExec(t *testing.T) {
	t.Parallel()

	Convey("TestExec", t, func() {
		ctx := context.Background()

		tmpdir := t.TempDir()

		errCh := make(chan error, 1)

		Convey("exit", func() {
			testBinary, err := build(filepath.Join("testdata", "exit.go"), tmpdir)
			So(err, ShouldBeNil)

			Convey("exit 0", func() {
				cmd := CommandContext(ctx, testBinary)
				So(cmd.Start(), ShouldBeNil)

				So(cmd.Wait(), ShouldBeNil)

				So(cmd.ProcessState.ExitCode(), ShouldEqual, 0)
			})

			Convey("exit 42", func() {
				cmd := CommandContext(ctx, testBinary, "42")
				So(cmd.Start(), ShouldBeNil)

				So(cmd.Wait(), ShouldBeError, "exit status 42")

				So(cmd.ProcessState.ExitCode(), ShouldEqual, 42)
			})
		})

		Convey("timeout", func() {
			testBinary, err := build(filepath.Join("testdata", "timeout.go"), tmpdir)
			So(err, ShouldBeNil)

			cmd := CommandContext(ctx, testBinary)
			rc, err := cmd.StdoutPipe()
			So(err, ShouldBeNil)

			So(cmd.Start(), ShouldBeNil)

			expected := []byte("I'm alive!")
			buf := make([]byte, len(expected))
			n, err := rc.Read(buf)
			So(err, ShouldBeNil)
			So(n, ShouldEqual, len(expected))
			So(buf, ShouldResemble, expected)

			So(rc.Close(), ShouldBeNil)

			go func() {
				errCh <- cmd.Wait()
			}()

			select {
			case err := <-errCh:
				Print(err)
				So("should not reach here", ShouldBeNil)
			case <-time.After(time.Millisecond):
			}

			So(cmd.Terminate(), ShouldBeNil)

			select {
			case err := <-errCh:
				if runtime.GOOS == "windows" {
					So(err, ShouldBeError, "exit status 2")
				} else {
					So(err, ShouldBeError, "signal: terminated")
				}
			case <-time.After(time.Minute):
				Print(err)
				So("should not reach here", ShouldBeNil)
			}

			if runtime.GOOS == "windows" {
				So(cmd.ProcessState.ExitCode(), ShouldEqual, 2)
			} else {
				So(cmd.ProcessState.ExitCode(), ShouldEqual, -1)
			}
		})

		Convey("context timeout", func() {
			testBinary, err := build(filepath.Join("testdata", "timeout.go"), tmpdir)
			So(err, ShouldBeNil)

			if runtime.GOOS == "windows" {
				// TODO(tikuta): support context timeout on windows
				return
			}

			ctx, cancel := context.WithTimeout(ctx, time.Millisecond)
			defer cancel()

			cmd := CommandContext(ctx, testBinary)

			So(cmd.Start(), ShouldBeNil)

			So(cmd.Wait(), ShouldBeError, "signal: killed")

			So(cmd.ProcessState.ExitCode(), ShouldEqual, -1)
		})

	})
}

func TestSetEnv(t *testing.T) {
	t.Parallel()

	Convey("TestSetEnv", t, func() {
		ctx := context.Background()

		tmpdir := t.TempDir()

		testBinary, err := build(filepath.Join("testdata", "env.go"), tmpdir)
		So(err, ShouldBeNil)

		cmd := CommandContext(ctx, testBinary)
		env := environ.System()
		env.Set("envvar", "envvar")
		cmd.Env = env.Sorted()

		So(cmd.Start(), ShouldBeNil)

		So(cmd.Wait(), ShouldBeNil)

		So(cmd.ProcessState.ExitCode(), ShouldEqual, 0)
	})
}
<commit_msg>[exec2] Make the test more resilient.<commit_after>// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package exec2

import (
	"context"
	"os/exec"
	"path/filepath"
	"runtime"
	"testing"
	"time"

	"go.chromium.org/luci/common/system/environ"

	. "github.com/smartystreets/goconvey/convey"
	. "go.chromium.org/luci/common/testing/assertions"
)

func build(src, tmpdir string) (string, error) {
	binary := filepath.Join(tmpdir, "exe.exe")
	cmd := exec.Command("go", "build", "-o", binary, src)
	if err := cmd.Run(); err != nil {
		return "", err
	}
	return binary, nil
}

func TestExec(t *testing.T) {
	t.Parallel()

	Convey("TestExec", t, func() {
		ctx := context.Background()

		tmpdir := t.TempDir()

		errCh := make(chan error, 1)

		Convey("exit", func() {
			testBinary, err := build(filepath.Join("testdata", "exit.go"), tmpdir)
			So(err, ShouldBeNil)

			Convey("exit 0", func() {
				cmd := CommandContext(ctx, testBinary)
				So(cmd.Start(), ShouldBeNil)

				So(cmd.Wait(), ShouldBeNil)

				So(cmd.ProcessState.ExitCode(), ShouldEqual, 0)
			})

			Convey("exit 42", func() {
				cmd := CommandContext(ctx, testBinary, "42")
				So(cmd.Start(), ShouldBeNil)

				So(cmd.Wait(), ShouldBeError, "exit status 42")

				So(cmd.ProcessState.ExitCode(), ShouldEqual, 42)
			})
		})

		Convey("timeout", func() {
			testBinary, err := build(filepath.Join("testdata", "timeout.go"), tmpdir)
			So(err, ShouldBeNil)

			cmd := CommandContext(ctx, testBinary)
			rc, err := cmd.StdoutPipe()
			So(err, ShouldBeNil)

			So(cmd.Start(), ShouldBeNil)

			expected := []byte("I'm alive!")
			buf := make([]byte, len(expected))
			n, err := rc.Read(buf)
			So(err, ShouldBeNil)
			So(n, ShouldEqual, len(expected))
			So(buf, ShouldResemble, expected)

			So(rc.Close(), ShouldBeNil)

			go func() {
				errCh <- cmd.Wait()
			}()

			select {
			case err := <-errCh:
				Print(err)
				So("should not reach here", ShouldBeNil)
			case <-time.After(time.Millisecond):
			}

			So(cmd.Terminate(), ShouldBeNil)

			select {
			case err := <-errCh:
				if runtime.GOOS == "windows" {
					So(err, ShouldErrLike, "exit status")
				} else {
					So(err, ShouldErrLike, "signal: terminated")
				}
				// The value of the exit code depends on GOOS and version of Go runtime.
				So(cmd.ProcessState.ExitCode(), ShouldNotEqual, 0)
			case <-time.After(time.Minute):
				Print(err)
				So("should not reach here", ShouldBeNil)
			}
		})

		Convey("context timeout", func() {
			testBinary, err := build(filepath.Join("testdata", "timeout.go"), tmpdir)
			So(err, ShouldBeNil)

			if runtime.GOOS == "windows" {
				// TODO(tikuta): support context timeout on windows
				return
			}

			ctx, cancel := context.WithTimeout(ctx, time.Millisecond)
			defer cancel()

			cmd := CommandContext(ctx, testBinary)

			So(cmd.Start(), ShouldBeNil)

			So(cmd.Wait(), ShouldBeError, "signal: killed")

			So(cmd.ProcessState.ExitCode(), ShouldEqual, -1)
		})

	})
}

func TestSetEnv(t *testing.T) {
	t.Parallel()

	Convey("TestSetEnv", t, func() {
		ctx := context.Background()

		tmpdir := t.TempDir()

		testBinary, err := build(filepath.Join("testdata", "env.go"), tmpdir)
		So(err, ShouldBeNil)

		cmd := CommandContext(ctx, testBinary)
		env := environ.System()
		env.Set("envvar", "envvar")
		cmd.Env = env.Sorted()

		So(cmd.Start(), ShouldBeNil)

		So(cmd.Wait(), ShouldBeNil)

		So(cmd.ProcessState.ExitCode(), ShouldEqual, 0)
	})
}
<|endoftext|>


<commit_before>package profitbricks

import (
	"fmt"
	"net/http"
)

type KubernetesClusters struct {
	// URL to the collection representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// Unique representation for Kubernetes Cluster as a collection on a resource.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// Slice of items in that collection
	// Read Only: true
	Items []KubernetesCluster `json:"items"`

	// The type of resource within a collection
	// Read Only: true
	// Enum: [collection]
	PBType string `json:"type,omitempty"`
}

type KubernetesCluster struct {
	// URL to the object representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// The resource's unique identifier.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// metadata
	Metadata *Metadata `json:"metadata,omitempty"`

	// properties
	// Required: true
	Properties *KubernetesClusterProperties `json:"properties"`

	// The type of object
	// Read Only: true
	// Enum: [k8s]
	PBType string `json:"type,omitempty"`

	// Entities of a cluster
	Entities KubernetesClusterEntities `json:"entities,omitempty"`
}

type UpdatedKubernetesCluster struct {
	// URL to the object representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// The resource's unique identifier.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// metadata
	Metadata *Metadata `json:"metadata,omitempty"`

	// properties
	// Required: true
	Properties *KubernetesClusterProperties `json:"properties"`

	// The type of object
	// Read Only: true
	// Enum: [k8s]
	PBType string `json:"type,omitempty"`

	// Entities of a cluster
	Entities KubernetesClusterEntities `json:"-"`
}

type KubernetesClusterEntities struct {
	// NodePools of a cluster
	NodePools *KubernetesNodePools `json:"nodepools,omitempty"`
}

type MaintenanceWindow struct {
	// The english name of the day of the week
	// Required: false
	DayOfTheWeek string `json:"dayOfTheWeek,omitempty"`
	// A string of the following format: 08:00:00
	// Required: false
	Time string `json:"time,omitempty"`
}

type KubernetesClusterProperties struct {
	// A Kubernetes Cluster Name. Valid Kubernetes Cluster name must be 63 characters or less and must not be empty
	// and begin and end with an alphanumeric character ([a-z0-9]) with dashes (-), dots (.) and alphanumerics
	// between.
	// Required: true
	Name string `json:"name"`
	// The desired Kubernetes Version
	// Please consult the API documentation for supported versions
	// Required: false
	K8sVersion string `json:"k8sVersion,omitempty"`
	// The desired Maintanance Window
	// Required: false
	MaintenanceWindow *MaintenanceWindow `json:"maintenanceWindow,omitempty"`
}

type KubernetesConfig struct {
	// URL to the object representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// The resource's unique identifier.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// properties
	// Required: true
	Properties KubernetesConfigProperties `json:"properties"`

	// The type of object
	// Read Only: true
	// Enum: [kubeconfig]
	PBType string `json:"type,omitempty"`
}

type KubernetesConfigProperties struct {
	// A Kubernetes Config file data
	KubeConfig string `json:"kubeconfig,omitempty"`
}

type KubernetesNodePool struct {
	// URL to the object representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// The resource's unique identifier.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// metadata
	Metadata *Metadata `json:"metadata,omitempty"`

	// properties
	// Required: true
	Properties *KubernetesNodePoolProperties `json:"properties"`

	// The type of object
	// Read Only: true
	// Enum: [nodepool]
	PBType string `json:"type,omitempty"`
}

type KubernetesNodePoolProperties struct {
	// The availability zone in which the servers should exist
	// Required: true
	// Enum: [AUTO ZONE_1 ZONE_2]
	AvailabilityZone string `json:"availabilityZone,omitempty"`

	// Number of cores for node
	// Required: true
	CoresCount uint32 `json:"coresCount,omitempty"`

	// A valid cpu family name
	// Required: true
	CPUFamily string `json:"cpuFamily,omitempty"`

	// The unique identifier of the data center where the worker nodes of the node pool will be provisioned.
	// Required: true
	DatacenterID string `json:"datacenterId,omitempty"`

	// A Kubernetes Node Pool Name. Valid Kubernetes Node Pool name must be 63 characters or less and must not be
	// empty or begin and end with an alphanumeric character ([a-z0-9]) with dashes (-), dots (.) and alphanumerics
	// between.
	// Required: true
	Name string `json:"name,omitempty"`

	// Number of nodes part of the Node Pool
	// Required: true
	NodeCount uint32 `json:"nodeCount,omitempty"`

	// RAM size for node, minimum size 2048MB is recommended
	// Required: true
	RAMSize uint32 `json:"ramSize,omitempty"`

	// The size of the volume in GB. The size should be greater than 10GB.
	// Required: true
	StorageSize uint32 `json:"storageSize,omitempty"`

	// Hardware type of the volume
	// Required: true
	// Enum: [HDD SSD]
	StorageType string `json:"storageType,omitempty"`
}

type KubernetesNodePools struct {
	// URL to the collection representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// Unique representation for Kubernetes Nodes as a collection on a resource.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// Slice of items in that collection
	// Read Only: true
	Items []KubernetesNodePool `json:"items"`

	// The type of resource within a collection
	// Read Only: true
	// Enum: [nodepool]
	Type string `json:"type,omitempty"`
}

type KubernetesNodes struct {
	// URL to the collection representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// Unique representation for Kubernetes Node Pool as a collection on a resource.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// Slice of items in that collection
	// Read Only: true
	Items []KubernetesNode `json:"items"`

	// The type of resource within a collection
	// Read Only: true
	// Enum: [nodepool]
	Type string `json:"type,omitempty"`
}

type KubernetesNode struct {
	// URL to the object representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// The resource's unique identifier.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// metadata
	Metadata *Metadata `json:"metadata,omitempty"`

	// The properties of the node
	Properties *KubernetesNodeProperties `json:"properties"`

	// The type of object
	// Read Only: true
	// Enum: [nodepool]
	PBType string `json:"type,omitempty"`
}

type KubernetesNodeProperties struct {
	// The generated unique name of the node.
	// Read Only: true
	Name string `json:"name,omitempty"`

	// The assigned public IP of the node.
	// Read Only: true
	PublicIP string `json:"publicIP,omitempty"`

	// The k8s version that the node has.
	// Read Only: true
	K8sVersion string `json:"k8sVersion,omitempty"`
}

// ListKubernetesClusters gets a list of all clusters
func (c *Client) ListKubernetesClusters() (*KubernetesClusters, error) {
	rsp := &KubernetesClusters{}
	return rsp, c.GetOK(kubernetesClustersPath(), rsp)
}

// GetKubernetesCluster gets cluster with given id
func (c *Client) GetKubernetesCluster(clusterID string) (*KubernetesCluster, error) {
	rsp := &KubernetesCluster{}
	return rsp, c.GetOK(kubernetesClusterPath(clusterID), rsp)
}

// CreateKubernetesCluster creates a cluster
func (c *Client) CreateKubernetesCluster(cluster KubernetesCluster) (*KubernetesCluster, error) {
	rsp := &KubernetesCluster{}
	return rsp, c.PostAcc(kubernetesClustersPath(), cluster, rsp)
}

// DeleteKubernetesCluster deletes cluster
func (c *Client) DeleteKubernetesCluster(clusterId string) (*http.Header, error) {
	h := &http.Header{}
	return h, c.Delete(kubernetesClusterPath(clusterId), h, http.StatusOK)
}

// UpdateKubernetesCluster updates cluster
func (c *Client) UpdateKubernetesCluster(clusterID string, cluster UpdatedKubernetesCluster) (*KubernetesCluster, error) {
	rsp := &KubernetesCluster{}
	return rsp, c.Put(kubernetesClusterPath(clusterID), cluster, rsp, http.StatusOK)
}

// GetKubeconfig returns the kubeconfig of cluster
func (c *Client) GetKubeconfig(clusterID string) (string, error) {
	rsp := &KubernetesConfig{}
	if err := c.GetOK(kubeConfigPath(clusterID), rsp); err != nil {
		return "", err
	}
	return rsp.Properties.KubeConfig, nil
}

// ListKubernetesNodePools gets a list of all node pools of a cluster
func (c *Client) ListKubernetesNodePools(clusterID string) (*KubernetesNodePools, error) {
	rsp := &KubernetesNodePools{}
	return rsp, c.GetOK(kubernetesNodePoolsPath(clusterID), rsp)
}

// CreateKubernetesNodePool creates a new node pool for cluster
func (c *Client) CreateKubernetesNodePool(clusterID string, nodePool KubernetesNodePool) (*KubernetesNodePool, error) {
	rsp := &KubernetesNodePool{}
	return rsp, c.PostAcc(kubernetesNodePoolsPath(clusterID), nodePool, rsp)
}

// DeleteKubernetesNodePool deletes node pool from cluster
func (c *Client) DeleteKubernetesNodePool(clusterID, nodePoolID string) (*http.Header, error) {
	return c.DeleteAcc(kubernetesNodePoolPath(clusterID, nodePoolID))
}

// GetKubernetesNodePool gets node pool of the cluster
func (c *Client) GetKubernetesNodePool(clusterID, nodePoolID string) (*KubernetesNodePool, error) {
	rsp := &KubernetesNodePool{}
	return rsp, c.GetOK(kubernetesNodePoolPath(clusterID, nodePoolID), rsp)
}

// Update KubernetesNodePool updates node pool
func (c *Client) UpdateKubernetesNodePool(clusterID, nodePoolID string, nodePool KubernetesNodePool) (*KubernetesNodePool, error) {
	rsp := &KubernetesNodePool{}
	return rsp, c.PutAcc(kubernetesNodePoolPath(clusterID, nodePoolID), nodePool, rsp)
}

// ListKubernetesNodes gets a list of all nodes of a node pool
func (c *Client) ListKubernetesNodes(clusterID, nodePoolID string) (*KubernetesNodes, error) {
	rsp := &KubernetesNodes{}
	return rsp, c.GetOK(kubernetesNodesPath(clusterID, nodePoolID), rsp)
}

// GetKubernetesNode gets node of a node pool
func (c *Client) GetKubernetesNode(clusterID, nodePoolID, nodeID string) (*KubernetesNode, error) {
	rsp := &KubernetesNode{}
	return rsp, c.GetOK(kubernetesNodePath(clusterID, nodePoolID, nodeID), rsp)
}

// DeleteKubernetesNode deletes a node from a node pool, decreasing its size by 1.
func (c *Client) DeleteKubernetesNode(clusterID, nodePoolID, nodeID string) (*http.Header, error) {
	return c.DeleteAcc(kubernetesNodePath(clusterID, nodePoolID, nodeID))
}

// ReplaceKubernetesNode replaces a node of a node pool.
func (c *Client) ReplaceKubernetesNode(clusterID, nodePoolID, nodeID string) (*http.Header, error) {
	url := kubernetesNodeReplacePath(clusterID, nodePoolID, nodeID)
	rsp, err := c.R().SetError(ApiError{}).Post(url)
	if err != nil {
		return nil, NewClientError(HttpClientError, fmt.Sprintf("[POST] %s: Client error: %s", url, err))
	}
	h := rsp.Header()
	return &h, validateResponse(rsp, http.StatusAccepted)
}
<commit_msg>Fix: Expect accepted status for k8s cluster deletion requests<commit_after>package profitbricks

import (
	"fmt"
	"net/http"
)

type KubernetesClusters struct {
	// URL to the collection representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// Unique representation for Kubernetes Cluster as a collection on a resource.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// Slice of items in that collection
	// Read Only: true
	Items []KubernetesCluster `json:"items"`

	// The type of resource within a collection
	// Read Only: true
	// Enum: [collection]
	PBType string `json:"type,omitempty"`
}

type KubernetesCluster struct {
	// URL to the object representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// The resource's unique identifier.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// metadata
	Metadata *Metadata `json:"metadata,omitempty"`

	// properties
	// Required: true
	Properties *KubernetesClusterProperties `json:"properties"`

	// The type of object
	// Read Only: true
	// Enum: [k8s]
	PBType string `json:"type,omitempty"`

	// Entities of a cluster
	Entities KubernetesClusterEntities `json:"entities,omitempty"`
}

type UpdatedKubernetesCluster struct {
	// URL to the object representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// The resource's unique identifier.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// metadata
	Metadata *Metadata `json:"metadata,omitempty"`

	// properties
	// Required: true
	Properties *KubernetesClusterProperties `json:"properties"`

	// The type of object
	// Read Only: true
	// Enum: [k8s]
	PBType string `json:"type,omitempty"`

	// Entities of a cluster
	Entities KubernetesClusterEntities `json:"-"`
}

type KubernetesClusterEntities struct {
	// NodePools of a cluster
	NodePools *KubernetesNodePools `json:"nodepools,omitempty"`
}

type MaintenanceWindow struct {
	// The english name of the day of the week
	// Required: false
	DayOfTheWeek string `json:"dayOfTheWeek,omitempty"`
	// A string of the following format: 08:00:00
	// Required: false
	Time string `json:"time,omitempty"`
}

type KubernetesClusterProperties struct {
	// A Kubernetes Cluster Name. Valid Kubernetes Cluster name must be 63 characters or less and must not be empty
	// and begin and end with an alphanumeric character ([a-z0-9]) with dashes (-), dots (.) and alphanumerics
	// between.
	// Required: true
	Name string `json:"name"`
	// The desired Kubernetes Version
	// Please consult the API documentation for supported versions
	// Required: false
	K8sVersion string `json:"k8sVersion,omitempty"`
	// The desired Maintanance Window
	// Required: false
	MaintenanceWindow *MaintenanceWindow `json:"maintenanceWindow,omitempty"`
}

type KubernetesConfig struct {
	// URL to the object representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// The resource's unique identifier.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// properties
	// Required: true
	Properties KubernetesConfigProperties `json:"properties"`

	// The type of object
	// Read Only: true
	// Enum: [kubeconfig]
	PBType string `json:"type,omitempty"`
}

type KubernetesConfigProperties struct {
	// A Kubernetes Config file data
	KubeConfig string `json:"kubeconfig,omitempty"`
}

type KubernetesNodePool struct {
	// URL to the object representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// The resource's unique identifier.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// metadata
	Metadata *Metadata `json:"metadata,omitempty"`

	// properties
	// Required: true
	Properties *KubernetesNodePoolProperties `json:"properties"`

	// The type of object
	// Read Only: true
	// Enum: [nodepool]
	PBType string `json:"type,omitempty"`
}

type KubernetesNodePoolProperties struct {
	// The availability zone in which the servers should exist
	// Required: true
	// Enum: [AUTO ZONE_1 ZONE_2]
	AvailabilityZone string `json:"availabilityZone,omitempty"`

	// Number of cores for node
	// Required: true
	CoresCount uint32 `json:"coresCount,omitempty"`

	// A valid cpu family name
	// Required: true
	CPUFamily string `json:"cpuFamily,omitempty"`

	// The unique identifier of the data center where the worker nodes of the node pool will be provisioned.
	// Required: true
	DatacenterID string `json:"datacenterId,omitempty"`

	// A Kubernetes Node Pool Name. Valid Kubernetes Node Pool name must be 63 characters or less and must not be
	// empty or begin and end with an alphanumeric character ([a-z0-9]) with dashes (-), dots (.) and alphanumerics
	// between.
	// Required: true
	Name string `json:"name,omitempty"`

	// Number of nodes part of the Node Pool
	// Required: true
	NodeCount uint32 `json:"nodeCount,omitempty"`

	// RAM size for node, minimum size 2048MB is recommended
	// Required: true
	RAMSize uint32 `json:"ramSize,omitempty"`

	// The size of the volume in GB. The size should be greater than 10GB.
	// Required: true
	StorageSize uint32 `json:"storageSize,omitempty"`

	// Hardware type of the volume
	// Required: true
	// Enum: [HDD SSD]
	StorageType string `json:"storageType,omitempty"`
}

type KubernetesNodePools struct {
	// URL to the collection representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// Unique representation for Kubernetes Nodes as a collection on a resource.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// Slice of items in that collection
	// Read Only: true
	Items []KubernetesNodePool `json:"items"`

	// The type of resource within a collection
	// Read Only: true
	// Enum: [nodepool]
	Type string `json:"type,omitempty"`
}

type KubernetesNodes struct {
	// URL to the collection representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// Unique representation for Kubernetes Node Pool as a collection on a resource.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// Slice of items in that collection
	// Read Only: true
	Items []KubernetesNode `json:"items"`

	// The type of resource within a collection
	// Read Only: true
	// Enum: [nodepool]
	Type string `json:"type,omitempty"`
}

type KubernetesNode struct {
	// URL to the object representation (absolute path)
	// Read Only: true
	// Format: uri
	Href string `json:"href,omitempty"`

	// The resource's unique identifier.
	// Read Only: true
	ID string `json:"id,omitempty"`

	// metadata
	Metadata *Metadata `json:"metadata,omitempty"`

	// The properties of the node
	Properties *KubernetesNodeProperties `json:"properties"`

	// The type of object
	// Read Only: true
	// Enum: [nodepool]
	PBType string `json:"type,omitempty"`
}

type KubernetesNodeProperties struct {
	// The generated unique name of the node.
	// Read Only: true
	Name string `json:"name,omitempty"`

	// The assigned public IP of the node.
	// Read Only: true
	PublicIP string `json:"publicIP,omitempty"`

	// The k8s version that the node has.
	// Read Only: true
	K8sVersion string `json:"k8sVersion,omitempty"`
}

// ListKubernetesClusters gets a list of all clusters
func (c *Client) ListKubernetesClusters() (*KubernetesClusters, error) {
	rsp := &KubernetesClusters{}
	return rsp, c.GetOK(kubernetesClustersPath(), rsp)
}

// GetKubernetesCluster gets cluster with given id
func (c *Client) GetKubernetesCluster(clusterID string) (*KubernetesCluster, error) {
	rsp := &KubernetesCluster{}
	return rsp, c.GetOK(kubernetesClusterPath(clusterID), rsp)
}

// CreateKubernetesCluster creates a cluster
func (c *Client) CreateKubernetesCluster(cluster KubernetesCluster) (*KubernetesCluster, error) {
	rsp := &KubernetesCluster{}
	return rsp, c.PostAcc(kubernetesClustersPath(), cluster, rsp)
}

// DeleteKubernetesCluster deletes cluster
func (c *Client) DeleteKubernetesCluster(clusterID string) (*http.Header, error) {
	h := &http.Header{}
	return h, c.Delete(kubernetesClusterPath(clusterID), h, http.StatusAccepted)
}

// UpdateKubernetesCluster updates cluster
func (c *Client) UpdateKubernetesCluster(clusterID string, cluster UpdatedKubernetesCluster) (*KubernetesCluster, error) {
	rsp := &KubernetesCluster{}
	return rsp, c.Put(kubernetesClusterPath(clusterID), cluster, rsp, http.StatusOK)
}

// GetKubeconfig returns the kubeconfig of cluster
func (c *Client) GetKubeconfig(clusterID string) (string, error) {
	rsp := &KubernetesConfig{}
	if err := c.GetOK(kubeConfigPath(clusterID), rsp); err != nil {
		return "", err
	}
	return rsp.Properties.KubeConfig, nil
}

// ListKubernetesNodePools gets a list of all node pools of a cluster
func (c *Client) ListKubernetesNodePools(clusterID string) (*KubernetesNodePools, error) {
	rsp := &KubernetesNodePools{}
	return rsp, c.GetOK(kubernetesNodePoolsPath(clusterID), rsp)
}

// CreateKubernetesNodePool creates a new node pool for cluster
func (c *Client) CreateKubernetesNodePool(clusterID string, nodePool KubernetesNodePool) (*KubernetesNodePool, error) {
	rsp := &KubernetesNodePool{}
	return rsp, c.PostAcc(kubernetesNodePoolsPath(clusterID), nodePool, rsp)
}

// DeleteKubernetesNodePool deletes node pool from cluster
func (c *Client) DeleteKubernetesNodePool(clusterID, nodePoolID string) (*http.Header, error) {
	return c.DeleteAcc(kubernetesNodePoolPath(clusterID, nodePoolID))
}

// GetKubernetesNodePool gets node pool of the cluster
func (c *Client) GetKubernetesNodePool(clusterID, nodePoolID string) (*KubernetesNodePool, error) {
	rsp := &KubernetesNodePool{}
	return rsp, c.GetOK(kubernetesNodePoolPath(clusterID, nodePoolID), rsp)
}

// Update KubernetesNodePool updates node pool
func (c *Client) UpdateKubernetesNodePool(clusterID, nodePoolID string, nodePool KubernetesNodePool) (*KubernetesNodePool, error) {
	rsp := &KubernetesNodePool{}
	return rsp, c.PutAcc(kubernetesNodePoolPath(clusterID, nodePoolID), nodePool, rsp)
}

// ListKubernetesNodes gets a list of all nodes of a node pool
func (c *Client) ListKubernetesNodes(clusterID, nodePoolID string) (*KubernetesNodes, error) {
	rsp := &KubernetesNodes{}
	return rsp, c.GetOK(kubernetesNodesPath(clusterID, nodePoolID), rsp)
}

// GetKubernetesNode gets node of a node pool
func (c *Client) GetKubernetesNode(clusterID, nodePoolID, nodeID string) (*KubernetesNode, error) {
	rsp := &KubernetesNode{}
	return rsp, c.GetOK(kubernetesNodePath(clusterID, nodePoolID, nodeID), rsp)
}

// DeleteKubernetesNode deletes a node from a node pool, decreasing its size by 1.
func (c *Client) DeleteKubernetesNode(clusterID, nodePoolID, nodeID string) (*http.Header, error) {
	return c.DeleteAcc(kubernetesNodePath(clusterID, nodePoolID, nodeID))
}

// ReplaceKubernetesNode replaces a node of a node pool.
func (c *Client) ReplaceKubernetesNode(clusterID, nodePoolID, nodeID string) (*http.Header, error) {
	url := kubernetesNodeReplacePath(clusterID, nodePoolID, nodeID)
	rsp, err := c.R().SetError(ApiError{}).Post(url)
	if err != nil {
		return nil, NewClientError(HttpClientError, fmt.Sprintf("[POST] %s: Client error: %s", url, err))
	}
	h := rsp.Header()
	return &h, validateResponse(rsp, http.StatusAccepted)
}
<|endoftext|>


<commit_before>// Package kht provides an implementation of a keyed hash tree.
//
// A keyed hash tree is a hash tree which uses a keyed hash algorithm (e.g.,
// HMAC), used to derive block-level keys for encrypting large files.
//
// The notion of a keyed hash tree comes from Li et al's papers on Horus, a
// large-scale encrypted storage system:
//
// http://www.ssrc.ucsc.edu/pub/rajendran11-pdsw.html
// https://www.usenix.org/conference/fast13/technical-sessions/presentation/li_yan
//
// This implementation derives keys from the level and the level offset, both
// encoded at little-Endian 64-bit unsigned integers.
package kht

import (
	"crypto/hmac"
	"encoding/binary"
	"hash"
	"math"
)

// A KeyedHash is a hash algorithm which depends on a secret key.
type KeyedHash func(key []byte) hash.Hash

// HMAC returns a keyed hash implementation using the HMAC of the given hash
// algorithm.
func HMAC(alg func() hash.Hash) KeyedHash {
	return func(key []byte) hash.Hash {
		return hmac.New(alg, key)
	}
}

// A KeyedHashTree is a tree of keyed hashes, used to derive keys.
type KeyedHashTree struct {
	root                      []byte
	alg                       KeyedHash
	blockSize, maxSize, depth uint64
	factor                    float64
}

// New returns a KeyedHashTree with the given root key, keyed hash algorithm,
// block size, maximum size, and branching factor.
func New(key []byte, alg KeyedHash, blockSize, maxSize uint64, factor float64) *KeyedHashTree {
	return &KeyedHashTree{
		root:      key,
		alg:       alg,
		blockSize: blockSize,
		maxSize:   maxSize,
		factor:    factor,

		depth: uint64(
			math.Ceil(
				math.Log(float64(maxSize)/float64(blockSize)) /
					math.Log(factor),
			),
		),
	}
}

// Key returns the derived key at the given offset.
func (t *KeyedHashTree) Key(offset uint64) []byte {
	if offset > t.maxSize {
		panic("offset greater than maximum size")
	}

	buf := make([]byte, 16)
	k := make([]byte, len(t.root))
	copy(k, t.root)
	for i := uint64(0); i < t.depth; i++ {
		level := t.depth - i
		blockSize := uint64(math.Pow(t.factor, float64(level-1))) * t.blockSize
		y := offset / blockSize

		binary.LittleEndian.PutUint64(buf, uint64(i))
		binary.LittleEndian.PutUint64(buf[8:], uint64(y))

		h := t.alg(k)
		_, _ = h.Write(buf)
		k = h.Sum(k[:0])
	}
	return k
}
<commit_msg>Docfix<commit_after>// Package kht provides an implementation of a keyed hash tree.
//
// A keyed hash tree is a hash tree which uses a keyed hash algorithm (e.g.,
// HMAC), used to derive block-level keys for encrypting large files.
//
// The notion of a keyed hash tree comes from Li et al's papers on Horus, a
// large-scale encrypted storage system:
//
// http://www.ssrc.ucsc.edu/pub/rajendran11-pdsw.html
//
// https://www.usenix.org/conference/fast13/technical-sessions/presentation/li_yan
//
// This implementation derives keys from the level and the level offset, both
// encoded as little-endian 64-bit unsigned integers.
package kht

import (
	"crypto/hmac"
	"encoding/binary"
	"hash"
	"math"
)

// A KeyedHash is a hash algorithm which depends on a secret key.
type KeyedHash func(key []byte) hash.Hash

// HMAC returns a keyed hash implementation using the HMAC of the given hash
// algorithm.
func HMAC(alg func() hash.Hash) KeyedHash {
	return func(key []byte) hash.Hash {
		return hmac.New(alg, key)
	}
}

// A KeyedHashTree is a tree of keyed hashes, used to derive keys.
type KeyedHashTree struct {
	root                      []byte
	alg                       KeyedHash
	blockSize, maxSize, depth uint64
	factor                    float64
}

// New returns a KeyedHashTree with the given root key, keyed hash algorithm,
// block size, maximum size, and branching factor.
func New(key []byte, alg KeyedHash, blockSize, maxSize uint64, factor float64) *KeyedHashTree {
	return &KeyedHashTree{
		root:      key,
		alg:       alg,
		blockSize: blockSize,
		maxSize:   maxSize,
		factor:    factor,

		depth: uint64(
			math.Ceil(
				math.Log(float64(maxSize)/float64(blockSize)) /
					math.Log(factor),
			),
		),
	}
}

// Key returns the derived key at the given offset.
func (t *KeyedHashTree) Key(offset uint64) []byte {
	if offset > t.maxSize {
		panic("offset greater than maximum size")
	}

	buf := make([]byte, 16)
	k := make([]byte, len(t.root))
	copy(k, t.root)
	for i := uint64(0); i < t.depth; i++ {
		level := t.depth - i
		blockSize := uint64(math.Pow(t.factor, float64(level-1))) * t.blockSize
		y := offset / blockSize

		binary.LittleEndian.PutUint64(buf, uint64(i))
		binary.LittleEndian.PutUint64(buf[8:], uint64(y))

		h := t.alg(k)
		_, _ = h.Write(buf)
		k = h.Sum(k[:0])
	}
	return k
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
	"github.com/stianeikeland/go-rpio"
	"net/http"
	"os"
	"strings"
	"time"
)

var (
	mode = 0 // 0=go-rpio,1=embd

	// LEDs
	ledRedPin    = 4
	ledYellowPin = 17
	ledGreenPin  = 27
	ledToColor   = map[int]string{}
	ledMapEmbd = map[int]embd.DigitalPin{}
	ledMap     = map[int]rpio.Pin{}

	// Buttons
	buttonPin = 22
	buttonMap = map[int]embd.DigitalPin{}
)

func getLEDString(color string) string {
	return "Toggle " + strings.ToUpper(color)
}

func getToggledValue(pin embd.DigitalPin) int {
	val, _ := pin.Read()
	if val == embd.High {
		return embd.Low
	} else {
		return embd.High
	}
}

func toggleLEDEmbd(pin embd.DigitalPin, color string) {
	fmt.Println(getLEDString(color))
	toggledValue := getToggledValue(pin)
	fmt.Println("Val to write", toggledValue)
	embd.DigitalWrite(pin.N(), toggledValue)
}

func toggleLED(pin rpio.Pin, color string) {
	fmt.Println(getLEDString(color))
	fmt.Println("Current value:", pin.Read())
	pin.Toggle()
}

func initButtons() {
	buttonMap[0],_ = embd.NewDigitalPin(buttonPin)
	buttonMap[0].SetDirection(embd.In)
	buttonMap[0].ActiveLow(false)
	quit := make(chan interface{})
	err := buttonMap[0].Watch(embd.EdgeFalling, func(btn embd.DigitalPin) {
		fmt.Println("Pressed", btn)
		quit <- btn
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("Button %v was pressed.\n", <-quit)
}

func initLEDs() {
	if mode == 1 {
		ledMapEmbd[0],_ = embd.NewDigitalPin(ledRedPin)
		ledMapEmbd[1],_ = embd.NewDigitalPin(ledYellowPin)
		ledMapEmbd[2],_ = embd.NewDigitalPin(ledGreenPin)
		for i := 0; i < len(ledMapEmbd); i++ {
			ledMapEmbd[i].SetDirection(embd.Out)
			embd.DigitalWrite(ledMapEmbd[i].N(), embd.Low)
		}
	} else {
		ledMap[0] = rpio.Pin(ledRedPin)
		ledMap[1] = rpio.Pin(ledYellowPin)
		ledMap[2] = rpio.Pin(ledGreenPin)
		for i := 0; i < len(ledMap); i++ {
			ledMap[i].Output()
			ledMap[i].Low()
		}
	}
	ledToColor[0] = "red"
	ledToColor[1] = "yellow"
	ledToColor[2] = "green"
}

func initGPIO() error {
	if mode == 1 {
		return embd.InitGPIO()
	} else {
		return rpio.Open()
	}
}

func doLedToggling(i int) {
	if mode == 1 {
		toggleLEDEmbd(ledMapEmbd[i%3], ledToColor[i%3])
	} else {
		toggleLED(ledMap[i%3], ledToColor[i%3])
	}
	time.Sleep(time.Second)
}

type Gpio struct {
	ID int
	Name string
	Value int
}

func gpios (w http.ResponseWriter, r *http.Request) {

    m := Gpio{1, "GPIO2", 1}
    b, err := json.Marshal(m)

    if err != nil {
        panic(err)
    }

     w.Write(b)
}

func initWebServer() {
	http.HandleFunc("/gpios", gpios)
    http.ListenAndServe(":8080", nil)
}

func main() {
	fmt.Println("Parsing parameters")
	num := flag.Int("num", 3, "number of blinks")
	modeFromCli := flag.Int("mode", 0, "mode")
	button := flag.Bool("button", false, "button mode")
	api := flag.Bool("api", true, "API enabled")
	flag.Parse()
	mode = *modeFromCli
	if mode == 1 {
		fmt.Println("Running using Embd.io")
	} else {
		fmt.Println("Running using go-rpio")
	}
	fmt.Println("Number of blinks:", *num)

	if *api {
		go initWebServer()
	}

	var err = initGPIO()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	initLEDs()
	if mode == 1 && *button {
		initButtons()
	}

	if mode == 1 {
		defer embd.CloseGPIO()
	} else {
		defer rpio.Close()
	}

	if !*button {
		if *num == 0 {
			var counter = 0
			for {
				doLedToggling(counter)
				counter++
			}
		} else {
			for i := 0; i < *num; i++ {
				doLedToggling(i)
			}
		}
	}
}
<commit_msg>Using embd pins<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
	"github.com/stianeikeland/go-rpio"
)

var (
	mode = 0 // 0=go-rpio,1=embd

	// LEDs
	ledRedPin    = 4
	ledYellowPin = 17
	ledGreenPin  = 27
	ledToColor   = map[int]string{}
	ledMapEmbd   = map[int]embd.DigitalPin{}
	ledMap       = map[int]rpio.Pin{}

	// Buttons
	buttonPin = 22
	buttonMap = map[int]embd.DigitalPin{}
)

func getLEDString(color string) string {
	return "Toggle " + strings.ToUpper(color)
}

func getToggledValue(pin embd.DigitalPin) int {
	val, _ := pin.Read()
	if val == embd.High {
		return embd.Low
	}
	return embd.High
}

func toggleLEDEmbd(pin embd.DigitalPin, color string) {
	fmt.Println(getLEDString(color))
	toggledValue := getToggledValue(pin)
	fmt.Println("Val to write", toggledValue)
	embd.DigitalWrite(pin.N(), toggledValue)
}

func toggleLED(pin rpio.Pin, color string) {
	fmt.Println(getLEDString(color))
	fmt.Println("Current value:", pin.Read())
	pin.Toggle()
}

func initButtons() {
	buttonMap[0], _ = embd.NewDigitalPin(buttonPin)
	buttonMap[0].SetDirection(embd.In)
	buttonMap[0].ActiveLow(false)
	quit := make(chan interface{})
	err := buttonMap[0].Watch(embd.EdgeFalling, func(btn embd.DigitalPin) {
		fmt.Println("Pressed", btn)
		quit <- btn
	})
	if err != nil {
		panic(err)
	}
	fmt.Printf("Button %v was pressed.\n", <-quit)
}

func initLEDs() {
	if mode == 1 {
		ledMapEmbd[0], _ = embd.NewDigitalPin(ledRedPin)
		ledMapEmbd[1], _ = embd.NewDigitalPin(ledYellowPin)
		ledMapEmbd[2], _ = embd.NewDigitalPin(ledGreenPin)
		for i := 0; i < len(ledMapEmbd); i++ {
			ledMapEmbd[i].SetDirection(embd.Out)
			embd.DigitalWrite(ledMapEmbd[i].N(), embd.Low)
		}
	} else {
		ledMap[0] = rpio.Pin(ledRedPin)
		ledMap[1] = rpio.Pin(ledYellowPin)
		ledMap[2] = rpio.Pin(ledGreenPin)
		for i := 0; i < len(ledMap); i++ {
			ledMap[i].Output()
			ledMap[i].Low()
		}
	}
	ledToColor[0] = "red"
	ledToColor[1] = "yellow"
	ledToColor[2] = "green"
}

func initGPIO() error {
	if mode == 1 {
		return embd.InitGPIO()
	}
	return rpio.Open()
}

func doLedToggling(i int) {
	if mode == 1 {
		toggleLEDEmbd(ledMapEmbd[i%3], ledToColor[i%3])
	} else {
		toggleLED(ledMap[i%3], ledToColor[i%3])
	}
	time.Sleep(time.Second)
}

type Gpio struct {
	ID    int
	Name  string
	Value int
}

func gpios(w http.ResponseWriter, r *http.Request) {
	pins := ledMapEmbd
	gpios := []Gpio{}
	for i := 0; i < len(pins); i++ {
		pin := pins[i]
		pinValue, _ := pin.Read()
		gpio := Gpio{i, "GPIO" + strconv.Itoa(pin.N()), pinValue}
		gpios[i] = gpio
	}
	json, err := json.Marshal(gpios)

	if err != nil {
		panic(err)
	}

	w.Write(json)
}

func initWebServer() {
	http.HandleFunc("/gpios", gpios)
	http.ListenAndServe(":8080", nil)
}

func main() {
	fmt.Println("Parsing parameters")
	num := flag.Int("num", 3, "number of blinks")
	modeFromCli := flag.Int("mode", 0, "mode")
	button := flag.Bool("button", false, "button mode")
	api := flag.Bool("api", true, "API enabled")
	flag.Parse()
	mode = *modeFromCli
	if mode == 1 {
		fmt.Println("Running using Embd.io")
	} else {
		fmt.Println("Running using go-rpio")
	}
	fmt.Println("Number of blinks:", *num)

	if *api && mode == 1 {
		go initWebServer()
	}

	var err = initGPIO()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	initLEDs()
	if mode == 1 && *button {
		initButtons()
	}

	if mode == 1 {
		defer embd.CloseGPIO()
	} else {
		defer rpio.Close()
	}

	if !*button {
		if *num == 0 {
			var counter = 0
			for {
				doLedToggling(counter)
				counter++
			}
		} else {
			for i := 0; i < *num; i++ {
				doLedToggling(i)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	flags "github.com/jessevdk/go-flags"
	"github.com/mit-dci/lit/coinparam"
	"github.com/mit-dci/lit/litbamf"
	"github.com/mit-dci/lit/litrpc"
	"github.com/mit-dci/lit/lnutil"
	"github.com/mit-dci/lit/qln"
)

type config struct { // define a struct for usage with go-flags
	Tn3host     string `long:"tn3" description:"Connect to bitcoin testnet3."`
	Bc2host     string `long:"bc2" description:"bc2 full node."`
	Lt4host     string `long:"lt4" description:"Connect to litecoin testnet4."`
	Reghost     string `long:"reg" description:"Connect to bitcoin regtest."`
	Litereghost string `long:"litereg" description:"Connect to litecoin regtest."`
	Tvtchost    string `long:"tvtc" description:"Connect to Vertcoin test node."`
	Vtchost     string `long:"vtc" description:"Connect to Vertcoin."`
	LitHomeDir  string `long:"dir" description:"Specify Home Directory of lit as an absolute path."`
	TrackerURL  string `long:"tracker" description:"LN address tracker URL http|https://host:port"`
	ConfigFile  string

	ReSync  bool `short:"r" long:"reSync" description:"Resync from the given tip."`
	Tower   bool `long:"tower" description:"Watchtower: Run a watching node"`
	Hard    bool `short:"t" long:"hard" description:"Flag to set networks."`
	Verbose bool `short:"v" long:"verbose" description:"Set verbosity to true."`

	Rpcport uint16 `short:"p" long:"rpcport" description:"Set RPC port to connect to"`

	Params *coinparam.Params
}

var (
	defaultLitHomeDirName = os.Getenv("HOME") + "/.lit"
	defaultTrackerURL     = "http://ni.media.mit.edu:46580"
	defaultKeyFileName    = "privkey.hex"
	defaultConfigFilename = "lit.conf"
	defaultHomeDir        = os.Getenv("HOME")
	defaultConfigFile     = filepath.Join(os.Getenv("HOME"), "/.lit/lit.conf")
	defaultRpcport        = uint16(8001)
)

func fileExists(name string) bool {
	if _, err := os.Stat(name); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}

// newConfigParser returns a new command line flags parser.
func newConfigParser(conf *config, options flags.Options) *flags.Parser {
	parser := flags.NewParser(conf, options)
	return parser
}
func linkWallets(node *qln.LitNode, key *[32]byte, conf *config) error {
	// for now, wallets are linked to the litnode on startup, and
	// can't appear / disappear while it's running.  Later
	// could support dynamically adding / removing wallets

	// order matters; the first registered wallet becomes the default

	var err error
	// try regtest
	if conf.Reghost != "" {
		p := &coinparam.RegressionNetParams
		fmt.Printf("reg: %s\n", conf.Reghost)
		err = node.LinkBaseWallet(key, 120, conf.ReSync, conf.Tower, conf.Reghost, p)
		if err != nil {
			return err
		}
	}
	// try testnet3
	if conf.Tn3host != "" {
		p := &coinparam.TestNet3Params
		err = node.LinkBaseWallet(
			key, 1210000, conf.ReSync, conf.Tower,
			conf.Tn3host, p)
		if err != nil {
			return err
		}
	}
	// try litecoin regtest
	if conf.Litereghost != "" {
		p := &coinparam.LiteRegNetParams
		err = node.LinkBaseWallet(key, 120, conf.ReSync, conf.Tower, conf.Litereghost, p)
		if err != nil {
			return err
		}
	}

	// try litecoin testnet4
	if conf.Lt4host != "" {
		p := &coinparam.LiteCoinTestNet4Params
		err = node.LinkBaseWallet(
			key, p.StartHeight, conf.ReSync, conf.Tower,
			conf.Lt4host, p)
		if err != nil {
			return err
		}
	}
	// try vertcoin testnet
	if conf.Tvtchost != "" {
		p := &coinparam.VertcoinTestNetParams
		err = node.LinkBaseWallet(
			key, 0, conf.ReSync, conf.Tower,
			conf.Tvtchost, p)
		if err != nil {
			return err
		}
	}
	// try vertcoin mainnet
	if conf.Vtchost != "" {
		p := &coinparam.VertcoinParams
		err = node.LinkBaseWallet(
			key, p.StartHeight, conf.ReSync, conf.Tower,
			conf.Vtchost, p)
		if err != nil {
			return err
		}

	}
	return nil
}

func main() {

	conf := config{
		LitHomeDir: defaultLitHomeDirName,
		ConfigFile: defaultConfigFile,
		Rpcport:    defaultRpcport,
		TrackerURL: defaultTrackerURL,
	}

	// Pre-parse the command line options to see if an alternative config
	// file or the version flag was specified.  Any errors aside from the
	// help message error can be ignored here since they will be caught by
	// the final parse below.
	usageMessage := fmt.Sprintf("Use %s -h to show usage", "./lit")
	preconf := conf
	preParser := newConfigParser(&preconf, flags.HelpFlag)
	_, err := preParser.ParseArgs(os.Args)
	if err != nil {
		if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
			fmt.Println(err)
			return
		}
	}

	// Load config from file
	parser := newConfigParser(&conf, flags.Default) //parse

	_, err = os.Stat(preconf.LitHomeDir) // create directory
	if err != nil {
		log.Println("Error while creating a directory")
		log.Fatal(err)
	}
	if os.IsNotExist(err) {
		os.Mkdir(preconf.LitHomeDir, 0700)
		log.Println("Creating a new config file")
		err := createDefaultConfigFile(preconf.LitHomeDir) // Source of error
		if err != nil {
			log.Println("Error creating a default config file: %v\n")
			log.Fatal(err)
		}
	}

	if !(preconf.ConfigFile != defaultConfigFile) {
		if _, err := os.Stat(filepath.Join(filepath.Join(preconf.LitHomeDir), "lit.conf")); os.IsNotExist(err) {
			if err != nil {
				log.Fatal(err)
			}
			log.Println("Creating a new config file")
			err := createDefaultConfigFile(filepath.Join(preconf.LitHomeDir)) // Source of error
			if err != nil {
				log.Fatal(err)
				return
			}
		}

		preconf.ConfigFile = filepath.Join(filepath.Join(preconf.LitHomeDir), "lit.conf")
		// lets parse the config file provided, if any
		err := flags.NewIniParser(parser).ParseFile(preconf.ConfigFile)
		if err != nil {
			_, ok := err.(*os.PathError)
			if !ok {
				log.Fatal(err)
			}
		}
	}
	// Parse command line options again to ensure they take precedence.
	_, err = parser.ParseArgs(os.Args) // returns invalid flags
	if err != nil {
		fmt.Println(usageMessage)
		// no need to print the error as we already have
		return
	}

	logFilePath := filepath.Join(conf.LitHomeDir, "lit.log")

	logfile, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	defer logfile.Close()

	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	if conf.Verbose {
		logOutput := io.MultiWriter(os.Stdout, logfile)
		log.SetOutput(logOutput)
	} else {
		log.SetOutput(logfile)
	}

	// Allow node with no linked wallets, for testing.
	// TODO Should update tests and disallow nodes without wallets later.
	//	if conf.Tn3host == "" && conf.Lt4host == "" && conf.Reghost == "" {
	//		log.Fatal("error: no network specified; use -tn3, -reg, -lt4")
	//	}

	// Keys: the litNode, and wallits, all get 32 byte keys.
	// Right now though, they all get the *same* key.  For lit as a single binary
	// now, all using the same key makes sense; could split up later.

	keyFilePath := filepath.Join(conf.LitHomeDir, defaultKeyFileName)

	// read key file (generate if not found)
	key, err := lnutil.ReadKeyFile(keyFilePath)
	if err != nil {
		log.Fatal(err)
	}

	// Setup LN node.  Activate Tower if in hard mode.
	// give node and below file pathof lit home directoy
	node, err := qln.NewLitNode(key, conf.LitHomeDir, conf.TrackerURL)
	if err != nil {
		log.Fatal(err)
	}

	// node is up; link wallets based on args
	err = linkWallets(node, key, &conf)
	if err != nil {
		log.Fatal(err)
	}

	rpcl := new(litrpc.LitRPC)
	rpcl.Node = node
	rpcl.OffButton = make(chan bool, 1)

	go litrpc.RPCListen(rpcl, conf.Rpcport)
	litbamf.BamfListen(conf.Rpcport, conf.LitHomeDir)

	<-rpcl.OffButton
	fmt.Printf("Got stop request\n")
	time.Sleep(time.Second)

	return
	// New directory being created over at PWD
	// conf file being created at /
}

func createDefaultConfigFile(destinationPath string) error {

	// We assume sample config file path is same as binary TODO: change to ~/.lit/config/
	path, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		return err
	}
	sampleConfigPath := filepath.Join(path, defaultConfigFilename)

	// We generate a random user and password
	randomBytes := make([]byte, 20)
	_, err = rand.Read(randomBytes)
	if err != nil {
		return err
	}
	generatedRPCUser := base64.StdEncoding.EncodeToString(randomBytes)

	_, err = rand.Read(randomBytes)
	if err != nil {
		return err
	}
	generatedRPCPass := base64.StdEncoding.EncodeToString(randomBytes)

	src, err := os.Open(sampleConfigPath)
	if err != nil {
		return err
	}
	defer src.Close()

	dest, err := os.OpenFile(filepath.Join(destinationPath, defaultConfigFilename),
		os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	defer dest.Close()

	// We copy every line from the sample config file to the destination,
	// only replacing the two lines for rpcuser and rpcpass
	reader := bufio.NewReader(src)
	for err != io.EOF {
		var line string
		line, err = reader.ReadString('\n')
		if err != nil && err != io.EOF {
			return err
		}

		if strings.Contains(line, "rpcuser=") {
			line = "rpcuser=" + generatedRPCUser + "\n"
		} else if strings.Contains(line, "rpcpass=") {
			line = "rpcpass=" + generatedRPCPass + "\n"
		}

		if _, err := dest.WriteString(line); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Create config file without sample file<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"time"

	flags "github.com/jessevdk/go-flags"
	"github.com/mit-dci/lit/coinparam"
	"github.com/mit-dci/lit/litbamf"
	"github.com/mit-dci/lit/litrpc"
	"github.com/mit-dci/lit/lnutil"
	"github.com/mit-dci/lit/qln"
)

type config struct { // define a struct for usage with go-flags
	Tn3host     string `long:"tn3" description:"Connect to bitcoin testnet3."`
	Bc2host     string `long:"bc2" description:"bc2 full node."`
	Lt4host     string `long:"lt4" description:"Connect to litecoin testnet4."`
	Reghost     string `long:"reg" description:"Connect to bitcoin regtest."`
	Litereghost string `long:"litereg" description:"Connect to litecoin regtest."`
	Tvtchost    string `long:"tvtc" description:"Connect to Vertcoin test node."`
	Vtchost     string `long:"vtc" description:"Connect to Vertcoin."`
	LitHomeDir  string `long:"dir" description:"Specify Home Directory of lit as an absolute path."`
	TrackerURL  string `long:"tracker" description:"LN address tracker URL http|https://host:port"`
	ConfigFile  string

	ReSync  bool `short:"r" long:"reSync" description:"Resync from the given tip."`
	Tower   bool `long:"tower" description:"Watchtower: Run a watching node"`
	Hard    bool `short:"t" long:"hard" description:"Flag to set networks."`
	Verbose bool `short:"v" long:"verbose" description:"Set verbosity to true."`

	Rpcport uint16 `short:"p" long:"rpcport" description:"Set RPC port to connect to"`

	Params *coinparam.Params
}

var (
	defaultLitHomeDirName = os.Getenv("HOME") + "/.lit"
	defaultTrackerURL     = "http://ni.media.mit.edu:46580"
	defaultKeyFileName    = "privkey.hex"
	defaultConfigFilename = "lit.conf"
	defaultHomeDir        = os.Getenv("HOME")
	defaultRpcport        = uint16(8001)
)

func fileExists(name string) bool {
	if _, err := os.Stat(name); err != nil {
		if os.IsNotExist(err) {
			return false
		}
	}
	return true
}

// newConfigParser returns a new command line flags parser.
func newConfigParser(conf *config, options flags.Options) *flags.Parser {
	parser := flags.NewParser(conf, options)
	return parser
}
func linkWallets(node *qln.LitNode, key *[32]byte, conf *config) error {
	// for now, wallets are linked to the litnode on startup, and
	// can't appear / disappear while it's running.  Later
	// could support dynamically adding / removing wallets

	// order matters; the first registered wallet becomes the default

	var err error
	// try regtest
	if conf.Reghost != "" {
		p := &coinparam.RegressionNetParams
		fmt.Printf("reg: %s\n", conf.Reghost)
		err = node.LinkBaseWallet(key, 120, conf.ReSync, conf.Tower, conf.Reghost, p)
		if err != nil {
			return err
		}
	}
	// try testnet3
	if conf.Tn3host != "" {
		p := &coinparam.TestNet3Params
		err = node.LinkBaseWallet(
			key, 1210000, conf.ReSync, conf.Tower,
			conf.Tn3host, p)
		if err != nil {
			return err
		}
	}
	// try litecoin regtest
	if conf.Litereghost != "" {
		p := &coinparam.LiteRegNetParams
		err = node.LinkBaseWallet(key, 120, conf.ReSync, conf.Tower, conf.Litereghost, p)
		if err != nil {
			return err
		}
	}

	// try litecoin testnet4
	if conf.Lt4host != "" {
		p := &coinparam.LiteCoinTestNet4Params
		err = node.LinkBaseWallet(
			key, p.StartHeight, conf.ReSync, conf.Tower,
			conf.Lt4host, p)
		if err != nil {
			return err
		}
	}
	// try vertcoin testnet
	if conf.Tvtchost != "" {
		p := &coinparam.VertcoinTestNetParams
		err = node.LinkBaseWallet(
			key, 0, conf.ReSync, conf.Tower,
			conf.Tvtchost, p)
		if err != nil {
			return err
		}
	}
	// try vertcoin mainnet
	if conf.Vtchost != "" {
		p := &coinparam.VertcoinParams
		err = node.LinkBaseWallet(
			key, p.StartHeight, conf.ReSync, conf.Tower,
			conf.Vtchost, p)
		if err != nil {
			return err
		}

	}
	return nil
}

func main() {

	conf := config{
		LitHomeDir: defaultLitHomeDirName,
		Rpcport:    defaultRpcport,
		TrackerURL: defaultTrackerURL,
	}

	// Pre-parse the command line options to see if an alternative config
	// file or the version flag was specified.  Any errors aside from the
	// help message error can be ignored here since they will be caught by
	// the final parse below.
	usageMessage := fmt.Sprintf("Use %s -h to show usage", "./lit")
	preconf := conf
	preParser := newConfigParser(&preconf, flags.HelpFlag)
	_, err := preParser.ParseArgs(os.Args)
	if err != nil {
		if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
			fmt.Println(err)
			return
		}
	}

	// Load config from file
	parser := newConfigParser(&conf, flags.Default) //parse

	_, err = os.Stat(preconf.LitHomeDir) // create directory
	if err != nil {
		log.Println("Error while creating a directory")
	}
	if os.IsNotExist(err) {
		// first time the guy is running lit, lets set tn3 to true
		os.Mkdir(preconf.LitHomeDir, 0700)
		log.Println("Creating a new config file")
		err := createDefaultConfigFile(preconf.LitHomeDir) // Source of error
		if err != nil {
			log.Println("Error creating a default config file: %v\n")
			log.Fatal(err)
		}
	}

	if _, err := os.Stat(filepath.Join(filepath.Join(preconf.LitHomeDir), "lit.conf")); os.IsNotExist(err) {
		// if there is no config file found over at the directory, create one
		if err != nil {
			log.Fatal(err)
		}
		log.Println("Creating a new config file")
		err := createDefaultConfigFile(filepath.Join(preconf.LitHomeDir)) // Source of error
		if err != nil {
			log.Fatal(err)
			return
		}
	}

	preconf.ConfigFile = filepath.Join(filepath.Join(preconf.LitHomeDir), "lit.conf")
	// lets parse the config file provided, if any
	err = flags.NewIniParser(parser).ParseFile(preconf.ConfigFile)
	if err != nil {
		_, ok := err.(*os.PathError)
		if !ok {
			log.Fatal(err)
		}
	}
	// Parse command line options again to ensure they take precedence.
	_, err = parser.ParseArgs(os.Args) // returns invalid flags
	if err != nil {
		fmt.Println(usageMessage)
		// no need to print the error as we already have
		return
	}

	logFilePath := filepath.Join(conf.LitHomeDir, "lit.log")

	logfile, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	defer logfile.Close()

	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	if conf.Verbose {
		logOutput := io.MultiWriter(os.Stdout, logfile)
		log.SetOutput(logOutput)
	} else {
		log.SetOutput(logfile)
	}

	// Allow node with no linked wallets, for testing.
	// TODO Should update tests and disallow nodes without wallets later.
	//	if conf.Tn3host == "" && conf.Lt4host == "" && conf.Reghost == "" {
	//		log.Fatal("error: no network specified; use -tn3, -reg, -lt4")
	//	}

	// Keys: the litNode, and wallits, all get 32 byte keys.
	// Right now though, they all get the *same* key.  For lit as a single binary
	// now, all using the same key makes sense; could split up later.

	keyFilePath := filepath.Join(conf.LitHomeDir, defaultKeyFileName)

	// read key file (generate if not found)
	key, err := lnutil.ReadKeyFile(keyFilePath)
	if err != nil {
		log.Fatal(err)
	}

	// Setup LN node.  Activate Tower if in hard mode.
	// give node and below file pathof lit home directoy
	node, err := qln.NewLitNode(key, conf.LitHomeDir, conf.TrackerURL)
	if err != nil {
		log.Fatal(err)
	}

	// node is up; link wallets based on args
	err = linkWallets(node, key, &conf)
	if err != nil {
		log.Fatal(err)
	}

	rpcl := new(litrpc.LitRPC)
	rpcl.Node = node
	rpcl.OffButton = make(chan bool, 1)

	go litrpc.RPCListen(rpcl, conf.Rpcport)
	litbamf.BamfListen(conf.Rpcport, conf.LitHomeDir)

	<-rpcl.OffButton
	fmt.Printf("Got stop request\n")
	time.Sleep(time.Second)

	return
	// New directory being created over at PWD
	// conf file being created at /
}

func createDefaultConfigFile(destinationPath string) error {

	dest, err := os.OpenFile(filepath.Join(destinationPath, defaultConfigFilename),
		os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return err
	}
	defer dest.Close()

	writer := bufio.NewWriter(dest)
	defaultArgs := []byte("tn3=1")
	_, err = writer.Write(defaultArgs)
	if err != nil {
		return err
	}
	writer.Flush()
	return nil
}
<|endoftext|>


<commit_before>package main
import "fmt"
import "math"

func main() {

fmt.Println(math.Log2(5.0))

}
<commit_msg>Delete log.go<commit_after><|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2014 AT&T
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package communication

import (
	"bytes"
	"fmt"
	"net"
	"net/http"

	"github.com/golang/glog"
	"code.google.com/p/gogoprotobuf/proto"

	"github.com/att-innovate/charmander-scheduler/upid"
)

func MesosMasterReachable(masterAddress string) bool {
	_, err := http.Get("http://" + masterAddress)
	return err == nil
}

func SendMessageToMesos(sender *upid.UPID, msg *Message) error {
	glog.Infof("Sending message to %v from %v\n", msg.UPID, sender)
	// marshal payload
	b, err := proto.Marshal(msg.ProtoMessage)
	if err != nil {
		glog.Errorf("Failed to marshal message %v: %v\n", msg, err)
		return err
	}
	msg.Bytes = b
	// create request
	req, err := makeLibprocessRequest(sender, msg)
	if err != nil {
		glog.Errorf("Failed to make libprocess request: %v\n", err)
		return err
	}
	// send it
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		glog.Errorf("Failed to POST: %v %v\n", err, resp)
		return err
	}
	resp.Body.Close()
	// ensure master acknowledgement.
	if (resp.StatusCode != http.StatusOK) &&
		(resp.StatusCode != http.StatusAccepted) {
		msg := fmt.Sprintf("Master %s rejected %s.  Returned status %s.", msg.UPID, msg.RequestURI(), resp.Status)
		glog.Errorln(msg)
		return fmt.Errorf(msg)
	}

	return nil
}

func makeLibprocessRequest(sender *upid.UPID, msg *Message) (*http.Request, error) {
	hostport := net.JoinHostPort(msg.UPID.Host, msg.UPID.Port)
	targetURL := fmt.Sprintf("http://%s%s", hostport, msg.RequestURI())
	glog.Infof("Target URL %s", targetURL)
	req, err := http.NewRequest("POST", targetURL, bytes.NewReader(msg.Bytes))
	if err != nil {
		glog.Errorf("Failed to create request: %v\n", err)
		return nil, err
	}
	req.Header.Add("Libprocess-From", sender.String())
	req.Header.Add("Content-Type", "application/x-protobuf")
	req.Header.Add("Connection", "Keep-Alive")

	return req, nil
}

<commit_msg>Additional error message to MesosMasterReachable<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2014 AT&T
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package communication

import (
	"bytes"
	"fmt"
	"net"
	"net/http"

	"github.com/golang/glog"
	"code.google.com/p/gogoprotobuf/proto"

	"github.com/att-innovate/charmander-scheduler/upid"
)

func MesosMasterReachable(masterAddress string) bool {
	_, err := http.Get("http://" + masterAddress)
	if err != nil {
		glog.Errorf("Failed to connect to mesos %v: %v\n", masterAddress, err)
		return err
	}

	return true
}

func SendMessageToMesos(sender *upid.UPID, msg *Message) error {
	glog.Infof("Sending message to %v from %v\n", msg.UPID, sender)
	// marshal payload
	b, err := proto.Marshal(msg.ProtoMessage)
	if err != nil {
		glog.Errorf("Failed to marshal message %v: %v\n", msg, err)
		return err
	}
	msg.Bytes = b
	// create request
	req, err := makeLibprocessRequest(sender, msg)
	if err != nil {
		glog.Errorf("Failed to make libprocess request: %v\n", err)
		return err
	}
	// send it
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		glog.Errorf("Failed to POST: %v %v\n", err, resp)
		return err
	}
	resp.Body.Close()
	// ensure master acknowledgement.
	if (resp.StatusCode != http.StatusOK) &&
		(resp.StatusCode != http.StatusAccepted) {
		msg := fmt.Sprintf("Master %s rejected %s.  Returned status %s.", msg.UPID, msg.RequestURI(), resp.Status)
		glog.Errorln(msg)
		return fmt.Errorf(msg)
	}

	return nil
}

func makeLibprocessRequest(sender *upid.UPID, msg *Message) (*http.Request, error) {
	hostport := net.JoinHostPort(msg.UPID.Host, msg.UPID.Port)
	targetURL := fmt.Sprintf("http://%s%s", hostport, msg.RequestURI())
	glog.Infof("Target URL %s", targetURL)
	req, err := http.NewRequest("POST", targetURL, bytes.NewReader(msg.Bytes))
	if err != nil {
		glog.Errorf("Failed to create request: %v\n", err)
		return nil, err
	}
	req.Header.Add("Libprocess-From", sender.String())
	req.Header.Add("Content-Type", "application/x-protobuf")
	req.Header.Add("Connection", "Keep-Alive")

	return req, nil
}

<|endoftext|>


<commit_before>package mask

import (
	"errors"
	"net"
	"strings"
)

func NewMask(a string) (*net.IPNet, error) {
	parts := strings.Split(a, "/")
	if len(parts) == 5 && parts[1] == "ip4" && parts[3] == "ipcidr" {
		_, ipn, err := net.ParseCIDR(parts[2] + "/" + parts[4])
		if err != nil {
			return nil, err
		}
		return ipn, nil
	}
	return nil, errors.New("invalid format")
}
<commit_msg>support ipv6<commit_after>package mask

import (
	"errors"
	"net"
	"strings"
)

var ErrInvalidFormat = errors.New("invalid multiaddr-filter format")

func NewMask(a string) (*net.IPNet, error) {
	parts := strings.Split(a, "/")
	if len(parts) != 5 {
		return nil, ErrInvalidFormat
	}

	// check it's a valid filter address. ip + cidr
	isip := parts[1] == "ip4" || parts[1] == "ip6"
	iscidr := parts[3] == "ipcidr"
	if !isip || !iscidr {
		return nil, ErrInvalidFormat
	}

	_, ipn, err := net.ParseCIDR(parts[2] + "/" + parts[4])
	if err != nil {
		return nil, err
	}
	return ipn, nil
}
<|endoftext|>


<commit_before>package graval

import (
	"log"
	"net"
	"strconv"
	"strings"
)

type FTPServer struct {
	name          string
	listenTo      string
	driverFactory FTPDriverFactory
}

func NewFTPServer(hostname string, port int, factory FTPDriverFactory) *FTPServer {
	s := new(FTPServer)
	s.listenTo = buildTcpString(hostname, port)
	s.name = "Go FTP Server"
	s.driverFactory = factory
	return s
}

func (ftpServer *FTPServer) ListenAndServe() (err error) {
	laddr, err := net.ResolveTCPAddr("tcp", ftpServer.listenTo)
	if err != nil {
		log.Fatal(err)
	}
	listener, err := net.ListenTCP("tcp", laddr)
	if err != nil {
		log.Fatal(err)
	}
	for {
		ftpConn, err := ftpServer.Accept(listener)
		if err != nil {
			break
		}
		go ftpConn.Serve()
	}
	return
}

func (ftpServer *FTPServer) Accept(listener *net.TCPListener) (ftpConn *ftpConn, err error) {
	tcpConn, err := listener.AcceptTCP()
	if err == nil {
		ftpConn = newftpConn(tcpConn, ftpServer.driverFactory.NewDriver())
	}
	return
}

func buildTcpString(hostname string, port int) (result string) {
	if strings.Contains(hostname, ":") {
		// ipv6
		if port == 0 {
			result = "["+hostname+"]"
		} else {
			result = "["+hostname +"]:" + strconv.Itoa(port)
		}
	} else {
		// ipv4
		if port == 0 {
			result = hostname
		} else {
			result = hostname +":" + strconv.Itoa(port)
		}
	}
	return
}
<commit_msg>the accept() method should not be exported from the package<commit_after>package graval

import (
	"log"
	"net"
	"strconv"
	"strings"
)

type FTPServer struct {
	name          string
	listenTo      string
	driverFactory FTPDriverFactory
}

func NewFTPServer(hostname string, port int, factory FTPDriverFactory) *FTPServer {
	s := new(FTPServer)
	s.listenTo = buildTcpString(hostname, port)
	s.name = "Go FTP Server"
	s.driverFactory = factory
	return s
}

func (ftpServer *FTPServer) ListenAndServe() (err error) {
	laddr, err := net.ResolveTCPAddr("tcp", ftpServer.listenTo)
	if err != nil {
		log.Fatal(err)
	}
	listener, err := net.ListenTCP("tcp", laddr)
	if err != nil {
		log.Fatal(err)
	}
	for {
		ftpConn, err := ftpServer.accept(listener)
		if err != nil {
			break
		}
		go ftpConn.Serve()
	}
	return
}

func (ftpServer *FTPServer) accept(listener *net.TCPListener) (ftpConn *ftpConn, err error) {
	tcpConn, err := listener.AcceptTCP()
	if err == nil {
		ftpConn = newftpConn(tcpConn, ftpServer.driverFactory.NewDriver())
	}
	return
}

func buildTcpString(hostname string, port int) (result string) {
	if strings.Contains(hostname, ":") {
		// ipv6
		if port == 0 {
			result = "["+hostname+"]"
		} else {
			result = "["+hostname +"]:" + strconv.Itoa(port)
		}
	} else {
		// ipv4
		if port == 0 {
			result = hostname
		} else {
			result = hostname +":" + strconv.Itoa(port)
		}
	}
	return
}
<|endoftext|>


<commit_before>package mux

import (
	"net/http"

	errors "github.com/gogolfing/mux/errors"
	muxpath "github.com/gogolfing/mux/path"
)

type Mux struct {
	trie                    *trie
	MethodNotAllowedHandler http.Handler
	NotFoundHandler         http.Handler
}

func New() *Mux {
	return NewHandlers(nil, nil)
}

func NewHandlers(notFound, methodNotAllowed http.Handler) *Mux {
	return &Mux{
		newTrie(),
		notFound,
		methodNotAllowed,
	}
}

func (m *Mux) Root() *Route {
	return m.trie.subRoute("/")
}

func (m *Mux) HandleFunc(path string, handlerFunc http.HandlerFunc, methods ...string) *Route {
	return m.Handle(path, http.HandlerFunc(handlerFunc), methods...)
}

func (m *Mux) Handle(path string, handler http.Handler, methods ...string) *Route {
	return m.trie.handle(muxpath.EnsureRootSlash(path), handler, methods...)
}

func (m *Mux) SubRoute(path string) *Route {
	return m.trie.subRoute(muxpath.EnsureRootSlash(path))
}

func (m *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := muxpath.Clean(r.URL.Path)
	handler, err := m.trie.getHandler(r, path)
	if err != nil {
		m.serveError(w, r, err)
		return
	}
	if handler != nil {
		handler.ServeHTTP(w, r)
		return
	}
}

func (m *Mux) serveError(w http.ResponseWriter, r *http.Request, err error) {
	handler := m.getErrorHandler(err)
	if handler != nil {
		handler.ServeHTTP(w, r)
	}
}

func (m *Mux) getErrorHandler(err error) http.Handler {
	if handler, ok := err.(errors.ErrMethodNotAllowed); ok {
		if m.MethodNotAllowedHandler != nil {
			return m.MethodNotAllowedHandler
		}
		return handler
	}
	if err == errors.ErrNotFound {
		if m.NotFoundHandler != nil {
			return m.NotFoundHandler
		}
		return errors.ErrNotFound
	}
	return nil
}
<commit_msg>Fixes *Mux.Root().<commit_after>package mux

import (
	"net/http"

	errors "github.com/gogolfing/mux/errors"
	muxpath "github.com/gogolfing/mux/path"
)

type Mux struct {
	trie                    *trie
	MethodNotAllowedHandler http.Handler
	NotFoundHandler         http.Handler
}

func New() *Mux {
	return NewHandlers(nil, nil)
}

func NewHandlers(notFound, methodNotAllowed http.Handler) *Mux {
	return &Mux{
		newTrie(),
		notFound,
		methodNotAllowed,
	}
}

func (m *Mux) Root() *Route {
	return m.SubRoute("/")
}

func (m *Mux) HandleFunc(path string, handlerFunc http.HandlerFunc, methods ...string) *Route {
	return m.Handle(path, http.HandlerFunc(handlerFunc), methods...)
}

func (m *Mux) Handle(path string, handler http.Handler, methods ...string) *Route {
	return m.trie.handle(muxpath.EnsureRootSlash(path), handler, methods...)
}

func (m *Mux) SubRoute(path string) *Route {
	return m.trie.subRoute(muxpath.EnsureRootSlash(path))
}

func (m *Mux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := muxpath.Clean(r.URL.Path)
	handler, err := m.trie.getHandler(r, path)
	if err != nil {
		m.serveError(w, r, err)
		return
	}
	if handler != nil {
		handler.ServeHTTP(w, r)
		return
	}
}

func (m *Mux) serveError(w http.ResponseWriter, r *http.Request, err error) {
	handler := m.getErrorHandler(err)
	if handler != nil {
		handler.ServeHTTP(w, r)
	}
}

func (m *Mux) getErrorHandler(err error) http.Handler {
	if handler, ok := err.(errors.ErrMethodNotAllowed); ok {
		if m.MethodNotAllowedHandler != nil {
			return m.MethodNotAllowedHandler
		}
		return handler
	}
	if err == errors.ErrNotFound {
		if m.NotFoundHandler != nil {
			return m.NotFoundHandler
		}
		return errors.ErrNotFound
	}
	return nil
}
<|endoftext|>