<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package file

import (
	"fmt"
	"strings"

	"github.com/admpub/nging/application/dbschema"
	"github.com/admpub/nging/application/library/fileupdater"
	"github.com/admpub/nging/application/model/base"
	"github.com/webx-top/com"
	"github.com/webx-top/db"
	"github.com/webx-top/echo"
)

func NewEmbedded(ctx echo.Context, fileMdls ...*File) *Embedded {
	if ctx == nil {
		panic(`ctx is nil`)
	}
	var fileM *File
	if len(fileMdls) > 0 {
		fileM = fileMdls[0]
	} else {
		fileM = NewFile(ctx)
	}
	m := &Embedded{
		FileEmbedded: &dbschema.FileEmbedded{},
		base:         base.New(ctx),
		File:         fileM,
	}
	m.FileEmbedded.SetContext(ctx)
	return m
}

type Embedded struct {
	*dbschema.FileEmbedded
	base    *base.Base
	File    *File
	updater *fileupdater.FileUpdater
}

func (f *Embedded) Updater(table string, field string, tableID string) *fileupdater.FileUpdater {
	if f.updater == nil {
		f.updater = fileupdater.New(f)
	}
	f.updater.Set(table, field, tableID)
	return f.updater
}

// DeleteByTableID 删除嵌入文件
func (f *Embedded) DeleteByTableID(project string, table string, tableID string) error {
	_, err := f.ListByOffset(nil, nil, 0, -1, db.And(
		db.Cond{`table_id`: tableID},
		db.Cond{`table_name`: table},
		db.Cond{`project`: project},
	))
	if err != nil {
		return err
	}
	var ids []uint64
	for _, row := range f.Objects() {
		err = f.File.SetField(nil, `used_times`, db.Raw(`used_times-1`), db.And(
			db.Cond{`used_times`: db.Gt(0)},
			db.Cond{`id`: db.In(strings.Split(row.FileIds, `,`))},
		))
		if err != nil {
			return err
		}
		ids = append(ids, row.Id)
	}
	if len(ids) > 0 {
		err = f.Delete(nil, db.Cond{`id`: db.In(ids)})
	}
	return err
}

func (f *Embedded) UpdateByFileID(project string, table string, field string, tableID string, fileID uint64) error {
	err := f.File.UpdateUnrelation(project, table, field, tableID, fileID)
	if err != nil {
		return err
	}
	err = f.File.Incr(fileID)
	if err != nil {
		return err
	}
	m := &dbschema.FileEmbedded{}
	err = m.Get(nil, db.And(
		db.Cond{`table_id`: tableID},
		db.Cond{`table_name`: table},
		db.Cond{`field_name`: field},
	))
	if err != nil {
		if err != db.ErrNoMoreRows {
			return err
		}
		m.Reset()
		m.FieldName = field
		m.TableName = table
		m.Project = project
		m.TableId = tableID
		m.FileIds = fmt.Sprint(fileID)
		_, err = m.Add()
	}
	return err
}

func (f *Embedded) UpdateEmbedded(embedded bool, project string, table string, field string, tableID string, fileIds ...interface{}) (err error) {
	f.base.Begin()
	defer func() {
		f.base.End(err == nil)
	}()
	f.Use(f.base.Tx())
	f.File.Use(f.Trans())

	err = f.File.UpdateUnrelation(project, table, field, tableID, fileIds...)
	if err != nil {
		return err
	}

	m := &dbschema.FileEmbedded{}
	err = m.Use(f.Trans()).Get(nil, db.And(
		db.Cond{`table_id`: tableID},
		db.Cond{`table_name`: table},
		db.Cond{`field_name`: field},
	))
	if err != nil {
		if err != db.ErrNoMoreRows {
			return err
		}
		if len(fileIds) < 1 {
			return nil
		}
		// 不存在时，添加
		m.Reset()
		m.FieldName = field
		m.TableName = table
		m.Project = project
		m.TableId = tableID
		if embedded {
			m.Embedded = `Y`
		} else {
			m.Embedded = `N`
		}
		m.FileIds = ""
		err = f.File.Incr(fileIds...)
		if err != nil {
			return err
		}
		for _, v := range fileIds {
			m.FileIds += fmt.Sprintf("%v,", v)
		}
		m.FileIds = strings.TrimSuffix(m.FileIds, ",")
		_, err = m.Add()
		return err
	}
	isEmpty := len(fileIds) < 1
	if isEmpty { // 删除关联记录
		return f.DeleteByInstance(m)
	}
	var fidsString string
	fidList := make([]string, len(fileIds))
	for k, v := range fileIds {
		s := fmt.Sprint(v)
		fidList[k] = s
		fidsString += s + `,`
	}
	fidsString = strings.TrimSuffix(fidsString, ",")
	if m.FileIds == fidsString {
		return nil
	}
	ids := strings.Split(m.FileIds, ",")
	//已删除引用
	err = f.DeleteFileByIds(ids, fidList...)
	if err != nil {
		return err
	}
	//新增引用
	err = f.AddFileByIds(fidList, ids...)
	if err != nil {
		return err
	}
	m.FileIds = fidsString
	err = f.SetField(nil, `file_ids`, m.FileIds, db.Cond{`id`: m.Id})
	return err
}

// RelationEmbeddedFiles 关联嵌入的文件
// @param project 项目名称
// @param table 表名称
// @param field 被嵌入的字段名
// @param tableID 表中行主键ID
// @param v 内容
// @return
// @author AdamShen <swh@admpub.com>
func (f *Embedded) RelationEmbeddedFiles(project string, table string, field string, tableID string, v string) error {
	var (
		files []interface{}
		fids  []interface{} //旧文件ID
	)
	EmbeddedRes(v, func(file string, fid int64) {
		var exists bool
		if fid > 0 {
			exists = com.InSliceIface(fid, fids)
		} else {
			exists = com.InSliceIface(file, files)
		}
		if exists {
			return
		}
		files = append(files, file)
		if fid > 0 {
			fids = append(fids, fid)
		}
	})
	if len(fids) < 1 && len(files) > 0 {
		fids = f.File.GetIDByViewURLs(files)
	}
	err := f.UpdateEmbedded(true, project, table, field, tableID, fids...)
	return err
}

func (f *Embedded) RelationFiles(project string, table string, field string, tableID string, v string, seperator ...string) error {
	var (
		files []interface{}
		fids  []interface{} //旧文件ID
	)
	RelatedRes(v, func(file string, fid int64) {
		var exists bool
		if fid > 0 {
			exists = com.InSliceIface(fid, fids)
		} else {
			exists = com.InSliceIface(file, files)
		}
		if exists {
			return
		}
		files = append(files, file)
		if fid > 0 {
			fids = append(fids, fid)
		}
	}, seperator...)
	if len(fids) < 1 && len(files) > 0 {
		fids = f.File.GetIDByViewURLs(files)
	}
	err := f.UpdateEmbedded(false, project, table, field, tableID, fids...)
	return err
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package file

import (
	"fmt"
	"strings"

	"github.com/webx-top/com"
	"github.com/webx-top/db"
	"github.com/webx-top/echo"

	"github.com/admpub/nging/application/dbschema"
	"github.com/admpub/nging/application/library/fileupdater"
	"github.com/admpub/nging/application/model/base"
)

func NewEmbedded(ctx echo.Context, fileMdls ...*File) *Embedded {
	if ctx == nil {
		panic(`ctx is nil`)
	}
	var fileM *File
	if len(fileMdls) > 0 {
		fileM = fileMdls[0]
	} else {
		fileM = NewFile(ctx)
	}
	m := &Embedded{
		FileEmbedded: &dbschema.FileEmbedded{},
		base:         base.New(ctx),
		File:         fileM,
	}
	m.FileEmbedded.SetContext(ctx)
	return m
}

type Embedded struct {
	*dbschema.FileEmbedded
	base    *base.Base
	File    *File
	updater *fileupdater.FileUpdater
}

func (f *Embedded) Updater(table string, field string, tableID string) *fileupdater.FileUpdater {
	if f.updater == nil {
		f.updater = fileupdater.New(f)
	}
	f.updater.Set(table, field, tableID)
	return f.updater
}

// DeleteByTableID 删除嵌入文件
func (f *Embedded) DeleteByTableID(project string, table string, tableID string) error {
	_, err := f.ListByOffset(nil, nil, 0, -1, db.And(
		db.Cond{`table_id`: tableID},
		db.Cond{`table_name`: table},
		db.Cond{`project`: project},
	))
	if err != nil {
		return err
	}
	var ids []uint64
	for _, row := range f.Objects() {
		err = f.File.SetField(nil, `used_times`, db.Raw(`used_times-1`), db.And(
			db.Cond{`used_times`: db.Gt(0)},
			db.Cond{`id`: db.In(strings.Split(row.FileIds, `,`))},
		))
		if err != nil {
			return err
		}
		ids = append(ids, row.Id)
	}
	if len(ids) > 0 {
		err = f.Delete(nil, db.Cond{`id`: db.In(ids)})
	}
	return err
}

func (f *Embedded) UpdateByFileID(project string, table string, field string, tableID string, fileID uint64) error {
	err := f.File.UpdateUnrelation(project, table, field, tableID, fileID)
	if err != nil {
		return err
	}
	err = f.File.Incr(fileID)
	if err != nil {
		return err
	}
	m := &dbschema.FileEmbedded{}
	err = m.Get(nil, db.And(
		db.Cond{`table_id`: tableID},
		db.Cond{`table_name`: table},
		db.Cond{`field_name`: field},
	))
	if err != nil {
		if err != db.ErrNoMoreRows {
			return err
		}
		m.Reset()
		m.FieldName = field
		m.TableName = table
		m.Project = project
		m.TableId = tableID
		m.FileIds = fmt.Sprint(fileID)
		_, err = m.Add()
	}
	return err
}

func (f *Embedded) UpdateEmbedded(embedded bool, project string, table string, field string, tableID string, fileIds ...interface{}) (err error) {
	f.base.Begin()
	defer func() {
		f.base.End(err == nil)
	}()
	f.Use(f.base.Tx())
	f.File.Use(f.Trans())

	err = f.File.UpdateUnrelation(project, table, field, tableID, fileIds...)
	if err != nil {
		return err
	}

	m := &dbschema.FileEmbedded{}
	err = m.Use(f.Trans()).Get(nil, db.And(
		db.Cond{`table_id`: tableID},
		db.Cond{`table_name`: table},
		db.Cond{`field_name`: field},
	))
	if err != nil {
		if err != db.ErrNoMoreRows {
			return err
		}
		if len(fileIds) < 1 {
			return nil
		}
		// 不存在时，添加
		m.Reset()
		m.FieldName = field
		m.TableName = table
		m.Project = project
		m.TableId = tableID
		if embedded {
			m.Embedded = `Y`
		} else {
			m.Embedded = `N`
		}
		m.FileIds = ""
		err = f.File.Incr(fileIds...)
		if err != nil {
			return err
		}
		for _, v := range fileIds {
			m.FileIds += fmt.Sprintf("%v,", v)
		}
		m.FileIds = strings.TrimSuffix(m.FileIds, ",")
		_, err = m.Add()
		return err
	}
	isEmpty := len(fileIds) < 1
	if isEmpty { // 删除关联记录
		return f.DeleteByInstance(m)
	}
	var fidsString string
	fidList := make([]string, len(fileIds))
	for k, v := range fileIds {
		s := fmt.Sprint(v)
		fidList[k] = s
		fidsString += s + `,`
	}
	fidsString = strings.TrimSuffix(fidsString, ",")
	if m.FileIds == fidsString {
		return nil
	}
	ids := strings.Split(m.FileIds, ",")
	//新增引用
	err = f.AddFileByIds(fidList, ids...)
	if err != nil {
		return err
	}
	//已删除引用
	err = f.DeleteFileByIds(ids, fidList...)
	if err != nil {
		return err
	}
	m.FileIds = fidsString
	err = f.SetField(nil, `file_ids`, m.FileIds, db.Cond{`id`: m.Id})
	return err
}

// RelationEmbeddedFiles 关联嵌入的文件
// @param project 项目名称
// @param table 表名称
// @param field 被嵌入的字段名
// @param tableID 表中行主键ID
// @param v 内容
// @return
// @author AdamShen <swh@admpub.com>
func (f *Embedded) RelationEmbeddedFiles(project string, table string, field string, tableID string, v string) error {
	var (
		files []interface{}
		fids  []interface{} //旧文件ID
	)
	EmbeddedRes(v, func(file string, fid int64) {
		var exists bool
		if fid > 0 {
			exists = com.InSliceIface(fid, fids)
		} else {
			exists = com.InSliceIface(file, files)
		}
		if exists {
			return
		}
		files = append(files, file)
		if fid > 0 {
			fids = append(fids, fid)
		}
	})
	if len(fids) < 1 && len(files) > 0 {
		fids = f.File.GetIDByViewURLs(files)
	}
	err := f.UpdateEmbedded(true, project, table, field, tableID, fids...)
	return err
}

func (f *Embedded) RelationFiles(project string, table string, field string, tableID string, v string, seperator ...string) error {
	var (
		files []interface{}
		fids  []interface{} //旧文件ID
	)
	RelatedRes(v, func(file string, fid int64) {
		var exists bool
		if fid > 0 {
			exists = com.InSliceIface(fid, fids)
		} else {
			exists = com.InSliceIface(file, files)
		}
		if exists {
			return
		}
		files = append(files, file)
		if fid > 0 {
			fids = append(fids, fid)
		}
	}, seperator...)
	if len(fids) < 1 && len(files) > 0 {
		fids = f.File.GetIDByViewURLs(files)
	}
	err := f.UpdateEmbedded(false, project, table, field, tableID, fids...)
	return err
}
<|endoftext|>


<commit_before>package pbWriter

import (
	"github.com/dh1tw/remoteAudio/audiocodec"
)

// Option is the type for a function option
type Option func(*Options)

// Options is the data structure holding the optional values. The
// values are typcially set by calling the functional options.
type Options struct {
	DeviceName      string
	Encoder         audiocodec.Encoder
	Channels        int
	Samplerate      float64
	FramesPerBuffer int
	UserID          string
	ToWireCb        func([]byte)
}

// Channels is a functional option to set the amount of channels to be used
// with the audio device. Typically this is either Mono (1) or Stereo (2).
// Make sure that your audio device supports the specified amount of channels.
func Channels(chs int) Option {
	return func(args *Options) {
		args.Channels = chs
	}
}

// Samplerate is a functional option to set the sampling rate of the
// audio device. Make sure your audio device supports the specified sampling
// rate.
func Samplerate(s float64) Option {
	return func(args *Options) {
		args.Samplerate = s
	}
}

// FramesPerBuffer is a functional option which sets the amount of sample frames
// our audio device will request / provide when executing the callback.
// Example: A buffer with 960 frames at 48000kHz / stereo contains
// 1920 samples and results in 20ms Audio.
func FramesPerBuffer(s int) Option {
	return func(args *Options) {
		args.FramesPerBuffer = s
	}
}

// Encoder is a functional option to set a specific decoder. By default,
// the opus decoder is used.
func Encoder(enc audiocodec.Encoder) Option {
	return func(args *Options) {
		args.Encoder = enc
	}
}

// UserID is a functional option to set the UserID. All serialized protobufs
// contain the UserID to mark their origin.
func UserID(s string) Option {
	return func(args *Options) {
		args.UserID = s
	}
}

// ToWireCb is a functional option to set the callback which will be executed
// when data has been serialized and is ready to be send to the network.
func ToWireCb(cb func([]byte)) Option {
	return func(args *Options) {
		args.ToWireCb = cb
	}
}
<commit_msg>fixed typo<commit_after>package pbWriter

import (
	"github.com/dh1tw/remoteAudio/audiocodec"
)

// Option is the type for a function option
type Option func(*Options)

// Options is the data structure holding the optional values. The
// values are typically set by calling the functional options.
type Options struct {
	DeviceName      string
	Encoder         audiocodec.Encoder
	Channels        int
	Samplerate      float64
	FramesPerBuffer int
	UserID          string
	ToWireCb        func([]byte)
}

// Channels is a functional option to set the amount of channels to be used
// with the audio device. Typically this is either Mono (1) or Stereo (2).
// Make sure that your audio device supports the specified amount of channels.
func Channels(chs int) Option {
	return func(args *Options) {
		args.Channels = chs
	}
}

// Samplerate is a functional option to set the sampling rate of the
// audio device. Make sure your audio device supports the specified sampling
// rate.
func Samplerate(s float64) Option {
	return func(args *Options) {
		args.Samplerate = s
	}
}

// FramesPerBuffer is a functional option which sets the amount of sample frames
// our audio device will request / provide when executing the callback.
// Example: A buffer with 960 frames at 48000kHz / stereo contains
// 1920 samples and results in 20ms Audio.
func FramesPerBuffer(s int) Option {
	return func(args *Options) {
		args.FramesPerBuffer = s
	}
}

// Encoder is a functional option to set a specific decoder. By default,
// the opus decoder is used.
func Encoder(enc audiocodec.Encoder) Option {
	return func(args *Options) {
		args.Encoder = enc
	}
}

// UserID is a functional option to set the UserID. All serialized protobufs
// contain the UserID to mark their origin.
func UserID(s string) Option {
	return func(args *Options) {
		args.UserID = s
	}
}

// ToWireCb is a functional option to set the callback which will be executed
// when data has been serialized and is ready to be send to the network.
func ToWireCb(cb func([]byte)) Option {
	return func(args *Options) {
		args.ToWireCb = cb
	}
}
<|endoftext|>


<commit_before>package loda

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/lodastack/agent/agent/common"
	"github.com/lodastack/agent/agent/goplugin"
	"github.com/lodastack/agent/agent/plugins"
	"github.com/lodastack/log"
	"github.com/lodastack/models"
)

var Zerotimes = 0

func Get(url string) (b []byte, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		err = fmt.Errorf("Get url failed: %s  not found", url)
		return
	}
	b, err = ioutil.ReadAll(resp.Body)
	return
}

func Post(url string, data []byte) ([]byte, error) {
	body := bytes.NewBuffer([]byte(data))
	client := &http.Client{}
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json;charset=utf-8")

	res, err := client.Do(req)

	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		err = fmt.Errorf("Post url failed: %s code: %d", url, res.StatusCode)
		return nil, err
	}

	result, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	return result, nil
}

func Ns() ([]string, error) {
	var res []string
	host, err := common.Hostname()
	if err != nil {
		return res, err
	}
	url := fmt.Sprintf("http://%s/api/v1/agent/ns", common.Conf.RegistryAddr)
	data := make(map[string]string)
	data["hostname"] = host
	data["ips"] = strings.Join(common.GetIpList(), ",")
	byteData, err := json.Marshal(data)
	b, err := Post(url, byteData)
	if err != nil {
		return res, err
	}
	var response models.Response
	err = json.Unmarshal(b, &response)
	if err != nil {
		log.Warning("json.Marshal Ns failed: ", err)
		return res, err
	}

	resp, ok := response.Data.(map[string]string)
	if !ok {
		return res, fmt.Errorf("response data is not a map type")
	}

	var ids []string
	for ns, id := range resp {
		res = append(res, ns)
		ids = append(ids, id)
	}
	common.SetUUID(ids)
	return res, nil
}

func pullResources(ns string) (res []map[string]interface{}, err error) {
	url := fmt.Sprintf("http://%s/api/v1/resource/%s/collect", common.Conf.RegistryAddr, ns)
	b, err := Get(url)
	if err != nil {
		return
	}
	var response models.Response
	err = json.Unmarshal(b, &response)
	if err != nil {
		return
	}
	res, ok := response.Data.([]map[string]interface{})
	if !ok {
		err = fmt.Errorf("response data is not a map slice type")
		return
	}

	if len(res) == 0 {
		err = fmt.Errorf("no items under this namespace")
		Zerotimes++
	} else {
		Zerotimes = 0
	}
	return
}

func MonitorItems() (ports []common.PortMonitor,
	procs []common.ProcMonitor,
	pluginCollectors map[string]plugins.Collector,
	gopluginCollectors map[string]goplugin.Collector,
	intervals map[string]int, err error) {
	nss := common.GetNamespaces()
	pluginCollectors = make(map[string]plugins.Collector)
	pluginInfo := make(map[string]bool)
	gopluginCollectors = make(map[string]goplugin.Collector)
	intervals = make(map[string]int)

	for _, ns := range nss {
		err = nil
		var items []map[string]interface{}
		items, err = pullResources(ns)
		if err != nil {
			log.Error("failed to get resources from registry, ns: ", ns, " err: ", err)
			continue
		}

		for _, item := range items {
			monitorType, ok := item["measurement_type"].(string)
			if !ok {
				log.Warning("measurement_type is not a string: ", item["measurement_type"])
				continue
			}
			b, err := json.Marshal(item)
			if err != nil {
				log.Warning("json.Marshal item failed: ", err)
				continue
			}
			switch monitorType {
			case common.TYPE_PORT:
				if port, err := parsePort(b); err == nil {
					port.Namespace = ns
					ports = append(ports, port)
				} else {
					log.Warning("parsePort failed: ", err)
				}
			case common.TYPE_PROC:
				if proc, err := parseProc(b); err == nil {
					proc.Namespace = ns
					procs = append(procs, proc)
				} else {
					log.Warning("parseProc failed: ", err)
				}
			case common.TYPE_PLUGIN:
				if col, err := parsePlugin(b); err == nil {
					col.Namespace = ns
					pluginCollectors[ns+"|"+col.ProjectName] = col
					pluginInfo[ns+"|"+col.Repo] = true
				} else {
					log.Warning("get plugin collection failed: ", err)
				}
			case common.TYPE_GOPLUGIN:
				if col, err := parseGoPlugin(b); err == nil {
					col.Namespace = ns
					gopluginCollectors[ns+"|"+col.Name] = col
				} else {
					log.Warning("parse goplugin collection failed: ", err)
				}
			default:
				if interval, ok := item["interval"].(int); ok {
					intervals[monitorType] = interval
				}
			}
		}
		getAlarmPlugin(ns, pluginCollectors, pluginInfo)
	}

	common.SetPluginInfo(pluginInfo)
	for _, t := range append(common.SYS_TYPES, common.TYPE_PORT, common.TYPE_PROC) {
		if intervals[t] == 0 {
			intervals[t] = common.DEFAULT_INTERVAL[t]
		}
	}
	return
}

func getAlarmPlugin(ns string, pluginCollectors map[string]plugins.Collector, pluginInfo map[string]bool) {
	url := fmt.Sprintf("http://%s/api/v1/resource?ns=%s&resource=alarm", common.Conf.RegistryAddr, ns)
	b, err := Get(url)
	if err != nil {
		return
	}
	var response models.Response
	err = json.Unmarshal(b, &response)
	if err != nil {
		log.Error("Unmarshal from alarm failed: ", err)
		return
	}

	alarms, ok := response.Data.([]map[string]interface{})
	if !ok {
		err = fmt.Errorf("response data is not a map slice type")
		return
	}

	for _, alarm := range alarms {
		ac, ok := alarm["actions"]
		if !ok {
			continue
		}
		actions, ok := ac.([]interface{})
		if !ok {
			continue
		}
		for _, a := range actions {
			action, ok := a.(map[string]interface{})
			if !ok {
				continue
			}
			t, ok := action["type"]
			if !ok {
				continue
			}
			if t1, ok := t.(string); ok && t1 == "AGENT" {
				b, err := json.Marshal(action)
				if err != nil {
					log.Error("json.Marshal action failed: ", err)
					continue
				}
				var col plugins.Collector
				err = json.Unmarshal(b, &col)
				if err != nil {
					log.Error("json.Unmarshal to plugins.Collector from alarm failed: ", err)
					continue
				}
				col, err = formatPlugin(col)
				if err != nil {
					continue
				}
				col.Cycle = 0
				pluginInfo[ns+"|"+col.Repo] = true
				//col.Namespace = ns
				//pluginCollectors[ns+"|"+col.ProjectName] = col
			}
		}
	}
}

func parsePlugin(b []byte) (res plugins.Collector, err error) {
	if err = json.Unmarshal(b, &res); err != nil {
		return
	}
	var cycle int
	if cycle, err = strconv.Atoi(res.StrCycle); err != nil {
		return
	}
	res.Cycle = cycle
	res, err = formatPlugin(res)
	return
}

func formatPlugin(p plugins.Collector) (plugins.Collector, error) {
	if strings.Contains(p.Repo, ":") {
		s := strings.Split(p.Repo, ":")[1]
		p.Repo = s[:len(s)-4]
	}
	if strings.Count(p.Repo, "/") == 1 {
		p.ProjectName = strings.Split(p.Repo, "/")[1]
	}
	if p.Parameters != "" {
		for _, s := range strings.Split(p.Parameters, " ") {
			if s != "" {
				if strings.ContainsAny(s, ";|&<>`") {
					return p, errors.New("Invalid parameter")
				}
				p.Param = append(p.Param, s)
			}
		}
	}
	return p, nil
}

func parseProc(b []byte) (res common.ProcMonitor, err error) {
	err = json.Unmarshal(b, &res)
	return
}

func parsePort(b []byte) (res common.PortMonitor, err error) {
	err = json.Unmarshal(b, &res)
	return
}

func parseGoPlugin(b []byte) (res goplugin.Collector, err error) {
	err = json.Unmarshal(b, &res)
	return
}
<commit_msg>print data when registry failed<commit_after>package loda

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/lodastack/agent/agent/common"
	"github.com/lodastack/agent/agent/goplugin"
	"github.com/lodastack/agent/agent/plugins"
	"github.com/lodastack/log"
	"github.com/lodastack/models"
)

var Zerotimes = 0

func Get(url string) (b []byte, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		err = fmt.Errorf("Get url failed: %s  not found", url)
		return
	}
	b, err = ioutil.ReadAll(resp.Body)
	return
}

func Post(url string, data []byte) ([]byte, error) {
	body := bytes.NewBuffer([]byte(data))
	client := &http.Client{}
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json;charset=utf-8")

	res, err := client.Do(req)

	if err != nil {
		return nil, err
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		err = fmt.Errorf("Post url failed: %s code: %d", url, res.StatusCode)
		return nil, err
	}

	result, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	return result, nil
}

func Ns() ([]string, error) {
	var res []string
	host, err := common.Hostname()
	if err != nil {
		return res, err
	}
	url := fmt.Sprintf("http://%s/api/v1/agent/ns", common.Conf.RegistryAddr)
	data := make(map[string]string)
	data["hostname"] = host
	data["ips"] = strings.Join(common.GetIpList(), ",")
	byteData, err := json.Marshal(data)
	b, err := Post(url, byteData)
	if err != nil {
		return res, err
	}
	var response models.Response
	err = json.Unmarshal(b, &response)
	if err != nil {
		log.Warning("json.Marshal Ns failed: ", err)
		return res, err
	}

	resp, ok := response.Data.(map[string]string)
	if !ok {
		return res, fmt.Errorf("response data is not a map type: %v", response.Data)
	}

	var ids []string
	for ns, id := range resp {
		res = append(res, ns)
		ids = append(ids, id)
	}
	common.SetUUID(ids)
	return res, nil
}

func pullResources(ns string) (res []map[string]interface{}, err error) {
	url := fmt.Sprintf("http://%s/api/v1/resource/%s/collect", common.Conf.RegistryAddr, ns)
	b, err := Get(url)
	if err != nil {
		return
	}
	var response models.Response
	err = json.Unmarshal(b, &response)
	if err != nil {
		return
	}
	res, ok := response.Data.([]map[string]interface{})
	if !ok {
		err = fmt.Errorf("response data is not a map slice type: %v", response.Data)
		return
	}

	if len(res) == 0 {
		err = fmt.Errorf("no items under this namespace")
		Zerotimes++
	} else {
		Zerotimes = 0
	}
	return
}

func MonitorItems() (ports []common.PortMonitor,
	procs []common.ProcMonitor,
	pluginCollectors map[string]plugins.Collector,
	gopluginCollectors map[string]goplugin.Collector,
	intervals map[string]int, err error) {
	nss := common.GetNamespaces()
	pluginCollectors = make(map[string]plugins.Collector)
	pluginInfo := make(map[string]bool)
	gopluginCollectors = make(map[string]goplugin.Collector)
	intervals = make(map[string]int)

	for _, ns := range nss {
		err = nil
		var items []map[string]interface{}
		items, err = pullResources(ns)
		if err != nil {
			log.Error("failed to get resources from registry, ns: ", ns, " err: ", err)
			continue
		}

		for _, item := range items {
			monitorType, ok := item["measurement_type"].(string)
			if !ok {
				log.Warning("measurement_type is not a string: ", item["measurement_type"])
				continue
			}
			b, err := json.Marshal(item)
			if err != nil {
				log.Warning("json.Marshal item failed: ", err)
				continue
			}
			switch monitorType {
			case common.TYPE_PORT:
				if port, err := parsePort(b); err == nil {
					port.Namespace = ns
					ports = append(ports, port)
				} else {
					log.Warning("parsePort failed: ", err)
				}
			case common.TYPE_PROC:
				if proc, err := parseProc(b); err == nil {
					proc.Namespace = ns
					procs = append(procs, proc)
				} else {
					log.Warning("parseProc failed: ", err)
				}
			case common.TYPE_PLUGIN:
				if col, err := parsePlugin(b); err == nil {
					col.Namespace = ns
					pluginCollectors[ns+"|"+col.ProjectName] = col
					pluginInfo[ns+"|"+col.Repo] = true
				} else {
					log.Warning("get plugin collection failed: ", err)
				}
			case common.TYPE_GOPLUGIN:
				if col, err := parseGoPlugin(b); err == nil {
					col.Namespace = ns
					gopluginCollectors[ns+"|"+col.Name] = col
				} else {
					log.Warning("parse goplugin collection failed: ", err)
				}
			default:
				if interval, ok := item["interval"].(int); ok {
					intervals[monitorType] = interval
				}
			}
		}
		getAlarmPlugin(ns, pluginCollectors, pluginInfo)
	}

	common.SetPluginInfo(pluginInfo)
	for _, t := range append(common.SYS_TYPES, common.TYPE_PORT, common.TYPE_PROC) {
		if intervals[t] == 0 {
			intervals[t] = common.DEFAULT_INTERVAL[t]
		}
	}
	return
}

func getAlarmPlugin(ns string, pluginCollectors map[string]plugins.Collector, pluginInfo map[string]bool) {
	url := fmt.Sprintf("http://%s/api/v1/resource?ns=%s&resource=alarm", common.Conf.RegistryAddr, ns)
	b, err := Get(url)
	if err != nil {
		return
	}
	var response models.Response
	err = json.Unmarshal(b, &response)
	if err != nil {
		log.Error("Unmarshal from alarm failed: ", err)
		return
	}

	alarms, ok := response.Data.([]map[string]interface{})
	if !ok {
		err = fmt.Errorf("response data is not a map slice type")
		return
	}

	for _, alarm := range alarms {
		ac, ok := alarm["actions"]
		if !ok {
			continue
		}
		actions, ok := ac.([]interface{})
		if !ok {
			continue
		}
		for _, a := range actions {
			action, ok := a.(map[string]interface{})
			if !ok {
				continue
			}
			t, ok := action["type"]
			if !ok {
				continue
			}
			if t1, ok := t.(string); ok && t1 == "AGENT" {
				b, err := json.Marshal(action)
				if err != nil {
					log.Error("json.Marshal action failed: ", err)
					continue
				}
				var col plugins.Collector
				err = json.Unmarshal(b, &col)
				if err != nil {
					log.Error("json.Unmarshal to plugins.Collector from alarm failed: ", err)
					continue
				}
				col, err = formatPlugin(col)
				if err != nil {
					continue
				}
				col.Cycle = 0
				pluginInfo[ns+"|"+col.Repo] = true
				//col.Namespace = ns
				//pluginCollectors[ns+"|"+col.ProjectName] = col
			}
		}
	}
}

func parsePlugin(b []byte) (res plugins.Collector, err error) {
	if err = json.Unmarshal(b, &res); err != nil {
		return
	}
	var cycle int
	if cycle, err = strconv.Atoi(res.StrCycle); err != nil {
		return
	}
	res.Cycle = cycle
	res, err = formatPlugin(res)
	return
}

func formatPlugin(p plugins.Collector) (plugins.Collector, error) {
	if strings.Contains(p.Repo, ":") {
		s := strings.Split(p.Repo, ":")[1]
		p.Repo = s[:len(s)-4]
	}
	if strings.Count(p.Repo, "/") == 1 {
		p.ProjectName = strings.Split(p.Repo, "/")[1]
	}
	if p.Parameters != "" {
		for _, s := range strings.Split(p.Parameters, " ") {
			if s != "" {
				if strings.ContainsAny(s, ";|&<>`") {
					return p, errors.New("Invalid parameter")
				}
				p.Param = append(p.Param, s)
			}
		}
	}
	return p, nil
}

func parseProc(b []byte) (res common.ProcMonitor, err error) {
	err = json.Unmarshal(b, &res)
	return
}

func parsePort(b []byte) (res common.PortMonitor, err error) {
	err = json.Unmarshal(b, &res)
	return
}

func parseGoPlugin(b []byte) (res goplugin.Collector, err error) {
	err = json.Unmarshal(b, &res)
	return
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clientv3

import (
	"sync"

	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
)

//
// Tx.If(
//  Compare(Value(k1), ">", v1),
//  Compare(Version(k1), "=", 2)
// ).Then(
//  OpPut(k2,v2), OpPut(k3,v3)
// ).Else(
//  OpPut(k4,v4), OpPut(k5,v5)
// ).Commit()
type Txn interface {
	// If takes a list of comparison. If all comparisons passed in succeed,
	// the operations passed into Then() will be executed. Or the operations
	// passed into Else() will be executed.
	If(cs ...Cmp) Txn

	// Then takes a list of operations. The Ops list will be executed, if the
	// comparisons passed in If() succeed.
	Then(ops ...Op) Txn

	// Else takes a list of operations. The Ops list will be executed, if the
	// comparisons passed in If() fail.
	Else(ops ...Op) Txn

	// Commit tries to commit the transaction.
	Commit() (*TxnResponse, error)

	// TODO: add a Do for shortcut the txn without any condition?
}

type txn struct {
	kv  *kv
	ctx context.Context

	mu    sync.Mutex
	cif   bool
	cthen bool
	celse bool

	isWrite bool

	cmps []*pb.Compare

	sus []*pb.RequestUnion
	fas []*pb.RequestUnion
}

func (txn *txn) If(cs ...Cmp) Txn {
	txn.mu.Lock()
	defer txn.mu.Unlock()

	if txn.cif {
		panic("cannot call If twice!")
	}

	if txn.cthen {
		panic("cannot call If after Then!")
	}

	if txn.celse {
		panic("cannot call If after Else!")
	}

	txn.cif = true

	for _, cmp := range cs {
		txn.cmps = append(txn.cmps, (*pb.Compare)(&cmp))
	}

	return txn
}

func (txn *txn) Then(ops ...Op) Txn {
	txn.mu.Lock()
	defer txn.mu.Unlock()

	if txn.cthen {
		panic("cannot call Then twice!")
	}
	if txn.celse {
		panic("cannot call Then after Else!")
	}

	txn.cthen = true

	for _, op := range ops {
		txn.isWrite = txn.isWrite || op.isWrite()
		txn.sus = append(txn.sus, op.toRequestUnion())
	}

	return txn
}

func (txn *txn) Else(ops ...Op) Txn {
	txn.mu.Lock()
	defer txn.mu.Unlock()

	if txn.celse {
		panic("cannot call Else twice!")
	}

	txn.celse = true

	for _, op := range ops {
		txn.isWrite = txn.isWrite || op.isWrite()
		txn.fas = append(txn.fas, op.toRequestUnion())
	}

	return txn
}

func (txn *txn) Commit() (*TxnResponse, error) {
	txn.mu.Lock()
	defer txn.mu.Unlock()

	kv := txn.kv

	for {
		r := &pb.TxnRequest{Compare: txn.cmps, Success: txn.sus, Failure: txn.fas}
		resp, err := kv.getRemote().Txn(txn.ctx, r)
		if err == nil {
			return (*TxnResponse)(resp), nil
		}

		if isRPCError(err) {
			return nil, err
		}

		if txn.isWrite {
			go kv.switchRemote(err)
			return nil, err
		}

		if nerr := kv.switchRemote(err); nerr != nil {
			return nil, nerr
		}
	}
}
<commit_msg>clientv3: copy correct pointers into txn comparisons<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clientv3

import (
	"sync"

	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/net/context"
	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
)

//
// Tx.If(
//  Compare(Value(k1), ">", v1),
//  Compare(Version(k1), "=", 2)
// ).Then(
//  OpPut(k2,v2), OpPut(k3,v3)
// ).Else(
//  OpPut(k4,v4), OpPut(k5,v5)
// ).Commit()
type Txn interface {
	// If takes a list of comparison. If all comparisons passed in succeed,
	// the operations passed into Then() will be executed. Or the operations
	// passed into Else() will be executed.
	If(cs ...Cmp) Txn

	// Then takes a list of operations. The Ops list will be executed, if the
	// comparisons passed in If() succeed.
	Then(ops ...Op) Txn

	// Else takes a list of operations. The Ops list will be executed, if the
	// comparisons passed in If() fail.
	Else(ops ...Op) Txn

	// Commit tries to commit the transaction.
	Commit() (*TxnResponse, error)

	// TODO: add a Do for shortcut the txn without any condition?
}

type txn struct {
	kv  *kv
	ctx context.Context

	mu    sync.Mutex
	cif   bool
	cthen bool
	celse bool

	isWrite bool

	cmps []*pb.Compare

	sus []*pb.RequestUnion
	fas []*pb.RequestUnion
}

func (txn *txn) If(cs ...Cmp) Txn {
	txn.mu.Lock()
	defer txn.mu.Unlock()

	if txn.cif {
		panic("cannot call If twice!")
	}

	if txn.cthen {
		panic("cannot call If after Then!")
	}

	if txn.celse {
		panic("cannot call If after Else!")
	}

	txn.cif = true

	for i := range cs {
		txn.cmps = append(txn.cmps, (*pb.Compare)(&cs[i]))
	}

	return txn
}

func (txn *txn) Then(ops ...Op) Txn {
	txn.mu.Lock()
	defer txn.mu.Unlock()

	if txn.cthen {
		panic("cannot call Then twice!")
	}
	if txn.celse {
		panic("cannot call Then after Else!")
	}

	txn.cthen = true

	for _, op := range ops {
		txn.isWrite = txn.isWrite || op.isWrite()
		txn.sus = append(txn.sus, op.toRequestUnion())
	}

	return txn
}

func (txn *txn) Else(ops ...Op) Txn {
	txn.mu.Lock()
	defer txn.mu.Unlock()

	if txn.celse {
		panic("cannot call Else twice!")
	}

	txn.celse = true

	for _, op := range ops {
		txn.isWrite = txn.isWrite || op.isWrite()
		txn.fas = append(txn.fas, op.toRequestUnion())
	}

	return txn
}

func (txn *txn) Commit() (*TxnResponse, error) {
	txn.mu.Lock()
	defer txn.mu.Unlock()

	kv := txn.kv

	for {
		r := &pb.TxnRequest{Compare: txn.cmps, Success: txn.sus, Failure: txn.fas}
		resp, err := kv.getRemote().Txn(txn.ctx, r)
		if err == nil {
			return (*TxnResponse)(resp), nil
		}

		if isRPCError(err) {
			return nil, err
		}

		if txn.isWrite {
			go kv.switchRemote(err)
			return nil, err
		}

		if nerr := kv.switchRemote(err); nerr != nil {
			return nil, nerr
		}
	}
}
<|endoftext|>


<commit_before>package signatory

import (
	"crypto/hmac"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"time"

	"github.com/IABTechLab/adscert/internal/adscerterrors"
	"github.com/IABTechLab/adscert/internal/formats"
	"github.com/IABTechLab/adscert/internal/logger"
	"github.com/IABTechLab/adscert/pkg/adscert/api"
	"github.com/IABTechLab/adscert/pkg/adscert/discovery"
	"github.com/IABTechLab/adscert/pkg/adscert/metrics"
	"github.com/benbjohnson/clock"
)

func NewLocalAuthenticatedConnectionsSignatory(
	originCallsign string,
	reader io.Reader,
	clock clock.Clock,
	dnsResolver discovery.DNSResolver,
	domainStore discovery.DomainStore,
	domainCheckInterval time.Duration,
	domainRenewalInterval time.Duration,
	base64PrivateKeys []string) AuthenticatedConnectionsSignatory {

	return &localAuthenticatedConnectionsSignatory{
		originCallsign:      originCallsign,
		secureRandom:        reader,
		clock:               clock,
		counterpartyManager: discovery.NewDefaultDomainIndexer(dnsResolver, domainStore, domainCheckInterval, domainRenewalInterval, base64PrivateKeys),
	}
}

type localAuthenticatedConnectionsSignatory struct {
	originCallsign string
	secureRandom   io.Reader
	clock          clock.Clock

	counterpartyManager discovery.DomainIndexer
}

func (s *localAuthenticatedConnectionsSignatory) SignAuthenticatedConnection(request *api.AuthenticatedConnectionSignatureRequest) (*api.AuthenticatedConnectionSignatureResponse, error) {

	var err error
	startTime := time.Now()
	response := &api.AuthenticatedConnectionSignatureResponse{RequestInfo: request.RequestInfo}

	if request.RequestInfo == nil || request.RequestInfo.InvokingDomain == "" || len(request.RequestInfo.UrlHash) == 0 {
		response.SignatureOperationStatus = api.SignatureOperationStatus_SIGNATURE_OPERATION_STATUS_MALFORMED_REQUEST
		return response, errors.New("required parameters are missing")
	}

	// add nonce and timestamp if not already provided in the request
	// this is the typical case to keep the client's usage simple
	if request.Timestamp == "" {
		request.Timestamp = s.clock.Now().UTC().Format("060102T150405")
	}
	if request.Nonce == "" {
		if request.Nonce, err = s.generateNonce(); err != nil {
			metrics.RecordSigning(adscerterrors.ErrSigningGenerateNonce)
		}
	}

	domainInfos, err := s.counterpartyManager.LookupIdentitiesForDomain(request.RequestInfo.InvokingDomain)
	if err != nil || len(domainInfos) == 0 {
		logger.Infof("counterparty lookup error")
		metrics.RecordSigning(adscerterrors.ErrSigningCounterpartyLookup)
		response.SignatureOperationStatus = api.SignatureOperationStatus_SIGNATURE_OPERATION_STATUS_SIGNATORY_INTERNAL_ERROR
		return response, err
	}

	for _, domainInfo := range domainInfos {
		signatureInfo, err := s.signSingleMessage(request, domainInfo)
		if err != nil {
			metrics.RecordSigning(adscerterrors.ErrSigningEmbossMessage)
			response.SignatureOperationStatus = api.SignatureOperationStatus_SIGNATURE_OPERATION_STATUS_SIGNATORY_INTERNAL_ERROR
			return response, err
		}
		response.RequestInfo.SignatureInfo = append(response.RequestInfo.SignatureInfo, signatureInfo)
	}

	metrics.RecordSigning(nil)
	metrics.RecordSigningTime(time.Since(startTime))
	response.SignatureOperationStatus = api.SignatureOperationStatus_SIGNATURE_OPERATION_STATUS_OK
	return response, nil
}

func (s *localAuthenticatedConnectionsSignatory) signSingleMessage(request *api.AuthenticatedConnectionSignatureRequest, domainInfo discovery.DomainInfo) (sigInfo *api.SignatureInfo, err error) {

	acs, err := formats.NewAuthenticatedConnectionSignature(formats.StatusOK, s.originCallsign, request.RequestInfo.InvokingDomain)
	if err != nil {
		acs.SetStatus(formats.StatusErrorOnSignature)
		setSignatureInfoFromAuthenticatedConnection(sigInfo, acs)
		return sigInfo, fmt.Errorf("error constructing authenticated connection signature format: %v", err)
	}

	if domainInfo.GetStatus() != discovery.DomainStatusOK {
		acs.SetStatus(formats.StatusErrorOnSignature)
		setSignatureInfoFromAuthenticatedConnection(sigInfo, acs)
		return sigInfo, fmt.Errorf("domain info is not available: %v", err)
	}

	sharedSecret, hasSecret := domainInfo.GetSharedSecret()
	if hasSecret {
		err = acs.AddParametersForSignature(sharedSecret.LocalKeyID(), domainInfo.GetAdsCertIdentityDomain(), sharedSecret.RemoteKeyID(), request.Timestamp, request.Nonce)
		if err != nil {
			acs.SetStatus(formats.StatusErrorOnSignature)
			setSignatureInfoFromAuthenticatedConnection(sigInfo, acs)
			return sigInfo, fmt.Errorf("error adding signature params: %v", err)
		}

	} else {
		acs.SetStatus(formats.StatusErrorOnSignature)
		setSignatureInfoFromAuthenticatedConnection(sigInfo, acs)
		return sigInfo, nil
	}

	acs.SetStatus(formats.StatusOK)
	setSignatureInfoFromAuthenticatedConnection(sigInfo, acs)
	message := acs.EncodeMessage()
	bodyHMAC, urlHMAC := generateSignatures(domainInfo, []byte(message), request.RequestInfo.BodyHash[:], request.RequestInfo.UrlHash[:])
	sigInfo.SignatureMessage = message + formats.EncodeSignatureSuffix(bodyHMAC, urlHMAC)

	return sigInfo, nil
}

func (s *localAuthenticatedConnectionsSignatory) VerifyAuthenticatedConnection(request *api.AuthenticatedConnectionVerificationRequest) (*api.AuthenticatedConnectionVerificationResponse, error) {

	startTime := time.Now()
	response := &api.AuthenticatedConnectionVerificationResponse{}

	for _, requestInfo := range request.RequestInfo {
		verificationInfo := &api.RequestVerificationInfo{}

		for _, signatureInfo := range requestInfo.SignatureInfo {
			decodeStatus := s.checkSingleSignature(requestInfo, signatureInfo)
			logger.Infof("%v", decodeStatus)
			verificationInfo.SignatureDecodeStatus = append(verificationInfo.SignatureDecodeStatus, decodeStatus)
		}

		response.VerificationInfo = append(response.VerificationInfo, verificationInfo)
	}

	metrics.RecordVerifyTime(time.Since(startTime))
	response.VerificationOperationStatus = api.VerificationOperationStatus_VERIFICATION_OPERATION_STATUS_OK
	return response, nil
}

func (s *localAuthenticatedConnectionsSignatory) checkSingleSignature(requestInfo *api.RequestInfo, signatureInfo *api.SignatureInfo) api.SignatureDecodeStatus {

	acs, err := formats.DecodeAuthenticatedConnectionSignature(signatureInfo.SignatureMessage)
	if err != nil {
		metrics.RecordVerify(adscerterrors.ErrVerifyDecodeSignature)
		return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_SIGNATURE_MALFORMED
	}

	// Validate invocation hostname matches request
	if acs.GetAttributeInvoking() != requestInfo.InvokingDomain {
		logger.Infof("unrelated signature %s versus %s", acs.GetAttributeInvoking(), requestInfo.InvokingDomain)
		metrics.RecordVerify(adscerterrors.ErrVerifySignatureRequestHostMismatch)
		return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_UNRELATED_SIGNATURE
	}

	domainInfos, err := s.counterpartyManager.LookupIdentitiesForDomain(acs.GetAttributeFrom())
	if err != nil || len(domainInfos) == 0 {
		logger.Infof("counterparty lookup error")
		metrics.RecordVerify(adscerterrors.ErrVerifyCounterpartyLookup)
		return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_COUNTERPARTY_LOOKUP_ERROR
	}

	for _, domainInfo := range domainInfos {
		if _, hasSecret := domainInfo.GetSharedSecret(); !hasSecret {
			logger.Infof("no shared secret")
			metrics.RecordVerify(adscerterrors.ErrVerifyMissingSharedSecret)
			return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_NO_SHARED_SECRET_AVAILABLE
		}

		bodyHMAC, urlHMAC := generateSignatures(domainInfo, []byte(acs.EncodeMessage()), requestInfo.BodyHash[:], requestInfo.UrlHash[:])
		bodyValid, urlValid := acs.CompareSignatures(bodyHMAC, urlHMAC)
		if bodyValid && urlValid {
			metrics.RecordVerify(nil)
			return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_BODY_AND_URL_VALID
		} else if bodyValid {
			metrics.RecordVerify(nil)
			return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_BODY_VALID
		}
	}

	metrics.RecordVerify(adscerterrors.ErrVerifyInvalidSignature)
	return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_INVALID_SIGNATURE
}

func generateSignatures(domainInfo discovery.DomainInfo, message []byte, bodyHash []byte, urlHash []byte) ([]byte, []byte) {

	sharedSecret, _ := domainInfo.GetSharedSecret()
	h := hmac.New(sha256.New, sharedSecret.Secret()[:])

	h.Write([]byte(message))
	h.Write(bodyHash)
	bodyHMAC := h.Sum(nil)

	h.Write(urlHash)
	urlHMAC := h.Sum(nil)

	return bodyHMAC, urlHMAC
}

func (s *localAuthenticatedConnectionsSignatory) generateNonce() (string, error) {
	var nonce [32]byte
	n, err := io.ReadFull(s.secureRandom, nonce[:])
	if err != nil {
		return "", fmt.Errorf("error generating random: %v", err)
	}
	if n != 32 {
		return "", fmt.Errorf("unexpected number of random values: %d", n)
	}
	return formats.B64truncate(nonce[:], 12), nil
}
<commit_msg>Fix a SIGSEGV in signSingleMessage - sigInfo always nil (#49)<commit_after>package signatory

import (
	"crypto/hmac"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"time"

	"github.com/IABTechLab/adscert/internal/adscerterrors"
	"github.com/IABTechLab/adscert/internal/formats"
	"github.com/IABTechLab/adscert/internal/logger"
	"github.com/IABTechLab/adscert/pkg/adscert/api"
	"github.com/IABTechLab/adscert/pkg/adscert/discovery"
	"github.com/IABTechLab/adscert/pkg/adscert/metrics"
	"github.com/benbjohnson/clock"
)

func NewLocalAuthenticatedConnectionsSignatory(
	originCallsign string,
	reader io.Reader,
	clock clock.Clock,
	dnsResolver discovery.DNSResolver,
	domainStore discovery.DomainStore,
	domainCheckInterval time.Duration,
	domainRenewalInterval time.Duration,
	base64PrivateKeys []string) AuthenticatedConnectionsSignatory {

	return &localAuthenticatedConnectionsSignatory{
		originCallsign:      originCallsign,
		secureRandom:        reader,
		clock:               clock,
		counterpartyManager: discovery.NewDefaultDomainIndexer(dnsResolver, domainStore, domainCheckInterval, domainRenewalInterval, base64PrivateKeys),
	}
}

type localAuthenticatedConnectionsSignatory struct {
	originCallsign string
	secureRandom   io.Reader
	clock          clock.Clock

	counterpartyManager discovery.DomainIndexer
}

func (s *localAuthenticatedConnectionsSignatory) SignAuthenticatedConnection(request *api.AuthenticatedConnectionSignatureRequest) (*api.AuthenticatedConnectionSignatureResponse, error) {

	var err error
	startTime := time.Now()
	response := &api.AuthenticatedConnectionSignatureResponse{RequestInfo: request.RequestInfo}

	if request.RequestInfo == nil || request.RequestInfo.InvokingDomain == "" || len(request.RequestInfo.UrlHash) == 0 {
		response.SignatureOperationStatus = api.SignatureOperationStatus_SIGNATURE_OPERATION_STATUS_MALFORMED_REQUEST
		return response, errors.New("required parameters are missing")
	}

	// add nonce and timestamp if not already provided in the request
	// this is the typical case to keep the client's usage simple
	if request.Timestamp == "" {
		request.Timestamp = s.clock.Now().UTC().Format("060102T150405")
	}
	if request.Nonce == "" {
		if request.Nonce, err = s.generateNonce(); err != nil {
			metrics.RecordSigning(adscerterrors.ErrSigningGenerateNonce)
		}
	}

	domainInfos, err := s.counterpartyManager.LookupIdentitiesForDomain(request.RequestInfo.InvokingDomain)
	if err != nil || len(domainInfos) == 0 {
		logger.Infof("counterparty lookup error")
		metrics.RecordSigning(adscerterrors.ErrSigningCounterpartyLookup)
		response.SignatureOperationStatus = api.SignatureOperationStatus_SIGNATURE_OPERATION_STATUS_SIGNATORY_INTERNAL_ERROR
		return response, err
	}

	for _, domainInfo := range domainInfos {
		signatureInfo, err := s.signSingleMessage(request, domainInfo)
		if err != nil {
			metrics.RecordSigning(adscerterrors.ErrSigningEmbossMessage)
			response.SignatureOperationStatus = api.SignatureOperationStatus_SIGNATURE_OPERATION_STATUS_SIGNATORY_INTERNAL_ERROR
			return response, err
		}
		response.RequestInfo.SignatureInfo = append(response.RequestInfo.SignatureInfo, signatureInfo)
	}

	metrics.RecordSigning(nil)
	metrics.RecordSigningTime(time.Since(startTime))
	response.SignatureOperationStatus = api.SignatureOperationStatus_SIGNATURE_OPERATION_STATUS_OK
	return response, nil
}

func (s *localAuthenticatedConnectionsSignatory) signSingleMessage(request *api.AuthenticatedConnectionSignatureRequest, domainInfo discovery.DomainInfo) (*api.SignatureInfo, error) {

	sigInfo := &api.SignatureInfo{}
	acs, err := formats.NewAuthenticatedConnectionSignature(formats.StatusOK, s.originCallsign, request.RequestInfo.InvokingDomain)
	if err != nil {
		acs.SetStatus(formats.StatusErrorOnSignature)
		setSignatureInfoFromAuthenticatedConnection(sigInfo, acs)
		return sigInfo, fmt.Errorf("error constructing authenticated connection signature format: %v", err)
	}

	if domainInfo.GetStatus() != discovery.DomainStatusOK {
		acs.SetStatus(formats.StatusErrorOnSignature)
		setSignatureInfoFromAuthenticatedConnection(sigInfo, acs)
		return sigInfo, fmt.Errorf("domain info is not available: %v", err)
	}

	sharedSecret, hasSecret := domainInfo.GetSharedSecret()
	if hasSecret {
		err = acs.AddParametersForSignature(sharedSecret.LocalKeyID(), domainInfo.GetAdsCertIdentityDomain(), sharedSecret.RemoteKeyID(), request.Timestamp, request.Nonce)
		if err != nil {
			acs.SetStatus(formats.StatusErrorOnSignature)
			setSignatureInfoFromAuthenticatedConnection(sigInfo, acs)
			return sigInfo, fmt.Errorf("error adding signature params: %v", err)
		}

	} else {
		acs.SetStatus(formats.StatusErrorOnSignature)
		setSignatureInfoFromAuthenticatedConnection(sigInfo, acs)
		return sigInfo, nil
	}

	acs.SetStatus(formats.StatusOK)
	setSignatureInfoFromAuthenticatedConnection(sigInfo, acs)
	message := acs.EncodeMessage()
	bodyHMAC, urlHMAC := generateSignatures(domainInfo, []byte(message), request.RequestInfo.BodyHash[:], request.RequestInfo.UrlHash[:])
	sigInfo.SignatureMessage = message + formats.EncodeSignatureSuffix(bodyHMAC, urlHMAC)

	return sigInfo, nil
}

func (s *localAuthenticatedConnectionsSignatory) VerifyAuthenticatedConnection(request *api.AuthenticatedConnectionVerificationRequest) (*api.AuthenticatedConnectionVerificationResponse, error) {

	startTime := time.Now()
	response := &api.AuthenticatedConnectionVerificationResponse{}

	for _, requestInfo := range request.RequestInfo {
		verificationInfo := &api.RequestVerificationInfo{}

		for _, signatureInfo := range requestInfo.SignatureInfo {
			decodeStatus := s.checkSingleSignature(requestInfo, signatureInfo)
			logger.Infof("%v", decodeStatus)
			verificationInfo.SignatureDecodeStatus = append(verificationInfo.SignatureDecodeStatus, decodeStatus)
		}

		response.VerificationInfo = append(response.VerificationInfo, verificationInfo)
	}

	metrics.RecordVerifyTime(time.Since(startTime))
	response.VerificationOperationStatus = api.VerificationOperationStatus_VERIFICATION_OPERATION_STATUS_OK
	return response, nil
}

func (s *localAuthenticatedConnectionsSignatory) checkSingleSignature(requestInfo *api.RequestInfo, signatureInfo *api.SignatureInfo) api.SignatureDecodeStatus {

	acs, err := formats.DecodeAuthenticatedConnectionSignature(signatureInfo.SignatureMessage)
	if err != nil {
		metrics.RecordVerify(adscerterrors.ErrVerifyDecodeSignature)
		return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_SIGNATURE_MALFORMED
	}

	// Validate invocation hostname matches request
	if acs.GetAttributeInvoking() != requestInfo.InvokingDomain {
		logger.Infof("unrelated signature %s versus %s", acs.GetAttributeInvoking(), requestInfo.InvokingDomain)
		metrics.RecordVerify(adscerterrors.ErrVerifySignatureRequestHostMismatch)
		return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_UNRELATED_SIGNATURE
	}

	domainInfos, err := s.counterpartyManager.LookupIdentitiesForDomain(acs.GetAttributeFrom())
	if err != nil || len(domainInfos) == 0 {
		logger.Infof("counterparty lookup error")
		metrics.RecordVerify(adscerterrors.ErrVerifyCounterpartyLookup)
		return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_COUNTERPARTY_LOOKUP_ERROR
	}

	for _, domainInfo := range domainInfos {
		if _, hasSecret := domainInfo.GetSharedSecret(); !hasSecret {
			logger.Infof("no shared secret")
			metrics.RecordVerify(adscerterrors.ErrVerifyMissingSharedSecret)
			return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_NO_SHARED_SECRET_AVAILABLE
		}

		bodyHMAC, urlHMAC := generateSignatures(domainInfo, []byte(acs.EncodeMessage()), requestInfo.BodyHash[:], requestInfo.UrlHash[:])
		bodyValid, urlValid := acs.CompareSignatures(bodyHMAC, urlHMAC)
		if bodyValid && urlValid {
			metrics.RecordVerify(nil)
			return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_BODY_AND_URL_VALID
		} else if bodyValid {
			metrics.RecordVerify(nil)
			return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_BODY_VALID
		}
	}

	metrics.RecordVerify(adscerterrors.ErrVerifyInvalidSignature)
	return api.SignatureDecodeStatus_SIGNATURE_DECODE_STATUS_INVALID_SIGNATURE
}

func generateSignatures(domainInfo discovery.DomainInfo, message []byte, bodyHash []byte, urlHash []byte) ([]byte, []byte) {

	sharedSecret, _ := domainInfo.GetSharedSecret()
	h := hmac.New(sha256.New, sharedSecret.Secret()[:])

	h.Write([]byte(message))
	h.Write(bodyHash)
	bodyHMAC := h.Sum(nil)

	h.Write(urlHash)
	urlHMAC := h.Sum(nil)

	return bodyHMAC, urlHMAC
}

func (s *localAuthenticatedConnectionsSignatory) generateNonce() (string, error) {
	var nonce [32]byte
	n, err := io.ReadFull(s.secureRandom, nonce[:])
	if err != nil {
		return "", fmt.Errorf("error generating random: %v", err)
	}
	if n != 32 {
		return "", fmt.Errorf("unexpected number of random values: %d", n)
	}
	return formats.B64truncate(nonce[:], 12), nil
}
<|endoftext|>


<commit_before>package cluster

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rancher/norman/api/access"
	"github.com/rancher/norman/httperror"
	"github.com/rancher/norman/types"
	"github.com/rancher/rancher/pkg/controllers/management/etcdbackup"
	"github.com/rancher/rancher/pkg/ref"
	mgmtv3 "github.com/rancher/types/apis/management.cattle.io/v3"
	v3 "github.com/rancher/types/apis/management.cattle.io/v3"
	client "github.com/rancher/types/client/management/v3"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func (a ActionHandler) BackupEtcdHandler(actionName string, action *types.Action, apiContext *types.APIContext) error {
	response := map[string]interface{}{
		"message": "starting ETCD backup",
	}
	// checking access
	var mgmtCluster mgmtv3.Cluster
	if err := access.ByID(apiContext, apiContext.Version, apiContext.Type, apiContext.ID, &mgmtCluster); err != nil {
		response["message"] = "none existent Cluster"
		apiContext.WriteResponse(http.StatusBadRequest, response)
		return errors.Wrapf(err, "failed to get Cluster by ID %s", apiContext.ID)
	}

	cluster, err := a.ClusterClient.Get(apiContext.ID, v1.GetOptions{})
	if err != nil {
		response["message"] = "none existent Cluster"
		apiContext.WriteResponse(http.StatusBadRequest, response)
		return errors.Wrapf(err, "failed to get Cluster by ID %s", apiContext.ID)
	}

	newBackup, err := etcdbackup.NewBackupObject(cluster, true)
	if err != nil {
		response["message"] = "failed to initialize etcdbackup object"
		apiContext.WriteResponse(http.StatusInternalServerError, response)
		return errors.Wrapf(err, "failed to initialize etcdbackup object")
	}

	backup, err := a.BackupClient.Create(newBackup)
	if err != nil {
		response["message"] = "failed to create etcdbackup object"
		apiContext.WriteResponse(http.StatusInternalServerError, response)
		return errors.Wrapf(err, "failed to cteate etcdbackup object")
	}
	backupJSON, err := json.Marshal(backup)
	if err != nil {
		return err
	}
	apiContext.Response.Header().Set("Content-Type", "application/json")
	http.ServeContent(apiContext.Response, apiContext.Request, "backupEtcd", time.Now(), bytes.NewReader(backupJSON))
	return nil
}

func (a ActionHandler) RestoreFromEtcdBackupHandler(actionName string, action *types.Action, apiContext *types.APIContext) error {
	response := map[string]interface{}{
		"message": "restoring etcdbackup for the cluster",
	}

	data, err := ioutil.ReadAll(apiContext.Request.Body)
	if err != nil {
		response["message"] = "reading request body error"
		apiContext.WriteResponse(http.StatusInternalServerError, response)
		return errors.Wrap(err, "failed to read request body")
	}

	input := client.RestoreFromEtcdBackupInput{}
	if err = json.Unmarshal(data, &input); err != nil {
		response["message"] = "failed to parse request content"
		apiContext.WriteResponse(http.StatusBadRequest, response)
		return errors.Wrap(err, "unmarshaling input error")
	}
	// checking access
	var mgmtCluster client.Cluster
	if err := access.ByID(apiContext, apiContext.Version, apiContext.Type, apiContext.ID, &mgmtCluster); err != nil {
		response["message"] = "nonexistent Cluster"
		apiContext.WriteResponse(http.StatusBadRequest, response)
		return errors.Wrapf(err, "failed to get Cluster by ID %s", apiContext.ID)
	}

	cluster, err := a.ClusterClient.Get(apiContext.ID, v1.GetOptions{})
	if err != nil {
		response["message"] = "nonexistent Cluster"
		apiContext.WriteResponse(http.StatusBadRequest, response)
		return errors.Wrapf(err, "failed to get Cluster by ID %s", apiContext.ID)
	}

	var backup *v3.EtcdBackup
	clusterBackupConfig := cluster.Spec.RancherKubernetesEngineConfig.Services.Etcd.BackupConfig
	if clusterBackupConfig != nil && clusterBackupConfig.S3BackupConfig == nil {
		ns, name := ref.Parse(input.EtcdBackupID)
		if ns == "" || name == "" {
			return httperror.NewAPIError(httperror.InvalidFormat, fmt.Sprintf("invalid input id %s", input.EtcdBackupID))
		}
		backup, err = a.BackupClient.GetNamespaced(ns, name, v1.GetOptions{})
		if err != nil {
			response["message"] = "error getting backup config"
			apiContext.WriteResponse(http.StatusInternalServerError, response)
			return errors.Wrapf(err, "failed to get backup config by ID %s", input.EtcdBackupID)
		}
		if backup.Spec.BackupConfig.S3BackupConfig != nil {
			return httperror.NewAPIError(httperror.MethodNotAllowed,
				fmt.Sprintf("restoring S3 backups with no cluster level S3 configuration is not supported %s", input.EtcdBackupID))
		}
	}

	if input.RestoreRkeConfig != "" && backup.Status.ClusterObject == "" {
		// attempting to restore rke config and the backup does not contain data, probably pre 2.4 backup
		return httperror.NewAPIError(httperror.MethodNotAllowed,
			fmt.Sprintf("unable to restore RKE config, backup contains no cluster object: %s", input.EtcdBackupID))
	}

	// backup was taken in 2.4+ and has content
	switch strings.ToLower(input.RestoreRkeConfig) {
	case "kubernetesversion":
		// restore from copy stored inline to not have to decompress object
		cluster.Spec.RancherKubernetesEngineConfig.Version = backup.Status.KubernetesVersion
	case "all":
		clusterBackup, err := etcdbackup.DecompressCluster(backup.Status.ClusterObject)
		if err != nil {
			response["message"] = "error decompressing cluster object"
			apiContext.WriteResponse(http.StatusInternalServerError, response)
			return errors.Wrap(err,
				fmt.Sprintf("error decompressing cluster object for backupid %s: %s", input.EtcdBackupID, err))
		}
		cluster.Spec.RancherKubernetesEngineConfig = clusterBackup.Spec.RancherKubernetesEngineConfig
	}

	// flag cluster for restore
	cluster.Spec.RancherKubernetesEngineConfig.Restore.SnapshotName = input.EtcdBackupID
	cluster.Spec.RancherKubernetesEngineConfig.Restore.Restore = true

	if _, err = a.ClusterClient.Update(cluster); err != nil {
		response["message"] = "failed to update cluster object"
		apiContext.WriteResponse(http.StatusInternalServerError, response)
		return errors.Wrapf(err, "unable to update Cluster %s", cluster.Name)
	}
	apiContext.WriteResponse(http.StatusCreated, response)
	return nil
}
<commit_msg>fixing nil backup during s3 restore<commit_after>package cluster

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rancher/norman/api/access"
	"github.com/rancher/norman/httperror"
	"github.com/rancher/norman/types"
	"github.com/rancher/rancher/pkg/controllers/management/etcdbackup"
	"github.com/rancher/rancher/pkg/ref"
	mgmtv3 "github.com/rancher/types/apis/management.cattle.io/v3"
	client "github.com/rancher/types/client/management/v3"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func (a ActionHandler) BackupEtcdHandler(actionName string, action *types.Action, apiContext *types.APIContext) error {
	response := map[string]interface{}{
		"message": "starting ETCD backup",
	}
	// checking access
	var mgmtCluster mgmtv3.Cluster
	if err := access.ByID(apiContext, apiContext.Version, apiContext.Type, apiContext.ID, &mgmtCluster); err != nil {
		response["message"] = "none existent Cluster"
		apiContext.WriteResponse(http.StatusBadRequest, response)
		return errors.Wrapf(err, "failed to get Cluster by ID %s", apiContext.ID)
	}

	cluster, err := a.ClusterClient.Get(apiContext.ID, v1.GetOptions{})
	if err != nil {
		response["message"] = "none existent Cluster"
		apiContext.WriteResponse(http.StatusBadRequest, response)
		return errors.Wrapf(err, "failed to get Cluster by ID %s", apiContext.ID)
	}

	newBackup, err := etcdbackup.NewBackupObject(cluster, true)
	if err != nil {
		response["message"] = "failed to initialize etcdbackup object"
		apiContext.WriteResponse(http.StatusInternalServerError, response)
		return errors.Wrapf(err, "failed to initialize etcdbackup object")
	}

	backup, err := a.BackupClient.Create(newBackup)
	if err != nil {
		response["message"] = "failed to create etcdbackup object"
		apiContext.WriteResponse(http.StatusInternalServerError, response)
		return errors.Wrapf(err, "failed to cteate etcdbackup object")
	}
	backupJSON, err := json.Marshal(backup)
	if err != nil {
		return err
	}
	apiContext.Response.Header().Set("Content-Type", "application/json")
	http.ServeContent(apiContext.Response, apiContext.Request, "backupEtcd", time.Now(), bytes.NewReader(backupJSON))
	return nil
}

func (a ActionHandler) RestoreFromEtcdBackupHandler(actionName string, action *types.Action, apiContext *types.APIContext) error {
	response := map[string]interface{}{
		"message": "restoring etcdbackup for the cluster",
	}

	data, err := ioutil.ReadAll(apiContext.Request.Body)
	if err != nil {
		response["message"] = "reading request body error"
		apiContext.WriteResponse(http.StatusInternalServerError, response)
		return errors.Wrap(err, "failed to read request body")
	}

	input := client.RestoreFromEtcdBackupInput{}
	if err = json.Unmarshal(data, &input); err != nil {
		response["message"] = "failed to parse request content"
		apiContext.WriteResponse(http.StatusBadRequest, response)
		return errors.Wrap(err, "unmarshaling input error")
	}
	// checking access
	var mgmtCluster client.Cluster
	if err := access.ByID(apiContext, apiContext.Version, apiContext.Type, apiContext.ID, &mgmtCluster); err != nil {
		response["message"] = "nonexistent Cluster"
		apiContext.WriteResponse(http.StatusBadRequest, response)
		return errors.Wrapf(err, "failed to get Cluster by ID %s", apiContext.ID)
	}

	cluster, err := a.ClusterClient.Get(apiContext.ID, v1.GetOptions{})
	if err != nil {
		response["message"] = "nonexistent Cluster"
		apiContext.WriteResponse(http.StatusBadRequest, response)
		return errors.Wrapf(err, "failed to get Cluster by ID %s", apiContext.ID)
	}

	ns, name := ref.Parse(input.EtcdBackupID)
	if ns == "" || name == "" {
		return httperror.NewAPIError(httperror.InvalidFormat, fmt.Sprintf("invalid input id %s", input.EtcdBackupID))
	}

	backup, err := a.BackupClient.GetNamespaced(ns, name, v1.GetOptions{})
	if err != nil {
		response["message"] = "error getting backup config"
		apiContext.WriteResponse(http.StatusInternalServerError, response)
		return errors.Wrapf(err, "failed to get backup config by ID %s", input.EtcdBackupID)
	}

	clusterBackupConfig := cluster.Spec.RancherKubernetesEngineConfig.Services.Etcd.BackupConfig
	if clusterBackupConfig != nil &&
		clusterBackupConfig.S3BackupConfig == nil &&
		backup.Spec.BackupConfig.S3BackupConfig != nil {
		return httperror.NewAPIError(httperror.MethodNotAllowed,
			fmt.Sprintf(
				"restoring S3 backups with no cluster level S3 configuration is not supported %s",
				input.EtcdBackupID))
	}

	if input.RestoreRkeConfig != "" && backup.Status.ClusterObject == "" {
		// attempting to restore rke config and the backup does not contain data, probably pre 2.4 backup
		return httperror.NewAPIError(httperror.MethodNotAllowed,
			fmt.Sprintf("unable to restore RKE config, backup contains no cluster object: %s", input.EtcdBackupID))
	}

	// backup was taken in 2.4+ and has content
	switch strings.ToLower(input.RestoreRkeConfig) {
	case "kubernetesversion":
		// restore from copy stored inline to not have to decompress object
		cluster.Spec.RancherKubernetesEngineConfig.Version = backup.Status.KubernetesVersion
	case "all":
		clusterBackup, err := etcdbackup.DecompressCluster(backup.Status.ClusterObject)
		if err != nil {
			response["message"] = "error decompressing cluster object"
			apiContext.WriteResponse(http.StatusInternalServerError, response)
			return errors.Wrap(err,
				fmt.Sprintf("error decompressing cluster object for backupid %s: %s", input.EtcdBackupID, err))
		}
		cluster.Spec.RancherKubernetesEngineConfig = clusterBackup.Spec.RancherKubernetesEngineConfig
	}

	// flag cluster for restore
	cluster.Spec.RancherKubernetesEngineConfig.Restore.SnapshotName = input.EtcdBackupID
	cluster.Spec.RancherKubernetesEngineConfig.Restore.Restore = true

	if _, err = a.ClusterClient.Update(cluster); err != nil {
		response["message"] = "failed to update cluster object"
		apiContext.WriteResponse(http.StatusInternalServerError, response)
		return errors.Wrapf(err, "unable to update Cluster %s", cluster.Name)
	}
	apiContext.WriteResponse(http.StatusCreated, response)
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vtgate

import (
	"context"
	"flag"
	"fmt"
	"os"
	"strings"
	"testing"

	"github.com/google/go-cmp/cmp"

	"github.com/stretchr/testify/require"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/test/endtoend/cluster"
)

var (
	clusterInstance *cluster.LocalProcessCluster
	vtParams        mysql.ConnParams
	KeyspaceName    = "ks"
	Cell            = "test"
	SchemaSQL       = `create table t1(
	id1 bigint,
	id2 bigint,
	primary key(id1)
) Engine=InnoDB;`

	VSchema = `
{
  "sharded": true,
  "vindexes": {
    "hash": {
      "type": "hash"
    }
  },
  "tables": {
    "t1": {
      "column_vindexes": [
        {
          "column": "id1",
          "name": "hash"
        }
      ]
    }
  }
}`
)

func TestMain(m *testing.M) {
	defer cluster.PanicHandler(nil)
	flag.Parse()

	exitCode := func() int {
		clusterInstance = cluster.NewCluster(Cell, "localhost")
		defer clusterInstance.Teardown()

		// Start topo server
		err := clusterInstance.StartTopo()
		if err != nil {
			return 1
		}

		// Start keyspace
		keyspace := &cluster.Keyspace{
			Name:      KeyspaceName,
			SchemaSQL: SchemaSQL,
			VSchema:   VSchema,
		}
		if err := clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 1, false); err != nil {
			return 1
		}

		// Start vtgate
		if err := clusterInstance.StartVtgate(); err != nil {
			return 1
		}

		vtParams = mysql.ConnParams{
			Host: clusterInstance.Hostname,
			Port: clusterInstance.VtgateMySQLPort,
		}
		return m.Run()
	}()
	os.Exit(exitCode)
}

func TestScatterErrsAsWarns(t *testing.T) {
	oltp, err := mysql.Connect(context.Background(), &vtParams)
	require.NoError(t, err)
	defer oltp.Close()

	olap, err := mysql.Connect(context.Background(), &vtParams)
	require.NoError(t, err)
	defer olap.Close()

	checkedExec(t, oltp, `insert into t1(id1, id2) values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)`)
	defer func() {
		checkedExec(t, oltp, "use @master")
		checkedExec(t, oltp, `delete from t1`)
	}()

	query1 := `select /*vt+ SCATTER_ERRORS_AS_WARNINGS */ id1 from t1`
	query2 := `select /*vt+ SCATTER_ERRORS_AS_WARNINGS */ id1 from t1 order by id1`
	showQ := "show warnings"

	// stop the mysql on one tablet, query will fail at vttablet level
	require.NoError(t,
		clusterInstance.Keyspaces[0].Shards[0].Replica().MysqlctlProcess.Stop())

	modes := []struct {
		conn *mysql.Conn
		m    string
	}{
		{m: "oltp", conn: oltp},
		{m: "olap", conn: olap},
	}

	for _, mode := range modes {
		t.Run(mode.m, func(t *testing.T) {
			// connection setup
			checkedExec(t, mode.conn, "use @replica")
			checkedExec(t, mode.conn, fmt.Sprintf("set workload = %s", mode.m))

			assertMatches(t, mode.conn, query1, `[[INT64(4)]]`)
			assertContainsOneOf(t, mode.conn, showQ, "no valid tablet", "no healthy tablet")
			assertMatches(t, mode.conn, query2, `[[INT64(4)]]`)
			assertContainsOneOf(t, mode.conn, showQ, "no valid tablet", "no healthy tablet")

			// invalid_field should throw error and not warning
			_, err = mode.conn.ExecuteFetch("SELECT /*vt+ SCATTER_ERRORS_AS_WARNINGS */ invalid_field from t1;", 1, false)
			require.EqualError(t, err, "target: test_keyspace.0.master: vttablet: rpc error: code = NotFound desc = Unknown column 'invalid_field' in 'field list' (errno 1054) (sqlstate 42S22) (CallerID: vtgate client 1): Sql: \"select /*vt+ SCATTER_ERRORS_AS_WARNINGS */ invalid_field from vt_insert_test\", BindVars: {} (errno 1054) (sqlstate 42S22) during query: SELECT /*vt+ SCATTER_ERRORS_AS_WARNINGS */ invalid_field from vt_insert_test")
		})
	}
}

func checkedExec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
	t.Helper()
	qr, err := conn.ExecuteFetch(query, 1000, true)
	require.NoError(t, err)
	return qr
}

func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) {
	t.Helper()
	qr := checkedExec(t, conn, query)
	got := fmt.Sprintf("%v", qr.Rows)
	diff := cmp.Diff(expected, got)
	if diff != "" {
		t.Errorf("Query: %s (-want +got):\n%s\n%s", query, diff, got)
	}
}

func assertContainsOneOf(t *testing.T, conn *mysql.Conn, query string, expected ...string) {
	t.Helper()
	qr := checkedExec(t, conn, query)
	got := fmt.Sprintf("%v", qr.Rows)
	for _, s := range expected {
		if strings.Contains(got, s) {
			return
		}
	}

	t.Errorf("%s\n did not match any of %v", got, expected)
}
<commit_msg> fix error checking<commit_after>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vtgate

import (
	"context"
	"flag"
	"fmt"
	"os"
	"strings"
	"testing"

	"github.com/google/go-cmp/cmp"

	"github.com/stretchr/testify/require"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/test/endtoend/cluster"
)

var (
	clusterInstance *cluster.LocalProcessCluster
	vtParams        mysql.ConnParams
	KeyspaceName    = "ks"
	Cell            = "test"
	SchemaSQL       = `create table t1(
	id1 bigint,
	id2 bigint,
	primary key(id1)
) Engine=InnoDB;`

	VSchema = `
{
  "sharded": true,
  "vindexes": {
    "hash": {
      "type": "hash"
    }
  },
  "tables": {
    "t1": {
      "column_vindexes": [
        {
          "column": "id1",
          "name": "hash"
        }
      ]
    }
  }
}`
)

func TestMain(m *testing.M) {
	defer cluster.PanicHandler(nil)
	flag.Parse()

	exitCode := func() int {
		clusterInstance = cluster.NewCluster(Cell, "localhost")
		defer clusterInstance.Teardown()

		// Start topo server
		err := clusterInstance.StartTopo()
		if err != nil {
			return 1
		}

		// Start keyspace
		keyspace := &cluster.Keyspace{
			Name:      KeyspaceName,
			SchemaSQL: SchemaSQL,
			VSchema:   VSchema,
		}
		if err := clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 1, false); err != nil {
			return 1
		}

		// Start vtgate
		if err := clusterInstance.StartVtgate(); err != nil {
			return 1
		}

		vtParams = mysql.ConnParams{
			Host: clusterInstance.Hostname,
			Port: clusterInstance.VtgateMySQLPort,
		}
		return m.Run()
	}()
	os.Exit(exitCode)
}

func TestScatterErrsAsWarns(t *testing.T) {
	oltp, err := mysql.Connect(context.Background(), &vtParams)
	require.NoError(t, err)
	defer oltp.Close()

	olap, err := mysql.Connect(context.Background(), &vtParams)
	require.NoError(t, err)
	defer olap.Close()

	checkedExec(t, oltp, `insert into t1(id1, id2) values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5)`)
	defer func() {
		checkedExec(t, oltp, "use @master")
		checkedExec(t, oltp, `delete from t1`)
	}()

	query1 := `select /*vt+ SCATTER_ERRORS_AS_WARNINGS */ id1 from t1`
	query2 := `select /*vt+ SCATTER_ERRORS_AS_WARNINGS */ id1 from t1 order by id1`
	showQ := "show warnings"

	// stop the mysql on one tablet, query will fail at vttablet level
	require.NoError(t,
		clusterInstance.Keyspaces[0].Shards[0].Replica().MysqlctlProcess.Stop())

	modes := []struct {
		conn *mysql.Conn
		m    string
	}{
		{m: "oltp", conn: oltp},
		{m: "olap", conn: olap},
	}

	for _, mode := range modes {
		t.Run(mode.m, func(t *testing.T) {
			// connection setup
			checkedExec(t, mode.conn, "use @replica")
			checkedExec(t, mode.conn, fmt.Sprintf("set workload = %s", mode.m))

			assertMatches(t, mode.conn, query1, `[[INT64(4)]]`)
			assertContainsOneOf(t, mode.conn, showQ, "no valid tablet", "no healthy tablet", "mysql.sock: connect: no such file or directory")
			assertMatches(t, mode.conn, query2, `[[INT64(4)]]`)
			assertContainsOneOf(t, mode.conn, showQ, "no valid tablet", "no healthy tablet", "mysql.sock: connect: no such file or directory")

			// invalid_field should throw error and not warning
			_, err = mode.conn.ExecuteFetch("SELECT /*vt+ SCATTER_ERRORS_AS_WARNINGS */ invalid_field from t1;", 1, false)
			require.Error(t, err)
			serr := mysql.NewSQLErrorFromError(err).(*mysql.SQLError)
			require.Equal(t, 1054, serr.Number())
		})
	}
}

func checkedExec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
	t.Helper()
	qr, err := conn.ExecuteFetch(query, 1000, true)
	require.NoError(t, err)
	return qr
}

func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) {
	t.Helper()
	qr := checkedExec(t, conn, query)
	got := fmt.Sprintf("%v", qr.Rows)
	diff := cmp.Diff(expected, got)
	if diff != "" {
		t.Errorf("Query: %s (-want +got):\n%s\n%s", query, diff, got)
	}
}

func assertContainsOneOf(t *testing.T, conn *mysql.Conn, query string, expected ...string) {
	t.Helper()
	qr := checkedExec(t, conn, query)
	got := fmt.Sprintf("%v", qr.Rows)
	for _, s := range expected {
		if strings.Contains(got, s) {
			return
		}
	}

	t.Errorf("%s\n did not match any of %v", got, expected)
}
<|endoftext|>