<commit_before>package mk2if

import (
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"sync"
	"time"
)

type scaling struct {
	scale  float64
	offset float64
}

type mk2Ser struct {
	info   *Mk2Info
	report *Mk2Info
	p      io.ReadWriter
	sc     []scaling
	scN    int
	run    chan struct{}
	locked bool
	sync.RWMutex
	infochan chan *Mk2Info
	wg sync.WaitGroup
}

func NewMk2Connection(dev io.ReadWriter) (Mk2If, error) {
	mk2 := &mk2Ser{}
	mk2.p = dev
	mk2.info = &Mk2Info{}
	mk2.report = &Mk2Info{}
	mk2.scN = 0
	mk2.locked = false
	mk2.sc = make([]scaling, 0)
	mk2.setTarget()
	mk2.run = make(chan struct{})
	mk2.infochan = make(chan *Mk2Info)
	mk2.wg.Add(1)
	go mk2.frameLock()
	return mk2, nil
}

// Locks to incoming frame.
func (mk2 *mk2Ser) frameLock() {

	frame := make([]byte, 256)
	var size byte
	for {
		select {
		case <-mk2.run:
			mk2.wg.Done()
			return
		default:
		}
		if mk2.locked {
			size = mk2.readByte()
			l, err := io.ReadFull(mk2.p, frame[0:int(size)+1])
			if err != nil {
				mk2.addError(fmt.Errorf("Read Error: %v", err))
				mk2.locked = false
			} else if l != int(size)+1 {
				mk2.addError(errors.New("Read Length Error"))
				mk2.locked = false
			} else {
				mk2.handleFrame(size, frame[0:int(size+1)])
			}
		} else {
			tmp := mk2.readByte()
			if tmp == 0xff || tmp == 0x20 {
				l, err := io.ReadFull(mk2.p, frame[0:int(size)])
				if err != nil {
					mk2.addError(fmt.Errorf("Read Error: %v", err))
					time.Sleep(1 * time.Second)
				} else if l != int(size) {
					mk2.addError(errors.New("Read Length Error"))
				} else {
					if checkChecksum(size, tmp, frame[0:int(size)]) {
						mk2.locked = true
						log.Printf("Locked")
					}
				}
			}
			size = tmp
		}
	}
}

// Close Mk2
func (mk2 *mk2Ser) Close() {
	close(mk2.run)
	mk2.wg.Wait()
}

// Returns last known state with all reported errors since previous poll.
// Mk2Info.Valid will be false if no polling has completed.
func (mk2 *mk2Ser) GetMk2Info() *Mk2Info {
	mk2.RLock()
	defer mk2.RUnlock()
	return mk2.report
}

func (mk2 *mk2Ser) C() chan *Mk2Info {
	return mk2.infochan
}

func (mk2 *mk2Ser) readByte() byte {
	buffer := make([]byte, 1)
	_, err := io.ReadFull(mk2.p, buffer)
	if err != nil {
		mk2.addError(fmt.Errorf("Read error: %v", err))
		return 0
	}
	return buffer[0]
}

// Adds error to error slice.
func (mk2 *mk2Ser) addError(err error) {
	if mk2.info.Errors == nil {
		mk2.info.Errors = make([]error, 0)
	}
	mk2.info.Errors = append(mk2.info.Errors, err)
	mk2.info.Valid = false
}

// Updates report.
func (mk2 *mk2Ser) updateReport() {
	mk2.Lock()
	defer mk2.Unlock()
	mk2.report = mk2.info
	select {
	case mk2.infochan <- mk2.info:
	default:
	}
	mk2.info = &Mk2Info{}
}

// Checks for valid frame and chooses decoding.
func (mk2 *mk2Ser) handleFrame(l byte, frame []byte) {
	if checkChecksum(l, frame[0], frame[1:]) {
		switch frame[0] {
		case 0xff:
			switch frame[1] {
			case 0x56: // V
				mk2.versionDecode(frame[2:])
			case 0x57:
				switch frame[2] {
				case 0x8e:
					mk2.scaleDecode(frame[2:])
				case 0x85:
					mk2.stateDecode(frame[2:])
				}

			case 0x4C: // L
				mk2.ledDecode(frame[2:])
			}

		case 0x20:
			switch frame[5] {
			case 0x0C:
				mk2.dcDecode(frame[1:])
			case 0x08:
				mk2.acDecode(frame[1:])
			}
		}
	} else {
		log.Printf("Failed")
		mk2.locked = false
	}
}

// Set the target VBus device.
func (mk *mk2Ser) setTarget() {
	cmd := make([]byte, 3)
	cmd[0] = 0x41 // A
	cmd[1] = 0x01
	cmd[2] = 0x00
	mk.sendCommand(cmd)
}

// Request the scaling factor for entry 'in'.
func (mk *mk2Ser) reqScaleFactor(in byte) {
	cmd := make([]byte, 4)
	cmd[0] = 0x57 // W
	cmd[1] = 0x36
	cmd[2] = in
	mk.sendCommand(cmd)
}

// Decode the scale factor frame.
func (mk *mk2Ser) scaleDecode(frame []byte) {
	scl := uint16(frame[2])<<8 + uint16(frame[1])
	ofs := int16(uint16(frame[5])<<8 + uint16(frame[4]))

	tmp := scaling{}
	tmp.offset = float64(ofs)
	if scl >= 0x4000 {
		tmp.scale = math.Abs(1 / (0x8000 - float64(scl)))
	} else {
		tmp.scale = math.Abs(float64(scl))
	}
	mk.sc = append(mk.sc, tmp)

	mk.scN++
	if mk.scN < 14 {
		mk.reqScaleFactor(byte(mk.scN))
	} else {
		log.Print("Monitoring starting.")
	}

}

// Decode the version number
func (mk *mk2Ser) versionDecode(frame []byte) {
	mk.info.Version = 0
	mk.info.Valid = true
	for i := 0; i < 4; i++ {
		mk.info.Version += uint32(frame[i]) << uint(i) * 8
	}

	if mk.scN < 14 {
		log.Print("Get scaling factors.")
		mk.reqScaleFactor(byte(mk.scN))
	} else {
		// Send DC status request
		cmd := make([]byte, 2)
		cmd[0] = 0x46 //F
		cmd[1] = 0
		mk.sendCommand(cmd)
	}
}

// Apply scaling to float
func (mk *mk2Ser) applyScale(value float64, scale int) float64 {
	return mk.sc[scale].scale * (value + mk.sc[scale].offset)
}

// Convert bytes->int16->float
func getSigned(data []byte) float64 {
	return float64(int16(data[0]) + int16(data[1])<<8)
}

// Convert bytes->uint32->float
func getUnsigned(data []byte) float64 {
	return float64(uint32(data[0]) + uint32(data[1])<<8 + uint32(data[2])<<16)
}

// Decodes DC frame.
func (mk *mk2Ser) dcDecode(frame []byte) {
	mk.info.BatVoltage = mk.applyScale(getSigned(frame[5:7]), 4)

	usedC := mk.applyScale(getUnsigned(frame[7:10]), 5)
	chargeC := mk.applyScale(getUnsigned(frame[10:13]), 5)
	mk.info.BatCurrent = usedC - chargeC

	mk.info.OutFrequency = 10 / (mk.applyScale(float64(frame[13]), 7))

	// Send L1 status request
	cmd := make([]byte, 2)
	cmd[0] = 0x46 //F
	cmd[1] = 1
	mk.sendCommand(cmd)
}

// Decodes AC frame.
func (mk *mk2Ser) acDecode(frame []byte) {
	mk.info.InVoltage = mk.applyScale(getSigned(frame[5:7]), 0)
	mk.info.InCurrent = mk.applyScale(getSigned(frame[7:9]), 1)
	mk.info.OutVoltage = mk.applyScale(getSigned(frame[9:11]), 2)
	mk.info.OutCurrent = mk.applyScale(getSigned(frame[11:13]), 3)

	if frame[13] == 0xff {
		mk.info.InFrequency = 0
	} else {
		mk.info.InFrequency = 10 / (mk.applyScale(float64(frame[13]), 8))
	}

	// Send status request
	cmd := make([]byte, 1)
	cmd[0] = 0x4C //F
	mk.sendCommand(cmd)
}

// Decode charge state of battery.
func (mk *mk2Ser) stateDecode(frame []byte) {
	mk.info.ChargeState = mk.applyScale(getSigned(frame[1:3]), 13)
	mk.updateReport()
}

// Decode the LED state frame.
func (mk *mk2Ser) ledDecode(frame []byte) {
	mk.info.LedListOn = getLeds(frame[0])
	mk.info.LedListBlink = getLeds(frame[1])

	// Send charge state request
	cmd := make([]byte, 4)
	cmd[0] = 0x57 //W
	cmd[1] = 0x30
	cmd[2] = 13
	mk.sendCommand(cmd)
}

// Adds active LEDs to list.
func getLeds(led byte) []int {
	leds := make([]int, 0)
	for i := 0; i < 8; i++ {
		t := (led >> uint(i)) & 1
		if t == 1 {
			leds = append(leds, int(1<<uint(i)))
		}
	}
	return leds
}

// Adds header and trailing crc for frame to send.
func (mk2 *mk2Ser) sendCommand(data []byte) {
	l := len(data)
	dataOut := make([]byte, l+3)
	dataOut[0] = byte(l + 1)
	dataOut[1] = 0xff
	cr := -dataOut[0] - dataOut[1]
	for i := 0; i < len(data); i++ {
		cr = cr - data[i]
		dataOut[i+2] = data[i]
	}
	dataOut[l+2] = cr

	_, err := mk2.p.Write(dataOut)
	if err != nil {
		mk2.addError(fmt.Errorf("Write error: %v", err))
	}
}

// Checks the frame crc.
func checkChecksum(l, t byte, d []byte) bool {
	cr := (uint16(l) + uint16(t)) % 256
	for i := 0; i < len(d); i++ {
		cr = (cr + uint16(d[i])) % 256
	}
	if cr == 0 {
		return true
	}
	return false
}
<commit_msg>gofmt<commit_after>package mk2if

import (
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"sync"
	"time"
)

type scaling struct {
	scale  float64
	offset float64
}

type mk2Ser struct {
	info   *Mk2Info
	report *Mk2Info
	p      io.ReadWriter
	sc     []scaling
	scN    int
	run    chan struct{}
	locked bool
	sync.RWMutex
	infochan chan *Mk2Info
	wg       sync.WaitGroup
}

func NewMk2Connection(dev io.ReadWriter) (Mk2If, error) {
	mk2 := &mk2Ser{}
	mk2.p = dev
	mk2.info = &Mk2Info{}
	mk2.report = &Mk2Info{}
	mk2.scN = 0
	mk2.locked = false
	mk2.sc = make([]scaling, 0)
	mk2.setTarget()
	mk2.run = make(chan struct{})
	mk2.infochan = make(chan *Mk2Info)
	mk2.wg.Add(1)
	go mk2.frameLock()
	return mk2, nil
}

// Locks to incoming frame.
func (mk2 *mk2Ser) frameLock() {

	frame := make([]byte, 256)
	var size byte
	for {
		select {
		case <-mk2.run:
			mk2.wg.Done()
			return
		default:
		}
		if mk2.locked {
			size = mk2.readByte()
			l, err := io.ReadFull(mk2.p, frame[0:int(size)+1])
			if err != nil {
				mk2.addError(fmt.Errorf("Read Error: %v", err))
				mk2.locked = false
			} else if l != int(size)+1 {
				mk2.addError(errors.New("Read Length Error"))
				mk2.locked = false
			} else {
				mk2.handleFrame(size, frame[0:int(size+1)])
			}
		} else {
			tmp := mk2.readByte()
			if tmp == 0xff || tmp == 0x20 {
				l, err := io.ReadFull(mk2.p, frame[0:int(size)])
				if err != nil {
					mk2.addError(fmt.Errorf("Read Error: %v", err))
					time.Sleep(1 * time.Second)
				} else if l != int(size) {
					mk2.addError(errors.New("Read Length Error"))
				} else {
					if checkChecksum(size, tmp, frame[0:int(size)]) {
						mk2.locked = true
						log.Printf("Locked")
					}
				}
			}
			size = tmp
		}
	}
}

// Close Mk2
func (mk2 *mk2Ser) Close() {
	close(mk2.run)
	mk2.wg.Wait()
}

// Returns last known state with all reported errors since previous poll.
// Mk2Info.Valid will be false if no polling has completed.
func (mk2 *mk2Ser) GetMk2Info() *Mk2Info {
	mk2.RLock()
	defer mk2.RUnlock()
	return mk2.report
}

func (mk2 *mk2Ser) C() chan *Mk2Info {
	return mk2.infochan
}

func (mk2 *mk2Ser) readByte() byte {
	buffer := make([]byte, 1)
	_, err := io.ReadFull(mk2.p, buffer)
	if err != nil {
		mk2.addError(fmt.Errorf("Read error: %v", err))
		return 0
	}
	return buffer[0]
}

// Adds error to error slice.
func (mk2 *mk2Ser) addError(err error) {
	if mk2.info.Errors == nil {
		mk2.info.Errors = make([]error, 0)
	}
	mk2.info.Errors = append(mk2.info.Errors, err)
	mk2.info.Valid = false
}

// Updates report.
func (mk2 *mk2Ser) updateReport() {
	mk2.Lock()
	defer mk2.Unlock()
	mk2.report = mk2.info
	select {
	case mk2.infochan <- mk2.info:
	default:
	}
	mk2.info = &Mk2Info{}
}

// Checks for valid frame and chooses decoding.
func (mk2 *mk2Ser) handleFrame(l byte, frame []byte) {
	if checkChecksum(l, frame[0], frame[1:]) {
		switch frame[0] {
		case 0xff:
			switch frame[1] {
			case 0x56: // V
				mk2.versionDecode(frame[2:])
			case 0x57:
				switch frame[2] {
				case 0x8e:
					mk2.scaleDecode(frame[2:])
				case 0x85:
					mk2.stateDecode(frame[2:])
				}

			case 0x4C: // L
				mk2.ledDecode(frame[2:])
			}

		case 0x20:
			switch frame[5] {
			case 0x0C:
				mk2.dcDecode(frame[1:])
			case 0x08:
				mk2.acDecode(frame[1:])
			}
		}
	} else {
		log.Printf("Failed")
		mk2.locked = false
	}
}

// Set the target VBus device.
func (mk *mk2Ser) setTarget() {
	cmd := make([]byte, 3)
	cmd[0] = 0x41 // A
	cmd[1] = 0x01
	cmd[2] = 0x00
	mk.sendCommand(cmd)
}

// Request the scaling factor for entry 'in'.
func (mk *mk2Ser) reqScaleFactor(in byte) {
	cmd := make([]byte, 4)
	cmd[0] = 0x57 // W
	cmd[1] = 0x36
	cmd[2] = in
	mk.sendCommand(cmd)
}

// Decode the scale factor frame.
func (mk *mk2Ser) scaleDecode(frame []byte) {
	scl := uint16(frame[2])<<8 + uint16(frame[1])
	ofs := int16(uint16(frame[5])<<8 + uint16(frame[4]))

	tmp := scaling{}
	tmp.offset = float64(ofs)
	if scl >= 0x4000 {
		tmp.scale = math.Abs(1 / (0x8000 - float64(scl)))
	} else {
		tmp.scale = math.Abs(float64(scl))
	}
	mk.sc = append(mk.sc, tmp)

	mk.scN++
	if mk.scN < 14 {
		mk.reqScaleFactor(byte(mk.scN))
	} else {
		log.Print("Monitoring starting.")
	}

}

// Decode the version number
func (mk *mk2Ser) versionDecode(frame []byte) {
	mk.info.Version = 0
	mk.info.Valid = true
	for i := 0; i < 4; i++ {
		mk.info.Version += uint32(frame[i]) << uint(i) * 8
	}

	if mk.scN < 14 {
		log.Print("Get scaling factors.")
		mk.reqScaleFactor(byte(mk.scN))
	} else {
		// Send DC status request
		cmd := make([]byte, 2)
		cmd[0] = 0x46 //F
		cmd[1] = 0
		mk.sendCommand(cmd)
	}
}

// Apply scaling to float
func (mk *mk2Ser) applyScale(value float64, scale int) float64 {
	return mk.sc[scale].scale * (value + mk.sc[scale].offset)
}

// Convert bytes->int16->float
func getSigned(data []byte) float64 {
	return float64(int16(data[0]) + int16(data[1])<<8)
}

// Convert bytes->uint32->float
func getUnsigned(data []byte) float64 {
	return float64(uint32(data[0]) + uint32(data[1])<<8 + uint32(data[2])<<16)
}

// Decodes DC frame.
func (mk *mk2Ser) dcDecode(frame []byte) {
	mk.info.BatVoltage = mk.applyScale(getSigned(frame[5:7]), 4)

	usedC := mk.applyScale(getUnsigned(frame[7:10]), 5)
	chargeC := mk.applyScale(getUnsigned(frame[10:13]), 5)
	mk.info.BatCurrent = usedC - chargeC

	mk.info.OutFrequency = 10 / (mk.applyScale(float64(frame[13]), 7))

	// Send L1 status request
	cmd := make([]byte, 2)
	cmd[0] = 0x46 //F
	cmd[1] = 1
	mk.sendCommand(cmd)
}

// Decodes AC frame.
func (mk *mk2Ser) acDecode(frame []byte) {
	mk.info.InVoltage = mk.applyScale(getSigned(frame[5:7]), 0)
	mk.info.InCurrent = mk.applyScale(getSigned(frame[7:9]), 1)
	mk.info.OutVoltage = mk.applyScale(getSigned(frame[9:11]), 2)
	mk.info.OutCurrent = mk.applyScale(getSigned(frame[11:13]), 3)

	if frame[13] == 0xff {
		mk.info.InFrequency = 0
	} else {
		mk.info.InFrequency = 10 / (mk.applyScale(float64(frame[13]), 8))
	}

	// Send status request
	cmd := make([]byte, 1)
	cmd[0] = 0x4C //F
	mk.sendCommand(cmd)
}

// Decode charge state of battery.
func (mk *mk2Ser) stateDecode(frame []byte) {
	mk.info.ChargeState = mk.applyScale(getSigned(frame[1:3]), 13)
	mk.updateReport()
}

// Decode the LED state frame.
func (mk *mk2Ser) ledDecode(frame []byte) {
	mk.info.LedListOn = getLeds(frame[0])
	mk.info.LedListBlink = getLeds(frame[1])

	// Send charge state request
	cmd := make([]byte, 4)
	cmd[0] = 0x57 //W
	cmd[1] = 0x30
	cmd[2] = 13
	mk.sendCommand(cmd)
}

// Adds active LEDs to list.
func getLeds(led byte) []int {
	leds := make([]int, 0)
	for i := 0; i < 8; i++ {
		t := (led >> uint(i)) & 1
		if t == 1 {
			leds = append(leds, int(1<<uint(i)))
		}
	}
	return leds
}

// Adds header and trailing crc for frame to send.
func (mk2 *mk2Ser) sendCommand(data []byte) {
	l := len(data)
	dataOut := make([]byte, l+3)
	dataOut[0] = byte(l + 1)
	dataOut[1] = 0xff
	cr := -dataOut[0] - dataOut[1]
	for i := 0; i < len(data); i++ {
		cr = cr - data[i]
		dataOut[i+2] = data[i]
	}
	dataOut[l+2] = cr

	_, err := mk2.p.Write(dataOut)
	if err != nil {
		mk2.addError(fmt.Errorf("Write error: %v", err))
	}
}

// Checks the frame crc.
func checkChecksum(l, t byte, d []byte) bool {
	cr := (uint16(l) + uint16(t)) % 256
	for i := 0; i < len(d); i++ {
		cr = (cr + uint16(d[i])) % 256
	}
	if cr == 0 {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package mmic

import (
	log "github.com/Sirupsen/logrus"

	"fmt"
	"github.com/eternnoir/mmic/mmic/config"
	"github.com/eternnoir/mmic/mmic/handlers"
	"github.com/eternnoir/mmic/mmic/handlers/trello"
	"github.com/labstack/echo"
	"github.com/labstack/echo/engine/standard"
	"github.com/labstack/echo/middleware"
)

type Mmic struct {
	Port       string
	HttpServer *echo.Echo
	RouteMap   map[string]handlers.Handler
}

func NewMmic(config *config.MmicConfig) (*Mmic, error) {
	log.Infof("Create Mnic by mmic config: %#v", config)
	mmic := &Mmic{Port: config.Port, HttpServer: echo.New()}
	mmic.RouteMap = make(map[string]handlers.Handler)
	err := mmic.bindHandlers(config)
	if err != nil {
		return nil, err
	}

	return mmic, nil
}

func (mmic *Mmic) bindHandlers(config *config.MmicConfig) error {
	err := mmic.bindTrelloHandlers(config.TrelloConfigs)
	if err != nil {
		log.Errorf("Bind handler fail. %s", err)
		return err
	}

	return nil
}

func (mmic *Mmic) bindTrelloHandlers(configs []config.TrelloConfig) error {
	for _, config := range configs {
		if _, inmap := mmic.RouteMap[config.RoutePath]; inmap {
			return fmt.Errorf("Rout path %s already exist.", config.RoutePath)
		}
		th := trello.NewTrelloHandler(config)
		log.Debugf("Bind %#v", config)
		mmic.HttpServer.Post(config.RoutePath, th)
		mmic.RouteMap[config.RoutePath] = th
	}
	return nil
}

func (mmic *Mmic) Start() {
	mmic.HttpServer.Use(middleware.Logger())
	mmic.HttpServer.Use(middleware.Recover())
	mmic.HttpServer.Run(standard.New(":" + mmic.Port))
}
<commit_msg>Add empty handler.<commit_after>package mmic

import (
	log "github.com/Sirupsen/logrus"

	"fmt"
	"github.com/eternnoir/mmic/mmic/config"
	"github.com/eternnoir/mmic/mmic/handlers"
	"github.com/eternnoir/mmic/mmic/handlers/trello"
	"github.com/labstack/echo"
	"github.com/labstack/echo/engine/standard"
	"github.com/labstack/echo/middleware"
	"net/http"
)

type Mmic struct {
	Port       string
	HttpServer *echo.Echo
	RouteMap   map[string]handlers.Handler
}

func NewMmic(config *config.MmicConfig) (*Mmic, error) {
	log.Infof("Create Mnic by mmic config: %#v", config)
	mmic := &Mmic{Port: config.Port, HttpServer: echo.New()}
	mmic.RouteMap = make(map[string]handlers.Handler)
	err := mmic.bindHandlers(config)
	if err != nil {
		return nil, err
	}

	return mmic, nil
}

func (mmic *Mmic) bindHandlers(config *config.MmicConfig) error {
	err := mmic.bindTrelloHandlers(config.TrelloConfigs)
	if err != nil {
		log.Errorf("Bind handler fail. %s", err)
		return err
	}

	return nil
}

func (mmic *Mmic) bindTrelloHandlers(configs []config.TrelloConfig) error {
	for _, config := range configs {
		if _, inmap := mmic.RouteMap[config.RoutePath]; inmap {
			return fmt.Errorf("Rout path %s already exist.", config.RoutePath)
		}
		th := trello.NewTrelloHandler(config)
		log.Debugf("Bind %#v", config)
		mmic.HttpServer.Post(config.RoutePath, th)
		mmic.HttpServer.Head(config.RoutePath, emptyHandler())
		mmic.RouteMap[config.RoutePath] = th
	}
	return nil
}

func (mmic *Mmic) Start() {
	mmic.HttpServer.Use(middleware.Logger())
	mmic.HttpServer.Use(middleware.Recover())
	mmic.HttpServer.Run(standard.New(":" + mmic.Port))
}

func emptyHandler() echo.HandlerFunc {
	return func(c echo.Context) error {
		return c.String(http.StatusOK, "Hi")
	}
}
<|endoftext|>


<commit_before>package cs

import (
	"fmt"
	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/ecs"
	"net/http"
	"time"
)

type NodePoolInfo struct {
	NodePoolId      string        `json:"nodepool_id"`
	RegionId        common.Region `json:"region_id"`
	Name            string        `json:"name"`
	Created         time.Time     `json:"created"`
	Updated         time.Time     `json:"updated"`
	IsDefault       bool          `json:"is_default"`
	NodePoolType    string        `json:"type"`
	ResourceGroupId string        `json:"resource_group_id"`
}

type ScalingGroup struct {
	VpcId              string             `json:"vpc_id"`
	VswitchIds         []string           `json:"vswitch_ids"`
	InstanceTypes      []string           `json:"instance_types"`
	LoginPassword      string             `json:"login_password"`
	KeyPair            string             `json:"key_pair"`
	SecurityGroupId    string             `json:"security_group_id"`
	SystemDiskCategory ecs.DiskCategory   `json:"system_disk_category"`
	SystemDiskSize     int64              `json:"system_disk_size"`
	DataDisks          []NodePoolDataDisk `json:"data_disks"` //支持多个数据盘
	Tags               []Tag              `json:"tags"`
	ImageId            string             `json:"image_id"`
	Platform           string             `json:"platform"`
	// 支持包年包月
	InstanceChargeType string   `json:"instance_charge_type"`
	Period             int      `json:"period"`
	PeriodUnit         string   `json:"period_unit"`
	AutoRenew          bool     `json:"auto_renew"`
	AutoRenewPeriod    int      `json:"auto_renew_period"`
	RdsInstances       []string ` json:"rds_instances"`
	ScalingPolicy      string   `json:"scaling_policy"`
	ScalingGroupId     string   `json:"scaling_group_id"`
}

type AutoScaling struct {
	Enable      bool   `json:"enable"`
	MaxInstance int64  `json:"max_instance"`
	MinInstance int64  `json:"min_instance"`
	Type        string `json:"type"`
	// eip
	IsBindEip *bool `json:"is_bond_eip"`
	// value: PayByBandwidth / PayByTraffic
	EipInternetChargeType string `json:"eip_internet_charge_type"`
	// default 5
	EipBandWidth int64 `json:"eip_bandwidth"`
}

type KubernetesConfig struct {
	NodeNameMode string  `json:"node_name_mode"`
	Taints       []Taint `json:"taints"`
	Labels       []Label `json:"labels"`
	CpuPolicy    string  `json:"cpu_policy"`
	UserData     string  `json:"user_data"`

	Runtime           string `json:"runtime,omitempty"`
	RuntimeVersion    string `json:"runtime_version"`
	CmsEnabled        bool   `json:"cms_enabled"`
	OverwriteHostname bool   `json:"overwrite_hostname"`
}

// 加密计算节点池
type TEEConfig struct {
	TEEType   string `json:"tee_type"`
	TEEEnable bool   `json:"tee_enable"`
}

type CreateNodePoolRequest struct {
	RegionId         common.Region `json:"region_id"`
	Count            int64         `json:"count"`
	NodePoolInfo     `json:"nodepool_info"`
	ScalingGroup     `json:"scaling_group"`
	KubernetesConfig `json:"kubernetes_config"`
	AutoScaling      `json:"auto_scaling"`
	TEEConfig        `json:"tee_config"`
}

type BasicNodePool struct {
	NodePoolInfo   `json:"nodepool_info"`
	NodePoolStatus `json:"status"`
}

type NodePoolStatus struct {
	TotalNodes   int `json:"total_nodes"`
	OfflineNodes int `json:"offline_nodes"`
	ServingNodes int `json:"serving_nodes"`
	//DesiredNodes int  `json:"desired_nodes"`
	RemovingNodes int    `json:"removing_nodes"`
	FailedNodes   int    `json:"failed_nodes"`
	InitialNodes  int    `json:"initial_nodes"`
	HealthyNodes  int    `json:"healthy_nodes"`
	State         string `json:"state"`
}

type NodePoolDetail struct {
	BasicNodePool
	KubernetesConfig `json:"kubernetes_config"`
	ScalingGroup     `json:"scaling_group"`
	AutoScaling      `json:"auto_scaling"`
	TEEConfig        `json:"tee_config"`
}

type CreateNodePoolResponse struct {
	Response
	NodePoolID string `json:"nodepool_id"`
	Message    string `json:"Message"`
}

type UpdateNodePoolRequest struct {
	RegionId         common.Region `json:"region_id"`
	Count            int64         `json:"count"`
	NodePoolInfo     `json:"nodepool_info"`
	ScalingGroup     `json:"scaling_group"`
	KubernetesConfig `json:"kubernetes_config"`
	AutoScaling      `json:"auto_scaling"`
}

type NodePoolsDetail struct {
	Response
	NodePools []NodePoolDetail `json:"nodepools"`
}

func (client *Client) CreateNodePool(request *CreateNodePoolRequest, clusterId string) (*CreateNodePoolResponse, error) {
	response := &CreateNodePoolResponse{}
	err := client.Invoke(request.RegionId, http.MethodPost, fmt.Sprintf("/clusters/%s/nodepools", clusterId), nil, request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (client *Client) DescribeNodePoolDetail(clusterId, nodePoolId string) (*NodePoolDetail, error) {
	nodePool := &NodePoolDetail{}
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/clusters/%s/nodepools/%s", clusterId, nodePoolId), nil, nil, nodePool)
	if err != nil {
		return nil, err
	}
	return nodePool, nil
}

func (client *Client) DescribeClusterNodePools(clusterId string) (*[]NodePoolDetail, error) {
	nodePools := &NodePoolsDetail{}
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/clusters/%s/nodepools", clusterId), nil, nil, nodePools)
	if err != nil {
		return nil, err
	}
	return &nodePools.NodePools, nil
}

func (client *Client) UpdateNodePool(clusterId string, nodePoolId string, request *UpdateNodePoolRequest) (*Response, error) {
	response := &Response{}
	err := client.Invoke(request.RegionId, http.MethodPut, fmt.Sprintf("/clusters/%s/nodepools/%s", clusterId, nodePoolId), nil, request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (client *Client) DeleteNodePool(clusterId, nodePoolId string) error {
	return client.Invoke("", http.MethodDelete, fmt.Sprintf("/clusters/%s/nodepools/%s", clusterId, nodePoolId), nil, nil, nil)
}
<commit_msg>Support<commit_after>package cs

import (
	"fmt"
	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/ecs"
	"net/http"
	"time"
)

type NodePoolInfo struct {
	NodePoolId      string        `json:"nodepool_id"`
	RegionId        common.Region `json:"region_id"`
	Name            string        `json:"name"`
	Created         time.Time     `json:"created"`
	Updated         time.Time     `json:"updated"`
	IsDefault       bool          `json:"is_default"`
	NodePoolType    string        `json:"type"`
	ResourceGroupId string        `json:"resource_group_id"`
}

type ScalingGroup struct {
	VpcId              string             `json:"vpc_id"`
	VswitchIds         []string           `json:"vswitch_ids"`
	InstanceTypes      []string           `json:"instance_types"`
	LoginPassword      string             `json:"login_password"`
	KeyPair            string             `json:"key_pair"`
	SecurityGroupId    string             `json:"security_group_id"`
	SystemDiskCategory ecs.DiskCategory   `json:"system_disk_category"`
	SystemDiskSize     int64              `json:"system_disk_size"`
	DataDisks          []NodePoolDataDisk `json:"data_disks"` //支持多个数据盘
	Tags               []Tag              `json:"tags"`
	ImageId            string             `json:"image_id"`
	Platform           string             `json:"platform"`
	// 支持包年包月
	InstanceChargeType string   `json:"instance_charge_type"`
	Period             int      `json:"period"`
	PeriodUnit         string   `json:"period_unit"`
	AutoRenew          bool     `json:"auto_renew"`
	AutoRenewPeriod    int      `json:"auto_renew_period"`
	RdsInstances       []string ` json:"rds_instances"`
	ScalingPolicy      string   `json:"scaling_policy"`
	ScalingGroupId     string   `json:"scaling_group_id"`
}

type AutoScaling struct {
	Enable      bool   `json:"enable"`
	MaxInstance int64  `json:"max_instance"`
	MinInstance int64  `json:"min_instance"`
	Type        string `json:"type"`
	// eip
	IsBindEip *bool `json:"is_bond_eip"`
	// value: PayByBandwidth / PayByTraffic
	EipInternetChargeType string `json:"eip_internet_charge_type"`
	// default 5
	EipBandWidth int64 `json:"eip_bandwidth"`
}

type KubernetesConfig struct {
	NodeNameMode string  `json:"node_name_mode"`
	Taints       []Taint `json:"taints"`
	Labels       []Label `json:"labels"`
	CpuPolicy    string  `json:"cpu_policy"`
	UserData     string  `json:"user_data"`

	Runtime           string `json:"runtime,omitempty"`
	RuntimeVersion    string `json:"runtime_version"`
	CmsEnabled        bool   `json:"cms_enabled"`
	OverwriteHostname bool   `json:"overwrite_hostname"`
}

// 加密计算节点池
type TEEConfig struct {
	TEEType   string `json:"tee_type"`
	TEEEnable bool   `json:"tee_enable"`
}

// 托管节点池配置
type Management struct {
	Enable      bool        `json:"enable"`
	AutoRepair  bool        `json:"auto_repair"`
	UpgradeConf UpgradeConf `json:"upgrade_config"`
}

type UpgradeConf struct {
	AutoUpgrade       bool  `json:"auto_upgrade"`
	Surge             int64 `json:"surge"`
	SurgePercentage   int64 `json:"surge_percentage,omitempty"`
	MaxUnavailable    int64 `json:"max_unavailable"`
	KeepSurgeOnFailed bool  `json:"keep_surge_on_failed"`
}

type CreateNodePoolRequest struct {
	RegionId         common.Region `json:"region_id"`
	Count            int64         `json:"count"`
	NodePoolInfo     `json:"nodepool_info"`
	ScalingGroup     `json:"scaling_group"`
	KubernetesConfig `json:"kubernetes_config"`
	AutoScaling      `json:"auto_scaling"`
	TEEConfig        `json:"tee_config"`
	Management       `json:"management"`
}

type BasicNodePool struct {
	NodePoolInfo   `json:"nodepool_info"`
	NodePoolStatus `json:"status"`
}

type NodePoolStatus struct {
	TotalNodes   int `json:"total_nodes"`
	OfflineNodes int `json:"offline_nodes"`
	ServingNodes int `json:"serving_nodes"`
	//DesiredNodes int  `json:"desired_nodes"`
	RemovingNodes int    `json:"removing_nodes"`
	FailedNodes   int    `json:"failed_nodes"`
	InitialNodes  int    `json:"initial_nodes"`
	HealthyNodes  int    `json:"healthy_nodes"`
	State         string `json:"state"`
}

type NodePoolDetail struct {
	BasicNodePool
	KubernetesConfig `json:"kubernetes_config"`
	ScalingGroup     `json:"scaling_group"`
	AutoScaling      `json:"auto_scaling"`
	TEEConfig        `json:"tee_config"`
	Management       `json:"management"`
}

type CreateNodePoolResponse struct {
	Response
	NodePoolID string `json:"nodepool_id"`
	Message    string `json:"Message"`
}

type UpdateNodePoolRequest struct {
	RegionId         common.Region `json:"region_id"`
	Count            int64         `json:"count"`
	NodePoolInfo     `json:"nodepool_info"`
	ScalingGroup     `json:"scaling_group"`
	KubernetesConfig `json:"kubernetes_config"`
	AutoScaling      `json:"auto_scaling"`
	Management       `json:"management"`
}

type NodePoolsDetail struct {
	Response
	NodePools []NodePoolDetail `json:"nodepools"`
}

func (client *Client) CreateNodePool(request *CreateNodePoolRequest, clusterId string) (*CreateNodePoolResponse, error) {
	response := &CreateNodePoolResponse{}
	err := client.Invoke(request.RegionId, http.MethodPost, fmt.Sprintf("/clusters/%s/nodepools", clusterId), nil, request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (client *Client) DescribeNodePoolDetail(clusterId, nodePoolId string) (*NodePoolDetail, error) {
	nodePool := &NodePoolDetail{}
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/clusters/%s/nodepools/%s", clusterId, nodePoolId), nil, nil, nodePool)
	if err != nil {
		return nil, err
	}
	return nodePool, nil
}

func (client *Client) DescribeClusterNodePools(clusterId string) (*[]NodePoolDetail, error) {
	nodePools := &NodePoolsDetail{}
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/clusters/%s/nodepools", clusterId), nil, nil, nodePools)
	if err != nil {
		return nil, err
	}
	return &nodePools.NodePools, nil
}

func (client *Client) UpdateNodePool(clusterId string, nodePoolId string, request *UpdateNodePoolRequest) (*Response, error) {
	response := &Response{}
	err := client.Invoke(request.RegionId, http.MethodPut, fmt.Sprintf("/clusters/%s/nodepools/%s", clusterId, nodePoolId), nil, request, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

func (client *Client) DeleteNodePool(clusterId, nodePoolId string) error {
	return client.Invoke("", http.MethodDelete, fmt.Sprintf("/clusters/%s/nodepools/%s", clusterId, nodePoolId), nil, nil, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/kyma-project/test-infra/development/tools/pkg/common"
	"github.com/kyma-project/test-infra/development/tools/pkg/dnscollector"
	log "github.com/sirupsen/logrus"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"
	dns "google.golang.org/api/dns/v1"
)

const defaultAddressRegexpList = "(remoteenvs-)?gkeint-(pr|commit)-.*,(remoteenvs-)?gke-upgrade-(pr|commit)-.*,(remoteenvs-)?(load-test|e2etest),gke-central-(pr|commit)-.*,gke-backup-(pr|commit)-.*"
const minAgeInHours = 1
const minPatternLength = 5

var (
	project               = flag.String("project", "", "project id [required]")
	regions               = flag.String("regions", "", "comma-separted list of GCP regions [required]")
	dnsZone               = flag.String("dnsZone", "", "Name of the DNS Managed Zone [Required]")
	dryRun                = flag.Bool("dryRun", true, "Dry Run enabled, nothing is deleted")
	ageInHours            = flag.Int("ageInHours", 2, "IP Address age in hours. Addresses older than: now()-ageInHours are considered for removal.")
	addressNameRegexpList = flag.String("addressRegexpList", defaultAddressRegexpList, "Address name regexp list. Separate items with commas, spaces are trimmed. Matching addresses are considered for removal.")
)

func main() {
	flag.Parse()

	if *project == "" {
		fmt.Fprint(os.Stderr, "missing -project flag\n\n")
		flag.Usage()
		os.Exit(2)
	}

	if *dnsZone == "" {
		fmt.Fprint(os.Stderr, "missing -dnsZone flag\n\n")
		flag.Usage()
		os.Exit(2)
	}

	regionsList := splitPatterns(*regions)
	for _, region := range regionsList {
		if len(region) == 0 {
			fmt.Fprint(os.Stderr, "invalid region: \"\"\n\n")
			flag.Usage()
			os.Exit(2)
		}
	}

	patterns := splitPatterns(*addressNameRegexpList)
	regexpList := []*regexp.Regexp{}
	for _, pattern := range patterns {
		if len(pattern) < minPatternLength {
			fmt.Fprintf(os.Stderr, "invalid pattern: \"%s\". Value must not be shorter than %d characters.\n\n", pattern, minPatternLength)
			flag.Usage()
			os.Exit(2)
		}
		regexpList = append(regexpList, regexp.MustCompile(pattern))
	}

	if *ageInHours < minAgeInHours {
		fmt.Fprintf(os.Stderr, "invalid ageInHours. Value must not be smaller than %d\n\n", minAgeInHours)
		flag.Usage()
		os.Exit(2)
	}

	common.ShoutFirst("Running with arguments: project: \"%s\", regions: \"%s\", dnsZone: \"%s\", dryRun: %t, ageInHours: %d, addressRegexpList: %s", *project, quoteElems(regionsList), *dnsZone, *dryRun, *ageInHours, quoteElems(patterns))
	ctx := context.Background()

	computeConn, err := google.DefaultClient(ctx, compute.CloudPlatformScope)
	if err != nil {
		log.Fatalf("Could not get authenticated client: %v", err)
	}

	computeSvc, err := compute.New(computeConn)
	if err != nil {
		log.Fatalf("Could not initialize gke client for Compute API: %v", err)
	}

	dnsSvc, err := dns.New(computeConn)
	if err != nil {
		log.Fatalf("Could not initialize gke client for Compute API: %v", err)
	}

	computeAPI := &dnscollector.ComputeServiceWrapper{Context: ctx, Compute: computeSvc}
	dnsAPI := &dnscollector.DNSServiceWrapper{Context: ctx, DNS: dnsSvc}
	shouldRemoveFunc := dnscollector.DefaultIPAddressRemovalPredicate(regexpList, *ageInHours)

	cleaner := dnscollector.New(computeAPI, dnsAPI, shouldRemoveFunc)
	allSucceeded, err := cleaner.Run(*project, *dnsZone, regionsList, !(*dryRun))

	if err != nil {
		log.Fatalf("IP/DNS collector error: %v", err)
	}

	if !allSucceeded {
		log.Warn("Some operations failed.")
	}

	common.Shout("Finished")
}

func quoteElems(elems []string) string {

	res := "\"" + elems[0] + "\""
	for i := 1; i < len(elems); i++ {
		res = res + ",\"" + elems[i] + "\""
	}

	return "[" + res + "]"
}

func splitPatterns(commaSeparated string) []string {

	res := []string{}
	values := strings.Split(commaSeparated, ",")
	for _, pattern := range values {
		val := strings.Trim(pattern, " ")
		res = append(res, val)
	}

	return res
}
<commit_msg>extending default cleaner regex (#1404)<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/kyma-project/test-infra/development/tools/pkg/common"
	"github.com/kyma-project/test-infra/development/tools/pkg/dnscollector"
	log "github.com/sirupsen/logrus"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"
	dns "google.golang.org/api/dns/v1"
)

const defaultAddressRegexpList = "(remoteenvs-)?gkeint-(pr|commit)-.*,(remoteenvs-)?gke-upgrade-(pr|commit)-.*,(remoteenvs-)?(load-test|e2etest),gke-central-(pr|commit)-.*,gke-backup-(pr|commit)-.*,service-catalog-crd-periodic,gkecompint-(pr|commit)-.*,gke-minio-(pr|commit)-.*,gke-minio-min-(pr|commit).*"
const minAgeInHours = 1
const minPatternLength = 5

var (
	project               = flag.String("project", "", "project id [required]")
	regions               = flag.String("regions", "", "comma-separted list of GCP regions [required]")
	dnsZone               = flag.String("dnsZone", "", "Name of the DNS Managed Zone [Required]")
	dryRun                = flag.Bool("dryRun", true, "Dry Run enabled, nothing is deleted")
	ageInHours            = flag.Int("ageInHours", 2, "IP Address age in hours. Addresses older than: now()-ageInHours are considered for removal.")
	addressNameRegexpList = flag.String("addressRegexpList", defaultAddressRegexpList, "Address name regexp list. Separate items with commas, spaces are trimmed. Matching addresses are considered for removal.")
)

func main() {
	flag.Parse()

	if *project == "" {
		fmt.Fprint(os.Stderr, "missing -project flag\n\n")
		flag.Usage()
		os.Exit(2)
	}

	if *dnsZone == "" {
		fmt.Fprint(os.Stderr, "missing -dnsZone flag\n\n")
		flag.Usage()
		os.Exit(2)
	}

	regionsList := splitPatterns(*regions)
	for _, region := range regionsList {
		if len(region) == 0 {
			fmt.Fprint(os.Stderr, "invalid region: \"\"\n\n")
			flag.Usage()
			os.Exit(2)
		}
	}

	patterns := splitPatterns(*addressNameRegexpList)
	regexpList := []*regexp.Regexp{}
	for _, pattern := range patterns {
		if len(pattern) < minPatternLength {
			fmt.Fprintf(os.Stderr, "invalid pattern: \"%s\". Value must not be shorter than %d characters.\n\n", pattern, minPatternLength)
			flag.Usage()
			os.Exit(2)
		}
		regexpList = append(regexpList, regexp.MustCompile(pattern))
	}

	if *ageInHours < minAgeInHours {
		fmt.Fprintf(os.Stderr, "invalid ageInHours. Value must not be smaller than %d\n\n", minAgeInHours)
		flag.Usage()
		os.Exit(2)
	}

	common.ShoutFirst("Running with arguments: project: \"%s\", regions: \"%s\", dnsZone: \"%s\", dryRun: %t, ageInHours: %d, addressRegexpList: %s", *project, quoteElems(regionsList), *dnsZone, *dryRun, *ageInHours, quoteElems(patterns))
	ctx := context.Background()

	computeConn, err := google.DefaultClient(ctx, compute.CloudPlatformScope)
	if err != nil {
		log.Fatalf("Could not get authenticated client: %v", err)
	}

	computeSvc, err := compute.New(computeConn)
	if err != nil {
		log.Fatalf("Could not initialize gke client for Compute API: %v", err)
	}

	dnsSvc, err := dns.New(computeConn)
	if err != nil {
		log.Fatalf("Could not initialize gke client for Compute API: %v", err)
	}

	computeAPI := &dnscollector.ComputeServiceWrapper{Context: ctx, Compute: computeSvc}
	dnsAPI := &dnscollector.DNSServiceWrapper{Context: ctx, DNS: dnsSvc}
	shouldRemoveFunc := dnscollector.DefaultIPAddressRemovalPredicate(regexpList, *ageInHours)

	cleaner := dnscollector.New(computeAPI, dnsAPI, shouldRemoveFunc)
	allSucceeded, err := cleaner.Run(*project, *dnsZone, regionsList, !(*dryRun))

	if err != nil {
		log.Fatalf("IP/DNS collector error: %v", err)
	}

	if !allSucceeded {
		log.Warn("Some operations failed.")
	}

	common.Shout("Finished")
}

func quoteElems(elems []string) string {

	res := "\"" + elems[0] + "\""
	for i := 1; i < len(elems); i++ {
		res = res + ",\"" + elems[i] + "\""
	}

	return "[" + res + "]"
}

func splitPatterns(commaSeparated string) []string {

	res := []string{}
	values := strings.Split(commaSeparated, ",")
	for _, pattern := range values {
		val := strings.Trim(pattern, " ")
		res = append(res, val)
	}

	return res
}
<|endoftext|>


<commit_before>package cas

import (
	"crypto/sha1"
	"encoding/hex"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

// [Content-Addressed Storage](http://de.wikipedia.org/wiki/Content-Addressed_Storage)
type Storage struct {
	KeyToPath map[string]string
	PathToKey map[string]string
}

// imports a file
func (c *Storage) ImportFile(path string) (key string, err error) {
	path, err = filepath.Abs(path)
	if err != nil {
		return "", err
	}
	c.DelByPath(path)
	file, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer file.Close()
	hash := sha1.New()
	_, err = io.Copy(hash, file)
	if err != nil {
		return "", err
	}
	key = hex.EncodeToString(hash.Sum(nil))
	c.KeyToPath[key] = path
	c.PathToKey[path] = key
	return key, nil
}

// recursively imports a directory
func (c *Storage) ImportDir(path string) (err error) {
	fis, err := ioutil.ReadDir(path)
	if err != nil {
		return err
	}
	for _, fi := range fis {
		if fi.IsDir() {
			err = c.ImportDir(path + string(os.PathSeparator) + fi.Name())
		} else {
			_, err = c.ImportFile(path + string(os.PathSeparator) + fi.Name())
		}
		if err != nil {
			return err
		}
	}
	return nil
}

// retrieves a file by the key returned when importing it
func (c *Storage) Get(key string) (*os.File, error) {
	return os.Open(c.KeyToPath[key])
}

func (c *Storage) DelByPath(path string) (err error) {
	path, err = filepath.Abs(path)
	if err != nil {
		return err
	}
	key := c.PathToKey[path]
	delete(c.PathToKey, path)
	delete(c.KeyToPath, key)
	return nil
}

func (c *Storage) DelByKey(key string) {
	path := c.KeyToPath[key]
	delete(c.PathToKey, path)
	delete(c.KeyToPath, key)
}

func (c *Storage) Flush() {
	c.PathToKey = make(map[string]string)
	c.KeyToPath = make(map[string]string)
}

func (c *Storage) String() string {
	s := ""
	for key, path := range c.KeyToPath {
		s = s + key + " " + path + "\n"
	}
	return s
}

func (c *Storage) Update() {
	for _, path := range c.KeyToPath {
		c.ImportFile(path)
	}
}

func New() *Storage {
	return &Storage{
		KeyToPath: make(map[string]string),
		PathToKey: make(map[string]string),
	}
}
<commit_msg>Add Has, HasKey, HasPath<commit_after>package cas

import (
	"crypto/sha1"
	"encoding/hex"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

// [Content-Addressed Storage](http://de.wikipedia.org/wiki/Content-Addressed_Storage)
type Storage struct {
	KeyToPath map[string]string
	PathToKey map[string]string
}

// imports a file
func (c *Storage) ImportFile(path string) (key string, err error) {
	path, err = filepath.Abs(path)
	if err != nil {
		return "", err
	}
	c.DelByPath(path)
	file, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer file.Close()
	hash := sha1.New()
	_, err = io.Copy(hash, file)
	if err != nil {
		return "", err
	}
	key = hex.EncodeToString(hash.Sum(nil))
	c.KeyToPath[key] = path
	c.PathToKey[path] = key
	return key, nil
}

// recursively imports a directory
func (c *Storage) ImportDir(path string) (err error) {
	fis, err := ioutil.ReadDir(path)
	if err != nil {
		return err
	}
	for _, fi := range fis {
		if fi.IsDir() {
			err = c.ImportDir(path + string(os.PathSeparator) + fi.Name())
		} else {
			_, err = c.ImportFile(path + string(os.PathSeparator) + fi.Name())
		}
		if err != nil {
			return err
		}
	}
	return nil
}

// retrieves a file by the key returned when importing it
func (c *Storage) Get(key string) (*os.File, error) {
	return os.Open(c.KeyToPath[key])
}

func (c *Storage) Has(key string) bool {
	return c.HasKey(key)
}

func (c *Storage) HasKey(key string) bool {
	return c.KeyToPath[key] != ""
}

func (c *Storage) HasPath(path string) bool {
	return c.PathToKey[path] != ""
}

func (c *Storage) DelByPath(path string) (err error) {
	path, err = filepath.Abs(path)
	if err != nil {
		return err
	}
	key := c.PathToKey[path]
	delete(c.PathToKey, path)
	delete(c.KeyToPath, key)
	return nil
}

func (c *Storage) DelByKey(key string) {
	path := c.KeyToPath[key]
	delete(c.PathToKey, path)
	delete(c.KeyToPath, key)
}

func (c *Storage) Flush() {
	c.PathToKey = make(map[string]string)
	c.KeyToPath = make(map[string]string)
}

func (c *Storage) String() string {
	s := ""
	for key, path := range c.KeyToPath {
		s = s + key + " " + path + "\n"
	}
	return s
}

func (c *Storage) Update() {
	for _, path := range c.KeyToPath {
		c.ImportFile(path)
	}
}

func New() *Storage {
	return &Storage{
		KeyToPath: make(map[string]string),
		PathToKey: make(map[string]string),
	}
}
<|endoftext|>


<commit_before>package consul

import (
	"fmt"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/serf/serf"
	"log"
	"net"
	"net/rpc"
	"os"
	"path/filepath"
	"sync"
	"time"
)

const (
	serfLANSnapshot = "serf/local.snapshot"
	serfWANSnapshot = "serf/remote.snapshot"
	raftState       = "raft/"
)

// Server is Consul server which manages the service discovery,
// health checking, DC forwarding, Raft, and multiple Serf pools.
type Server struct {
	config *Config

	// Connection pool to other consul servers
	connPool *ConnPool

	// eventChLAN is used to receive events from the
	// serf cluster in the datacenter
	eventChLAN chan serf.Event

	// eventChWAN is used to receive events from the
	// serf cluster that spans datacenters
	eventChWAN chan serf.Event

	// fsm is the state machine used with Raft to provide
	// strong consistency.
	fsm *consulFSM

	// Logger uses the provided LogOutput
	logger *log.Logger

	// The raft instance is used among Consul nodes within the
	// DC to protect operations that require strong consistency
	raft          *raft.Raft
	raftLayer     *RaftLayer
	raftStore     *raft.SQLiteStore
	raftTransport *raft.NetworkTransport

	// rpcClients is used to track active clients
	rpcClients    map[net.Conn]struct{}
	rpcClientLock sync.Mutex

	// rpcListener is used to listen for incoming connections
	rpcListener net.Listener
	rpcServer   *rpc.Server

	// serfLAN is the Serf cluster maintained inside the DC
	// which contains all the DC nodes
	serfLAN *serf.Serf

	// serfWAN is the Serf cluster maintained between DC's
	// which SHOULD only consist of Consul servers
	serfWAN *serf.Serf

	shutdown     bool
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// NewServer is used to construct a new Consul server from the
// configuration, potentially returning an error
func NewServer(config *Config) (*Server, error) {
	// Check for a data directory!
	if config.DataDir == "" {
		return nil, fmt.Errorf("Config must provide a DataDir")
	}

	// Ensure we have a log output
	if config.LogOutput == nil {
		config.LogOutput = os.Stderr
	}

	// Create a logger
	logger := log.New(config.LogOutput, "", log.LstdFlags)

	// Create server
	s := &Server{
		config:     config,
		connPool:   NewPool(5),
		eventChLAN: make(chan serf.Event, 256),
		eventChWAN: make(chan serf.Event, 256),
		logger:     logger,
		rpcClients: make(map[net.Conn]struct{}),
		rpcServer:  rpc.NewServer(),
		shutdownCh: make(chan struct{}),
	}

	// Initialize the RPC layer
	if err := s.setupRPC(); err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start RPC layer: %v", err)
	}

	// Start the Serf listeners to prevent a deadlock
	go s.lanEventHandler()
	go s.wanEventHandler()

	// Initialize the lan Serf
	var err error
	s.serfLAN, err = s.setupSerf(config.SerfLANConfig,
		s.eventChLAN, serfLANSnapshot)
	if err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start lan serf: %v", err)
	}

	// Initialize the wan Serf
	s.serfWAN, err = s.setupSerf(config.SerfWANConfig,
		s.eventChWAN, serfWANSnapshot)
	if err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start wan serf: %v", err)
	}

	// Initialize the Raft server
	if err := s.setupRaft(); err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start Raft: %v", err)
	}

	return s, nil
}

// ensurePath is used to make sure a path exists
func (s *Server) ensurePath(path string, dir bool) error {
	if !dir {
		path = filepath.Dir(path)
	}
	return os.MkdirAll(path, 0755)
}

// setupSerf is used to setup and initialize a Serf
func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string) (*serf.Serf, error) {
	conf.NodeName = s.config.NodeName
	conf.Role = fmt.Sprintf("consul:%s:%s", s.config.Datacenter, s.rpcListener.Addr().String())
	conf.MemberlistConfig.LogOutput = s.config.LogOutput
	conf.LogOutput = s.config.LogOutput
	conf.EventCh = ch
	conf.SnapshotPath = filepath.Join(s.config.DataDir, path)
	if err := s.ensurePath(conf.SnapshotPath, false); err != nil {
		return nil, err
	}
	return serf.Create(conf)
}

// setupRaft is used to setup and initialize Raft
func (s *Server) setupRaft() error {
	// Create the base path
	path := filepath.Join(s.config.DataDir, raftState)
	if err := s.ensurePath(path, true); err != nil {
		return err
	}

	// Create the SQLite store for logs and stable storage
	store, err := raft.NewSQLiteStore(path)
	if err != nil {
		return err
	}
	s.raftStore = store

	// Create the snapshot store
	snapshots, err := raft.NewFileSnapshotStore(path, 3)
	if err != nil {
		store.Close()
		return err
	}

	// Create a transport layer
	trans := raft.NewNetworkTransport(s.raftLayer, 3, 10*time.Second)
	s.raftTransport = trans

	// Setup the peer store
	peers := raft.NewJSONPeers(path, trans)

	// Create the FSM
	s.fsm = &consulFSM{server: s}

	// Setup the Raft store
	s.raft, err = raft.NewRaft(s.config.RaftConfig, s.fsm, store, store, snapshots,
		peers, trans)
	if err != nil {
		store.Close()
		trans.Close()
		return err
	}
	return nil
}

// setupRPC is used to setup the RPC listener
func (s *Server) setupRPC() error {
	// Register the handlers
	s.rpcServer.Register(&Raft{server: s})

	list, err := net.Listen("tcp", s.config.RPCAddr)
	if err != nil {
		return err
	}
	s.rpcListener = list
	s.raftLayer = NewRaftLayer(s.rpcListener.Addr())
	go s.listen()
	return nil
}

// Shutdown is used to shutdown the server
func (s *Server) Shutdown() error {
	s.logger.Printf("[INFO] Shutting down Consul server")
	s.shutdownLock.Lock()
	defer s.shutdownLock.Unlock()

	if s.shutdown {
		return nil
	}

	s.shutdown = true
	close(s.shutdownCh)

	if s.serfLAN != nil {
		s.serfLAN.Shutdown()
		s.serfLAN = nil
	}

	if s.serfWAN != nil {
		s.serfWAN.Shutdown()
		s.serfWAN = nil
	}

	if s.raft != nil {
		s.raftTransport.Close()
		s.raftLayer.Close()
		s.raft.Shutdown()
		s.raftStore.Close()
		s.raft = nil
		s.raftLayer = nil
		s.raftStore = nil
	}

	if s.rpcListener != nil {
		s.rpcListener.Close()
		s.rpcListener = nil
	}

	// Close all the RPC connections
	s.rpcClientLock.Lock()
	for conn := range s.rpcClients {
		conn.Close()
	}
	s.rpcClientLock.Unlock()

	// Close the connection pool
	s.connPool.Shutdown()

	return nil
}

// Leave is used to prepare for a graceful shutdown of the server
func (s *Server) Leave() error {
	s.logger.Printf("[INFO] Consul server starting leave")

	// Leave the WAN pool
	if s.serfWAN != nil {
		if err := s.serfWAN.Leave(); err != nil {
			s.logger.Printf("[ERR] Failed to leave WAN Serf cluster: %v", err)
		}
	}

	// Leave the LAN pool
	if s.serfLAN != nil {
		if err := s.serfLAN.Leave(); err != nil {
			s.logger.Printf("[ERR] Failed to leave LAN Serf cluster: %v", err)
		}
	}

	// Leave the Raft cluster
	if s.raft != nil {
		// Get the leader
		leader := s.raft.Leader()
		if leader == nil {
			s.logger.Printf("[ERR] Failed to leave Raft cluster: no leader")
			goto AFTER_LEAVE
		}

		// Request that we are removed
		// TODO: Properly forward to leader
		future := s.raft.RemovePeer(s.rpcListener.Addr())

		// Wait for the future
		ch := make(chan error, 1)
		go func() {
			ch <- future.Error()
		}()

		// Wait for the commit
		select {
		case err := <-ch:
			if err != nil {
				s.logger.Printf("[ERR] Failed to leave Raft cluster: %v", err)
			}
		case <-time.After(3 * time.Second):
			s.logger.Printf("[ERR] Timedout leaving Raft cluster")
		}
	}
AFTER_LEAVE:
	return nil
}

// JoinLAN is used to have Consul join the inner-DC pool
// The target address should be another node inside the DC
// listening on the Serf LAN address
func (s *Server) JoinLAN(addr string) error {
	_, err := s.serfLAN.Join([]string{addr}, false)
	return err
}

// JoinWAN is used to have Consul join the cross-WAN Consul ring
// The target address should be another node listening on the
// Serf WAN address
func (s *Server) JoinWAN(addr string) error {
	_, err := s.serfWAN.Join([]string{addr}, false)
	return err
}
<commit_msg>consul: Leave does a RemovePeer RPC call<commit_after>package consul

import (
	"fmt"
	"github.com/hashicorp/raft"
	"github.com/hashicorp/serf/serf"
	"log"
	"net"
	"net/rpc"
	"os"
	"path/filepath"
	"sync"
	"time"
)

const (
	serfLANSnapshot = "serf/local.snapshot"
	serfWANSnapshot = "serf/remote.snapshot"
	raftState       = "raft/"
)

// Server is Consul server which manages the service discovery,
// health checking, DC forwarding, Raft, and multiple Serf pools.
type Server struct {
	config *Config

	// Connection pool to other consul servers
	connPool *ConnPool

	// eventChLAN is used to receive events from the
	// serf cluster in the datacenter
	eventChLAN chan serf.Event

	// eventChWAN is used to receive events from the
	// serf cluster that spans datacenters
	eventChWAN chan serf.Event

	// fsm is the state machine used with Raft to provide
	// strong consistency.
	fsm *consulFSM

	// Logger uses the provided LogOutput
	logger *log.Logger

	// The raft instance is used among Consul nodes within the
	// DC to protect operations that require strong consistency
	raft          *raft.Raft
	raftLayer     *RaftLayer
	raftStore     *raft.SQLiteStore
	raftTransport *raft.NetworkTransport

	// rpcClients is used to track active clients
	rpcClients    map[net.Conn]struct{}
	rpcClientLock sync.Mutex

	// rpcListener is used to listen for incoming connections
	rpcListener net.Listener
	rpcServer   *rpc.Server

	// serfLAN is the Serf cluster maintained inside the DC
	// which contains all the DC nodes
	serfLAN *serf.Serf

	// serfWAN is the Serf cluster maintained between DC's
	// which SHOULD only consist of Consul servers
	serfWAN *serf.Serf

	shutdown     bool
	shutdownCh   chan struct{}
	shutdownLock sync.Mutex
}

// NewServer is used to construct a new Consul server from the
// configuration, potentially returning an error
func NewServer(config *Config) (*Server, error) {
	// Check for a data directory!
	if config.DataDir == "" {
		return nil, fmt.Errorf("Config must provide a DataDir")
	}

	// Ensure we have a log output
	if config.LogOutput == nil {
		config.LogOutput = os.Stderr
	}

	// Create a logger
	logger := log.New(config.LogOutput, "", log.LstdFlags)

	// Create server
	s := &Server{
		config:     config,
		connPool:   NewPool(5),
		eventChLAN: make(chan serf.Event, 256),
		eventChWAN: make(chan serf.Event, 256),
		logger:     logger,
		rpcClients: make(map[net.Conn]struct{}),
		rpcServer:  rpc.NewServer(),
		shutdownCh: make(chan struct{}),
	}

	// Initialize the RPC layer
	if err := s.setupRPC(); err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start RPC layer: %v", err)
	}

	// Start the Serf listeners to prevent a deadlock
	go s.lanEventHandler()
	go s.wanEventHandler()

	// Initialize the lan Serf
	var err error
	s.serfLAN, err = s.setupSerf(config.SerfLANConfig,
		s.eventChLAN, serfLANSnapshot)
	if err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start lan serf: %v", err)
	}

	// Initialize the wan Serf
	s.serfWAN, err = s.setupSerf(config.SerfWANConfig,
		s.eventChWAN, serfWANSnapshot)
	if err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start wan serf: %v", err)
	}

	// Initialize the Raft server
	if err := s.setupRaft(); err != nil {
		s.Shutdown()
		return nil, fmt.Errorf("Failed to start Raft: %v", err)
	}

	return s, nil
}

// ensurePath is used to make sure a path exists
func (s *Server) ensurePath(path string, dir bool) error {
	if !dir {
		path = filepath.Dir(path)
	}
	return os.MkdirAll(path, 0755)
}

// setupSerf is used to setup and initialize a Serf
func (s *Server) setupSerf(conf *serf.Config, ch chan serf.Event, path string) (*serf.Serf, error) {
	conf.NodeName = s.config.NodeName
	conf.Role = fmt.Sprintf("consul:%s:%s", s.config.Datacenter, s.rpcListener.Addr().String())
	conf.MemberlistConfig.LogOutput = s.config.LogOutput
	conf.LogOutput = s.config.LogOutput
	conf.EventCh = ch
	conf.SnapshotPath = filepath.Join(s.config.DataDir, path)
	if err := s.ensurePath(conf.SnapshotPath, false); err != nil {
		return nil, err
	}
	return serf.Create(conf)
}

// setupRaft is used to setup and initialize Raft
func (s *Server) setupRaft() error {
	// Create the base path
	path := filepath.Join(s.config.DataDir, raftState)
	if err := s.ensurePath(path, true); err != nil {
		return err
	}

	// Create the SQLite store for logs and stable storage
	store, err := raft.NewSQLiteStore(path)
	if err != nil {
		return err
	}
	s.raftStore = store

	// Create the snapshot store
	snapshots, err := raft.NewFileSnapshotStore(path, 3)
	if err != nil {
		store.Close()
		return err
	}

	// Create a transport layer
	trans := raft.NewNetworkTransport(s.raftLayer, 3, 10*time.Second)
	s.raftTransport = trans

	// Setup the peer store
	peers := raft.NewJSONPeers(path, trans)

	// Create the FSM
	s.fsm = &consulFSM{server: s}

	// Setup the Raft store
	s.raft, err = raft.NewRaft(s.config.RaftConfig, s.fsm, store, store, snapshots,
		peers, trans)
	if err != nil {
		store.Close()
		trans.Close()
		return err
	}
	return nil
}

// setupRPC is used to setup the RPC listener
func (s *Server) setupRPC() error {
	// Register the handlers
	s.rpcServer.Register(&Raft{server: s})

	list, err := net.Listen("tcp", s.config.RPCAddr)
	if err != nil {
		return err
	}
	s.rpcListener = list
	s.raftLayer = NewRaftLayer(s.rpcListener.Addr())
	go s.listen()
	return nil
}

// Shutdown is used to shutdown the server
func (s *Server) Shutdown() error {
	s.logger.Printf("[INFO] Shutting down Consul server")
	s.shutdownLock.Lock()
	defer s.shutdownLock.Unlock()

	if s.shutdown {
		return nil
	}

	s.shutdown = true
	close(s.shutdownCh)

	if s.serfLAN != nil {
		s.serfLAN.Shutdown()
		s.serfLAN = nil
	}

	if s.serfWAN != nil {
		s.serfWAN.Shutdown()
		s.serfWAN = nil
	}

	if s.raft != nil {
		s.raftTransport.Close()
		s.raftLayer.Close()
		s.raft.Shutdown()
		s.raftStore.Close()
		s.raft = nil
		s.raftLayer = nil
		s.raftStore = nil
	}

	if s.rpcListener != nil {
		s.rpcListener.Close()
		s.rpcListener = nil
	}

	// Close all the RPC connections
	s.rpcClientLock.Lock()
	for conn := range s.rpcClients {
		conn.Close()
	}
	s.rpcClientLock.Unlock()

	// Close the connection pool
	s.connPool.Shutdown()

	return nil
}

// Leave is used to prepare for a graceful shutdown of the server
func (s *Server) Leave() error {
	s.logger.Printf("[INFO] Consul server starting leave")

	// Leave the WAN pool
	if s.serfWAN != nil {
		if err := s.serfWAN.Leave(); err != nil {
			s.logger.Printf("[ERR] Failed to leave WAN Serf cluster: %v", err)
		}
	}

	// Leave the LAN pool
	if s.serfLAN != nil {
		if err := s.serfLAN.Leave(); err != nil {
			s.logger.Printf("[ERR] Failed to leave LAN Serf cluster: %v", err)
		}
	}

	// Leave the Raft cluster
	if s.raft != nil {
		// Get the leader
		leader := s.raft.Leader()
		if leader == nil {
			s.logger.Printf("[ERR] Failed to leave Raft cluster: no leader")
			goto AFTER_LEAVE
		}

		// Request that we are removed
		ch := make(chan error, 1)
		go func() {
			var out struct{}
			peer := s.rpcListener.Addr().String()
			err := s.connPool.RPC(leader, "Raft.RemovePeer", peer, &out)
			ch <- err
		}()

		// Wait for the commit
		select {
		case err := <-ch:
			if err != nil {
				s.logger.Printf("[ERR] Failed to leave Raft cluster: %v", err)
			}
		case <-time.After(3 * time.Second):
			s.logger.Printf("[ERR] Timedout leaving Raft cluster")
		}
	}
AFTER_LEAVE:
	return nil
}

// JoinLAN is used to have Consul join the inner-DC pool
// The target address should be another node inside the DC
// listening on the Serf LAN address
func (s *Server) JoinLAN(addr string) error {
	_, err := s.serfLAN.Join([]string{addr}, false)
	return err
}

// JoinWAN is used to have Consul join the cross-WAN Consul ring
// The target address should be another node listening on the
// Serf WAN address
func (s *Server) JoinWAN(addr string) error {
	_, err := s.serfWAN.Join([]string{addr}, false)
	return err
}
<|endoftext|>


<commit_before>// Package defaults implements the Pipe interface providing default values
// for missing configuration.
package defaults

import (
	"fmt"
	"strings"

	"github.com/goreleaser/goreleaser/internal/client"
	"github.com/goreleaser/goreleaser/internal/middleware/errhandler"
	"github.com/goreleaser/goreleaser/internal/middleware/logging"
	"github.com/goreleaser/goreleaser/internal/middleware/skip"
	"github.com/goreleaser/goreleaser/internal/tmpl"
	"github.com/goreleaser/goreleaser/pkg/context"
	"github.com/goreleaser/goreleaser/pkg/defaults"
)

// Pipe that sets the defaults.
type Pipe struct{}

func (Pipe) String() string { return "setting defaults" }

// Run the pipe.
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Dist == "" {
		ctx.Config.Dist = "dist"
	}
	if ctx.Config.GitHubURLs.Download == "" {
		ctx.Config.GitHubURLs.Download = client.DefaultGitHubDownloadURL
	}
	if ctx.Config.GitLabURLs.Download == "" {
		ctx.Config.GitLabURLs.Download = client.DefaultGitLabDownloadURL
	}
	if ctx.Config.GiteaURLs.Download == "" {
		apiURL, err := tmpl.New(ctx).Apply(ctx.Config.GiteaURLs.API)
		if err != nil {
			return fmt.Errorf("templating Gitea API URL: %w", err)
		}

		ctx.Config.GiteaURLs.Download = strings.ReplaceAll(apiURL, "/api/v1", "")
	}
	for _, defaulter := range defaults.Defaulters {
		if err := skip.Maybe(
			defaulter,
			logging.Log(
				defaulter.String(),
				errhandler.Handle(defaulter.Default),
				logging.ExtraPadding,
			),
		)(ctx); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>fix: do not skip any defaults (#2521)<commit_after>// Package defaults implements the Pipe interface providing default values
// for missing configuration.
package defaults

import (
	"fmt"
	"strings"

	"github.com/goreleaser/goreleaser/internal/client"
	"github.com/goreleaser/goreleaser/internal/middleware/errhandler"
	"github.com/goreleaser/goreleaser/internal/middleware/logging"
	"github.com/goreleaser/goreleaser/internal/tmpl"
	"github.com/goreleaser/goreleaser/pkg/context"
	"github.com/goreleaser/goreleaser/pkg/defaults"
)

// Pipe that sets the defaults.
type Pipe struct{}

func (Pipe) String() string { return "setting defaults" }

// Run the pipe.
func (Pipe) Run(ctx *context.Context) error {
	if ctx.Config.Dist == "" {
		ctx.Config.Dist = "dist"
	}
	if ctx.Config.GitHubURLs.Download == "" {
		ctx.Config.GitHubURLs.Download = client.DefaultGitHubDownloadURL
	}
	if ctx.Config.GitLabURLs.Download == "" {
		ctx.Config.GitLabURLs.Download = client.DefaultGitLabDownloadURL
	}
	if ctx.Config.GiteaURLs.Download == "" {
		apiURL, err := tmpl.New(ctx).Apply(ctx.Config.GiteaURLs.API)
		if err != nil {
			return fmt.Errorf("templating Gitea API URL: %w", err)
		}

		ctx.Config.GiteaURLs.Download = strings.ReplaceAll(apiURL, "/api/v1", "")
	}
	for _, defaulter := range defaults.Defaulters {
		if err := logging.Log(
			defaulter.String(),
			errhandler.Handle(defaulter.Default),
			logging.ExtraPadding,
		)(ctx); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Source ranking is based on the following criteria in order:
// 1. More trusted source ranks higher
// 2. Latest data ranks higher
// 3. More data ranks higher

package ranking

import (
	pb "github.com/datacommonsorg/mixer/internal/proto"
	"github.com/datacommonsorg/mixer/internal/server/model"
)

// RankKey represents keys used for ranking.
// Import Name should be non-empty.
// Can use "*" for wildcard match for MeasurementMethod and ObservationPeriod
type RankKey struct {
	ImportName        string
	MeasurementMethod string
	ObservationPeriod string
}

// StatsRanking is used to rank multiple source series for the same
// StatisticalVariable, where lower value means higher ranking.
var StatsRanking = map[RankKey]int{
	{"CensusPEP", "CensusPEPSurvey", "*"}:                                0, // Population
	{"CensusACS5YearSurvey", "CensusACS5yrSurvey", "*"}:                  1, // Population
	{"CensusACS5YearSurvey_AggCountry", "CensusACS5yrSurvey", "*"}:       1, // Population
	{"CensusUSAMedianAgeIncome", "CensusACS5yrSurvey", "*"}:              1, // Population
	{"USDecennialCensus_RedistrictingRelease", "USDecennialCensus", "*"}: 2, // Population
	{"EurostatData", "EurostatRegionalPopulationData", "*"}:              3, // Population
	{"WorldDevelopmentIndicators", "*", "*"}:                             4, // Population
	// Prefer Indian Census population for Indian states, over something like OECD.
	{"IndiaCensus_Primary", "*", "*"}:                 5,    // Population
	{"WikipediaStatsData", "Wikipedia", "*"}:          1001, // Population
	{"HumanCuratedStats", "HumanCuratedStats", "*"}:   1002, // Population
	{"WikidataPopulation", "WikidataPopulation", "*"}: 1003, // Population

	{"BLS_LAUS", "BLSSeasonallyUnadjusted", "*"}: 0, // Unemployment Rate
	{"BLS_CPS", "BLSSeasonallyAdjusted", "*"}:    1, // Labor Force data ranked higher than WDI (above) or Eurostat
	{"EurostatData", "", "*"}:                    2, // Unemployment Rate

	{"NYT_COVID19", "NYT_COVID19_GitHub", "*"}: 0, // Covid

	{"CDC500", "AgeAdjustedPrevalence", "*"}: 0, // CDC500

	{"UNEnergy", "", "*"}:         0, // Electricity
	{"EIA_Electricity", "*", "*"}: 1, // Electricity

	// Prefer observational weather over gridded over projections
	{"NOAA_EPA_Observed_Historical_Weather", "*", "*"}: 0, // Observational
	{"Copernicus_ECMWF_ERA5_Monthly", "*", "*"}:        1, // Gridded reanalysis
	{"NASA_NEXDCP30", "NASA_Mean_CCSM4", "P1M"}:        2, // IPCC Projections
	{"NASA_NEXGDDP", "NASA_Mean_CMIP5_CCSM4", "P1M"}:   3, // IPCC Projections
	{"NASA_NEXDCP30_StatVarSeriesAggr", "*", "P1M"}:    4, // IPCC Projections
	{"NASA_NEXGDDP_StatVarSeriesAggr", "*", "P1M"}:     5, // IPCC Projections

	{"NASA_WetBulbComputation_Aggregation", "NASA_Mean_HadGEM2-AO", "*"}: 0, // Wet bulb year aggregation
	{"NASA_WetBulbComputation_Aggregation", "*", "*"}:                    1, // Wet bulb year aggregation
	{"NASA_WetBulbComputation", "NASA_Mean_HadGEM2-AO", "*"}:             2, // Wet bulb

	// Prefer FBI Hate Crime Publications over Data Commons Aggregate
	// Note: https://autopush.datacommons.org/tools/timeline#statsVar=Count_CriminalIncidents_IsHateCrime&place=country%2FUSA
	// Expected data 2010-2020: 6628, 6222, 5796, 5928, 5479, 5850, 6121, 7175, 7120, 7314, 8263
	// Note: https://autopush.datacommons.org/tools/timeline#place=geoId%2F06&statsVar=Count_CriminalIncidents_IsHateCrime
	// Expected data 2004-2010: 1393, 1379, 1297, 1400, 1381, 1015, 1092
	{"FBIHateCrimePublications", "*", "*"}: 0, // FBI Hate Crime Publications
	{"FBIHateCrime", "*", "*"}:             1, // FBI Hate Crime Aggregations
}

// BaseRank is the base ranking score for sources. If a source is prefered, it
// should be given a score lower than BaseRank in StatsRanking. If a source is not
// prefered, it should be given a score higher than BaseRank in StatsRanking
const BaseRank = 100

// CohortByRank implements sort.Interface for []*SourceSeries based on
// the rank score. Each source series data is keyed by the place dcid.
//
// Note this has the same data type as SeriesByRank but is used to compare
// cohort instead of time series.
type CohortByRank []*pb.SourceSeries

// GetScorePb derives the ranking score for a source series.
//
// The score depends on ImportName and other SVObs properties, by checking the
// StatsRanking dict. To get the score, ImportName is required, with optional
// properties:
// - MeasurementMethod
// - ObservationPeriod
//
// When there are exact match of the properties in StatsRanking, then use that
// score, otherwise can also match to wildcard options (indicated by *).
//
// If no entry is found, a BaseRank is assigned to the source series.
func GetScorePb(s *pb.SourceSeries) int {
	for _, propCombination := range []struct {
		mm string
		op string
	}{
		// Check exact match first
		{s.MeasurementMethod, s.ObservationPeriod},
		{s.MeasurementMethod, "*"},
		{"*", s.ObservationPeriod},
		{"*", "*"},
	} {
		key := RankKey{
			ImportName:        s.ImportName,
			MeasurementMethod: propCombination.mm,
			ObservationPeriod: propCombination.op,
		}
		score, ok := StatsRanking[key]
		if ok {
			return score
		}
	}
	return BaseRank
}

func (a CohortByRank) Len() int {
	return len(a)
}

func (a CohortByRank) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a CohortByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := GetScorePb(oi)
	oj := a[j]
	scorej := GetScorePb(oj)
	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}

	// Compare other fields to get consistent ranking.
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceUrl != oj.ProvenanceUrl {
		return oi.ProvenanceUrl < oj.ProvenanceUrl
	}
	return true
}

// SeriesByRank implements sort.Interface for []*SourceSeries based on
// the rank score. Each source series data is keyed by the observation date.
//
// This is the protobuf version of byRank.
type SeriesByRank []*pb.SourceSeries

func (a SeriesByRank) Len() int { return len(a) }

func (a SeriesByRank) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

func (a SeriesByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := GetScorePb(oi)
	oj := a[j]
	scorej := GetScorePb(oj)

	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}

	latesti := ""
	for date := range a[i].Val {
		if date > latesti {
			latesti = date
		}
	}

	latestj := ""
	for date := range a[j].Val {
		if date > latestj {
			latestj = date
		}
	}

	// Series with latest data is ranked higher
	if latesti != latestj {
		return latesti > latestj
	}

	// Series with more data is ranked higher
	if len(a[i].Val) != len(a[j].Val) {
		return len(a[i].Val) > len(a[j].Val)
	}

	// Compare other fields to get consistent ranking.
	if oi.ImportName != oj.ImportName {
		return oi.ImportName < oj.ImportName
	}
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceUrl != oj.ProvenanceUrl {
		return oi.ProvenanceUrl < oj.ProvenanceUrl
	}
	return true
}

// TODO(shifucun): Remove `SourceSeries` and use pb.SourceSeries everywhere.
// GetScore derives the ranking score for a source series.
//
// The score depends on ImportName and other SVObs properties, by checking the
// StatsRanking dict. To get the score, ImportName is required, with optional
// properties:
// - MeasurementMethod
// - ObservationPeriod
//
// When there are exact match of the properties in StatsRanking, then use that
// score, otherwise can also match to wildcard options (indicated by *).
//
// If no entry is found, a BaseRank is assigned to the source series.
func GetScore(s *model.SourceSeries) int {
	for _, propCombination := range []struct {
		mm string
		op string
	}{
		// Check exact match first
		{s.MeasurementMethod, s.ObservationPeriod},
		{s.MeasurementMethod, "*"},
		{"*", s.ObservationPeriod},
		{"*", "*"},
	} {
		key := RankKey{
			ImportName:        s.ImportName,
			MeasurementMethod: propCombination.mm,
			ObservationPeriod: propCombination.op,
		}
		score, ok := StatsRanking[key]
		if ok {
			return score
		}
	}
	return BaseRank
}

// ByRank implements sort.Interface for []*SourceSeries based on
// the rank score.
type ByRank []*model.SourceSeries

func (a ByRank) Len() int { return len(a) }

func (a ByRank) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

func (a ByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := GetScore(oi)
	oj := a[j]
	scorej := GetScore(oj)
	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}

	latesti := ""
	for date := range a[i].Val {
		if date > latesti {
			latesti = date
		}
	}

	latestj := ""
	for date := range a[j].Val {
		if date > latestj {
			latestj = date
		}
	}

	// Series with latest data is ranked higher
	if latesti != latestj {
		return latesti > latestj
	}

	// Series with more data is ranked higher
	if len(a[i].Val) != len(a[j].Val) {
		return len(a[i].Val) > len(a[j].Val)
	}

	// Compare other fields to get consistent ranking.
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceURL != oj.ProvenanceURL {
		return oi.ProvenanceURL < oj.ProvenanceURL
	}
	return true
}
<commit_msg>Update ranking imports. (#831)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Source ranking is based on the following criteria in order:
// 1. More trusted source ranks higher
// 2. Latest data ranks higher
// 3. More data ranks higher

package ranking

import (
	pb "github.com/datacommonsorg/mixer/internal/proto"
	"github.com/datacommonsorg/mixer/internal/server/model"
)

// RankKey represents keys used for ranking.
// Import Name should be non-empty.
// Can use "*" for wildcard match for MeasurementMethod and ObservationPeriod
type RankKey struct {
	ImportName        string
	MeasurementMethod string
	ObservationPeriod string
}

// StatsRanking is used to rank multiple source series for the same
// StatisticalVariable, where lower value means higher ranking.
var StatsRanking = map[RankKey]int{
	{"CensusPEP", "CensusPEPSurvey", "*"}:                                0, // Population
	{"CensusACS5YearSurvey", "CensusACS5yrSurvey", "*"}:                  1, // Population
	{"CensusACS5YearSurvey_AggCountry", "CensusACS5yrSurvey", "*"}:       1, // Population
	{"CensusUSAMedianAgeIncome", "CensusACS5yrSurvey", "*"}:              1, // Population
	{"USDecennialCensus_RedistrictingRelease", "USDecennialCensus", "*"}: 2, // Population
	{"EurostatData", "EurostatRegionalPopulationData", "*"}:              3, // Population
	{"WorldDevelopmentIndicators", "*", "*"}:                             4, // Population
	// Prefer Indian Census population for Indian states, over something like OECD.
	{"IndiaCensus_Primary", "*", "*"}:                 5,    // Population
	{"WikipediaStatsData", "Wikipedia", "*"}:          1001, // Population
	{"HumanCuratedStats", "HumanCuratedStats", "*"}:   1002, // Population
	{"WikidataPopulation", "WikidataPopulation", "*"}: 1003, // Population

	{"BLS_LAUS", "BLSSeasonallyUnadjusted", "*"}: 0, // Unemployment Rate
	{"BLS_CPS", "BLSSeasonallyAdjusted", "*"}:    1, // Labor Force data ranked higher than WDI (above) or Eurostat
	{"EurostatData", "", "*"}:                    2, // Unemployment Rate

	{"NYT_COVID19", "NYT_COVID19_GitHub", "*"}: 0, // Covid

	{"CDC500", "AgeAdjustedPrevalence", "*"}: 0, // CDC500

	{"UNEnergy", "", "*"}:         0, // Electricity
	{"EIA_Electricity", "*", "*"}: 1, // Electricity

	// Prefer observational weather over gridded over projections
	{"NOAA_EPA_Observed_Historical_Weather", "*", "*"}: 0, // Observational
	{"Copernicus_ECMWF_ERA5_Monthly", "*", "*"}:        1, // Gridded reanalysis
	{"NASA_NEXDCP30", "NASA_Mean_CCSM4", "P1M"}:        2, // IPCC Projections
	{"NASA_NEXDCP30_AggrDiffStats", "*", "P1M"}:        3, // IPCC Projections
	// TODO: Remove this once disppears from backend (replaced by NASA_NEXDCP30_AggrDiffStats).
	{"NASA_NEXDCP30_StatVarSeriesAggr", "*", "P1M"}: 4, // IPCC Projections

	{"NASA_WetBulbComputation_Aggregation", "NASA_Mean_HadGEM2-AO", "*"}: 0, // Wet bulb year aggregation
	{"NASA_WetBulbComputation_Aggregation", "*", "*"}:                    1, // Wet bulb year aggregation
	{"NASA_WetBulbComputation", "NASA_Mean_HadGEM2-AO", "*"}:             2, // Wet bulb

	// Prefer FBI Hate Crime Publications over Data Commons Aggregate
	// Note: https://autopush.datacommons.org/tools/timeline#statsVar=Count_CriminalIncidents_IsHateCrime&place=country%2FUSA
	// Expected data 2010-2020: 6628, 6222, 5796, 5928, 5479, 5850, 6121, 7175, 7120, 7314, 8263
	// Note: https://autopush.datacommons.org/tools/timeline#place=geoId%2F06&statsVar=Count_CriminalIncidents_IsHateCrime
	// Expected data 2004-2010: 1393, 1379, 1297, 1400, 1381, 1015, 1092
	{"FBIHateCrimePublications", "*", "*"}: 0, // FBI Hate Crime Publications
	{"FBIHateCrime", "*", "*"}:             1, // FBI Hate Crime Aggregations
}

// BaseRank is the base ranking score for sources. If a source is prefered, it
// should be given a score lower than BaseRank in StatsRanking. If a source is not
// prefered, it should be given a score higher than BaseRank in StatsRanking
const BaseRank = 100

// CohortByRank implements sort.Interface for []*SourceSeries based on
// the rank score. Each source series data is keyed by the place dcid.
//
// Note this has the same data type as SeriesByRank but is used to compare
// cohort instead of time series.
type CohortByRank []*pb.SourceSeries

// GetScorePb derives the ranking score for a source series.
//
// The score depends on ImportName and other SVObs properties, by checking the
// StatsRanking dict. To get the score, ImportName is required, with optional
// properties:
// - MeasurementMethod
// - ObservationPeriod
//
// When there are exact match of the properties in StatsRanking, then use that
// score, otherwise can also match to wildcard options (indicated by *).
//
// If no entry is found, a BaseRank is assigned to the source series.
func GetScorePb(s *pb.SourceSeries) int {
	for _, propCombination := range []struct {
		mm string
		op string
	}{
		// Check exact match first
		{s.MeasurementMethod, s.ObservationPeriod},
		{s.MeasurementMethod, "*"},
		{"*", s.ObservationPeriod},
		{"*", "*"},
	} {
		key := RankKey{
			ImportName:        s.ImportName,
			MeasurementMethod: propCombination.mm,
			ObservationPeriod: propCombination.op,
		}
		score, ok := StatsRanking[key]
		if ok {
			return score
		}
	}
	return BaseRank
}

func (a CohortByRank) Len() int {
	return len(a)
}

func (a CohortByRank) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a CohortByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := GetScorePb(oi)
	oj := a[j]
	scorej := GetScorePb(oj)
	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}

	// Compare other fields to get consistent ranking.
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceUrl != oj.ProvenanceUrl {
		return oi.ProvenanceUrl < oj.ProvenanceUrl
	}
	return true
}

// SeriesByRank implements sort.Interface for []*SourceSeries based on
// the rank score. Each source series data is keyed by the observation date.
//
// This is the protobuf version of byRank.
type SeriesByRank []*pb.SourceSeries

func (a SeriesByRank) Len() int { return len(a) }

func (a SeriesByRank) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

func (a SeriesByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := GetScorePb(oi)
	oj := a[j]
	scorej := GetScorePb(oj)

	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}

	latesti := ""
	for date := range a[i].Val {
		if date > latesti {
			latesti = date
		}
	}

	latestj := ""
	for date := range a[j].Val {
		if date > latestj {
			latestj = date
		}
	}

	// Series with latest data is ranked higher
	if latesti != latestj {
		return latesti > latestj
	}

	// Series with more data is ranked higher
	if len(a[i].Val) != len(a[j].Val) {
		return len(a[i].Val) > len(a[j].Val)
	}

	// Compare other fields to get consistent ranking.
	if oi.ImportName != oj.ImportName {
		return oi.ImportName < oj.ImportName
	}
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceUrl != oj.ProvenanceUrl {
		return oi.ProvenanceUrl < oj.ProvenanceUrl
	}
	return true
}

// TODO(shifucun): Remove `SourceSeries` and use pb.SourceSeries everywhere.
// GetScore derives the ranking score for a source series.
//
// The score depends on ImportName and other SVObs properties, by checking the
// StatsRanking dict. To get the score, ImportName is required, with optional
// properties:
// - MeasurementMethod
// - ObservationPeriod
//
// When there are exact match of the properties in StatsRanking, then use that
// score, otherwise can also match to wildcard options (indicated by *).
//
// If no entry is found, a BaseRank is assigned to the source series.
func GetScore(s *model.SourceSeries) int {
	for _, propCombination := range []struct {
		mm string
		op string
	}{
		// Check exact match first
		{s.MeasurementMethod, s.ObservationPeriod},
		{s.MeasurementMethod, "*"},
		{"*", s.ObservationPeriod},
		{"*", "*"},
	} {
		key := RankKey{
			ImportName:        s.ImportName,
			MeasurementMethod: propCombination.mm,
			ObservationPeriod: propCombination.op,
		}
		score, ok := StatsRanking[key]
		if ok {
			return score
		}
	}
	return BaseRank
}

// ByRank implements sort.Interface for []*SourceSeries based on
// the rank score.
type ByRank []*model.SourceSeries

func (a ByRank) Len() int { return len(a) }

func (a ByRank) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

func (a ByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := GetScore(oi)
	oj := a[j]
	scorej := GetScore(oj)
	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}

	latesti := ""
	for date := range a[i].Val {
		if date > latesti {
			latesti = date
		}
	}

	latestj := ""
	for date := range a[j].Val {
		if date > latestj {
			latestj = date
		}
	}

	// Series with latest data is ranked higher
	if latesti != latestj {
		return latesti > latestj
	}

	// Series with more data is ranked higher
	if len(a[i].Val) != len(a[j].Val) {
		return len(a[i].Val) > len(a[j].Val)
	}

	// Compare other fields to get consistent ranking.
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceURL != oj.ProvenanceURL {
		return oi.ProvenanceURL < oj.ProvenanceURL
	}
	return true
}
<|endoftext|>