<commit_before>package whisper

import (
	"encoding/binary"
	"io"
	"os"
	"time"
)

type Metadata struct {
	AggregationMethod uint32
	MaxRetention      uint32
	XFilesFactor      float32
	ArchiveCount      uint32
}

type ArchiveInfo struct {
	Offset          uint32
	SecondsPerPoint uint32
	Points          uint32
}

func (a ArchiveInfo) Retention() uint32 {
	return a.SecondsPerPoint * a.Points
}

func (a ArchiveInfo) Size() uint32 {
	return a.Points * pointSize
}

func (a ArchiveInfo) End() uint32 {
	return a.Offset + a.Size()
}

type Header struct {
	Metadata Metadata
	Archives []ArchiveInfo
}

type Archive []Point

type Point struct {
	Timestamp uint32
	Value     float64
}

type Whisper struct {
	Header Header
	file   *os.File
}

var pointSize, metadataSize, archiveSize uint32

// Aggregation type using averaging
const AGGREGATION_AVERAGE = 1

// Aggregation type using sum
const AGGREGATION_SUM = 2

// Aggregation type using the last value
const AGGREGATION_LAST = 3

// Aggregation type using the maximum value
const AGGREGATION_MAX = 4

// Aggregation type using the minimum value
const AGGREGATION_MIN = 5

func init() {
	pointSize = uint32(binary.Size(Point{}))
	metadataSize = uint32(binary.Size(Metadata{}))
	archiveSize = uint32(binary.Size(Archive{}))
}

// Read the header of a whisper database
func ReadHeader(buf io.ReadSeeker) (header Header, err error) {
	currentPos, err := buf.Seek(0, 1)
	if err != nil {
		return
	}
	defer func() {
		// Try to return to the original position when we exit
		_, e := buf.Seek(currentPos, 0)
		if e != nil {
			err = e
		}
		return
	}()

	// Start at the beginning of the file
	_, err = buf.Seek(0, 0)
	if err != nil {
		return
	}

	// Read metadata
	var metadata Metadata
	err = binary.Read(buf, binary.BigEndian, metadata)
	if err != nil {
		return
	}
	header.Metadata = metadata

	// Read archive info
	archives := make([]ArchiveInfo, metadata.ArchiveCount)
	for i := uint32(0); i < metadata.ArchiveCount; i++ {
		err = binary.Read(buf, binary.BigEndian, archives[i])
		if err != nil {
			return
		}
	}
	header.Archives = archives

	return
}

// Create a new whisper database at a given file path
func Create(path string, archives []ArchiveInfo, xFilesFactor float32, aggregationMethod uint32, sparse bool) (err error) {
	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)

	oldest := uint32(0)
	for _, archive := range archives {
		age := archive.SecondsPerPoint * archive.Points
		if age > oldest {
			oldest = age
		}
	}

	metadata := Metadata{
		AggregationMethod: aggregationMethod,
		XFilesFactor:      xFilesFactor,
		ArchiveCount:      uint32(len(archives)),
		MaxRetention:      oldest,
	}
	err = binary.Write(file, binary.BigEndian, metadata)
	if err != nil {
		return
	}

	headerSize := metadataSize + (archiveSize * uint32(len(archives)))
	archiveOffsetPointer := headerSize

	for _, archive := range archives {
		archive.Offset = archiveOffsetPointer
		err = binary.Write(file, binary.BigEndian, archive)
		if err != nil {
			return
		}
		archiveOffsetPointer += archive.Points * pointSize
	}

	if sparse {
		file.Seek(int64(archiveOffsetPointer-headerSize-1), 0)
		file.Write([]byte{0})
	} else {
		remaining := archiveOffsetPointer - headerSize
		chunkSize := uint32(16384)
		buf := make([]byte, chunkSize)
		for remaining > chunkSize {
			file.Write(buf)
			remaining -= chunkSize
		}
		file.Write(buf[:remaining])
	}

	return
}

func Open(path string) (whisper Whisper, err error) {
	file, err := os.OpenFile(path, os.O_RDWR, 0666)
	if err != nil {
		return
	}

	header, err := ReadHeader(file)
	if err != nil {
		return
	}
	whisper = Whisper{Header: header, file: file}
	return
}

func (w Whisper) Update(point Point) (err error) {
	now := uint32(time.Now().Unix())
	diff := now - point.Timestamp
	if !((diff < w.Header.Metadata.MaxRetention) && diff >= 0) {
		// TODO: Return an error
		return
	}

	// Find the higher-precision archive that covers the timestamp
	var lowerArchives []ArchiveInfo
	var currentArchive ArchiveInfo
	for i, currentArchive := range w.Header.Archives {
		if currentArchive.Retention() < diff {
			continue
		}
		lowerArchives = w.Header.Archives[i+1:]
	}

	// Normalize the point's timestamp to the current archive's precision
	point.Timestamp = point.Timestamp - (point.Timestamp % currentArchive.SecondsPerPoint)

	// Write the point
	offset, err := w.pointOffset(currentArchive, point.Timestamp)
	if err != nil {
		return
	}
	err = w.writePoint(offset, point)
	if err != nil {
		return
	}

	// Propagate data down to all the lower resolution archives
	higherArchive := currentArchive
	for _, lowerArchive := range lowerArchives {
		result, e := w.propagate(point.Timestamp, higherArchive, lowerArchive)
		if !result {
			break
		}
		if e != nil {
			err = e
			return
		}
		higherArchive = lowerArchive
	}

	return
}

func (w Whisper) propagate(timestamp uint32, higher ArchiveInfo, lower ArchiveInfo) (result bool, err error) {
	lowerIntervalStart := timestamp - (timestamp % lower.SecondsPerPoint)
	higherFirstOffset, err := w.pointOffset(higher, lowerIntervalStart)
	if err != nil {
		return
	}

	numHigherPoints := lower.SecondsPerPoint - higher.SecondsPerPoint
	higherSize := numHigherPoints * pointSize
	relativeFirstOffset := higherFirstOffset - higher.Offset
	relativeLastOffset := (relativeFirstOffset + higherSize) % higher.Size()
	higherLastOffset := relativeLastOffset + higher.Offset

	var points []Point
	if higherFirstOffset < higherLastOffset {
		// The selection is in the middle of the archive. eg: --####---
		points = make([]Point, (higherLastOffset-higherFirstOffset)/pointSize)
		err = w.readPoints(higherFirstOffset, points)
		if err != nil {
			return
		}
	} else {
		// The selection wraps over the end of the archive. eg: ##----###
		numEndPoints := (higher.End() - higherFirstOffset) / pointSize
		numBeginPoints := (higherLastOffset - higher.Offset) / pointSize
		points = make([]Point, numBeginPoints+numEndPoints)

		err = w.readPoints(higherFirstOffset, points[:numEndPoints])
		if err != nil {
			return
		}
		err = w.readPoints(higher.Offset, points[numEndPoints:])
		if err != nil {
			return
		}
	}

	neighborPoints := make([]Point, len(points))

	currentInterval := lowerIntervalStart
	for i := 0; i < len(points); i += 2 {
		if points[i].Timestamp == currentInterval {
			neighborPoints[i/2] = points[i+1]
		}
		currentInterval += higher.SecondsPerPoint
	}

	return
}

// Read a single point from an offset in the database
func (w Whisper) readPoint(offset uint32) (point Point, err error) {
	_, err = w.file.Seek(int64(offset), 0)
	if err != nil {
		return
	}
	err = binary.Read(w.file, binary.BigEndian, point)
	return
}

// Read a slice of points from an offset in the database
func (w Whisper) readPoints(offset uint32, points []Point) (err error) {
	w.file.Seek(int64(offset), 0)
	err = binary.Read(w.file, binary.BigEndian, points)
	return
}

// Write a point to an offset in the database
func (w Whisper) writePoint(offset uint32, point Point) (err error) {
	w.file.Seek(int64(offset), 0)
	err = binary.Write(w.file, binary.BigEndian, point)
	return
}

// Get the offset of a timestamp within an archive
func (w Whisper) pointOffset(archive ArchiveInfo, timestamp uint32) (offset uint32, err error) {
	basePoint, err := w.readPoint(0)
	if err != nil {
		return
	}
	if basePoint.Timestamp == 0 {
		// The archive has never been written, this will be the new base point
		offset = archive.Offset
	} else {
		timeDistance := timestamp - basePoint.Timestamp
		pointDistance := timeDistance / archive.SecondsPerPoint
		byteDistance := pointDistance * pointSize
		offset = archive.Offset + (byteDistance % archive.Size())
	}
	return
}

func aggregate(aggregationMethod int, points []Point) (point Point, err error) {
	switch aggregationMethod {
	case AGGREGATION_AVERAGE:
		for _, p := range points {
			point.Value += p.Value
		}
		point.Value /= float64(len(points))
	case AGGREGATION_SUM:
		for _, p := range points {
			point.Value += p.Value
		}
	case AGGREGATION_LAST:
		point.Value = points[len(points)-1].Value
	case AGGREGATION_MAX:
		point.Value = points[0].Value
		for _, p := range points {
			if p.Value > point.Value {
				point.Value = p.Value
			}
		}
	case AGGREGATION_MIN:
		point.Value = points[0].Value
		for _, p := range points {
			if p.Value < point.Value {
				point.Value = p.Value
			}
		}
	default:
		//TODO: Set err
	}
	return
}
<commit_msg>Finished implementing propagate, I think.<commit_after>package whisper

import (
	"encoding/binary"
	"io"
	"os"
	"time"
)

type Metadata struct {
	AggregationMethod uint32
	MaxRetention      uint32
	XFilesFactor      float32
	ArchiveCount      uint32
}

type ArchiveInfo struct {
	Offset          uint32
	SecondsPerPoint uint32
	Points          uint32
}

func (a ArchiveInfo) Retention() uint32 {
	return a.SecondsPerPoint * a.Points
}

func (a ArchiveInfo) Size() uint32 {
	return a.Points * pointSize
}

func (a ArchiveInfo) End() uint32 {
	return a.Offset + a.Size()
}

type Header struct {
	Metadata Metadata
	Archives []ArchiveInfo
}

type Archive []Point

type Point struct {
	Timestamp uint32
	Value     float64
}

type Whisper struct {
	Header Header
	file   *os.File
}

var pointSize, metadataSize, archiveSize uint32

// Aggregation type using averaging
const AGGREGATION_AVERAGE = 1

// Aggregation type using sum
const AGGREGATION_SUM = 2

// Aggregation type using the last value
const AGGREGATION_LAST = 3

// Aggregation type using the maximum value
const AGGREGATION_MAX = 4

// Aggregation type using the minimum value
const AGGREGATION_MIN = 5

func init() {
	pointSize = uint32(binary.Size(Point{}))
	metadataSize = uint32(binary.Size(Metadata{}))
	archiveSize = uint32(binary.Size(Archive{}))
}

// Read the header of a whisper database
func ReadHeader(buf io.ReadSeeker) (header Header, err error) {
	currentPos, err := buf.Seek(0, 1)
	if err != nil {
		return
	}
	defer func() {
		// Try to return to the original position when we exit
		_, e := buf.Seek(currentPos, 0)
		if e != nil {
			err = e
		}
		return
	}()

	// Start at the beginning of the file
	_, err = buf.Seek(0, 0)
	if err != nil {
		return
	}

	// Read metadata
	var metadata Metadata
	err = binary.Read(buf, binary.BigEndian, metadata)
	if err != nil {
		return
	}
	header.Metadata = metadata

	// Read archive info
	archives := make([]ArchiveInfo, metadata.ArchiveCount)
	for i := uint32(0); i < metadata.ArchiveCount; i++ {
		err = binary.Read(buf, binary.BigEndian, archives[i])
		if err != nil {
			return
		}
	}
	header.Archives = archives

	return
}

// Create a new whisper database at a given file path
func Create(path string, archives []ArchiveInfo, xFilesFactor float32, aggregationMethod uint32, sparse bool) (err error) {
	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)

	oldest := uint32(0)
	for _, archive := range archives {
		age := archive.SecondsPerPoint * archive.Points
		if age > oldest {
			oldest = age
		}
	}

	metadata := Metadata{
		AggregationMethod: aggregationMethod,
		XFilesFactor:      xFilesFactor,
		ArchiveCount:      uint32(len(archives)),
		MaxRetention:      oldest,
	}
	err = binary.Write(file, binary.BigEndian, metadata)
	if err != nil {
		return
	}

	headerSize := metadataSize + (archiveSize * uint32(len(archives)))
	archiveOffsetPointer := headerSize

	for _, archive := range archives {
		archive.Offset = archiveOffsetPointer
		err = binary.Write(file, binary.BigEndian, archive)
		if err != nil {
			return
		}
		archiveOffsetPointer += archive.Points * pointSize
	}

	if sparse {
		file.Seek(int64(archiveOffsetPointer-headerSize-1), 0)
		file.Write([]byte{0})
	} else {
		remaining := archiveOffsetPointer - headerSize
		chunkSize := uint32(16384)
		buf := make([]byte, chunkSize)
		for remaining > chunkSize {
			file.Write(buf)
			remaining -= chunkSize
		}
		file.Write(buf[:remaining])
	}

	return
}

func Open(path string) (whisper Whisper, err error) {
	file, err := os.OpenFile(path, os.O_RDWR, 0666)
	if err != nil {
		return
	}

	header, err := ReadHeader(file)
	if err != nil {
		return
	}
	whisper = Whisper{Header: header, file: file}
	return
}

func (w Whisper) Update(point Point) (err error) {
	now := uint32(time.Now().Unix())
	diff := now - point.Timestamp
	if !((diff < w.Header.Metadata.MaxRetention) && diff >= 0) {
		// TODO: Return an error
		return
	}

	// Find the higher-precision archive that covers the timestamp
	var lowerArchives []ArchiveInfo
	var currentArchive ArchiveInfo
	for i, currentArchive := range w.Header.Archives {
		if currentArchive.Retention() < diff {
			continue
		}
		lowerArchives = w.Header.Archives[i+1:]
	}

	// Normalize the point's timestamp to the current archive's precision
	point.Timestamp = point.Timestamp - (point.Timestamp % currentArchive.SecondsPerPoint)

	// Write the point
	offset, err := w.pointOffset(currentArchive, point.Timestamp)
	if err != nil {
		return
	}
	err = w.writePoint(offset, point)
	if err != nil {
		return
	}

	// Propagate data down to all the lower resolution archives
	higherArchive := currentArchive
	for _, lowerArchive := range lowerArchives {
		result, e := w.propagate(point.Timestamp, higherArchive, lowerArchive)
		if !result {
			break
		}
		if e != nil {
			err = e
			return
		}
		higherArchive = lowerArchive
	}

	return
}

func (w Whisper) propagate(timestamp uint32, higher ArchiveInfo, lower ArchiveInfo) (result bool, err error) {
	lowerIntervalStart := timestamp - (timestamp % lower.SecondsPerPoint)
	higherFirstOffset, err := w.pointOffset(higher, lowerIntervalStart)
	if err != nil {
		return
	}

	numHigherPoints := lower.SecondsPerPoint - higher.SecondsPerPoint
	higherSize := numHigherPoints * pointSize
	relativeFirstOffset := higherFirstOffset - higher.Offset
	relativeLastOffset := (relativeFirstOffset + higherSize) % higher.Size()
	higherLastOffset := relativeLastOffset + higher.Offset

	var points []Point
	if higherFirstOffset < higherLastOffset {
		// The selection is in the middle of the archive. eg: --####---
		points = make([]Point, (higherLastOffset-higherFirstOffset)/pointSize)
		err = w.readPoints(higherFirstOffset, points)
		if err != nil {
			return
		}
	} else {
		// The selection wraps over the end of the archive. eg: ##----###
		numEndPoints := (higher.End() - higherFirstOffset) / pointSize
		numBeginPoints := (higherLastOffset - higher.Offset) / pointSize
		points = make([]Point, numBeginPoints+numEndPoints)

		err = w.readPoints(higherFirstOffset, points[:numEndPoints])
		if err != nil {
			return
		}
		err = w.readPoints(higher.Offset, points[numEndPoints:])
		if err != nil {
			return
		}
	}

	neighborPoints := make([]Point, 0, len(points))

	currentInterval := lowerIntervalStart
	for i := 0; i < len(points); i += 2 {
		if points[i].Timestamp == currentInterval {
			neighborPoints = append(neighborPoints, points[i])
		}
		currentInterval += higher.SecondsPerPoint
	}

	knownPercent := float32(len(neighborPoints)) / float32(len(points)) < w.Header.Metadata.XFilesFactor
	if len(neighborPoints) == 0 || knownPercent {
		// There's nothing to propagate
		return false, nil
	}

	aggregatePoint, err := aggregate(w.Header.Metadata.AggregationMethod, neighborPoints)
	if err != nil {
		return
	}
	aggregatePoint.Timestamp = lowerIntervalStart
	aggregateOffset, err := w.pointOffset(lower, aggregatePoint.Timestamp)
	if err != nil {
		return
	}

	err = w.writePoint(aggregateOffset, aggregatePoint)
	if err != nil {
		return
	}

	return true, nil

}

// Read a single point from an offset in the database
func (w Whisper) readPoint(offset uint32) (point Point, err error) {
	_, err = w.file.Seek(int64(offset), 0)
	if err != nil {
		return
	}
	err = binary.Read(w.file, binary.BigEndian, point)
	return
}

// Read a slice of points from an offset in the database
func (w Whisper) readPoints(offset uint32, points []Point) (err error) {
	w.file.Seek(int64(offset), 0)
	err = binary.Read(w.file, binary.BigEndian, points)
	return
}

// Write a point to an offset in the database
func (w Whisper) writePoint(offset uint32, point Point) (err error) {
	w.file.Seek(int64(offset), 0)
	err = binary.Write(w.file, binary.BigEndian, point)
	return
}

// Get the offset of a timestamp within an archive
func (w Whisper) pointOffset(archive ArchiveInfo, timestamp uint32) (offset uint32, err error) {
	basePoint, err := w.readPoint(0)
	if err != nil {
		return
	}
	if basePoint.Timestamp == 0 {
		// The archive has never been written, this will be the new base point
		offset = archive.Offset
	} else {
		timeDistance := timestamp - basePoint.Timestamp
		pointDistance := timeDistance / archive.SecondsPerPoint
		byteDistance := pointDistance * pointSize
		offset = archive.Offset + (byteDistance % archive.Size())
	}
	return
}

func aggregate(aggregationMethod uint32, points []Point) (point Point, err error) {
	switch aggregationMethod {
	case AGGREGATION_AVERAGE:
		for _, p := range points {
			point.Value += p.Value
		}
		point.Value /= float64(len(points))
	case AGGREGATION_SUM:
		for _, p := range points {
			point.Value += p.Value
		}
	case AGGREGATION_LAST:
		point.Value = points[len(points)-1].Value
	case AGGREGATION_MAX:
		point.Value = points[0].Value
		for _, p := range points {
			if p.Value > point.Value {
				point.Value = p.Value
			}
		}
	case AGGREGATION_MIN:
		point.Value = points[0].Value
		for _, p := range points {
			if p.Value < point.Value {
				point.Value = p.Value
			}
		}
	default:
		//TODO: Set err
	}
	return
}
<|endoftext|>


<commit_before>package welove

import (
	"io"
	"os"
	"hash"
	"crypto/hmac"
	"crypto/sha1"
	"net/url"
	"encoding/base64"
)

type CustomOutput struct {
	out []io.Writer
}

func New(out ...io.Writer) CustomOutput {
	return CustomOutput{out}
}

func (c *CustomOutput) Add(out io.Writer) {
	c.out = append(c.out, out)
}

func (c *CustomOutput) Write(p []byte) (int, error) {
	var n int = 0
	var err error = nil
	for _, v := range c.out {
		n, err = v.Write(p)
	}
	return n, err
}

func DefaultLog(path string) CustomOutput {
	var file, _ = os.OpenFile(path, os.O_APPEND | os.O_CREATE, os.ModeAppend)
	return New(os.Stdout, file)
}

type Sig struct {
	key   []byte
	myMac hash.Hash
}

func NewSig(key []byte) *Sig {
	mac := hmac.New(sha1.New, key)
	love := new(Sig)
	love.myMac = mac
	return love
}

func (l *Sig)Encode(method, u string, data ...Data) string {
	var content string
	for _, v := range data {
		content = content + v.key + "=" + v.value + "&"
	}
	content = content[0:len(content) - 1]
	l.myMac.Write([]byte(method + "&" + url.QueryEscape(u) + "&" + url.QueryEscape(content)))
	return base64.StdEncoding.EncodeToString(l.myMac.Sum(nil))
}

type Data struct {
	key   string
	value string
}<commit_msg>日志文件读写权限<commit_after>package welove

import (
	"io"
	"os"
	"hash"
	"crypto/hmac"
	"crypto/sha1"
	"net/url"
	"encoding/base64"
)

type CustomOutput struct {
	out []io.Writer
}

func New(out ...io.Writer) CustomOutput {
	return CustomOutput{out}
}

func (c *CustomOutput) Add(out io.Writer) {
	c.out = append(c.out, out)
}

func (c *CustomOutput) Write(p []byte) (int, error) {
	var n int = 0
	var err error = nil
	for _, v := range c.out {
		n, err = v.Write(p)
	}
	return n, err
}

func DefaultLog(path string) CustomOutput {
	var file, _ = os.OpenFile(path, os.O_APPEND | os.O_CREATE | os.O_RDWR, os.ModeAppend)
	return New(os.Stdout, file)
}

type Sig struct {
	key   []byte
	myMac hash.Hash
}

func NewSig(key []byte) *Sig {
	mac := hmac.New(sha1.New, key)
	love := new(Sig)
	love.myMac = mac
	return love
}

func (l *Sig)Encode(method, u string, data ...Data) string {
	var content string
	for _, v := range data {
		content = content + v.key + "=" + v.value + "&"
	}
	content = content[0:len(content) - 1]
	l.myMac.Write([]byte(method + "&" + url.QueryEscape(u) + "&" + url.QueryEscape(content)))
	return base64.StdEncoding.EncodeToString(l.myMac.Sum(nil))
}

type Data struct {
	key   string
	value string
}<|endoftext|>


<commit_before>package cc_fb

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cripplet/clicker/db/config"
	"github.com/cripplet/clicker/firebase-db"
	"github.com/cripplet/clicker/lib"
	"net/http"
	"strconv"
	"strings"
)

type FBGameState struct {
	ID       string
	Exist    bool
	GameData cookie_clicker.GameStateData
}

type internalGameStateData struct {
	Version       string          `json:"version"`
	NCookies      float64         `json:"n_cookies"`
	NBuildings    map[string]int  `json:"n_buildings"`
	UpgradeStatus map[string]bool `json:"upgrade_status"`
}

type internalFBGameState struct {
	ID       string                `json:"ID"`
	Exist    bool                  `json:"exist"`
	GameData internalGameStateData `json:"data"`
}

func toInternalFBGameState(s FBGameState) internalFBGameState {
	internalData := internalGameStateData{
		Version:       s.GameData.Version,
		NCookies:      s.GameData.NCookies,
		NBuildings:    make(map[string]int),
		UpgradeStatus: make(map[string]bool),
	}
	for buildingType, nBuildings := range s.GameData.NBuildings {
		internalData.NBuildings[fmt.Sprintf("_%d", buildingType)] = nBuildings
	}
	for upgradeID, bought := range s.GameData.UpgradeStatus {
		internalData.UpgradeStatus[fmt.Sprintf("_%d", upgradeID)] = bought
	}
	return internalFBGameState{
		ID:       s.ID,
		Exist:    s.Exist,
		GameData: internalData,
	}
}

func fromInternalFBGameState(s internalFBGameState) FBGameState {
	gameData := cookie_clicker.GameStateData{
		Version:       s.GameData.Version,
		NCookies:      s.GameData.NCookies,
		NBuildings:    make(map[cookie_clicker.BuildingType]int),
		UpgradeStatus: make(map[cookie_clicker.UpgradeID]bool),
	}
	for buildingTypeString, nBuildings := range s.GameData.NBuildings {
		buildingTypeInt, _ := strconv.Atoi(strings.Replace(buildingTypeString, "_", "", -1))
		gameData.NBuildings[cookie_clicker.BuildingType(buildingTypeInt)] = nBuildings
	}
	for upgradeIDString, bought := range s.GameData.UpgradeStatus {
		upgradeIDInt, _ := strconv.Atoi(strings.Replace(upgradeIDString, "_", "", -1))
		gameData.UpgradeStatus[cookie_clicker.UpgradeID(upgradeIDInt)] = bought
	}
	return FBGameState{
		ID:       s.ID,
		Exist:    s.Exist,
		GameData: gameData,
	}
}

type PostID struct {
	Name string `json:"name"`
}

func newGameState() (FBGameState, error) {
	d := cookie_clicker.NewGameStateData()
	g := toInternalFBGameState(FBGameState{
		Exist:    true,
		GameData: *d,
	})
	gJSON, err := json.Marshal(&g)
	if err != nil {
		return FBGameState{}, err
	}

	p := PostID{}

	_, _, _, err = firebase_db.Post(
		cc_fb_config.CC_FIREBASE_CONFIG.Client,
		fmt.Sprintf("%s/game.json", cc_fb_config.CC_FIREBASE_CONFIG.ProjectPath),
		gJSON,
		false,
		map[string]string{},
		&p,
	)
	if err != nil {
		return FBGameState{}, err
	}

	g.ID = p.Name
	return fromInternalFBGameState(g), nil
}

func LoadGameState(id string) (FBGameState, string, error) {
	if id == "" {
		g, err := newGameState()
		if err != nil {
			return FBGameState{}, "", nil
		}
		id = g.ID
	}

	i := internalFBGameState{}
	_, _, eTag, err := firebase_db.Get(
		cc_fb_config.CC_FIREBASE_CONFIG.Client,
		fmt.Sprintf("%s/game/%s.json", cc_fb_config.CC_FIREBASE_CONFIG.ProjectPath, id),
		true,
		map[string]string{},
		&i,
	)
	if err != nil {
		return FBGameState{}, "", err
	}
	if i.Exist {
		i.ID = id
	}
	return fromInternalFBGameState(i), eTag, nil
}

func SaveGameState(g FBGameState, eTag string) error {
	i := toInternalFBGameState(g)
	iJSON, err := json.Marshal(&i)
	if err != nil {
		return err
	}

	_, statusCode, _, err := firebase_db.Put(
		cc_fb_config.CC_FIREBASE_CONFIG.Client,
		fmt.Sprintf("%s/game/%s.json", cc_fb_config.CC_FIREBASE_CONFIG.ProjectPath, g.ID),
		iJSON,
		false,
		eTag,
		map[string]string{},
		nil,
	)

	if err != nil {
		return err
	}

	if statusCode != http.StatusOK {
		return errors.New(fmt.Sprintf("HTTP error %d", statusCode))
	}

	return nil
}
<commit_msg>Add game observables to game state in DB<commit_after>package cc_fb

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/cripplet/clicker/db/config"
	"github.com/cripplet/clicker/firebase-db"
	"github.com/cripplet/clicker/lib"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type FBGameState struct {
	ID    string
	Exist bool

	// Imported Game data, used to initialize a game.
	GameData cookie_clicker.GameStateData

	// Calculated Game data.
	GameObservables FBGameObservableData
}

type FBGameObservableData struct {
	CookiesPerClick float64 `json:"cookies_per_click"`
	CPS             float64 `json:"cps"`
}

type internalGameStateData struct {
	Version       string          `json:"version"`
	NCookies      float64         `json:"n_cookies"`
	NBuildings    map[string]int  `json:"building"`
	UpgradeStatus map[string]bool `json:"upgrade"`
}

type internalFBGameState struct {
	ID              string                `json:"id"`
	Exist           bool                  `json:"exist"`
	GameData        internalGameStateData `json:"data"`
	GameObservables FBGameObservableData  `json:"observables"`
}

func toInternalFBGameState(s FBGameState) internalFBGameState {
	internalData := internalGameStateData{
		Version:       s.GameData.Version,
		NCookies:      s.GameData.NCookies,
		NBuildings:    make(map[string]int),
		UpgradeStatus: make(map[string]bool),
	}
	for buildingType, nBuildings := range s.GameData.NBuildings {
		internalData.NBuildings[fmt.Sprintf("_%d", buildingType)] = nBuildings
	}
	for upgradeID, bought := range s.GameData.UpgradeStatus {
		internalData.UpgradeStatus[fmt.Sprintf("_%d", upgradeID)] = bought
	}
	return internalFBGameState{
		ID:              s.ID,
		Exist:           s.Exist,
		GameData:        internalData,
		GameObservables: s.GameObservables,
	}
}

func fromInternalFBGameState(s internalFBGameState) FBGameState {
	gameData := cookie_clicker.GameStateData{
		Version:       s.GameData.Version,
		NCookies:      s.GameData.NCookies,
		NBuildings:    make(map[cookie_clicker.BuildingType]int),
		UpgradeStatus: make(map[cookie_clicker.UpgradeID]bool),
	}
	for buildingTypeString, nBuildings := range s.GameData.NBuildings {
		buildingTypeInt, _ := strconv.Atoi(strings.Replace(buildingTypeString, "_", "", -1))
		gameData.NBuildings[cookie_clicker.BuildingType(buildingTypeInt)] = nBuildings
	}
	for upgradeIDString, bought := range s.GameData.UpgradeStatus {
		upgradeIDInt, _ := strconv.Atoi(strings.Replace(upgradeIDString, "_", "", -1))
		gameData.UpgradeStatus[cookie_clicker.UpgradeID(upgradeIDInt)] = bought
	}
	return FBGameState{
		ID:              s.ID,
		Exist:           s.Exist,
		GameData:        gameData,
		GameObservables: s.GameObservables,
	}
}

type PostID struct {
	Name string `json:"name"`
}

func newGameState() (FBGameState, error) {
	n := time.Now()

	d := cookie_clicker.NewGameStateData()
	g := cookie_clicker.NewGameState()
	g.Load(*d)
	o := FBGameObservableData{
		CookiesPerClick: g.GetCookiesPerClick(),
		CPS:             g.GetCPS(n, n.Add(time.Second)),
	}

	s := FBGameState{
		Exist:           true,
		GameData:        *d,
		GameObservables: o,
	}

	i := toInternalFBGameState(s)
	iJSON, err := json.Marshal(&i)
	if err != nil {
		return FBGameState{}, err
	}

	p := PostID{}
	_, _, _, err = firebase_db.Post(
		cc_fb_config.CC_FIREBASE_CONFIG.Client,
		fmt.Sprintf("%s/game.json", cc_fb_config.CC_FIREBASE_CONFIG.ProjectPath),
		iJSON,
		false,
		map[string]string{},
		&p,
	)
	if err != nil {
		return FBGameState{}, err
	}

	i.ID = p.Name
	return fromInternalFBGameState(i), nil
}

func LoadGameState(id string) (FBGameState, string, error) {
	if id == "" {
		g, err := newGameState()
		if err != nil {
			return FBGameState{}, "", nil
		}
		id = g.ID
	}

	i := internalFBGameState{}
	_, _, eTag, err := firebase_db.Get(
		cc_fb_config.CC_FIREBASE_CONFIG.Client,
		fmt.Sprintf("%s/game/%s.json", cc_fb_config.CC_FIREBASE_CONFIG.ProjectPath, id),
		true,
		map[string]string{},
		&i,
	)
	if err != nil {
		return FBGameState{}, "", err
	}
	if i.Exist {
		i.ID = id
	}
	return fromInternalFBGameState(i), eTag, nil
}

func SaveGameState(g FBGameState, eTag string) error {
	i := toInternalFBGameState(g)
	iJSON, err := json.Marshal(&i)
	if err != nil {
		return err
	}

	_, statusCode, _, err := firebase_db.Put(
		cc_fb_config.CC_FIREBASE_CONFIG.Client,
		fmt.Sprintf("%s/game/%s.json", cc_fb_config.CC_FIREBASE_CONFIG.ProjectPath, g.ID),
		iJSON,
		false,
		eTag,
		map[string]string{},
		nil,
	)

	if err != nil {
		return err
	}

	if statusCode != http.StatusOK {
		return errors.New(fmt.Sprintf("HTTP error %d", statusCode))
	}

	return nil
}
<|endoftext|>


<commit_before>package whm

import "github.com/letsencrypt-cpanel/cpanelgo"

type ListAccountsApiResponse struct {
	BaseWhmApiResponse
	Data struct {
		Accounts []struct {
			User string `json:"user"`
		} `json:"acct"`
	} `json:"data"`
}

func (a WhmApi) ListAccounts() (ListAccountsApiResponse, error) {
	var out ListAccountsApiResponse

	err := a.WHMAPI1("listaccts", cpanelgo.Args{}, &out)
	if err == nil {
		err = out.Error()
	}

	return out, err
}

type AccountSummaryApiResponse struct {
	BaseWhmApiResponse
	Data struct {
		Account []struct {
			Email     string `json:"email"`
			Suspended int    `json:"suspended"`
		} `json:"acct"`
	} `json:"data"`
}

func (r AccountSummaryApiResponse) HasEmail() bool {
	e := r.Email()
	return e != "" && e != "*unknown*"
}

func (r AccountSummaryApiResponse) Email() string {
	for _, v := range r.Data.Account {
		if v.Email != "" {
			return v.Email
		}
	}
	return ""
}

func (r AccountSummaryApiResponse) Suspended() bool {
	for _, v := range r.Data.Account {
		if v.Suspended != 0 {
			return true
		}
	}
	return false
}

func (a WhmApi) AccountSummary(username string) (AccountSummaryApiResponse, error) {
	var out AccountSummaryApiResponse

	err := a.WHMAPI1("accountsummary", cpanelgo.Args{
		"user": username,
	}, &out)
	if err == nil {
		err = out.Error()
	}

	return out, err
}
<commit_msg>add theme to user struct<commit_after>package whm

import "github.com/letsencrypt-cpanel/cpanelgo"

type ListAccountsApiResponse struct {
	BaseWhmApiResponse
	Data struct {
		Accounts []struct {
			User  string `json:"user"`
			Theme string `json:"theme"`
		} `json:"acct"`
	} `json:"data"`
}

func (a WhmApi) ListAccounts() (ListAccountsApiResponse, error) {
	var out ListAccountsApiResponse

	err := a.WHMAPI1("listaccts", cpanelgo.Args{}, &out)
	if err == nil {
		err = out.Error()
	}

	return out, err
}

type AccountSummaryApiResponse struct {
	BaseWhmApiResponse
	Data struct {
		Account []struct {
			Email     string `json:"email"`
			Suspended int    `json:"suspended"`
		} `json:"acct"`
	} `json:"data"`
}

func (r AccountSummaryApiResponse) HasEmail() bool {
	e := r.Email()
	return e != "" && e != "*unknown*"
}

func (r AccountSummaryApiResponse) Email() string {
	for _, v := range r.Data.Account {
		if v.Email != "" {
			return v.Email
		}
	}
	return ""
}

func (r AccountSummaryApiResponse) Suspended() bool {
	for _, v := range r.Data.Account {
		if v.Suspended != 0 {
			return true
		}
	}
	return false
}

func (a WhmApi) AccountSummary(username string) (AccountSummaryApiResponse, error) {
	var out AccountSummaryApiResponse

	err := a.WHMAPI1("accountsummary", cpanelgo.Args{
		"user": username,
	}, &out)
	if err == nil {
		err = out.Error()
	}

	return out, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func main() {
	a := []int{3, 5, 7}
	b := []int{3, 3, 4}
	c := merge(a, b)
	fmt.Println(c)
}

func merge(a, b []int) []int {
	var combined []int
	x, y := 0, 0
	for x < len(a) && y < len(b) {
		if a[x] < b[y] {
			combined = append(combined, a[x])
			x++
		} else {
			combined = append(combined, b[y])
			y++
		}
	}
	if len(a)-x != 0 {
		combined = append(combined, a[x:]...)
	}
	if len(b)-y != 0 {
		combined = append(combined, b[y:]...)
	}
	return combined
}
<commit_msg>Remove unneccesary if statements<commit_after>package main

import (
	"fmt"
)

func main() {
	a := []int{3, 5, 7}
	b := []int{3, 3, 4, 6, 10, 10}
	c := merge(a, b)
	fmt.Println(c)
}

func merge(a, b []int) []int {
	var combined []int
	x, y := 0, 0
	for x < len(a) && y < len(b) {
		if a[x] < b[y] {
			combined = append(combined, a[x])
			x++
		} else {
			combined = append(combined, b[y])
			y++
		}
	}
	combined = append(combined, a[x:]...)
	combined = append(combined, b[y:]...)

	return combined
}
<|endoftext|>


<commit_before>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
This tool watches the Kubernetes API server for Pod (de)registration. New Pods are registered to
Vulcan by setting the correct etcd keys. A deleted Pod is deleted from Vulcan as well by removing it's key in etcd.
Pods will be registered using the following key pattern in etcd: /vulcan/backends/[pod label name]/servers/[pod IP]. Make sure
your Vulcan backend/frontend configuration is configured to use backend servers based on the pod name.
 */
package main

import (
	"github.com/gorilla/websocket"
	"net"
	"net/http"
	"net/url"
	"log"
	"encoding/json"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"flag"
	"github.com/coreos/go-etcd/etcd"
	"fmt"
)

var kubernetesEndpoint string
var etcdAddress string

type Registration struct {
	URL string
}

func init() {
	flag.StringVar(&kubernetesEndpoint, "pods", "", "Endpoint of Kubernetes pods API")
	flag.StringVar(&etcdAddress, "etcd", "", "etcd address")

	flag.Parse()

	if kubernetesEndpoint == "" || etcdAddress == "" {
		log.Fatal(`Missing required properties. Usage: Registrator -pods "ws://[kubernetes-server]/api/v1beta3/namespaces/default/pods?watch=true" -etcd "[etcd-address]"`)
	}
}

func main() {
	listenForPods()
}

/**
Open WS connection and start Go routines to listen for pods.
 */
func listenForPods() {

	wsConn := openConnection()

	var wsErrors chan string = make(chan string)
	go listen(wsConn, wsErrors)
	go reconnector(wsErrors)
	select {}

}

/**
Open WebSocket connection to Kubernetes API server
 */
func openConnection() *websocket.Conn {
	u, err := url.Parse(kubernetesEndpoint)
	if err != nil {
		log.Fatal(err)
	}

	rawConn, err := net.Dial("tcp", u.Host)
	if err != nil {
		log.Fatal(err)
	}

	wsHeaders := http.Header{
		"Origin":                   {kubernetesEndpoint},
		"Sec-WebSocket-Extensions": {"permessage-deflate; client_max_window_bits, x-webkit-deflate-frame"},
	}

	wsConn, resp, err := websocket.NewClient(rawConn, u, wsHeaders, 1024, 1024)
	if err != nil {
		log.Fatalf("websocket.NewClient Error: %s\nResp:%+v", err, resp)

	}

	return wsConn
}

/**
When the WebSocket connection disconnects for some reason, try to reconnect.
 */
func reconnector(wsErrors chan string) {
	for {
		_ = <- wsErrors
		log.Println("Reconnecting...")
		go listen(openConnection(), wsErrors)
	}
}

/**
Listen for Pods. We're only interested in MODIFIED and DELETED events.
 */
func listen(wsConn *websocket.Conn, wsErrors chan string) {
	log.Println("Listening for pods")

	for {
		_, r, err := wsConn.NextReader()

		if err != nil {
			log.Printf("Error getting reader: %v",err)
			wsErrors <- "Error"
			return
		}

		dec := json.NewDecoder(r)
		var objmap map[string]*json.RawMessage
		dec.Decode(&objmap)

		var actionType string
		json.Unmarshal(*objmap["type"], &actionType)

		var pod api.Pod
		err = json.Unmarshal(*objmap["object"], &pod)

		switch actionType {
		case "MODIFIED":
			register(pod)
		case "DELETED":
			deletePod(pod)
		}
	}
}

/**
Register a new backend server in Vulcan based on the new Pod
 */
func register(pod api.Pod) {
	if(pod.Status.Phase != "Running") {
		return
	}

	log.Printf("Registrating pod %v listening on %v to %v\n", pod.Name, pod.Status.PodIP, etcdAddress)

	machines := []string{etcdAddress}
	client := etcd.NewClient(machines)

	podUrl := fmt.Sprintf("http://%v:%v", pod.Status.PodIP, pod.Spec.Containers[0].Ports[0].HostPort)

	if _, err := client.Set("vulcan/backends/" + pod.Labels["name"] + "/servers/" + pod.Status.PodIP, `{"URL": "` + podUrl + `"}`, 0); err != nil {
		log.Fatal(err)
	}
}

/**
Delete a backend server from Vulcan when a Pod is deleted.
 */
func deletePod(pod api.Pod) {
	log.Printf("Deleting pod %v from %v\n", pod.Name, etcdAddress)

	machines := []string{etcdAddress}
	client := etcd.NewClient(machines)

	_, err := client.Delete("vulcan/backends/" + pod.Labels["name"] + "/servers/" + pod.Status.PodIP, false);

	if err != nil {
		log.Printf("Failed to delete backend '%v'", pod.Labels["name"]);
	}
}
<commit_msg>Using container port instead of host port<commit_after>/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/**
This tool watches the Kubernetes API server for Pod (de)registration. New Pods are registered to
Vulcan by setting the correct etcd keys. A deleted Pod is deleted from Vulcan as well by removing it's key in etcd.
Pods will be registered using the following key pattern in etcd: /vulcan/backends/[pod label name]/servers/[pod IP]. Make sure
your Vulcan backend/frontend configuration is configured to use backend servers based on the pod name.
 */
package main

import (
	"github.com/gorilla/websocket"
	"net"
	"net/http"
	"net/url"
	"log"
	"encoding/json"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"flag"
	"github.com/coreos/go-etcd/etcd"
	"fmt"
)

var kubernetesEndpoint string
var etcdAddress string

type Registration struct {
	URL string
}

func init() {
	flag.StringVar(&kubernetesEndpoint, "pods", "", "Endpoint of Kubernetes pods API")
	flag.StringVar(&etcdAddress, "etcd", "", "etcd address")

	flag.Parse()

	if kubernetesEndpoint == "" || etcdAddress == "" {
		log.Fatal(`Missing required properties. Usage: Registrator -pods "ws://[kubernetes-server]/api/v1beta3/namespaces/default/pods?watch=true" -etcd "[etcd-address]"`)
	}
}

func main() {
	listenForPods()
}

/**
Open WS connection and start Go routines to listen for pods.
 */
func listenForPods() {

	wsConn := openConnection()

	var wsErrors chan string = make(chan string)
	go listen(wsConn, wsErrors)
	go reconnector(wsErrors)
	select {}

}

/**
Open WebSocket connection to Kubernetes API server
 */
func openConnection() *websocket.Conn {
	u, err := url.Parse(kubernetesEndpoint)
	if err != nil {
		log.Fatal(err)
	}

	rawConn, err := net.Dial("tcp", u.Host)
	if err != nil {
		log.Fatal(err)
	}

	wsHeaders := http.Header{
		"Origin":                   {kubernetesEndpoint},
		"Sec-WebSocket-Extensions": {"permessage-deflate; client_max_window_bits, x-webkit-deflate-frame"},
	}

	wsConn, resp, err := websocket.NewClient(rawConn, u, wsHeaders, 1024, 1024)
	if err != nil {
		log.Fatalf("websocket.NewClient Error: %s\nResp:%+v", err, resp)

	}

	return wsConn
}

/**
When the WebSocket connection disconnects for some reason, try to reconnect.
 */
func reconnector(wsErrors chan string) {
	for {
		_ = <- wsErrors
		log.Println("Reconnecting...")
		go listen(openConnection(), wsErrors)
	}
}

/**
Listen for Pods. We're only interested in MODIFIED and DELETED events.
 */
func listen(wsConn *websocket.Conn, wsErrors chan string) {
	log.Println("Listening for pods")

	for {
		_, r, err := wsConn.NextReader()

		if err != nil {
			log.Printf("Error getting reader: %v",err)
			wsErrors <- "Error"
			return
		}

		dec := json.NewDecoder(r)
		var objmap map[string]*json.RawMessage
		dec.Decode(&objmap)

		var actionType string
		json.Unmarshal(*objmap["type"], &actionType)

		var pod api.Pod
		err = json.Unmarshal(*objmap["object"], &pod)

		switch actionType {
		case "MODIFIED":
			register(pod)
		case "DELETED":
			deletePod(pod)
		}
	}
}

/**
Register a new backend server in Vulcan based on the new Pod
 */
func register(pod api.Pod) {
	if(pod.Status.Phase != "Running") {
		return
	}

	log.Printf("Registrating pod %v listening on %v to %v\n", pod.Name, pod.Status.PodIP, etcdAddress)

	machines := []string{etcdAddress}
	client := etcd.NewClient(machines)

	podUrl := fmt.Sprintf("http://%v:%v", pod.Status.PodIP, pod.Spec.Containers[0].Ports[0].ContainerPort)

	if _, err := client.Set("vulcan/backends/" + pod.Labels["name"] + "/servers/" + pod.Status.PodIP, `{"URL": "` + podUrl + `"}`, 0); err != nil {
		log.Fatal(err)
	}
}

/**
Delete a backend server from Vulcan when a Pod is deleted.
 */
func deletePod(pod api.Pod) {
	log.Printf("Deleting pod %v from %v\n", pod.Name, etcdAddress)

	machines := []string{etcdAddress}
	client := etcd.NewClient(machines)

	_, err := client.Delete("vulcan/backends/" + pod.Labels["name"] + "/servers/" + pod.Status.PodIP, false);

	if err != nil {
		log.Printf("Failed to delete backend '%v'", pod.Labels["name"]);
	}
}
<|endoftext|>


<commit_before>package app

import (
	"context"
	"io"
	"net/url"

	"github.com/maxence-charriere/go-app/v7/pkg/errors"
)

type elem struct {
	attrs       map[string]string
	body        []UI
	ctx         context.Context
	ctxCancel   func()
	events      map[string]eventHandler
	jsvalue     Value
	parentElem  UI
	selfClosing bool
	tag         string
	this        UI
}

func (e *elem) Kind() Kind {
	return HTML
}

func (e *elem) JSValue() Value {
	return e.jsvalue
}

func (e *elem) Mounted() bool {
	return e.ctx != nil && e.ctx.Err() == nil &&
		e.self() != nil &&
		e.jsvalue != nil
}

func (e *elem) name() string {
	return e.tag
}

func (e *elem) self() UI {
	return e.this
}

func (e *elem) setSelf(n UI) {
	e.this = n
}

func (e *elem) context() context.Context {
	return e.ctx
}

func (e *elem) attributes() map[string]string {
	return e.attrs
}

func (e *elem) eventHandlers() map[string]eventHandler {
	return e.events
}

func (e *elem) parent() UI {
	return e.parentElem
}

func (e *elem) setParent(p UI) {
	e.parentElem = p
}

func (e *elem) children() []UI {
	return e.body
}

func (e *elem) mount() error {
	if e.Mounted() {
		return errors.New("mounting ui element failed").
			Tag("reason", "already mounted").
			Tag("name", e.name()).
			Tag("kind", e.Kind())
	}

	e.ctx, e.ctxCancel = context.WithCancel(context.Background())

	v := Window().Get("document").Call("createElement", e.tag)
	if !v.Truthy() {
		return errors.New("mounting ui element failed").
			Tag("reason", "create javascript node returned nil").
			Tag("name", e.name()).
			Tag("kind", e.Kind())
	}
	e.jsvalue = v

	for k, v := range e.attrs {
		e.setJsAttr(k, v)
	}

	for k, v := range e.events {
		e.setJsEventHandler(k, v)
	}

	for _, c := range e.children() {
		if err := e.appendChild(c, true); err != nil {
			return errors.New("mounting ui element failed").
				Tag("name", e.name()).
				Tag("kind", e.Kind()).
				Wrap(err)
		}
	}

	return nil
}

func (e *elem) dismount() {
	for _, c := range e.children() {
		dismount(c)
	}

	for k, v := range e.events {
		e.delJsEventHandler(k, v)
	}

	e.ctxCancel()
	e.jsvalue = nil
}

func (e *elem) update(n UI) error {
	if !e.Mounted() {
		return nil
	}

	if n.Kind() != e.Kind() || n.name() != e.name() {
		return errors.New("updating ui element failed").
			Tag("replace", true).
			Tag("reason", "different element types").
			Tag("current-kind", e.Kind()).
			Tag("current-name", e.name()).
			Tag("updated-kind", n.Kind()).
			Tag("updated-name", n.name())
	}

	e.updateAttrs(n.attributes())
	e.updateEventHandler(n.eventHandlers())

	achildren := e.children()
	bchildren := n.children()
	i := 0

	// Update children:
	for len(achildren) != 0 && len(bchildren) != 0 {
		a := achildren[0]
		b := bchildren[0]

		err := update(a, b)
		if isErrReplace(err) {
			err = e.replaceChildAt(i, b)
		}

		if err != nil {
			return errors.New("updating ui element failed").
				Tag("kind", e.Kind()).
				Tag("name", e.name()).
				Wrap(err)
		}

		achildren = achildren[1:]
		bchildren = bchildren[1:]
		i++
	}

	// Remove children:
	for len(achildren) != 0 {
		if err := e.removeChildAt(i); err != nil {
			return errors.New("updating ui element failed").
				Tag("kind", e.Kind()).
				Tag("name", e.name()).
				Wrap(err)
		}

		achildren = achildren[1:]
	}

	// Add children:
	for len(bchildren) != 0 {
		c := bchildren[0]

		if err := e.appendChild(c, false); err != nil {
			return errors.New("updating ui element failed").
				Tag("kind", e.Kind()).
				Tag("name", e.name()).
				Wrap(err)
		}

		bchildren = bchildren[1:]
	}

	return nil
}

func (e *elem) appendChild(c UI, onlyJsValue bool) error {
	if err := mount(c); err != nil {
		return errors.New("appending child failed").
			Tag("name", e.name()).
			Tag("kind", e.Kind()).
			Tag("child-name", c.name()).
			Tag("child-kind", c.Kind()).
			Wrap(err)
	}

	if !onlyJsValue {
		e.body = append(e.body, c)
	}

	c.setParent(e.self())
	e.JSValue().Call("appendChild", c)
	return nil
}

func (e *elem) replaceChildAt(idx int, new UI) error {
	old := e.body[idx]

	if err := mount(new); err != nil {
		return errors.New("replacing child failed").
			Tag("name", e.name()).
			Tag("kind", e.Kind()).
			Tag("index", idx).
			Tag("old-name", old.name()).
			Tag("old-kind", old.Kind()).
			Tag("new-name", new.name()).
			Tag("new-kind", new.Kind()).
			Wrap(err)
	}

	e.body[idx] = new
	new.setParent(e.self())
	e.JSValue().Call("replaceChild", new, old)

	dismount(old)
	return nil
}

func (e *elem) removeChildAt(idx int) error {
	body := e.body
	if idx < 0 || idx >= len(body) {
		return errors.New("removing child failed").
			Tag("reason", "index out of range").
			Tag("index", idx).
			Tag("name", e.name()).
			Tag("kind", e.Kind())
	}

	c := body[idx]

	copy(body[idx:], body[idx+1:])
	body[len(body)-1] = nil
	body = body[:len(body)-1]
	e.body = body

	e.JSValue().Call("removeChild", c)
	dismount(c)
	return nil
}

func (e *elem) updateAttrs(attrs map[string]string) {
	for k := range e.attrs {
		if _, exist := attrs[k]; !exist {
			e.delAttr(k)
		}
	}

	if e.attrs == nil && len(attrs) != 0 {
		e.attrs = make(map[string]string, len(attrs))
	}

	for k, v := range attrs {
		if curval := e.attrs[k]; curval != v {
			e.attrs[k] = v
			e.setJsAttr(k, v)
		}
	}
}

func (e *elem) setAttr(k string, v interface{}) {
	if e.attrs == nil {
		e.attrs = make(map[string]string)
	}

	switch k {
	case "style":
		s := e.attrs[k] + toString(v) + ";"
		e.attrs[k] = s
		return

	case "class":
		s := e.attrs[k]
		if s != "" {
			s += " "
		}
		s += toString(v)
		e.attrs[k] = s
		return
	}

	switch v := v.(type) {
	case bool:
		if !v {
			delete(e.attrs, k)
			return
		}
		e.attrs[k] = ""

	default:
		e.attrs[k] = toString(v)
	}
}

func (e *elem) setJsAttr(k, v string) {
	e.JSValue().Call("setAttribute", k, v)
}

func (e *elem) delAttr(k string) {
	e.JSValue().Call("removeAttribute", k)
	delete(e.attrs, k)
}

func (e *elem) updateEventHandler(handlers map[string]eventHandler) {
	for k, current := range e.events {
		if _, exists := handlers[k]; !exists {
			e.delJsEventHandler(k, current)
		}
	}

	if e.events == nil && len(handlers) != 0 {
		e.events = make(map[string]eventHandler, len(handlers))
	}

	for k, new := range handlers {
		if current, exists := e.events[k]; !current.equal(new) {
			if exists {
				e.delJsEventHandler(k, current)
			}

			e.events[k] = new
			e.setJsEventHandler(k, new)
		}
	}
}

func (e *elem) setEventHandler(k string, h EventHandler) {
	if e.events == nil {
		e.events = make(map[string]eventHandler)
	}

	e.events[k] = eventHandler{
		event: k,
		value: h,
	}
}

func (e *elem) setJsEventHandler(k string, h eventHandler) {
	jshandler := makeJsEventHandler(e.self(), h.value)
	h.jsvalue = jshandler
	e.events[k] = h
	e.JSValue().Call("addEventListener", k, jshandler)
}

func (e *elem) delJsEventHandler(k string, h eventHandler) {
	e.JSValue().Call("removeEventListener", k, h.jsvalue)
	h.jsvalue.Release()
	delete(e.events, k)
}

func (e *elem) setBody(body ...UI) {
	if e.selfClosing {
		panic(errors.New("setting html element body failed").
			Tag("reason", "self closing element can't have children").
			Tag("name", e.name()),
		)
	}

	e.body = FilterUIElems(body...)
}

func (e *elem) onNav(u *url.URL) {
	for _, c := range e.children() {
		c.onNav(u)
	}
}

func (e *elem) html(w io.Writer) {
	e.htmlWithIndent(w, 0)
}

func (e *elem) htmlWithIndent(w io.Writer, indent int) {
	writeIndent(w, indent)
	w.Write(stob("<"))
	w.Write(stob(e.tag))

	for k, v := range e.attrs {
		w.Write(stob(" "))
		w.Write(stob(k))

		if v != "" {
			w.Write(stob(`="`))
			w.Write(stob(v))
			w.Write(stob(`"`))
		}
	}

	w.Write(stob(">"))

	if e.selfClosing {
		return
	}

	for _, c := range e.body {
		w.Write(ln())
		c.(writableNode).htmlWithIndent(w, indent+1)
	}

	if len(e.body) != 0 {
		w.Write(ln())
		writeIndent(w, indent)
	}

	w.Write(stob("</"))
	w.Write(stob(e.tag))
	w.Write(stob(">"))
}
<commit_msg>Update element.go (#442)<commit_after>package app

import (
	"context"
	"io"
	"net/url"

	"github.com/maxence-charriere/go-app/v7/pkg/errors"
)

type elem struct {
	attrs       map[string]string
	body        []UI
	ctx         context.Context
	ctxCancel   func()
	events      map[string]eventHandler
	jsvalue     Value
	parentElem  UI
	selfClosing bool
	tag         string
	this        UI
}

func (e *elem) Kind() Kind {
	return HTML
}

func (e *elem) JSValue() Value {
	return e.jsvalue
}

func (e *elem) Mounted() bool {
	return e.ctx != nil && e.ctx.Err() == nil &&
		e.self() != nil &&
		e.jsvalue != nil
}

func (e *elem) name() string {
	return e.tag
}

func (e *elem) self() UI {
	return e.this
}

func (e *elem) setSelf(n UI) {
	e.this = n
}

func (e *elem) context() context.Context {
	return e.ctx
}

func (e *elem) attributes() map[string]string {
	return e.attrs
}

func (e *elem) eventHandlers() map[string]eventHandler {
	return e.events
}

func (e *elem) parent() UI {
	return e.parentElem
}

func (e *elem) setParent(p UI) {
	e.parentElem = p
}

func (e *elem) children() []UI {
	return e.body
}

func (e *elem) mount() error {
	if e.Mounted() {
		return errors.New("mounting ui element failed").
			Tag("reason", "already mounted").
			Tag("name", e.name()).
			Tag("kind", e.Kind())
	}

	e.ctx, e.ctxCancel = context.WithCancel(context.Background())

	v := Window().Get("document").Call("createElement", e.tag)
	if !v.Truthy() {
		return errors.New("mounting ui element failed").
			Tag("reason", "create javascript node returned nil").
			Tag("name", e.name()).
			Tag("kind", e.Kind())
	}
	e.jsvalue = v

	for k, v := range e.attrs {
		e.setJsAttr(k, v)
	}

	for k, v := range e.events {
		e.setJsEventHandler(k, v)
	}

	for _, c := range e.children() {
		if err := e.appendChild(c, true); err != nil {
			return errors.New("mounting ui element failed").
				Tag("name", e.name()).
				Tag("kind", e.Kind()).
				Wrap(err)
		}
	}

	return nil
}

func (e *elem) dismount() {
	for _, c := range e.children() {
		dismount(c)
	}

	for k, v := range e.events {
		e.delJsEventHandler(k, v)
	}

	e.ctxCancel()
	e.jsvalue = nil
}

func (e *elem) update(n UI) error {
	if !e.Mounted() {
		return nil
	}

	if n.Kind() != e.Kind() || n.name() != e.name() {
		return errors.New("updating ui element failed").
			Tag("replace", true).
			Tag("reason", "different element types").
			Tag("current-kind", e.Kind()).
			Tag("current-name", e.name()).
			Tag("updated-kind", n.Kind()).
			Tag("updated-name", n.name())
	}

	e.updateAttrs(n.attributes())
	e.updateEventHandler(n.eventHandlers())

	achildren := e.children()
	bchildren := n.children()
	i := 0

	// Update children:
	for len(achildren) != 0 && len(bchildren) != 0 {
		a := achildren[0]
		b := bchildren[0]

		err := update(a, b)
		if isErrReplace(err) {
			err = e.replaceChildAt(i, b)
		}

		if err != nil {
			return errors.New("updating ui element failed").
				Tag("kind", e.Kind()).
				Tag("name", e.name()).
				Wrap(err)
		}

		achildren = achildren[1:]
		bchildren = bchildren[1:]
		i++
	}

	// Remove children:
	for len(achildren) != 0 {
		if err := e.removeChildAt(i); err != nil {
			return errors.New("updating ui element failed").
				Tag("kind", e.Kind()).
				Tag("name", e.name()).
				Wrap(err)
		}

		achildren = achildren[1:]
	}

	// Add children:
	for len(bchildren) != 0 {
		c := bchildren[0]

		if err := e.appendChild(c, false); err != nil {
			return errors.New("updating ui element failed").
				Tag("kind", e.Kind()).
				Tag("name", e.name()).
				Wrap(err)
		}

		bchildren = bchildren[1:]
	}

	return nil
}

func (e *elem) appendChild(c UI, onlyJsValue bool) error {
	if err := mount(c); err != nil {
		return errors.New("appending child failed").
			Tag("name", e.name()).
			Tag("kind", e.Kind()).
			Tag("child-name", c.name()).
			Tag("child-kind", c.Kind()).
			Wrap(err)
	}

	if !onlyJsValue {
		e.body = append(e.body, c)
	}

	c.setParent(e.self())
	e.JSValue().Call("appendChild", c)
	return nil
}

func (e *elem) replaceChildAt(idx int, new UI) error {
	old := e.body[idx]

	if err := mount(new); err != nil {
		return errors.New("replacing child failed").
			Tag("name", e.name()).
			Tag("kind", e.Kind()).
			Tag("index", idx).
			Tag("old-name", old.name()).
			Tag("old-kind", old.Kind()).
			Tag("new-name", new.name()).
			Tag("new-kind", new.Kind()).
			Wrap(err)
	}

	e.body[idx] = new
	new.setParent(e.self())
	e.JSValue().Call("replaceChild", new, old)

	dismount(old)
	return nil
}

func (e *elem) removeChildAt(idx int) error {
	body := e.body
	if idx < 0 || idx >= len(body) {
		return errors.New("removing child failed").
			Tag("reason", "index out of range").
			Tag("index", idx).
			Tag("name", e.name()).
			Tag("kind", e.Kind())
	}

	c := body[idx]

	copy(body[idx:], body[idx+1:])
	body[len(body)-1] = nil
	body = body[:len(body)-1]
	e.body = body

	e.JSValue().Call("removeChild", c)
	dismount(c)
	return nil
}

func (e *elem) updateAttrs(attrs map[string]string) {
	for k := range e.attrs {
		if _, exists := attrs[k]; !exists {
			e.delAttr(k)
		}
	}

	if e.attrs == nil && len(attrs) != 0 {
		e.attrs = make(map[string]string, len(attrs))
	}

	for k, v := range attrs {
		if curval, exists := e.attrs[k]; !exists || curval != v {
			e.attrs[k] = v
			e.setJsAttr(k, v)
		}
	}
}

func (e *elem) setAttr(k string, v interface{}) {
	if e.attrs == nil {
		e.attrs = make(map[string]string)
	}

	switch k {
	case "style":
		s := e.attrs[k] + toString(v) + ";"
		e.attrs[k] = s
		return

	case "class":
		s := e.attrs[k]
		if s != "" {
			s += " "
		}
		s += toString(v)
		e.attrs[k] = s
		return
	}

	switch v := v.(type) {
	case bool:
		if !v {
			delete(e.attrs, k)
			return
		}
		e.attrs[k] = ""

	default:
		e.attrs[k] = toString(v)
	}
}

func (e *elem) setJsAttr(k, v string) {
	e.JSValue().Call("setAttribute", k, v)
}

func (e *elem) delAttr(k string) {
	e.JSValue().Call("removeAttribute", k)
	delete(e.attrs, k)
}

func (e *elem) updateEventHandler(handlers map[string]eventHandler) {
	for k, current := range e.events {
		if _, exists := handlers[k]; !exists {
			e.delJsEventHandler(k, current)
		}
	}

	if e.events == nil && len(handlers) != 0 {
		e.events = make(map[string]eventHandler, len(handlers))
	}

	for k, new := range handlers {
		if current, exists := e.events[k]; !current.equal(new) {
			if exists {
				e.delJsEventHandler(k, current)
			}

			e.events[k] = new
			e.setJsEventHandler(k, new)
		}
	}
}

func (e *elem) setEventHandler(k string, h EventHandler) {
	if e.events == nil {
		e.events = make(map[string]eventHandler)
	}

	e.events[k] = eventHandler{
		event: k,
		value: h,
	}
}

func (e *elem) setJsEventHandler(k string, h eventHandler) {
	jshandler := makeJsEventHandler(e.self(), h.value)
	h.jsvalue = jshandler
	e.events[k] = h
	e.JSValue().Call("addEventListener", k, jshandler)
}

func (e *elem) delJsEventHandler(k string, h eventHandler) {
	e.JSValue().Call("removeEventListener", k, h.jsvalue)
	h.jsvalue.Release()
	delete(e.events, k)
}

func (e *elem) setBody(body ...UI) {
	if e.selfClosing {
		panic(errors.New("setting html element body failed").
			Tag("reason", "self closing element can't have children").
			Tag("name", e.name()),
		)
	}

	e.body = FilterUIElems(body...)
}

func (e *elem) onNav(u *url.URL) {
	for _, c := range e.children() {
		c.onNav(u)
	}
}

func (e *elem) html(w io.Writer) {
	e.htmlWithIndent(w, 0)
}

func (e *elem) htmlWithIndent(w io.Writer, indent int) {
	writeIndent(w, indent)
	w.Write(stob("<"))
	w.Write(stob(e.tag))

	for k, v := range e.attrs {
		w.Write(stob(" "))
		w.Write(stob(k))

		if v != "" {
			w.Write(stob(`="`))
			w.Write(stob(v))
			w.Write(stob(`"`))
		}
	}

	w.Write(stob(">"))

	if e.selfClosing {
		return
	}

	for _, c := range e.body {
		w.Write(ln())
		c.(writableNode).htmlWithIndent(w, indent+1)
	}

	if len(e.body) != 0 {
		w.Write(ln())
		writeIndent(w, indent)
	}

	w.Write(stob("</"))
	w.Write(stob(e.tag))
	w.Write(stob(">"))
}
<|endoftext|>


<commit_before>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"

	"sigs.k8s.io/yaml"

	"istio.io/pkg/log"
	"istio.io/release-builder/pkg/model"
	"istio.io/release-builder/pkg/util"
)

// Build will create all artifacts required by the manifest
// This assumes the working directory has been setup and sources resolved.
func Build(manifest model.Manifest, githubToken string) error {
	if _, f := manifest.BuildOutputs[model.Docker]; f {
		if err := Docker(manifest); err != nil {
			return fmt.Errorf("failed to build Docker: %v", err)
		}
	}

	if err := SanitizeAllCharts(manifest); err != nil {
		return fmt.Errorf("failed to sanitize charts: %v", err)
	}
	if _, f := manifest.BuildOutputs[model.Helm]; f {
		if err := HelmCharts(manifest); err != nil {
			return fmt.Errorf("failed to build HelmCharts: %v", err)
		}
	}

	if _, f := manifest.BuildOutputs[model.Debian]; f {
		if err := Debian(manifest); err != nil {
			return fmt.Errorf("failed to build Debian: %v", err)
		}
	}

	if _, f := manifest.BuildOutputs[model.Rpm]; f {
		if err := Rpm(manifest); err != nil {
			return fmt.Errorf("failed to build Rpm: %v", err)
		}
	}

	if _, f := manifest.BuildOutputs[model.Archive]; f {
		if err := Archive(manifest); err != nil {
			return fmt.Errorf("failed to build Archive: %v", err)
		}
	}

	if _, f := manifest.BuildOutputs[model.Grafana]; f {
		if err := Grafana(manifest); err != nil {
			return fmt.Errorf("failed to build Grafana: %v", err)
		}
	}

	// Bundle all sources used in the build
	cmd := util.VerboseCommand("tar", "-czf", "out/sources.tar.gz", "sources")
	cmd.Dir = path.Join(manifest.Directory)
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to bundle sources: %v", err)
	}

	if err := writeManifest(manifest, manifest.OutDir()); err != nil {
		return fmt.Errorf("failed to write manifest: %v", err)
	}

	if err := writeLicense(manifest); err != nil {
		return fmt.Errorf("failed to package license file: %v", err)
	}

	return nil
}

// writeLicense copies the complete list of licenses for all dependant repos
func writeLicense(manifest model.Manifest) error {
	if err := os.MkdirAll(filepath.Join(manifest.OutDir(), "licenses"), 0o750); err != nil {
		return fmt.Errorf("failed to create license dir: %v", err)
	}
	for repo := range manifest.Dependencies.Get() {
		src := filepath.Join(manifest.RepoDir(repo), "licenses")
		// Just skip these, we can fail in the validation tests afterwards for repos we expect license for
		if _, err := os.Stat(src); os.IsNotExist(err) {
			log.Warnf("skipping license for %v", repo)
			continue
		}
		// Package as a tar.gz since there are hundreds of files
		cmd := util.VerboseCommand("tar", "-czf", filepath.Join(manifest.OutDir(), "licenses", repo+".tar.gz"), ".")
		cmd.Dir = src
		if err := cmd.Run(); err != nil {
			return fmt.Errorf("failed to compress license: %v", err)
		}
	}
	return nil
}

// writeManifest will output the manifest to yaml
func writeManifest(manifest model.Manifest, dir string) error {
	yml, err := yaml.Marshal(manifest)
	if err != nil {
		return fmt.Errorf("failed to marshal manifest: %v", err)
	}
	if err := ioutil.WriteFile(path.Join(dir, "manifest.yaml"), yml, 0o640); err != nil {
		return fmt.Errorf("failed to write manifest: %v", err)
	}
	return nil
}
<commit_msg>Skip helm charts for CI builds (#793)<commit_after>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"

	"sigs.k8s.io/yaml"

	"istio.io/pkg/log"
	"istio.io/release-builder/pkg/model"
	"istio.io/release-builder/pkg/util"
)

// Build will create all artifacts required by the manifest
// This assumes the working directory has been setup and sources resolved.
func Build(manifest model.Manifest, githubToken string) error {
	if _, f := manifest.BuildOutputs[model.Docker]; f {
		if err := Docker(manifest); err != nil {
			return fmt.Errorf("failed to build Docker: %v", err)
		}
	}

	if err := SanitizeAllCharts(manifest); err != nil {
		return fmt.Errorf("failed to sanitize charts: %v", err)
	}
	if vm, _ := regexp.Match(`\d+\.\d+\.\d+`, []byte(manifest.Version)); vm {
		if _, f := manifest.BuildOutputs[model.Helm]; f {
			if err := HelmCharts(manifest); err != nil {
				return fmt.Errorf("failed to build HelmCharts: %v", err)
			}
		}
	} else {
		log.Warnf("Invalid Semantic Version. Skipping Charts build")
	}

	if _, f := manifest.BuildOutputs[model.Debian]; f {
		if err := Debian(manifest); err != nil {
			return fmt.Errorf("failed to build Debian: %v", err)
		}
	}

	if _, f := manifest.BuildOutputs[model.Rpm]; f {
		if err := Rpm(manifest); err != nil {
			return fmt.Errorf("failed to build Rpm: %v", err)
		}
	}

	if _, f := manifest.BuildOutputs[model.Archive]; f {
		if err := Archive(manifest); err != nil {
			return fmt.Errorf("failed to build Archive: %v", err)
		}
	}

	if _, f := manifest.BuildOutputs[model.Grafana]; f {
		if err := Grafana(manifest); err != nil {
			return fmt.Errorf("failed to build Grafana: %v", err)
		}
	}

	// Bundle all sources used in the build
	cmd := util.VerboseCommand("tar", "-czf", "out/sources.tar.gz", "sources")
	cmd.Dir = path.Join(manifest.Directory)
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to bundle sources: %v", err)
	}

	if err := writeManifest(manifest, manifest.OutDir()); err != nil {
		return fmt.Errorf("failed to write manifest: %v", err)
	}

	if err := writeLicense(manifest); err != nil {
		return fmt.Errorf("failed to package license file: %v", err)
	}

	return nil
}

// writeLicense copies the complete list of licenses for all dependant repos
func writeLicense(manifest model.Manifest) error {
	if err := os.MkdirAll(filepath.Join(manifest.OutDir(), "licenses"), 0o750); err != nil {
		return fmt.Errorf("failed to create license dir: %v", err)
	}
	for repo := range manifest.Dependencies.Get() {
		src := filepath.Join(manifest.RepoDir(repo), "licenses")
		// Just skip these, we can fail in the validation tests afterwards for repos we expect license for
		if _, err := os.Stat(src); os.IsNotExist(err) {
			log.Warnf("skipping license for %v", repo)
			continue
		}
		// Package as a tar.gz since there are hundreds of files
		cmd := util.VerboseCommand("tar", "-czf", filepath.Join(manifest.OutDir(), "licenses", repo+".tar.gz"), ".")
		cmd.Dir = src
		if err := cmd.Run(); err != nil {
			return fmt.Errorf("failed to compress license: %v", err)
		}
	}
	return nil
}

// writeManifest will output the manifest to yaml
func writeManifest(manifest model.Manifest, dir string) error {
	yml, err := yaml.Marshal(manifest)
	if err != nil {
		return fmt.Errorf("failed to marshal manifest: %v", err)
	}
	if err := ioutil.WriteFile(path.Join(dir, "manifest.yaml"), yml, 0o640); err != nil {
		return fmt.Errorf("failed to write manifest: %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package config

// Host defines the configuration flags of a host
type Host struct {
	Host    string `yaml:"host,omitempty,flow"    json:"host,omitempty"`
	User    string `yaml:"user,omitempty,flow"    json:"user,omitempty"`
	Port    uint   `yaml:"port,omitempty,flow"    json:"port,omitempty"`
	Gateway string `yaml:"gateway,omitempty,flow" json:"gateway,omitempty"`
}

// ApplyDefaults ensures a Host is valid by filling the missing fields with defaults
func (h *Host) ApplyDefaults(defaults Host) {
	if h.Host == "" {
		h.Host = defaults.Host
	}
	if h.User == "" {
		h.User = defaults.User
	}
	if h.Port == 0 {
		h.Port = defaults.Port
	}
	if h.Gateway == "" {
		h.Gateway = defaults.Gateway
	}

	// Extra defaults
	if h.Port == 0 {
		h.Port = 22
	}
}
<commit_msg>Revert "Added Host.Gateway field"<commit_after>package config

// Host defines the configuration flags of a host
type Host struct {
	Host string `yaml:"host,omitempty,flow" json:"host,omitempty"`
	User string `yaml:"user,omitempty,flow" json:"user,omitempty"`
	Port uint   `yaml:"port,omitempty,flow" json:"port,omitempty"`
}

// ApplyDefaults ensures a Host is valid by filling the missing fields with defaults
func (h *Host) ApplyDefaults(defaults Host) {
	if h.Host == "" {
		h.Host = defaults.Host
	}
	if h.User == "" {
		h.User = defaults.User
	}
	if h.Port == 0 {
		h.Port = defaults.Port
	}

	// Extra defaults
	if h.Port == 0 {
		h.Port = 22
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ztls

import (
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	"github.com/zmap/zgrab/ztools/keys"
	"github.com/zmap/zgrab/ztools/x509"
)

var ErrUnimplementedCipher error = errors.New("unimplemented cipher suite")
var ErrNoMutualCipher error = errors.New("no mutual cipher suite")

type TLSVersion uint16

type CipherSuite uint16

type ClientHello struct {
	Random    []byte `json:"random"`
	SessionID []byte `json:"session_id,omitempty"`
}

type ServerHello struct {
	Version             TLSVersion  `json:"version"`
	Random              []byte      `json:"random"`
	SessionID           []byte      `json:"session_id"`
	CipherSuite         CipherSuite `json:"cipher_suite"`
	CompressionMethod   uint8       `json:"compression_method"`
	OcspStapling        bool        `json:"ocsp_stapling"`
	TicketSupported     bool        `json:"ticket"`
	SecureRenegotiation bool        `json:"secure_renegotiation"`
	HeartbeatSupported  bool        `json:"heartbeat"`
}

// SimpleCertificate holds a *x509.Certificate and a []byte for the certificate
type SimpleCertificate struct {
	Raw    []byte            `json:"raw"`
	Parsed *x509.Certificate `json:"parsed"`
}

// Certificates represents a TLS certificates message in a format friendly to the golang JSON library.
// ValidationError should be non-nil whenever Valid is false.
type Certificates struct {
	Certificate SimpleCertificate   `json:"certificate"`
	Chain       []SimpleCertificate `json:"chain"`
	Validation  *x509.Validation    `json:"validation"`
}

// ServerKeyExchange represents the raw key data sent by the server in TLS key exchange message
type ServerKeyExchange struct {
	Raw            []byte             `json:"-"`
	RSAParams      *keys.RSAPublicKey `json:"rsa_params,omitempty"`
	DHParams       *keys.DHParams     `json:"dh_params,omitempty"`
	ECDHParams     *keys.ECDHParams   `json:"ecdh_params,omitempty"`
	Signature      *DigitalSignature  `json:"signature,omitempty"`
	SignatureError string             `json:"signature_error,omitempty"`
}

// Finished represents a TLS Finished message
type Finished struct {
	VerifyData []byte `json:"verify_data"`
}

// ServerHandshake stores all of the messages sent by the server during a standard TLS Handshake.
// It implements zgrab.EventData interface
type ServerHandshake struct {
	ClientHello        *ClientHello       `json:"client_hello,omitempty"`
	ServerHello        *ServerHello       `json:"server_hello,omitempty"`
	ServerCertificates *Certificates      `json:"server_certificates,omitempty"`
	ServerKeyExchange  *ServerKeyExchange `json:"server_key_exchange,omitempty"`
	ServerFinished     *Finished          `json:"server_finished,omitempty"`
}

// MarshalJSON implements the json.Marshler interface
func (v *TLSVersion) MarshalJSON() ([]byte, error) {
	aux := struct {
		Name  string `json:"name"`
		Value int    `json:"value"`
	}{
		Name:  v.String(),
		Value: int(*v),
	}
	return json.Marshal(&aux)
}

// UnmarshalJSON implements the json.Unmarshaler interface
func (v *TLSVersion) UnmarshalJSON(b []byte) error {
	aux := struct {
		Name  string `json:"name"`
		Value int    `json:"value"`
	}{}
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	*v = TLSVersion(aux.Value)
	if expectedName := v.String(); expectedName != aux.Name {
		return fmt.Errorf("mismatched tls version and name: version: %d, name: %s, expected name: %s", aux.Value, aux.Name, expectedName)
	}
	return nil
}

// MarshalJSON implements the json.Marshler interface
func (cs *CipherSuite) MarshalJSON() ([]byte, error) {
	buf := make([]byte, 2)
	buf[0] = byte(*cs >> 8)
	buf[1] = byte(*cs)
	enc := strings.ToUpper(hex.EncodeToString(buf))
	aux := struct {
		Hex   string `json:"hex"`
		Name  string `json:"name"`
		Value int    `json:"value"`
	}{
		Hex:   fmt.Sprintf("0x%s", enc),
		Name:  cs.String(),
		Value: int(*cs),
	}
	return json.Marshal(&aux)
}

// UnmarshalJSON implements the json.Unmarshaler interface
func (cs *CipherSuite) UnmarshalJSON(b []byte) error {
	aux := struct {
		Hex   string `json:"hex"`
		Name  string `json:"name"`
		Value uint16 `json:"value"`
	}{}
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	if expectedName := nameForSuite(aux.Value); expectedName != aux.Name {
		return fmt.Errorf("mismatched cipher suite and name, suite: %d, name: %s, expected name: %s", aux.Value, aux.Name, expectedName)
	}
	*cs = CipherSuite(aux.Value)
	return nil
}

func (c *Conn) GetHandshakeLog() *ServerHandshake {
	return c.handshakeLog
}

func (m *clientHelloMsg) MakeLog() *ClientHello {
	ch := new(ClientHello)
	ch.Random = make([]byte, len(m.random))
	copy(ch.Random, m.random)
	ch.SessionID = make([]byte, len(m.sessionId))
	copy(ch.SessionID, m.sessionId)
	return ch
}

func (m *serverHelloMsg) MakeLog() *ServerHello {
	sh := new(ServerHello)
	sh.Version = TLSVersion(m.vers)
	sh.Random = make([]byte, len(m.random))
	copy(sh.Random, m.random)
	sh.SessionID = make([]byte, len(m.sessionId))
	copy(sh.SessionID, m.sessionId)
	sh.CipherSuite = CipherSuite(m.cipherSuite)
	sh.CompressionMethod = m.compressionMethod
	sh.OcspStapling = m.ocspStapling
	sh.TicketSupported = m.ticketSupported
	sh.SecureRenegotiation = m.secureRenegotiation
	sh.HeartbeatSupported = m.heartbeatEnabled
	return sh
}

func (m *certificateMsg) MakeLog() *Certificates {
	sc := new(Certificates)
	if len(m.certificates) >= 1 {
		cert := m.certificates[0]
		sc.Certificate.Raw = make([]byte, len(cert))
		copy(sc.Certificate.Raw, cert)
	}
	if len(m.certificates) >= 2 {
		chain := m.certificates[1:]
		sc.Chain = make([]SimpleCertificate, len(chain))
		for idx, cert := range chain {
			sc.Chain[idx].Raw = make([]byte, len(cert))
			copy(sc.Chain[idx].Raw, cert)
		}
	}
	return sc
}

// addParsed sets the parsed certificates and the validation. It assumes the
// chain slice has already been allocated.
func (c *Certificates) addParsed(certs []*x509.Certificate, validation *x509.Validation) {
	if len(certs) >= 1 {
		c.Certificate.Parsed = certs[0]
	}
	if len(certs) >= 2 {
		chain := certs[1:]
		for idx, cert := range chain {
			c.Chain[idx].Parsed = cert
		}
	}
	c.Validation = validation
}

func (m *serverKeyExchangeMsg) MakeLog(ka keyAgreement) *ServerKeyExchange {
	skx := new(ServerKeyExchange)
	skx.Raw = make([]byte, len(m.key))
	var auth keyAgreementAuthentication
	var errAuth error
	copy(skx.Raw, m.key)

	// Write out parameters
	switch ka := ka.(type) {
	case *rsaKeyAgreement:
		skx.RSAParams = ka.RSAParams()
		auth = ka.auth
		errAuth = ka.verifyError
	case *dheKeyAgreement:
		skx.DHParams = ka.DHParams()
		auth = ka.auth
		errAuth = ka.verifyError
	case *ecdheKeyAgreement:
		skx.ECDHParams = ka.ECDHParams()
		auth = ka.auth
		errAuth = ka.verifyError
	default:
		break
	}

	// Write out signature
	switch auth := auth.(type) {
	case *signedKeyAgreement:
		skx.Signature = auth.Signature()
	default:
		break
	}

	// Write the signature validation error
	if errAuth != nil {
		skx.SignatureError = errAuth.Error()
	}

	return skx
}

func (m *finishedMsg) MakeLog() *Finished {
	sf := new(Finished)
	sf.VerifyData = make([]byte, len(m.verifyData))
	copy(sf.VerifyData, m.verifyData)
	return sf
}
<commit_msg>Omit empty certificate chains<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ztls

import (
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	"github.com/zmap/zgrab/ztools/keys"
	"github.com/zmap/zgrab/ztools/x509"
)

var ErrUnimplementedCipher error = errors.New("unimplemented cipher suite")
var ErrNoMutualCipher error = errors.New("no mutual cipher suite")

type TLSVersion uint16

type CipherSuite uint16

type ClientHello struct {
	Random    []byte `json:"random"`
	SessionID []byte `json:"session_id,omitempty"`
}

type ServerHello struct {
	Version             TLSVersion  `json:"version"`
	Random              []byte      `json:"random"`
	SessionID           []byte      `json:"session_id"`
	CipherSuite         CipherSuite `json:"cipher_suite"`
	CompressionMethod   uint8       `json:"compression_method"`
	OcspStapling        bool        `json:"ocsp_stapling"`
	TicketSupported     bool        `json:"ticket"`
	SecureRenegotiation bool        `json:"secure_renegotiation"`
	HeartbeatSupported  bool        `json:"heartbeat"`
}

// SimpleCertificate holds a *x509.Certificate and a []byte for the certificate
type SimpleCertificate struct {
	Raw    []byte            `json:"raw"`
	Parsed *x509.Certificate `json:"parsed,omitempty"`
}

// Certificates represents a TLS certificates message in a format friendly to the golang JSON library.
// ValidationError should be non-nil whenever Valid is false.
type Certificates struct {
	Certificate SimpleCertificate   `json:"certificate"`
	Chain       []SimpleCertificate `json:"chain,omitempty"`
	Validation  *x509.Validation    `json:"validation"`
}

// ServerKeyExchange represents the raw key data sent by the server in TLS key exchange message
type ServerKeyExchange struct {
	Raw            []byte             `json:"-"`
	RSAParams      *keys.RSAPublicKey `json:"rsa_params,omitempty"`
	DHParams       *keys.DHParams     `json:"dh_params,omitempty"`
	ECDHParams     *keys.ECDHParams   `json:"ecdh_params,omitempty"`
	Signature      *DigitalSignature  `json:"signature,omitempty"`
	SignatureError string             `json:"signature_error,omitempty"`
}

// Finished represents a TLS Finished message
type Finished struct {
	VerifyData []byte `json:"verify_data"`
}

// ServerHandshake stores all of the messages sent by the server during a standard TLS Handshake.
// It implements zgrab.EventData interface
type ServerHandshake struct {
	ClientHello        *ClientHello       `json:"client_hello,omitempty"`
	ServerHello        *ServerHello       `json:"server_hello,omitempty"`
	ServerCertificates *Certificates      `json:"server_certificates,omitempty"`
	ServerKeyExchange  *ServerKeyExchange `json:"server_key_exchange,omitempty"`
	ServerFinished     *Finished          `json:"server_finished,omitempty"`
}

// MarshalJSON implements the json.Marshler interface
func (v *TLSVersion) MarshalJSON() ([]byte, error) {
	aux := struct {
		Name  string `json:"name"`
		Value int    `json:"value"`
	}{
		Name:  v.String(),
		Value: int(*v),
	}
	return json.Marshal(&aux)
}

// UnmarshalJSON implements the json.Unmarshaler interface
func (v *TLSVersion) UnmarshalJSON(b []byte) error {
	aux := struct {
		Name  string `json:"name"`
		Value int    `json:"value"`
	}{}
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	*v = TLSVersion(aux.Value)
	if expectedName := v.String(); expectedName != aux.Name {
		return fmt.Errorf("mismatched tls version and name: version: %d, name: %s, expected name: %s", aux.Value, aux.Name, expectedName)
	}
	return nil
}

// MarshalJSON implements the json.Marshler interface
func (cs *CipherSuite) MarshalJSON() ([]byte, error) {
	buf := make([]byte, 2)
	buf[0] = byte(*cs >> 8)
	buf[1] = byte(*cs)
	enc := strings.ToUpper(hex.EncodeToString(buf))
	aux := struct {
		Hex   string `json:"hex"`
		Name  string `json:"name"`
		Value int    `json:"value"`
	}{
		Hex:   fmt.Sprintf("0x%s", enc),
		Name:  cs.String(),
		Value: int(*cs),
	}
	return json.Marshal(&aux)
}

// UnmarshalJSON implements the json.Unmarshaler interface
func (cs *CipherSuite) UnmarshalJSON(b []byte) error {
	aux := struct {
		Hex   string `json:"hex"`
		Name  string `json:"name"`
		Value uint16 `json:"value"`
	}{}
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	if expectedName := nameForSuite(aux.Value); expectedName != aux.Name {
		return fmt.Errorf("mismatched cipher suite and name, suite: %d, name: %s, expected name: %s", aux.Value, aux.Name, expectedName)
	}
	*cs = CipherSuite(aux.Value)
	return nil
}

func (c *Conn) GetHandshakeLog() *ServerHandshake {
	return c.handshakeLog
}

func (m *clientHelloMsg) MakeLog() *ClientHello {
	ch := new(ClientHello)
	ch.Random = make([]byte, len(m.random))
	copy(ch.Random, m.random)
	ch.SessionID = make([]byte, len(m.sessionId))
	copy(ch.SessionID, m.sessionId)
	return ch
}

func (m *serverHelloMsg) MakeLog() *ServerHello {
	sh := new(ServerHello)
	sh.Version = TLSVersion(m.vers)
	sh.Random = make([]byte, len(m.random))
	copy(sh.Random, m.random)
	sh.SessionID = make([]byte, len(m.sessionId))
	copy(sh.SessionID, m.sessionId)
	sh.CipherSuite = CipherSuite(m.cipherSuite)
	sh.CompressionMethod = m.compressionMethod
	sh.OcspStapling = m.ocspStapling
	sh.TicketSupported = m.ticketSupported
	sh.SecureRenegotiation = m.secureRenegotiation
	sh.HeartbeatSupported = m.heartbeatEnabled
	return sh
}

func (m *certificateMsg) MakeLog() *Certificates {
	sc := new(Certificates)
	if len(m.certificates) >= 1 {
		cert := m.certificates[0]
		sc.Certificate.Raw = make([]byte, len(cert))
		copy(sc.Certificate.Raw, cert)
	}
	if len(m.certificates) >= 2 {
		chain := m.certificates[1:]
		sc.Chain = make([]SimpleCertificate, len(chain))
		for idx, cert := range chain {
			sc.Chain[idx].Raw = make([]byte, len(cert))
			copy(sc.Chain[idx].Raw, cert)
		}
	}
	return sc
}

// addParsed sets the parsed certificates and the validation. It assumes the
// chain slice has already been allocated.
func (c *Certificates) addParsed(certs []*x509.Certificate, validation *x509.Validation) {
	if len(certs) >= 1 {
		c.Certificate.Parsed = certs[0]
	}
	if len(certs) >= 2 {
		chain := certs[1:]
		for idx, cert := range chain {
			c.Chain[idx].Parsed = cert
		}
	}
	c.Validation = validation
}

func (m *serverKeyExchangeMsg) MakeLog(ka keyAgreement) *ServerKeyExchange {
	skx := new(ServerKeyExchange)
	skx.Raw = make([]byte, len(m.key))
	var auth keyAgreementAuthentication
	var errAuth error
	copy(skx.Raw, m.key)

	// Write out parameters
	switch ka := ka.(type) {
	case *rsaKeyAgreement:
		skx.RSAParams = ka.RSAParams()
		auth = ka.auth
		errAuth = ka.verifyError
	case *dheKeyAgreement:
		skx.DHParams = ka.DHParams()
		auth = ka.auth
		errAuth = ka.verifyError
	case *ecdheKeyAgreement:
		skx.ECDHParams = ka.ECDHParams()
		auth = ka.auth
		errAuth = ka.verifyError
	default:
		break
	}

	// Write out signature
	switch auth := auth.(type) {
	case *signedKeyAgreement:
		skx.Signature = auth.Signature()
	default:
		break
	}

	// Write the signature validation error
	if errAuth != nil {
		skx.SignatureError = errAuth.Error()
	}

	return skx
}

func (m *finishedMsg) MakeLog() *Finished {
	sf := new(Finished)
	sf.VerifyData = make([]byte, len(m.verifyData))
	copy(sf.VerifyData, m.verifyData)
	return sf
}
<|endoftext|>


<commit_before>package net

import (
	"io"
	"math"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/patrickmn/go-cache"

	"nimona.io/pkg/context"
	"nimona.io/pkg/discovery"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/log"
	"nimona.io/pkg/peer"
)

var UseUPNP = false

// TODO remove UseUPNP and replace with option
// nolint: gochecknoinits
func init() {
	UseUPNP, _ = strconv.ParseBool(os.Getenv("UPNP"))
}

// Network interface
type Network interface {
	Dial(ctx context.Context, peer *peer.Peer) (*Connection, error)
	Listen(ctx context.Context) (chan *Connection, error)

	AddMiddleware(handler MiddlewareHandler)
	AddTransport(tag string, tsp Transport)
}

// New creates a new p2p network using an address book
func New(
	discover discovery.Discoverer,
	local *peer.LocalPeer,
) (Network, error) {
	return &network{
		discoverer: discover,
		middleware: []MiddlewareHandler{},
		local:      local,
		midLock:    &sync.RWMutex{},
		transports: &sync.Map{},
		attempts:   newAttemptsMap(),
		blacklist:  cache.New(time.Second*5, time.Second*60),
	}, nil
}

// network allows dialing and listening for p2p connections
type network struct {
	discoverer discovery.Discoverer
	local      *peer.LocalPeer
	midLock    *sync.RWMutex
	transports *sync.Map
	middleware []MiddlewareHandler
	attempts   *attemptsMap
	blacklist  *cache.Cache
}

func (n *network) AddMiddleware(handler MiddlewareHandler) {
	n.midLock.Lock()
	defer n.midLock.Unlock()
	n.middleware = append(n.middleware, handler)
}

func (n *network) AddTransport(tag string, tsp Transport) {
	n.transports.Store(tag, tsp)
}

// Dial to a peer and return a net.Conn or error
func (n *network) Dial(
	ctx context.Context,
	p *peer.Peer,
) (*Connection, error) {
	logger := log.FromContext(ctx).With(
		log.String("peer", p.PublicKey().String()),
		log.Strings("addresses", p.Addresses),
	)

	logger.Debug("dialing")

	// keep a flag on whether all addresses where blacklisted so we can return
	// an ErrMissingSignature error
	allBlacklisted := true

	// go through all addresses and try to dial them
	for _, address := range p.Addresses {
		// check if address is currently blacklisted
		if _, blacklisted := n.blacklist.Get(address); blacklisted {
			logger.Debug("address is blacklisted, skipping")
			continue
		}
		// get protocol from address
		addressType := strings.Split(address, ":")[0]
		t, ok := n.transports.Load(addressType)
		if !ok {
			logger.Debug("not sure how to dial",
				log.String("type", addressType),
			)
			continue
		}

		// reset blacklist flag
		allBlacklisted = false

		// dial address
		trsp := t.(Transport)
		conn, err := trsp.Dial(ctx, address)
		if err != nil {
			// blacklist address
			attempts, backoff := n.exponentialyBlacklist(address)
			logger.Error("could not dial address, blacklisting",
				log.Int("failedAttempts", attempts),
				log.String("backoff", backoff.String()),
				log.String("type", addressType),
				log.Error(err),
			)
			continue
		}

		// pass connection to all middleware
		var merr error
		for _, mh := range n.middleware {
			conn, merr = mh(ctx, conn)
			if merr != nil {
				break
			}
		}
		if merr != nil {
			conn.Close() // nolint: errcheck
			logger.Info("could not handle middleware",
				log.String("type", addressType),
				log.Error(err),
			)
			continue
		}

		// at this point we consider the connection successful, so we can
		// reset the failed attempts
		n.attempts.Put(address, 0)
		n.attempts.Put(p.PublicKey().String(), 0)

		return conn, nil
	}

	err := ErrAllAddressesFailed
	if allBlacklisted {
		err = ErrAllAddressesBlacklisted
	}

	logger.Error("could not dial peer", log.Error(err))
	return nil, err
}

func (n *network) exponentialyBlacklist(k string) (int, time.Duration) {
	baseBackoff := float64(time.Second * 1)
	maxBackoff := float64(time.Minute * 10)
	attempts, _ := n.attempts.Get(k)
	attempts++
	backoff := baseBackoff * math.Pow(1.5, float64(attempts))
	if backoff > maxBackoff {
		backoff = maxBackoff
	}
	n.attempts.Put(k, attempts)
	n.blacklist.Set(k, attempts, time.Duration(backoff))
	return attempts, time.Duration(backoff)
}

// Listen
// TODO do we need to return a listener?
func (n *network) Listen(ctx context.Context) (chan *Connection, error) {
	logger := log.FromContext(ctx)
	cconn := make(chan *Connection, 10)

	n.transports.Range(func(key, value interface{}) bool {
		tsp := value.(Transport)
		chConn, err := tsp.Listen(ctx)
		if err != nil {
			// TODO log
			return true
		}
		go func() {
			for {
				conn := <-chConn
				n.midLock.RLock()
				failed := false

				for _, mh := range n.middleware {
					conn, err = mh(ctx, conn)
					if err != nil {
						if errors.CausedBy(err, io.EOF) {
							break
						}
						logger.Error(
							"middleware failure",
							log.Error(err),
						)

						if conn != nil {
							conn.conn.Close() // nolint: errcheck
						}
						failed = true
						break
					}
				}
				n.midLock.RUnlock()

				if !failed {
					cconn <- conn
				}
			}
		}()
		return true
	})

	return cconn, nil
}
<commit_msg>feat(net): add expvar based metrics<commit_after>package net

import (
	"expvar"
	"io"
	"math"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/patrickmn/go-cache"
	"github.com/zserge/metric"

	"nimona.io/pkg/context"
	"nimona.io/pkg/discovery"
	"nimona.io/pkg/errors"
	"nimona.io/pkg/log"
	"nimona.io/pkg/peer"
)

var UseUPNP = false

// TODO remove UseUPNP and replace with option
// nolint: gochecknoinits
func init() {
	UseUPNP, _ = strconv.ParseBool(os.Getenv("UPNP"))

	connConnOutCounter := metric.NewCounter("2m1s", "15m30s", "1h1m")
	expvar.Publish("nm:net.conn.out", connConnOutCounter)

	connConnIncCounter := metric.NewCounter("2m1s", "15m30s", "1h1m")
	expvar.Publish("nm:net.conn.in", connConnIncCounter)

	connDialCounter := metric.NewCounter("2m1s", "15m30s", "1h1m")
	expvar.Publish("nm:net.dial", connDialCounter)

	connBlacklistCounter := metric.NewCounter("2m1s", "15m30s", "1h1m")
	expvar.Publish("nm:net.conn.dial.blacklist", connBlacklistCounter)
}

// Network interface
type Network interface {
	Dial(ctx context.Context, peer *peer.Peer) (*Connection, error)
	Listen(ctx context.Context) (chan *Connection, error)

	AddMiddleware(handler MiddlewareHandler)
	AddTransport(tag string, tsp Transport)
}

// New creates a new p2p network using an address book
func New(
	discover discovery.Discoverer,
	local *peer.LocalPeer,
) (Network, error) {
	return &network{
		discoverer: discover,
		middleware: []MiddlewareHandler{},
		local:      local,
		midLock:    &sync.RWMutex{},
		transports: &sync.Map{},
		attempts:   newAttemptsMap(),
		blacklist:  cache.New(time.Second*5, time.Second*60),
	}, nil
}

// network allows dialing and listening for p2p connections
type network struct {
	discoverer discovery.Discoverer
	local      *peer.LocalPeer
	midLock    *sync.RWMutex
	transports *sync.Map
	middleware []MiddlewareHandler
	attempts   *attemptsMap
	blacklist  *cache.Cache
}

func (n *network) AddMiddleware(handler MiddlewareHandler) {
	n.midLock.Lock()
	defer n.midLock.Unlock()
	n.middleware = append(n.middleware, handler)
}

func (n *network) AddTransport(tag string, tsp Transport) {
	n.transports.Store(tag, tsp)
}

// Dial to a peer and return a net.Conn or error
func (n *network) Dial(
	ctx context.Context,
	p *peer.Peer,
) (*Connection, error) {
	logger := log.FromContext(ctx).With(
		log.String("peer", p.PublicKey().String()),
		log.Strings("addresses", p.Addresses),
	)

	logger.Debug("dialing")
	expvar.Get("nm:net.dial").(metric.Metric).Add(1)

	// keep a flag on whether all addresses where blacklisted so we can return
	// an ErrMissingSignature error
	allBlacklisted := true

	// go through all addresses and try to dial them
	for _, address := range p.Addresses {
		// check if address is currently blacklisted
		if _, blacklisted := n.blacklist.Get(address); blacklisted {
			logger.Debug("address is blacklisted, skipping")
			continue
		}
		// get protocol from address
		addressType := strings.Split(address, ":")[0]
		t, ok := n.transports.Load(addressType)
		if !ok {
			logger.Debug("not sure how to dial",
				log.String("type", addressType),
			)
			continue
		}

		// reset blacklist flag
		allBlacklisted = false

		// dial address
		trsp := t.(Transport)
		conn, err := trsp.Dial(ctx, address)
		if err != nil {
			// blacklist address
			expvar.Get("nm:net.conn.dial.blacklist").(metric.Metric).Add(1)
			attempts, backoff := n.exponentialyBlacklist(address)
			logger.Error("could not dial address, blacklisting",
				log.Int("failedAttempts", attempts),
				log.String("backoff", backoff.String()),
				log.String("type", addressType),
				log.Error(err),
			)
			continue
		}

		// pass connection to all middleware
		var merr error
		for _, mh := range n.middleware {
			conn, merr = mh(ctx, conn)
			if merr != nil {
				break
			}
		}
		if merr != nil {
			conn.Close() // nolint: errcheck
			logger.Info("could not handle middleware",
				log.String("type", addressType),
				log.Error(err),
			)
			continue
		}

		// at this point we consider the connection successful, so we can
		// reset the failed attempts
		n.attempts.Put(address, 0)
		n.attempts.Put(p.PublicKey().String(), 0)

		expvar.Get("nm:net.conn.out").(metric.Metric).Add(1)

		return conn, nil
	}

	err := ErrAllAddressesFailed
	if allBlacklisted {
		err = ErrAllAddressesBlacklisted
	}

	logger.Error("could not dial peer", log.Error(err))
	return nil, err
}

func (n *network) exponentialyBlacklist(k string) (int, time.Duration) {
	baseBackoff := float64(time.Second * 1)
	maxBackoff := float64(time.Minute * 10)
	attempts, _ := n.attempts.Get(k)
	attempts++
	backoff := baseBackoff * math.Pow(1.5, float64(attempts))
	if backoff > maxBackoff {
		backoff = maxBackoff
	}
	n.attempts.Put(k, attempts)
	n.blacklist.Set(k, attempts, time.Duration(backoff))
	return attempts, time.Duration(backoff)
}

// Listen
// TODO do we need to return a listener?
func (n *network) Listen(ctx context.Context) (chan *Connection, error) {
	logger := log.FromContext(ctx)
	cconn := make(chan *Connection, 10)

	n.transports.Range(func(key, value interface{}) bool {
		tsp := value.(Transport)
		chConn, err := tsp.Listen(ctx)
		if err != nil {
			// TODO log
			return true
		}
		go func() {
			for {
				conn := <-chConn
				n.midLock.RLock()
				failed := false

				expvar.Get("nm:net.conn.in").(metric.Metric).Add(1)

				for _, mh := range n.middleware {
					conn, err = mh(ctx, conn)
					if err != nil {
						if errors.CausedBy(err, io.EOF) {
							break
						}
						logger.Error(
							"middleware failure",
							log.Error(err),
						)

						if conn != nil {
							conn.conn.Close() // nolint: errcheck
						}
						failed = true
						break
					}
				}
				n.midLock.RUnlock()

				if !failed {
					cconn <- conn
				}
			}
		}()
		return true
	})

	return cconn, nil
}
<|endoftext|>


<commit_before>package object

import (
	"strconv"
	"strings"
)

// Generation represents object's "version" and starts from 0
type Generation uint64

// String returns generation as string to implement Stringer interface
func (generation Generation) String() string {
	return strconv.FormatUint(uint64(generation), 10)
}

// Next returns the next generation of the base object (current + 1)
func (generation Generation) Next() Generation {
	return generation + 1
}

// KeySeparator used to separate parts of the Key
const KeySeparator = ":"

/*
// Key represents human-readable unified object's key that can always identify any object in Aptomi.
// It consists of several parts - [<domain>:]<namespace>:<kind>:<name>:<rand_addon>:<generation>, where:
// * domain - Aptomi deployment name, optional
// * namespace
// * kind
// * name
// * rand_addon - random 6 letters added to the Key to be able to differentiate objects re-created with the same name, unique for any specific namespace:kind:name
// * generation - object "version", starts from 1
// So, it means that Key could be used to reference concrete object with concrete generation.
type Key string

type KeyParts struct {
	Domain     string
	Namespace  string
	Kind       string
	Name       string
	RandAddon  string
	Generation Generation
}

func (key Key) Parts() (*KeyParts, error) {
	parts := strings.Split(string(key), KeySeparator)
	partsLen := len(parts)

	domain := ""

	// todo(slukjanov): support non-namespaced objects like clusters? userproviders? etc

	if partsLen == 6 {
		domain = parts[0]
		parts = parts[1:]
	} else if partsLen != 5 {
		return nil, fmt.Errorf("Can't parse key: %s", key)
	}

	gen, err := strconv.ParseUint(parts[4], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("Can't parse generation of key %s with error: %s", key, err)
	}

	return &KeyParts{
		Domain:     domain,
		Namespace:  parts[0],
		Kind:       parts[1],
		Name:       parts[2],
		RandAddon:  parts[3],
		Generation: Generation(gen),
	}, nil
}

// KeyFromParts return uid and generation combined into Key
func KeyFromParts(domain string, namespace string, kind string, name string, randAddon string, generation Generation) Key {
	if len(domain) > 0 {
		return Key(fmt.Sprintf("%s:%s:%s:%s:%s:%s", domain, namespace, kind, name, randAddon, generation))
	}
	return Key(fmt.Sprintf("%s:%s:%s:%s:%s", namespace, kind, name, randAddon, generation))
}
*/

// Base interface represents unified object that could be stored in DB, accessed through API, etc.
type Base interface {
	GetNamespace() string
	GetKind() string
	GetName() string
	GetGeneration() Generation
	SetGeneration(Generation)
}

func GetKey(obj Base) string {
	return strings.Join([]string{obj.GetNamespace(), obj.GetKind(), obj.GetName()}, KeySeparator)
}
<commit_msg>Add ParseGeneration helper<commit_after>package object

import (
	"fmt"
	"strconv"
	"strings"
)

// Generation represents object's "version" and starts from 0
type Generation uint64

// String returns generation as string to implement Stringer interface
func (generation Generation) String() string {
	return strconv.FormatUint(uint64(generation), 10)
}

// Next returns the next generation of the base object (current + 1)
func (generation Generation) Next() Generation {
	return generation + 1
}

func ParseGeneration(gen string) Generation {
	val, err := strconv.ParseUint(gen, 10, 64)
	if err != nil {
		panic(fmt.Errorf("error while parsing generation from %s: %s", gen, err))
	}
	return Generation(val)
}

// KeySeparator used to separate parts of the Key
const KeySeparator = ":"

/*
// Key represents human-readable unified object's key that can always identify any object in Aptomi.
// It consists of several parts - [<domain>:]<namespace>:<kind>:<name>:<rand_addon>:<generation>, where:
// * domain - Aptomi deployment name, optional
// * namespace
// * kind
// * name
// * rand_addon - random 6 letters added to the Key to be able to differentiate objects re-created with the same name, unique for any specific namespace:kind:name
// * generation - object "version", starts from 1
// So, it means that Key could be used to reference concrete object with concrete generation.
type Key string

type KeyParts struct {
	Domain     string
	Namespace  string
	Kind       string
	Name       string
	RandAddon  string
	Generation Generation
}

func (key Key) Parts() (*KeyParts, error) {
	parts := strings.Split(string(key), KeySeparator)
	partsLen := len(parts)

	domain := ""

	// todo(slukjanov): support non-namespaced objects like clusters? userproviders? etc

	if partsLen == 6 {
		domain = parts[0]
		parts = parts[1:]
	} else if partsLen != 5 {
		return nil, fmt.Errorf("Can't parse key: %s", key)
	}

	gen, err := strconv.ParseUint(parts[4], 10, 64)
	if err != nil {
		return nil, fmt.Errorf("Can't parse generation of key %s with error: %s", key, err)
	}

	return &KeyParts{
		Domain:     domain,
		Namespace:  parts[0],
		Kind:       parts[1],
		Name:       parts[2],
		RandAddon:  parts[3],
		Generation: Generation(gen),
	}, nil
}

// KeyFromParts return uid and generation combined into Key
func KeyFromParts(domain string, namespace string, kind string, name string, randAddon string, generation Generation) Key {
	if len(domain) > 0 {
		return Key(fmt.Sprintf("%s:%s:%s:%s:%s:%s", domain, namespace, kind, name, randAddon, generation))
	}
	return Key(fmt.Sprintf("%s:%s:%s:%s:%s", namespace, kind, name, randAddon, generation))
}
*/

// Base interface represents unified object that could be stored in DB, accessed through API, etc.
type Base interface {
	GetNamespace() string
	GetKind() string
	GetName() string
	GetGeneration() Generation
	SetGeneration(Generation)
}

func GetKey(obj Base) string {
	return strings.Join([]string{obj.GetNamespace(), obj.GetKind(), obj.GetName()}, KeySeparator)
}
<|endoftext|>