<commit_before>package packager

import(
	tp "tritium/proto"
	proto "goprotobuf.googlecode.com/hg/proto"
	api "tritium/api"
	"strings"
	"path/filepath"
	"log4go"
	"os"
)

type Package struct {
	loaded []*PackageInfo
	location string
	LoadPath string
	FallbackPath string
	OutputFile string
	Log log4go.Logger
	*tp.Package
	Options PackageOptions
}

type PackageInfo struct {
	Name string
	Dependencies []string
	Types []string
}

var DefaultPackagePath = "packages"

func LoadDefaultPackage(path *string) (*Package) {
	if path == nil {
		path = &DefaultPackagePath
	}

	return buildPackage(*path, nil)
}

func OutputDefaultPackage(path string) (pkg *Package, newFilePath string) {
	pkg = BuildDefaultPackage()

	_, err := os.Stat(path)

	if err != nil {
		creationErr := os.MkdirAll(path, uint32(0777) )
		if creationErr != nil {
			panic("Could not make path(" + path + "). Error:" + creationErr.String())
		}
	}

	_, name := filepath.Split(pkg.OutputFile)
	newOutputFile := filepath.Join(path, name)

	os.Rename(pkg.OutputFile, newOutputFile)

	return pkg, newOutputFile
}

func BuildDefaultPackage() (*Package) {
	options := BuildOptions()
	return buildPackage(DefaultPackagePath, options)
}

func buildPackage(path string, options PackageOptions) (*Package) {
	// Terrible directory handling here... has to be executed from Tritium root

	pkg := NewPackage(path, options)
	pkg.Load("libxml")

	return pkg
}

func mergeOptions(options PackageOptions) PackageOptions {
	defaults := fetchDefaultOptions()

	if options == nil {
		return defaults
	}

	for k, _ := range defaults {
		_, ok := options[k]

		if !ok {
			options[k] = defaults[k]
		}
	}

	return options
}

func NewPackage(loadPath string, options PackageOptions) (*Package){
	options = mergeOptions(options)

	return &Package{
		Package: &tp.Package{
			Name: proto.String("combined"),
			Functions: make([]*tp.Function, 0),
			Types: make([]*tp.Type, 0),
		},
		loaded: make([]*PackageInfo, 0),
  	        Log: newLog(),
		LoadPath: loadPath,
	        Options: options,
	}
}

func (pkg *Package) BuildUserPackage(loadPath *string, fallbackPath *string) {
	userPackage := NewUserPackage(loadPath, fallbackPath)
	pkg.Merge(userPackage.Package)
}

func NewUserPackage(loadPath *string, fallbackPath *string) (*Package) {
	//TODO : Check for user-defined feature support

	userPackage := NewPackage(*loadPath, PackageOptions{"stdout" : false,"output_tpkg" : false,"use_tpkg" : false})

	userPackage.FallbackPath = *fallbackPath

	userPackages, _ := filepath.Glob(filepath.Join(userPackage.LoadPath, "*"))

	for _, path := range(userPackages) {
		components := strings.Split(path, "/")
		name := components[len(components)-1]
		userPackage.Load(name)
	}

	return userPackage
}

func newLog() (log4go.Logger) {
	pkgLog := make(log4go.Logger)
	os.Mkdir("tmp", uint32(0777) )

	pkgLog.AddFilter("file", log4go.FINE, log4go.NewFileLogWriter("tmp/packager.log", false))
	return pkgLog
}

func (pkg *Package)Load(packageName string) {

	user := api.FetchSessionUser()
	approved := user.RequestFeature("package:" + packageName)

	if !approved {
		panic("Package " + packageName + " not approved for use.")
	}

	err := pkg.loadFromPath(pkg.LoadPath, packageName)

	if err != nil && len(pkg.FallbackPath) != 0 {
		err = pkg.loadFromPath(pkg.FallbackPath, packageName)
	}

	if err != nil {
		panic(*err)
	}

}

func (pkg *Package)loadFromPath(path string, name string) (err *string) {
	pkg.Println(path + ":" + name)
	pkg.Log.Info("\n\n\n\nLoading:%v", path + ":" + name)

	if pkg.Options["use_tpkg"] {
		pkg.open(path, name)
		return nil
	}

	location := filepath.Join(path, name)

	old_location := pkg.location
	pkg.location = location
	info, err := readPackageInfoFile(location)

	if err != nil {
		return err
	}

	if len(info.Dependencies) > 0 {

		for _, dependency := range(info.Dependencies) {
			pkg.loadPackageDependency(dependency)
		}

	}

	for _, typeName := range(info.Types) {
		split := strings.Split(typeName, " < ")
		typeObj := &tp.Type{}
		if len(split) == 2 {
			typeName = split[0]
			index := pkg.findTypeIndex(split[1])

			typeObj.Implements = proto.Int32(int32(index))
		}
		typeObj.Name = proto.String(typeName)
		pkg.Types = append(pkg.Types, typeObj)
	}

	pkg.readHeaderFile(location)

	pkg.readPackageDefinitions(location)

	pkg.inheritFunctions()

	if pkg.Options["output_tpkg"] {
		pkg.write()
	}

	pkg.Println(" -- done")
	pkg.Log.Close()

	// TODO(SJ) : Kind of lame. Ideally I think we load other packages as whole packages and write a *.Merge method
	pkg.location = old_location

	return nil
}

<commit_msg>Allow override package source<commit_after>package packager

import(
	tp "tritium/proto"
	proto "goprotobuf.googlecode.com/hg/proto"
	api "tritium/api"
	"strings"
	"path/filepath"
	"log4go"
	"os"
)

type Package struct {
	loaded []*PackageInfo
	location string
	LoadPath string
	FallbackPath string
	OutputFile string
	Log log4go.Logger
	*tp.Package
	Options PackageOptions
}

type PackageInfo struct {
	Name string
	Dependencies []string
	Types []string
}

var DefaultPackagePath = "packages"

func LoadDefaultPackage(path *string) (*Package) {
	if path == nil {
		if os.ShellExpand("$TRITIUM_PACKAGES") != "" {
			path = &os.ShellExpand("$TRITIUM_PACKAGES")
		} else {
			path = &DefaultPackagePath
		}
	}

	return buildPackage(*path, nil)
}

func OutputDefaultPackage(path string) (pkg *Package, newFilePath string) {
	pkg = BuildDefaultPackage()

	_, err := os.Stat(path)

	if err != nil {
		creationErr := os.MkdirAll(path, uint32(0777) )
		if creationErr != nil {
			panic("Could not make path(" + path + "). Error:" + creationErr.String())
		}
	}

	_, name := filepath.Split(pkg.OutputFile)
	newOutputFile := filepath.Join(path, name)

	os.Rename(pkg.OutputFile, newOutputFile)

	return pkg, newOutputFile
}

func BuildDefaultPackage() (*Package) {
	options := BuildOptions()
	return buildPackage(DefaultPackagePath, options)
}

func buildPackage(path string, options PackageOptions) (*Package) {
	// Terrible directory handling here... has to be executed from Tritium root

	pkg := NewPackage(path, options)
	pkg.Load("libxml")

	return pkg
}

func mergeOptions(options PackageOptions) PackageOptions {
	defaults := fetchDefaultOptions()

	if options == nil {
		return defaults
	}

	for k, _ := range defaults {
		_, ok := options[k]

		if !ok {
			options[k] = defaults[k]
		}
	}

	return options
}

func NewPackage(loadPath string, options PackageOptions) (*Package){
	options = mergeOptions(options)

	return &Package{
		Package: &tp.Package{
			Name: proto.String("combined"),
			Functions: make([]*tp.Function, 0),
			Types: make([]*tp.Type, 0),
		},
		loaded: make([]*PackageInfo, 0),
  	        Log: newLog(),
		LoadPath: loadPath,
	        Options: options,
	}
}

func (pkg *Package) BuildUserPackage(loadPath *string, fallbackPath *string) {
	userPackage := NewUserPackage(loadPath, fallbackPath)
	pkg.Merge(userPackage.Package)
}

func NewUserPackage(loadPath *string, fallbackPath *string) (*Package) {
	//TODO : Check for user-defined feature support

	userPackage := NewPackage(*loadPath, PackageOptions{"stdout" : false,"output_tpkg" : false,"use_tpkg" : false})

	userPackage.FallbackPath = *fallbackPath

	userPackages, _ := filepath.Glob(filepath.Join(userPackage.LoadPath, "*"))

	for _, path := range(userPackages) {
		components := strings.Split(path, "/")
		name := components[len(components)-1]
		userPackage.Load(name)
	}

	return userPackage
}

func newLog() (log4go.Logger) {
	pkgLog := make(log4go.Logger)
	os.Mkdir("tmp", uint32(0777) )

	pkgLog.AddFilter("file", log4go.FINE, log4go.NewFileLogWriter("tmp/packager.log", false))
	return pkgLog
}

func (pkg *Package)Load(packageName string) {

	user := api.FetchSessionUser()
	approved := user.RequestFeature("package:" + packageName)

	if !approved {
		panic("Package " + packageName + " not approved for use.")
	}

	err := pkg.loadFromPath(pkg.LoadPath, packageName)

	if err != nil && len(pkg.FallbackPath) != 0 {
		err = pkg.loadFromPath(pkg.FallbackPath, packageName)
	}

	if err != nil {
		panic(*err)
	}

}

func (pkg *Package)loadFromPath(path string, name string) (err *string) {
	pkg.Println(path + ":" + name)
	pkg.Log.Info("\n\n\n\nLoading:%v", path + ":" + name)

	if pkg.Options["use_tpkg"] {
		pkg.open(path, name)
		return nil
	}

	location := filepath.Join(path, name)

	old_location := pkg.location
	pkg.location = location
	info, err := readPackageInfoFile(location)

	if err != nil {
		return err
	}

	if len(info.Dependencies) > 0 {

		for _, dependency := range(info.Dependencies) {
			pkg.loadPackageDependency(dependency)
		}

	}

	for _, typeName := range(info.Types) {
		split := strings.Split(typeName, " < ")
		typeObj := &tp.Type{}
		if len(split) == 2 {
			typeName = split[0]
			index := pkg.findTypeIndex(split[1])

			typeObj.Implements = proto.Int32(int32(index))
		}
		typeObj.Name = proto.String(typeName)
		pkg.Types = append(pkg.Types, typeObj)
	}

	pkg.readHeaderFile(location)

	pkg.readPackageDefinitions(location)

	pkg.inheritFunctions()

	if pkg.Options["output_tpkg"] {
		pkg.write()
	}

	pkg.Println(" -- done")
	pkg.Log.Close()

	// TODO(SJ) : Kind of lame. Ideally I think we load other packages as whole packages and write a *.Merge method
	pkg.location = old_location

	return nil
}

<|endoftext|>


<commit_before>/*
 * Copyright 2012 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package main

import (
	"testing"
	"io"
	"fmt"
)

func TestReadWrite(t *testing.T) {
	lb := NewListBuffer()
	for i := 0; i < 10; i++ {
		buf := []byte{1,2,3,4,5}
		lb.Write(buf)
	}
	buf := make([]byte, 3)
	_, err := lb.Read(buf)
	for err != io.EOF {
		fmt.Println(buf)
		_, err = lb.Read(buf)
	}
}

<commit_msg>More test cases<commit_after>/*
 * Copyright 2012 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package main

import (
	"testing"
	"io"
	"fmt"
)

func testReadWrite(size int, writeStep int, readStep int) error {
	lb := NewListBuffer()
	buf := make([]byte, size)
	var i int
	for i = 0; i < len(buf); i++ {
		buf[i] = byte(i)
	}
	for i = 0; i + writeStep < size; i += writeStep {
		lb.Write(buf[i:i+writeStep])
	}
	if i < size {
		lb.Write(buf[i:])
	}

	nbuf := make([]byte, size)

	for i = 0; i + readStep < size; i += readStep {
		_, err := lb.Read(nbuf[i:i+readStep])
		if size <= 100 {
			fmt.Println(nbuf[i:i+readStep])
		}
		if err != nil {
			return fmt.Errorf("Error: %v", err)
		}
	}
	if i < size {
		_, err := lb.Read(nbuf[i:])
		if size <= 100 {
			fmt.Println(nbuf[i:])
		}
		if err != nil {
			return fmt.Errorf("Error: %v", err)
		}
	}

	_, err := lb.Read(nbuf)
	if err != io.EOF {
		return fmt.Errorf("Should be EOF")
	}
	for i = 0; i < size; i++ {
		if nbuf[i] != buf[i] {
			return fmt.Errorf("@ %v: nbuf[i] = %v; buf[i] = %v",
				i, nbuf[i], buf[i])
		}
	}
	return nil
}

func TestReadWrite(t *testing.T) {
	testCases := [][]int {
		{100, 10, 20},
		{99, 10, 20},
		{100, 20, 10},
		{1024, 100, 10},
		{1024, 10, 10},
	}
	for _, c := range testCases {
		fmt.Printf("Test on %v\n", c)
		err := testReadWrite(c[0], c[1], c[2])
		if err != nil {
			t.Errorf("Error on %v: %v", c, err)
		}
	}
}

<|endoftext|>


<commit_before>package auth

import (
	"math"
	"time"

	"github.com/EndFirstCorp/onedb/redis"
	"github.com/pkg/errors"
)

type backendRedisSession struct {
	db     redis.Rediser
	prefix string
}

// NewBackendRedisSession returns a SessionBackender for Redis
func NewBackendRedisSession(server string, port int, password string, maxIdle, maxConnections int, keyPrefix string) sessionBackender {
	r := redis.New(server, port, password, maxIdle, maxConnections)
	return &backendRedisSession{db: r, prefix: keyPrefix}
}

// need to first check that this emailVerifyHash isn't being used, otherwise we'll clobber existing
func (r *backendRedisSession) CreateEmailSession(userID, email string, info map[string]interface{}, emailVerifyHash, csrfToken string) error {
	return r.saveEmailSession(&emailSession{userID, email, info, emailVerifyHash, csrfToken})
}

func (r *backendRedisSession) GetEmailSession(emailVerifyHash string) (*emailSession, error) {
	session := &emailSession{}
	return session, r.db.GetStruct(r.getEmailSessionKey(emailVerifyHash), session)
}

func (r *backendRedisSession) UpdateEmailSession(emailVerifyHash, userID string) error {
	s, err := r.GetEmailSession(emailVerifyHash)
	if err != nil {
		return err
	}
	s.UserID = userID
	return r.saveEmailSession(s)
}

func (r *backendRedisSession) DeleteEmailSession(emailVerifyHash string) error {
	return nil
}

func (r *backendRedisSession) CreateSession(userID, email string, info map[string]interface{}, sessionHash, csrfToken string, renewTimeUTC, expireTimeUTC time.Time) (*LoginSession, error) {
	session := LoginSession{userID, email, info, sessionHash, csrfToken, renewTimeUTC, expireTimeUTC}
	return &session, r.saveSession(&session)
}

func (r *backendRedisSession) CreateRememberMe(userID, email, selector, tokenHash string, renewTimeUTC, expireTimeUTC time.Time) (*rememberMeSession, error) {
	rememberMe := rememberMeSession{userID, email, selector, tokenHash, renewTimeUTC, expireTimeUTC}
	return &rememberMe, r.saveRememberMe(&rememberMe)
}

func (r *backendRedisSession) GetSession(sessionHash string) (*LoginSession, error) {
	session := &LoginSession{}
	return session, r.db.GetStruct(r.getSessionKey(sessionHash), session)
}

func (r *backendRedisSession) UpdateSession(sessionHash string, renewTimeUTC, expireTimeUTC time.Time) error {
	session, err := r.GetSession(sessionHash)
	if err != nil {
		return err
	}
	session.ExpireTimeUTC = expireTimeUTC
	session.RenewTimeUTC = renewTimeUTC
	return r.saveSession(session)
}

func (r *backendRedisSession) DeleteSession(sessionHash string) error {
	return r.db.Del(r.getSessionKey(sessionHash))
}

func (r *backendRedisSession) DeleteSessions(email string) error {
	return nil
}

func (r *backendRedisSession) InvalidateSessions(email string) error {
	return nil
}

func (r *backendRedisSession) GetRememberMe(selector string) (*rememberMeSession, error) {
	rememberMe := &rememberMeSession{}
	return rememberMe, r.db.GetStruct(r.getRememberMeKey(selector), rememberMe)
}

func (r *backendRedisSession) UpdateRememberMe(selector string, renewTimeUTC time.Time) error {
	rememberMe, err := r.GetRememberMe(selector)
	if err != nil {
		return err
	}
	rememberMe.RenewTimeUTC = renewTimeUTC
	return r.saveRememberMe(rememberMe)
}

func (r *backendRedisSession) DeleteRememberMe(selector string) error {
	return r.db.Del(r.getRememberMeKey(selector))
}

func (r *backendRedisSession) DeleteRememberMes(email string) error {
	return nil
}

func (r *backendRedisSession) Close() error {
	return r.db.Close()
}

func (r *backendRedisSession) saveEmailSession(session *emailSession) error {
	return r.save(r.getEmailSessionKey(session.EmailVerifyHash), session, emailExpireMins*60)
}

func (r *backendRedisSession) saveSession(session *LoginSession) error {
	if time.Since(session.ExpireTimeUTC).Seconds() >= 0 {
		return errors.New("Unable to save expired session")
	}
	return r.save(r.getSessionKey(session.SessionHash), session, round(rememberMeExpireDuration.Seconds()))
}

func (r *backendRedisSession) saveRememberMe(rememberMe *rememberMeSession) error {
	if time.Since(rememberMe.ExpireTimeUTC).Seconds() >= 0 {
		return errors.New("Unable to save expired rememberMe")
	}
	return r.save(r.getRememberMeKey(rememberMe.Selector), rememberMe, round(rememberMeExpireDuration.Seconds()))
}

func (r *backendRedisSession) getEmailSessionKey(emailVerifyHash string) string {
	return r.prefix + "/email/" + emailVerifyHash
}

func (r *backendRedisSession) getSessionKey(sessionHash string) string {
	return r.prefix + "/session/" + sessionHash
}

func (r *backendRedisSession) getRememberMeKey(selector string) string {
	return r.prefix + "/rememberMe/" + selector
}

func round(num float64) int {
	return int(math.Floor(0.5 + num))
}

func (r *backendRedisSession) save(key string, value interface{}, expireSeconds int) error {
	return r.db.SetWithExpire(key, value, expireSeconds)
}
<commit_msg>change interface return<commit_after>package auth

import (
	"math"
	"time"

	"github.com/EndFirstCorp/onedb/redis"
	"github.com/pkg/errors"
)

type backendRedisSession struct {
	db     redis.Rediser
	prefix string
}

// NewBackendRedisSession returns a SessionBackender for Redis
func NewBackendRedisSession(server string, port int, password string, maxIdle, maxConnections int, keyPrefix string) SessionBackender {
	r := redis.New(server, port, password, maxIdle, maxConnections)
	return &backendRedisSession{db: r, prefix: keyPrefix}
}

// need to first check that this emailVerifyHash isn't being used, otherwise we'll clobber existing
func (r *backendRedisSession) CreateEmailSession(userID, email string, info map[string]interface{}, emailVerifyHash, csrfToken string) error {
	return r.saveEmailSession(&emailSession{userID, email, info, emailVerifyHash, csrfToken})
}

func (r *backendRedisSession) GetEmailSession(emailVerifyHash string) (*emailSession, error) {
	session := &emailSession{}
	return session, r.db.GetStruct(r.getEmailSessionKey(emailVerifyHash), session)
}

func (r *backendRedisSession) UpdateEmailSession(emailVerifyHash, userID string) error {
	s, err := r.GetEmailSession(emailVerifyHash)
	if err != nil {
		return err
	}
	s.UserID = userID
	return r.saveEmailSession(s)
}

func (r *backendRedisSession) DeleteEmailSession(emailVerifyHash string) error {
	return nil
}

func (r *backendRedisSession) CreateSession(userID, email string, info map[string]interface{}, sessionHash, csrfToken string, renewTimeUTC, expireTimeUTC time.Time) (*LoginSession, error) {
	session := LoginSession{userID, email, info, sessionHash, csrfToken, renewTimeUTC, expireTimeUTC}
	return &session, r.saveSession(&session)
}

func (r *backendRedisSession) CreateRememberMe(userID, email, selector, tokenHash string, renewTimeUTC, expireTimeUTC time.Time) (*rememberMeSession, error) {
	rememberMe := rememberMeSession{userID, email, selector, tokenHash, renewTimeUTC, expireTimeUTC}
	return &rememberMe, r.saveRememberMe(&rememberMe)
}

func (r *backendRedisSession) GetSession(sessionHash string) (*LoginSession, error) {
	session := &LoginSession{}
	return session, r.db.GetStruct(r.getSessionKey(sessionHash), session)
}

func (r *backendRedisSession) UpdateSession(sessionHash string, renewTimeUTC, expireTimeUTC time.Time) error {
	session, err := r.GetSession(sessionHash)
	if err != nil {
		return err
	}
	session.ExpireTimeUTC = expireTimeUTC
	session.RenewTimeUTC = renewTimeUTC
	return r.saveSession(session)
}

func (r *backendRedisSession) DeleteSession(sessionHash string) error {
	return r.db.Del(r.getSessionKey(sessionHash))
}

func (r *backendRedisSession) DeleteSessions(email string) error {
	return nil
}

func (r *backendRedisSession) InvalidateSessions(email string) error {
	return nil
}

func (r *backendRedisSession) GetRememberMe(selector string) (*rememberMeSession, error) {
	rememberMe := &rememberMeSession{}
	return rememberMe, r.db.GetStruct(r.getRememberMeKey(selector), rememberMe)
}

func (r *backendRedisSession) UpdateRememberMe(selector string, renewTimeUTC time.Time) error {
	rememberMe, err := r.GetRememberMe(selector)
	if err != nil {
		return err
	}
	rememberMe.RenewTimeUTC = renewTimeUTC
	return r.saveRememberMe(rememberMe)
}

func (r *backendRedisSession) DeleteRememberMe(selector string) error {
	return r.db.Del(r.getRememberMeKey(selector))
}

func (r *backendRedisSession) DeleteRememberMes(email string) error {
	return nil
}

func (r *backendRedisSession) Close() error {
	return r.db.Close()
}

func (r *backendRedisSession) saveEmailSession(session *emailSession) error {
	return r.save(r.getEmailSessionKey(session.EmailVerifyHash), session, emailExpireMins*60)
}

func (r *backendRedisSession) saveSession(session *LoginSession) error {
	if time.Since(session.ExpireTimeUTC).Seconds() >= 0 {
		return errors.New("Unable to save expired session")
	}
	return r.save(r.getSessionKey(session.SessionHash), session, round(rememberMeExpireDuration.Seconds()))
}

func (r *backendRedisSession) saveRememberMe(rememberMe *rememberMeSession) error {
	if time.Since(rememberMe.ExpireTimeUTC).Seconds() >= 0 {
		return errors.New("Unable to save expired rememberMe")
	}
	return r.save(r.getRememberMeKey(rememberMe.Selector), rememberMe, round(rememberMeExpireDuration.Seconds()))
}

func (r *backendRedisSession) getEmailSessionKey(emailVerifyHash string) string {
	return r.prefix + "/email/" + emailVerifyHash
}

func (r *backendRedisSession) getSessionKey(sessionHash string) string {
	return r.prefix + "/session/" + sessionHash
}

func (r *backendRedisSession) getRememberMeKey(selector string) string {
	return r.prefix + "/rememberMe/" + selector
}

func round(num float64) int {
	return int(math.Floor(0.5 + num))
}

func (r *backendRedisSession) save(key string, value interface{}, expireSeconds int) error {
	return r.db.SetWithExpire(key, value, expireSeconds)
}
<|endoftext|>


<commit_before>package band

import "time"

func newRU864Band(repeaterCompatible bool) (Band, error) {
	var maxPayloadSize []MaxPayloadSize

	if repeaterCompatible {
		maxPayloadSize = []MaxPayloadSize{
			{M: 59, N: 51},
			{M: 59, N: 51},
			{M: 59, N: 51},
			{M: 123, N: 115},
			{M: 230, N: 222},
			{M: 230, N: 222},
			{M: 230, N: 222},
			{M: 230, N: 222},
		}
	} else {
		maxPayloadSize = []MaxPayloadSize{
			{M: 59, N: 51},
			{M: 59, N: 51},
			{M: 59, N: 51},
			{M: 123, N: 115},
			{M: 250, N: 242},
			{M: 250, N: 242},
			{M: 250, N: 242},
			{M: 250, N: 242},
		}
	}

	return Band{
		DefaultTXPower:   14,
		ImplementsCFlist: true,
		RX2Frequency:     869525000,
		RX2DataRate:      0,

		MaxFCntGap:       16384,
		ADRACKLimit:      64,
		ADRACKDelay:      32,
		ReceiveDelay1:    time.Second,
		ReceiveDelay2:    time.Second * 2,
		JoinAcceptDelay1: time.Second * 5,
		JoinAcceptDelay2: time.Second * 6,
		ACKTimeoutMin:    time.Second,
		ACKTimeoutMax:    time.Second * 3,

		DataRates: []DataRate{
			{Modulation: LoRaModulation, SpreadFactor: 12, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 11, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 10, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 9, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 8, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 7, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 7, Bandwidth: 250},
			{Modulation: FSKModulation, BitRate: 50000},
		},

		MaxPayloadSize: maxPayloadSize,

		rx1DataRate: [][]int{
			{0, 0, 0, 0, 0, 0},
			{1, 0, 0, 0, 0, 0},
			{2, 1, 0, 0, 0, 0},
			{3, 2, 1, 0, 0, 0},
			{4, 3, 2, 1, 0, 0},
			{5, 4, 3, 2, 1, 0},
			{6, 5, 4, 3, 2, 1},
			{7, 6, 5, 4, 3, 2},
		},

		TXPower: []int{
			20,
			14,
			11,
			8,
			5,
			2,
		},

		UplinkChannels: []Channel{
			{Frequency: 864100000, DataRates: []int{0, 1, 2, 3, 4, 5}},
			{Frequency: 864300000, DataRates: []int{0, 1, 2, 3, 4, 5}},
			{Frequency: 864500000, DataRates: []int{0, 1, 2, 3, 4, 5}},
		},

		DownlinkChannels: []Channel{
			{Frequency: 864100000, DataRates: []int{0, 1, 2, 3, 4, 5}},
			{Frequency: 864300000, DataRates: []int{0, 1, 2, 3, 4, 5}},
			{Frequency: 864500000, DataRates: []int{0, 1, 2, 3, 4, 5}},
		},

		getRX1ChannelFunc: func(txChannel int) int {
			return txChannel
		},

		getRX1FrequencyFunc: func(b *Band, txFrequency int) (int, error) {
			return txFrequency, nil
		},
	}, nil
}
<commit_msg>Update RU864 band config.<commit_after>package band

import "time"

func newRU864Band(repeaterCompatible bool) (Band, error) {
	var maxPayloadSize []MaxPayloadSize

	if repeaterCompatible {
		maxPayloadSize = []MaxPayloadSize{
			{M: 59, N: 51},
			{M: 59, N: 51},
			{M: 59, N: 51},
			{M: 123, N: 115},
			{M: 230, N: 222},
			{M: 230, N: 222},
			{M: 230, N: 222},
			{M: 230, N: 222},
		}
	} else {
		maxPayloadSize = []MaxPayloadSize{
			{M: 59, N: 51},
			{M: 59, N: 51},
			{M: 59, N: 51},
			{M: 123, N: 115},
			{M: 250, N: 242},
			{M: 250, N: 242},
			{M: 250, N: 242},
			{M: 250, N: 242},
		}
	}

	return Band{
		DefaultTXPower:   14,
		ImplementsCFlist: true,
		RX2Frequency:     869050000,
		RX2DataRate:      0,

		MaxFCntGap:       16384,
		ADRACKLimit:      64,
		ADRACKDelay:      32,
		ReceiveDelay1:    time.Second,
		ReceiveDelay2:    time.Second * 2,
		JoinAcceptDelay1: time.Second * 5,
		JoinAcceptDelay2: time.Second * 6,
		ACKTimeoutMin:    time.Second,
		ACKTimeoutMax:    time.Second * 3,

		DataRates: []DataRate{
			{Modulation: LoRaModulation, SpreadFactor: 12, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 11, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 10, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 9, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 8, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 7, Bandwidth: 125},
			{Modulation: LoRaModulation, SpreadFactor: 7, Bandwidth: 250},
			{Modulation: FSKModulation, BitRate: 50000},
		},

		MaxPayloadSize: maxPayloadSize,

		rx1DataRate: [][]int{
			{0, 0, 0, 0, 0, 0},
			{1, 0, 0, 0, 0, 0},
			{2, 1, 0, 0, 0, 0},
			{3, 2, 1, 0, 0, 0},
			{4, 3, 2, 1, 0, 0},
			{5, 4, 3, 2, 1, 0},
			{6, 5, 4, 3, 2, 1},
			{7, 6, 5, 4, 3, 2},
		},

		TXPower: []int{
			20,
			14,
			11,
			8,
			5,
			2,
		},

		UplinkChannels: []Channel{
			{Frequency: 864500000, DataRates: []int{0, 1, 2, 3, 4, 5}},
			{Frequency: 864700000, DataRates: []int{0, 1, 2, 3, 4, 5}},
			{Frequency: 864900000, DataRates: []int{0, 1, 2, 3, 4, 5}},
		},

		DownlinkChannels: []Channel{
			{Frequency: 864500000, DataRates: []int{0, 1, 2, 3, 4, 5}},
			{Frequency: 864700000, DataRates: []int{0, 1, 2, 3, 4, 5}},
			{Frequency: 864900000, DataRates: []int{0, 1, 2, 3, 4, 5}},
		},

		getRX1ChannelFunc: func(txChannel int) int {
			return txChannel
		},

		getRX1FrequencyFunc: func(b *Band, txFrequency int) (int, error) {
			return txFrequency, nil
		},
	}, nil
}
<|endoftext|>


<commit_before>package tsdb

import (
	"math/rand"
	"os"
	"path/filepath"
	"time"

	"github.com/coreos/etcd/pkg/fileutil"
	"github.com/fabxc/tsdb/labels"
	"github.com/oklog/ulid"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
)

// Compactor provides compaction against an underlying storage
// of time series data.
type Compactor interface {
	// Plan returns a set of non-overlapping directories that can
	// be compacted concurrently.
	// Results returned when compactions are in progress are undefined.
	Plan(dir string) ([][]string, error)

	// Write persists a Block into a directory.
	Write(dir string, b Block) error

	// Compact runs compaction against the provided directories. Must
	// only be called concurrently with results of Plan().
	Compact(dirs ...string) error
}

// compactor implements the Compactor interface.
type compactor struct {
	metrics *compactorMetrics
	opts    *compactorOptions
}

type compactorMetrics struct {
	ran      prometheus.Counter
	failed   prometheus.Counter
	duration prometheus.Histogram
}

func newCompactorMetrics(r prometheus.Registerer) *compactorMetrics {
	m := &compactorMetrics{}

	m.ran = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsdb_compactions_total",
		Help: "Total number of compactions that were executed for the partition.",
	})
	m.failed = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsdb_compactions_failed_total",
		Help: "Total number of compactions that failed for the partition.",
	})
	m.duration = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name: "tsdb_compaction_duration",
		Help: "Duration of compaction runs.",
	})

	if r != nil {
		r.MustRegister(
			m.ran,
			m.failed,
			m.duration,
		)
	}
	return m
}

type compactorOptions struct {
	maxBlockRange uint64
}

func newCompactor(r prometheus.Registerer, opts *compactorOptions) *compactor {
	return &compactor{
		opts:    opts,
		metrics: newCompactorMetrics(r),
	}
}

type compactionInfo struct {
	seq        int
	generation int
	mint, maxt int64
}

const compactionBlocksLen = 3

func (c *compactor) Plan(dir string) ([][]string, error) {
	dirs, err := blockDirs(dir)
	if err != nil {
		return nil, err
	}

	var bs []*BlockMeta

	for _, dir := range dirs {
		meta, err := readMetaFile(dir)
		if err != nil {
			return nil, err
		}
		if meta.Compaction.Generation > 0 {
			bs = append(bs, meta)
		}
	}

	if len(bs) == 0 {
		return nil, nil
	}

	sliceDirs := func(i, j int) [][]string {
		var res []string
		for k := i; k < j; k++ {
			res = append(res, dirs[k])
		}
		return [][]string{res}
	}

	// Then we care about compacting multiple blocks, starting with the oldest.
	for i := 0; i < len(bs)-compactionBlocksLen+1; i += compactionBlocksLen {
		if c.match(bs[i : i+3]) {
			return sliceDirs(i, i+compactionBlocksLen), nil
		}
	}

	return nil, nil
}

func (c *compactor) match(bs []*BlockMeta) bool {
	g := bs[0].Compaction.Generation

	for _, b := range bs {
		if b.Compaction.Generation != g {
			return false
		}
	}
	return uint64(bs[len(bs)-1].MaxTime-bs[0].MinTime) <= c.opts.maxBlockRange
}

var entropy = rand.New(rand.NewSource(time.Now().UnixNano()))

func mergeBlockMetas(blocks ...Block) (res BlockMeta) {
	m0 := blocks[0].Meta()

	res.Sequence = m0.Sequence
	res.MinTime = m0.MinTime
	res.MaxTime = blocks[len(blocks)-1].Meta().MaxTime
	res.ULID = ulid.MustNew(ulid.Now(), entropy)

	res.Compaction.Generation = m0.Compaction.Generation + 1

	for _, b := range blocks {
		res.Stats.NumSamples += b.Meta().Stats.NumSamples
	}
	return res
}

func (c *compactor) Compact(dirs ...string) (err error) {
	var blocks []Block

	for _, d := range dirs {
		b, err := newPersistedBlock(d)
		if err != nil {
			return err
		}
		blocks = append(blocks, b)
	}

	return c.write(dirs[0], blocks...)
}

func (c *compactor) Write(dir string, b Block) error {
	return c.write(dir, b)
}

// write creates a new block that is the union of the provided blocks into dir.
// It cleans up all files of the old blocks after completing successfully.
func (c *compactor) write(dir string, blocks ...Block) (err error) {
	defer func(t time.Time) {
		if err != nil {
			c.metrics.failed.Inc()
		}
		c.metrics.duration.Observe(time.Since(t).Seconds())
	}(time.Now())

	tmp := dir + ".tmp"

	if err = os.RemoveAll(tmp); err != nil {
		return err
	}

	if err = os.MkdirAll(tmp, 0777); err != nil {
		return err
	}

	// Populate chunk and index files into temporary directory with
	// data of all blocks.
	chunkw, err := newChunkWriter(chunkDir(tmp))
	if err != nil {
		return errors.Wrap(err, "open chunk writer")
	}
	indexw, err := newIndexWriter(tmp)
	if err != nil {
		return errors.Wrap(err, "open index writer")
	}

	meta, err := c.populate(blocks, indexw, chunkw)
	if err != nil {
		return errors.Wrap(err, "write compaction")
	}
	if err = writeMetaFile(tmp, meta); err != nil {
		return errors.Wrap(err, "write merged meta")
	}

	if err = chunkw.Close(); err != nil {
		return errors.Wrap(err, "close chunk writer")
	}
	if err = indexw.Close(); err != nil {
		return errors.Wrap(err, "close index writer")
	}

	// Block successfully written, make visible and remove old ones.
	if err := renameFile(tmp, dir); err != nil {
		return errors.Wrap(err, "rename block dir")
	}
	for _, b := range blocks[1:] {
		if err := os.RemoveAll(b.Dir()); err != nil {
			return err
		}
	}
	// Properly sync parent dir to ensure changes are visible.
	df, err := fileutil.OpenDir(dir)
	if err != nil {
		return errors.Wrap(err, "sync block dir")
	}
	if err := fileutil.Fsync(df); err != nil {
		return errors.Wrap(err, "sync block dir")
	}

	return nil
}

// populate fills the index and chunk writers with new data gathered as the union
// of the provided blocks. It returns meta information for the new block.
func (c *compactor) populate(blocks []Block, indexw IndexWriter, chunkw ChunkWriter) (*BlockMeta, error) {
	var set compactionSet

	for i, b := range blocks {
		all, err := b.Index().Postings("", "")
		if err != nil {
			return nil, err
		}
		// TODO(fabxc): find more transparent way of handling this.
		if hb, ok := b.(*headBlock); ok {
			all = hb.remapPostings(all)
		}
		s := newCompactionSeriesSet(b.Index(), b.Chunks(), all)

		if i == 0 {
			set = s
			continue
		}
		set, err = newCompactionMerger(set, s)
		if err != nil {
			return nil, err
		}
	}

	// We fully rebuild the postings list index from merged series.
	var (
		postings = &memPostings{m: make(map[term][]uint32, 512)}
		values   = map[string]stringset{}
		i        = uint32(0)
		meta     = mergeBlockMetas(blocks...)
	)

	for set.Next() {
		lset, chunks := set.At()
		if err := chunkw.WriteChunks(chunks...); err != nil {
			return nil, err
		}

		indexw.AddSeries(i, lset, chunks...)

		meta.Stats.NumChunks += uint64(len(chunks))
		meta.Stats.NumSeries++

		for _, l := range lset {
			valset, ok := values[l.Name]
			if !ok {
				valset = stringset{}
				values[l.Name] = valset
			}
			valset.set(l.Value)

			postings.add(i, term{name: l.Name, value: l.Value})
		}
		i++
	}
	if set.Err() != nil {
		return nil, set.Err()
	}

	s := make([]string, 0, 256)
	for n, v := range values {
		s = s[:0]

		for x := range v {
			s = append(s, x)
		}
		if err := indexw.WriteLabelIndex([]string{n}, s); err != nil {
			return nil, err
		}
	}

	for t := range postings.m {
		if err := indexw.WritePostings(t.name, t.value, postings.get(t)); err != nil {
			return nil, err
		}
	}
	// Write a postings list containing all series.
	all := make([]uint32, i)
	for i := range all {
		all[i] = uint32(i)
	}
	if err := indexw.WritePostings("", "", newListPostings(all)); err != nil {
		return nil, err
	}

	return &meta, nil
}

type compactionSet interface {
	Next() bool
	At() (labels.Labels, []ChunkMeta)
	Err() error
}

type compactionSeriesSet struct {
	p      Postings
	index  IndexReader
	chunks ChunkReader

	l   labels.Labels
	c   []ChunkMeta
	err error
}

func newCompactionSeriesSet(i IndexReader, c ChunkReader, p Postings) *compactionSeriesSet {
	return &compactionSeriesSet{
		index:  i,
		chunks: c,
		p:      p,
	}
}

func (c *compactionSeriesSet) Next() bool {
	if !c.p.Next() {
		return false
	}

	c.l, c.c, c.err = c.index.Series(c.p.At())
	if c.err != nil {
		return false
	}
	for i := range c.c {
		chk := &c.c[i]

		chk.Chunk, c.err = c.chunks.Chunk(chk.Ref)
		if c.err != nil {
			return false
		}
	}

	return true
}

func (c *compactionSeriesSet) Err() error {
	if c.err != nil {
		return c.err
	}
	return c.p.Err()
}

func (c *compactionSeriesSet) At() (labels.Labels, []ChunkMeta) {
	return c.l, c.c
}

type compactionMerger struct {
	a, b compactionSet

	aok, bok bool
	l        labels.Labels
	c        []ChunkMeta
}

type compactionSeries struct {
	labels labels.Labels
	chunks []ChunkMeta
}

func newCompactionMerger(a, b compactionSet) (*compactionMerger, error) {
	c := &compactionMerger{
		a: a,
		b: b,
	}
	// Initialize first elements of both sets as Next() needs
	// one element look-ahead.
	c.aok = c.a.Next()
	c.bok = c.b.Next()

	return c, c.Err()
}

func (c *compactionMerger) compare() int {
	if !c.aok {
		return 1
	}
	if !c.bok {
		return -1
	}
	a, _ := c.a.At()
	b, _ := c.b.At()
	return labels.Compare(a, b)
}

func (c *compactionMerger) Next() bool {
	if !c.aok && !c.bok || c.Err() != nil {
		return false
	}

	d := c.compare()
	// Both sets contain the current series. Chain them into a single one.
	if d > 0 {
		c.l, c.c = c.b.At()
		c.bok = c.b.Next()
	} else if d < 0 {
		c.l, c.c = c.a.At()
		c.aok = c.a.Next()
	} else {
		l, ca := c.a.At()
		_, cb := c.b.At()

		c.l = l
		c.c = append(ca, cb...)

		c.aok = c.a.Next()
		c.bok = c.b.Next()
	}
	return true
}

func (c *compactionMerger) Err() error {
	if c.a.Err() != nil {
		return c.a.Err()
	}
	return c.b.Err()
}

func (c *compactionMerger) At() (labels.Labels, []ChunkMeta) {
	return c.l, c.c
}

func renameFile(from, to string) error {
	if err := os.RemoveAll(to); err != nil {
		return err
	}
	if err := os.Rename(from, to); err != nil {
		return err
	}

	// Directory was renamed; sync parent dir to persist rename.
	pdir, err := fileutil.OpenDir(filepath.Dir(to))
	if err != nil {
		return err
	}
	if err = fileutil.Fsync(pdir); err != nil {
		return err
	}
	if err = pdir.Close(); err != nil {
		return err
	}
	return nil
}
<commit_msg>Fix compaction selection procedure<commit_after>package tsdb

import (
	"math/rand"
	"os"
	"path/filepath"
	"time"

	"github.com/coreos/etcd/pkg/fileutil"
	"github.com/fabxc/tsdb/labels"
	"github.com/oklog/ulid"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
)

// Compactor provides compaction against an underlying storage
// of time series data.
type Compactor interface {
	// Plan returns a set of non-overlapping directories that can
	// be compacted concurrently.
	// Results returned when compactions are in progress are undefined.
	Plan(dir string) ([][]string, error)

	// Write persists a Block into a directory.
	Write(dir string, b Block) error

	// Compact runs compaction against the provided directories. Must
	// only be called concurrently with results of Plan().
	Compact(dirs ...string) error
}

// compactor implements the Compactor interface.
type compactor struct {
	metrics *compactorMetrics
	opts    *compactorOptions
}

type compactorMetrics struct {
	ran      prometheus.Counter
	failed   prometheus.Counter
	duration prometheus.Histogram
}

func newCompactorMetrics(r prometheus.Registerer) *compactorMetrics {
	m := &compactorMetrics{}

	m.ran = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsdb_compactions_total",
		Help: "Total number of compactions that were executed for the partition.",
	})
	m.failed = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsdb_compactions_failed_total",
		Help: "Total number of compactions that failed for the partition.",
	})
	m.duration = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name: "tsdb_compaction_duration",
		Help: "Duration of compaction runs.",
	})

	if r != nil {
		r.MustRegister(
			m.ran,
			m.failed,
			m.duration,
		)
	}
	return m
}

type compactorOptions struct {
	maxBlockRange uint64
}

func newCompactor(r prometheus.Registerer, opts *compactorOptions) *compactor {
	return &compactor{
		opts:    opts,
		metrics: newCompactorMetrics(r),
	}
}

type compactionInfo struct {
	seq        int
	generation int
	mint, maxt int64
}

const compactionBlocksLen = 3

func (c *compactor) Plan(dir string) ([][]string, error) {
	dirs, err := blockDirs(dir)
	if err != nil {
		return nil, err
	}

	var bs []*BlockMeta

	for _, dir := range dirs {
		meta, err := readMetaFile(dir)
		if err != nil {
			return nil, err
		}
		if meta.Compaction.Generation > 0 {
			bs = append(bs, meta)
		}
	}

	if len(bs) == 0 {
		return nil, nil
	}

	sliceDirs := func(i, j int) [][]string {
		var res []string
		for k := i; k < j; k++ {
			res = append(res, dirs[k])
		}
		return [][]string{res}
	}

	// Then we care about compacting multiple blocks, starting with the oldest.
	for i := 0; i < len(bs)-compactionBlocksLen+1; i++ {
		if c.match(bs[i : i+3]) {
			return sliceDirs(i, i+compactionBlocksLen), nil
		}
	}

	return nil, nil
}

func (c *compactor) match(bs []*BlockMeta) bool {
	g := bs[0].Compaction.Generation

	for _, b := range bs {
		if b.Compaction.Generation != g {
			return false
		}
	}
	return uint64(bs[len(bs)-1].MaxTime-bs[0].MinTime) <= c.opts.maxBlockRange
}

var entropy = rand.New(rand.NewSource(time.Now().UnixNano()))

func mergeBlockMetas(blocks ...Block) (res BlockMeta) {
	m0 := blocks[0].Meta()

	res.Sequence = m0.Sequence
	res.MinTime = m0.MinTime
	res.MaxTime = blocks[len(blocks)-1].Meta().MaxTime
	res.ULID = ulid.MustNew(ulid.Now(), entropy)

	res.Compaction.Generation = m0.Compaction.Generation + 1

	for _, b := range blocks {
		res.Stats.NumSamples += b.Meta().Stats.NumSamples
	}
	return res
}

func (c *compactor) Compact(dirs ...string) (err error) {
	var blocks []Block

	for _, d := range dirs {
		b, err := newPersistedBlock(d)
		if err != nil {
			return err
		}
		blocks = append(blocks, b)
	}

	return c.write(dirs[0], blocks...)
}

func (c *compactor) Write(dir string, b Block) error {
	return c.write(dir, b)
}

// write creates a new block that is the union of the provided blocks into dir.
// It cleans up all files of the old blocks after completing successfully.
func (c *compactor) write(dir string, blocks ...Block) (err error) {
	defer func(t time.Time) {
		if err != nil {
			c.metrics.failed.Inc()
		}
		c.metrics.duration.Observe(time.Since(t).Seconds())
	}(time.Now())

	tmp := dir + ".tmp"

	if err = os.RemoveAll(tmp); err != nil {
		return err
	}

	if err = os.MkdirAll(tmp, 0777); err != nil {
		return err
	}

	// Populate chunk and index files into temporary directory with
	// data of all blocks.
	chunkw, err := newChunkWriter(chunkDir(tmp))
	if err != nil {
		return errors.Wrap(err, "open chunk writer")
	}
	indexw, err := newIndexWriter(tmp)
	if err != nil {
		return errors.Wrap(err, "open index writer")
	}

	meta, err := c.populate(blocks, indexw, chunkw)
	if err != nil {
		return errors.Wrap(err, "write compaction")
	}
	if err = writeMetaFile(tmp, meta); err != nil {
		return errors.Wrap(err, "write merged meta")
	}

	if err = chunkw.Close(); err != nil {
		return errors.Wrap(err, "close chunk writer")
	}
	if err = indexw.Close(); err != nil {
		return errors.Wrap(err, "close index writer")
	}

	// Block successfully written, make visible and remove old ones.
	if err := renameFile(tmp, dir); err != nil {
		return errors.Wrap(err, "rename block dir")
	}
	for _, b := range blocks[1:] {
		if err := os.RemoveAll(b.Dir()); err != nil {
			return err
		}
	}
	// Properly sync parent dir to ensure changes are visible.
	df, err := fileutil.OpenDir(dir)
	if err != nil {
		return errors.Wrap(err, "sync block dir")
	}
	if err := fileutil.Fsync(df); err != nil {
		return errors.Wrap(err, "sync block dir")
	}

	return nil
}

// populate fills the index and chunk writers with new data gathered as the union
// of the provided blocks. It returns meta information for the new block.
func (c *compactor) populate(blocks []Block, indexw IndexWriter, chunkw ChunkWriter) (*BlockMeta, error) {
	var set compactionSet

	for i, b := range blocks {
		all, err := b.Index().Postings("", "")
		if err != nil {
			return nil, err
		}
		// TODO(fabxc): find more transparent way of handling this.
		if hb, ok := b.(*headBlock); ok {
			all = hb.remapPostings(all)
		}
		s := newCompactionSeriesSet(b.Index(), b.Chunks(), all)

		if i == 0 {
			set = s
			continue
		}
		set, err = newCompactionMerger(set, s)
		if err != nil {
			return nil, err
		}
	}

	// We fully rebuild the postings list index from merged series.
	var (
		postings = &memPostings{m: make(map[term][]uint32, 512)}
		values   = map[string]stringset{}
		i        = uint32(0)
		meta     = mergeBlockMetas(blocks...)
	)

	for set.Next() {
		lset, chunks := set.At()
		if err := chunkw.WriteChunks(chunks...); err != nil {
			return nil, err
		}

		indexw.AddSeries(i, lset, chunks...)

		meta.Stats.NumChunks += uint64(len(chunks))
		meta.Stats.NumSeries++

		for _, l := range lset {
			valset, ok := values[l.Name]
			if !ok {
				valset = stringset{}
				values[l.Name] = valset
			}
			valset.set(l.Value)

			postings.add(i, term{name: l.Name, value: l.Value})
		}
		i++
	}
	if set.Err() != nil {
		return nil, set.Err()
	}

	s := make([]string, 0, 256)
	for n, v := range values {
		s = s[:0]

		for x := range v {
			s = append(s, x)
		}
		if err := indexw.WriteLabelIndex([]string{n}, s); err != nil {
			return nil, err
		}
	}

	for t := range postings.m {
		if err := indexw.WritePostings(t.name, t.value, postings.get(t)); err != nil {
			return nil, err
		}
	}
	// Write a postings list containing all series.
	all := make([]uint32, i)
	for i := range all {
		all[i] = uint32(i)
	}
	if err := indexw.WritePostings("", "", newListPostings(all)); err != nil {
		return nil, err
	}

	return &meta, nil
}

type compactionSet interface {
	Next() bool
	At() (labels.Labels, []ChunkMeta)
	Err() error
}

type compactionSeriesSet struct {
	p      Postings
	index  IndexReader
	chunks ChunkReader

	l   labels.Labels
	c   []ChunkMeta
	err error
}

func newCompactionSeriesSet(i IndexReader, c ChunkReader, p Postings) *compactionSeriesSet {
	return &compactionSeriesSet{
		index:  i,
		chunks: c,
		p:      p,
	}
}

func (c *compactionSeriesSet) Next() bool {
	if !c.p.Next() {
		return false
	}

	c.l, c.c, c.err = c.index.Series(c.p.At())
	if c.err != nil {
		return false
	}
	for i := range c.c {
		chk := &c.c[i]

		chk.Chunk, c.err = c.chunks.Chunk(chk.Ref)
		if c.err != nil {
			return false
		}
	}

	return true
}

func (c *compactionSeriesSet) Err() error {
	if c.err != nil {
		return c.err
	}
	return c.p.Err()
}

func (c *compactionSeriesSet) At() (labels.Labels, []ChunkMeta) {
	return c.l, c.c
}

type compactionMerger struct {
	a, b compactionSet

	aok, bok bool
	l        labels.Labels
	c        []ChunkMeta
}

type compactionSeries struct {
	labels labels.Labels
	chunks []ChunkMeta
}

func newCompactionMerger(a, b compactionSet) (*compactionMerger, error) {
	c := &compactionMerger{
		a: a,
		b: b,
	}
	// Initialize first elements of both sets as Next() needs
	// one element look-ahead.
	c.aok = c.a.Next()
	c.bok = c.b.Next()

	return c, c.Err()
}

func (c *compactionMerger) compare() int {
	if !c.aok {
		return 1
	}
	if !c.bok {
		return -1
	}
	a, _ := c.a.At()
	b, _ := c.b.At()
	return labels.Compare(a, b)
}

func (c *compactionMerger) Next() bool {
	if !c.aok && !c.bok || c.Err() != nil {
		return false
	}

	d := c.compare()
	// Both sets contain the current series. Chain them into a single one.
	if d > 0 {
		c.l, c.c = c.b.At()
		c.bok = c.b.Next()
	} else if d < 0 {
		c.l, c.c = c.a.At()
		c.aok = c.a.Next()
	} else {
		l, ca := c.a.At()
		_, cb := c.b.At()

		c.l = l
		c.c = append(ca, cb...)

		c.aok = c.a.Next()
		c.bok = c.b.Next()
	}
	return true
}

func (c *compactionMerger) Err() error {
	if c.a.Err() != nil {
		return c.a.Err()
	}
	return c.b.Err()
}

func (c *compactionMerger) At() (labels.Labels, []ChunkMeta) {
	return c.l, c.c
}

func renameFile(from, to string) error {
	if err := os.RemoveAll(to); err != nil {
		return err
	}
	if err := os.Rename(from, to); err != nil {
		return err
	}

	// Directory was renamed; sync parent dir to persist rename.
	pdir, err := fileutil.OpenDir(filepath.Dir(to))
	if err != nil {
		return err
	}
	if err = fileutil.Fsync(pdir); err != nil {
		return err
	}
	if err = pdir.Close(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package dnsr

import (
	lru "github.com/hashicorp/golang-lru"
	"github.com/miekg/dns"
)

type Resolver struct {
	cache *lru.Cache
}

func (r *Resolver) Resolve(qname string, qtype dns.Type) *Worker {
	w := &Worker{
		r: r,
		qname: qname,
		qtype: qtype,
		done: make(chan struct{}),
		RRs: make(chan dns.RR, 20),
	}
	go w.Start()
	return w
}

func (r *Resolver) recall(qname, qtype) ([]dns.RR, bool) {
	// FIXME: implement
	return []dns.RR{}, false
}

func (r *Resolver) remember(rr ...dns.RR) {
	// FIXME: implement
}

func (r *Resolver) rememberNX(qname, qtype) {
	// FIXME: implement
}

type Worker struct {
	r     *Resolver
	qname string
	qtype dns.Type
	done  chan struct{}
	RRs   chan dns.RR
}

func (w *Worker) Start() {
	defer w.Stop()
	rrs, ok := w.r.recall(w.qname, w.qtype)
	if ok {
		inject(w.RRs, rrs)
		return
	}
	w.resolveViaAuthorities()
}

func (w *Worker) Stop() {
	close(w.done)
	close(w.RRs)
}

func (w *Worker) resolveViaAuthorities() {
	pname, ok = parent(w.qname)
	if !ok {
		return
	}
	for rr := range w.r.Resolve(pname, dns.TypeNS).RRs {
		ns, ok := rr.(*dns.NS)
		if !ok {
			continue
		}
		w.resolveViaAuthority(ns.NS)
	}
}

func (w *Worker) resolveViaAuthority(nsname) {
	for rr := range w.r.Resolve(nsname, dns.TypeA).RRs {
		a, ok := rr.(*dns.A)
		if !ok {
			continue
		}
		addr := a.A.String() + ":53"
		w.resolveViaServer(addr)
	}
}

func (w *Worker) resolveViaServer(addr) {
	msg := &dns.Msg{}
	msg.SetQuestion(q.Name, q.Qtype)
	msg.MsgHdr.RecursionDesired = false
	client := &dns.Client{}
	rmsg, _, err := client.Exchange(msg, addr)
	if err != nil {
		return
	}

	// FIXME: cache NXDOMAIN responses responsibly
	if rmsg.Rcode == dns.RcodeNameError {
		w.r.rememberNX(qname, qtype)
	}

	// Cache responses
	w.r.remember(rMsg.Answer...)
	w.r.remember(rMsg.Ns...)
	w.r.remember(rMsg.Extra...)

	// Check cache again
	rrs, ok := w.r.recall(q)
	if ok {
		inject(c, rrs)
		return
	}
}

func inject(c chan<- dns.RR, rrs []dns.RR) {
	for _, rr := range rrs {
		c <- rr
	}
}
<commit_msg>tighter code<commit_after>package dnsr

import (
	"strings"

	lru "github.com/hashicorp/golang-lru"
	"github.com/miekg/dns"
)

type Resolver struct {
	cache  *lru.Cache
	client *dns.Client
}

func New(size int) *Resolver {
	if size < 0 {
		size = 10000
	}
	cache, _ := lru.New(size)
	r := &Resolver{
		client: &dns.Client{},
		cache:  cache,
	}
	r.cacheRoot()
	return r
}

func (r *Resolver) Resolve(qname string, qtype dns.Type) <-chan dns.RR {
	c = make(chan dns.RR, 20)
	go func() {
		defer close(c)
		if rrs := r.recall(qname, qtype); rrs != nil {
			inject(c, rrs)
			return
		}
		pname, ok := parent(qname)
		if !ok {
			return
		}
		for nrr := range r.Resolve(pname, dns.TypeNS) {
			ns, ok := nrr.(*dns.NS)
			if !ok {
				continue
			}
			for arr := range r.Resolve(ns.NS, dns.TypeA) {
				a, ok := arr.(*dns.A)
				if !ok {
					continue
				}
				addr := a.A.String() + ":53"
				qmsg := &dns.Msg{}
				qmsg.SetQuestion(q.Name, q.Qtype)
				qmsg.MsgHdr.RecursionDesired = false
				rmsg, _, err := r.client.Exchange(qmsg, addr)
				if err != nil {
					continue // FIXME: handle errors better from flaky/failing NS servers
				}
				if rmsg.Rcode == dns.RcodeNameError {
					r.rememberFor(qname, qtype) // FIXME: cache NXDOMAIN responses responsibly
				}
				r.remember(rmsg.Answer...)
				r.remember(rmsg.Ns...)
				r.remember(rmsg.Extra...)
				if r.recall(qname, qtype) {
					return
				}
				break
			}
			break
		}

		for _, crr := range r.recall(qname, dns.TypeCNAME) {
			cn, ok := rr.(*dns.CNAME)
			if !ok {
				continue
			}
			for rr := range r.Resolve(cn.Target, qtype) {
				r.rememberFor(qname, qtype, rr)
				c <- rr
			}
		}
	}()
	return c
}

func (r *Resolver) cacheRoot() {
	for t := range dns.ParseZone(strings.NewReader(root), "", "") {
		if t.Error == nil {
			r.remember(t.RR)
		}
	}
}

func (r *Resolver) recall(qname string, qtype dns.Type) []dns.RR {
	// FIXME: implement
	return nil
}

func (r *Resolver) remember(rrs ...dns.RR) {
	// FIXME: implement
}

func (r *Resolver) rememberFor(qname string, qtype dns.Type, rrs ...dns.RR) {
	// FIXME: implement
}

func inject(c chan<- dns.RR, rrs []dns.RR) {
	for _, rr := range rrs {
		c <- rr
	}
}
<|endoftext|>


<commit_before>// Package loggers provides objects that gathers resource data from a host.
package loggers

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log/syslog"
	"net"
	"net/http"
	"os"
	"reflect"
	"regexp"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/hpcloud/tail"
	"github.com/sethgrid/pester"

	resourced_config "github.com/resourced/resourced/config"
	"github.com/resourced/resourced/host"
	"github.com/resourced/resourced/libmap"
	"github.com/resourced/resourced/logline"
)

var loggerConstructors = make(map[string]func() ILogger)

func init() {
	Register("Base", NewBase)
}

// Register makes any logger constructor available by name.
func Register(name string, constructor func() ILogger) {
	if constructor == nil {
		panic("logger: Register logger constructor is nil")
	}
	if _, dup := loggerConstructors[name]; dup {
		panic("logger: Register called twice for logger constructor " + name)
	}
	loggerConstructors[name] = constructor
}

// NewGoStruct instantiates ILogger
func NewGoStruct(name string) (ILogger, error) {
	constructor, ok := loggerConstructors[name]
	if !ok {
		return nil, errors.New("GoStruct is undefined.")
	}

	return constructor(), nil
}

// NewGoStructByConfig instantiates ILogger given Config struct
func NewGoStructByConfig(config resourced_config.Config) (ILogger, error) {
	lgr, err := NewGoStruct(config.GoStruct)
	if err != nil {
		return nil, err
	}

	lgr.SetSource(config.Source)
	lgr.SetBufferSize(config.BufferSize)
	lgr.SetTargets(config.Targets)

	// Populate ILogger fields dynamically
	if len(config.GoStructFields) > 0 {
		for structFieldInString, value := range config.GoStructFields {
			goStructField := reflect.ValueOf(lgr).Elem().FieldByName(structFieldInString)

			if goStructField.IsValid() && goStructField.CanSet() {
				valueOfValue := reflect.ValueOf(value)
				goStructField.Set(valueOfValue)
			}
		}
	}

	return lgr, err
}

// ILogger is generic interface for all loggers.
type ILogger interface {
	SetSource(string)
	GetSource() string

	SetBufferSize(int64)
	GetBufferSize() int64

	SetTargets([]resourced_config.LogTargetConfig)
	GetTargets() []resourced_config.LogTargetConfig

	SetLoglines(string, []string)
	GetLoglines(string) []string
	GetLoglinesLength(string) int
	GetAndResetLoglines(string) []string
	ResetLoglines(string)
	ProcessOutgoingLoglines([]string, []string, []string) []string

	LogErrorAndResetLoglinesIfNeeded(string, error, string)

	SendLogToMaster(string, string, string, *host.Host, []string, string) error

	SendLogToAgent(string, int, []string, string) error

	SendLogToSyslog(string, string, syslog.Priority, string, []string, string) error

	WriteToFile(string, []string) error
}

type ILoggerChannel interface {
	ILogger
	PubSubKey(string) string
	RunBlockingChannel(string, <-chan interface{})
}

type ILoggerFile interface {
	ILogger
	RunBlockingFile(string)
}

func NewBase() ILogger {
	b := &Base{}
	b.Data = libmap.NewTSafeMapStrings(nil)
	b.BufferSize = 1000000

	return b
}

type Base struct {
	Source     string
	BufferSize int64
	Targets    []resourced_config.LogTargetConfig

	Data *libmap.TSafeMapStrings
}

// PubSubKey is used for forwarding live log line from TCP listener to every target's channel.
func (b *Base) PubSubKey(targetEndpoint string) string {
	return b.GetSource() + "->" + targetEndpoint
}

// RunBlockingChannel pulls log line from channel continuously.
func (b *Base) RunBlockingChannel(name string, ch <-chan interface{}) {
	for line := range ch {
		b.Data.Append(name, line.(string))
	}
}

// RunBlockingFile tails the file continuously.
func (b *Base) RunBlockingFile(file string) {
	t, err := tail.TailFile(file, tail.Config{
		Follow:   true,
		Location: &tail.SeekInfo{Offset: 0, Whence: os.SEEK_END},
	})
	if err == nil {
		if !b.Data.Exists(file) {
			b.Data.Set(file, make([]string, 0))
		}

		for line := range t.Lines {
			b.Data.Append(file, line.Text)
		}
	}
}

// SetSource
func (b *Base) SetSource(source string) {
	b.Source = source
}

// GetSource returns the source field.
func (b *Base) GetSource() string {
	return b.Source
}

// SetBufferSize sets BufferSize
func (b *Base) SetBufferSize(bufferSize int64) {
	b.BufferSize = bufferSize
}

// GetBufferSize returns BufferSize
func (b *Base) GetBufferSize() int64 {
	return b.BufferSize
}

// SetTargets sets []LogTargetConfig
func (b *Base) SetTargets(targets []resourced_config.LogTargetConfig) {
	b.Targets = targets
}

// SetLoglines sets loglines.
func (b *Base) SetLoglines(source string, loglines []string) {
	b.Data.Set(source, loglines)
}

// GetLoglines returns loglines.
func (b *Base) GetLoglines(source string) []string {
	return b.Data.Get(source)
}

// GetLoglinesLength returns the count of loglines.
func (b *Base) GetLoglinesLength(source string) int {
	return len(b.Data.Get(source))
}

// GetAndResetLoglines returns loglines.
func (b *Base) GetAndResetLoglines(source string) []string {
	loglines := b.Data.Get(source)
	b.ResetLoglines(source)
	return loglines
}

// ResetLoglines wipes it clean.
func (b *Base) ResetLoglines(source string) {
	b.Data.Reset(source)
}

// GetTargets returns slice of LogTargetConfig.
func (b *Base) GetTargets() []resourced_config.LogTargetConfig {
	return b.Targets
}

func (b *Base) LogErrorAndResetLoglinesIfNeeded(source string, err error, message string) {
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error": err.Error(),
		}).Error(message)

		// Check if we have to prune in-memory log lines.
		if int64(b.GetLoglinesLength(source)) > b.GetBufferSize() {
			b.ResetLoglines(source)
		}
	}
}

// denyLoglines filters out every log line that matches denyList regex.
func (b *Base) denyLoglines(loglines []string, denyList []string) []string {
	newLoglines := make([]string, 0)

	for _, logline := range loglines {
		for _, deny := range denyList {
			if deny == "" {
				newLoglines = append(newLoglines, logline)
				continue
			}

			match, err := regexp.MatchString(deny, logline)
			if err != nil || !match {
				newLoglines = append(newLoglines, logline)
			}
		}
	}

	return newLoglines
}

// allowLoglines filters out every log line that matches allowList regex.
func (b *Base) allowLoglines(loglines []string, allowList []string) []string {
	newLoglines := make([]string, 0)

	for _, logline := range loglines {
		for _, allow := range allowList {
			match, err := regexp.MatchString(allow, logline)
			if err == nil && match {
				newLoglines = append(newLoglines, logline)
			}
		}
	}

	return newLoglines
}

// ProcessOutgoingLoglines before forwarding to targets.
func (b *Base) ProcessOutgoingLoglines(loglines []string, allowList []string, denyList []string) []string {
	if len(allowList) > 0 {
		return b.allowLoglines(loglines, allowList)
	}
	if len(denyList) > 0 {
		return b.denyLoglines(loglines, denyList)
	}

	return loglines
}

// logPayloadForMaster packages the log data before sending to master.
func (b *Base) logPayloadForMaster(hostData *host.Host, loglines []string, source string) AgentLogPayload {
	toSend := AgentLogPayload{}
	toSend.Host.Name = hostData.Name
	toSend.Host.Tags = hostData.Tags
	toSend.Data.Filename = source
	toSend.Data.Loglines = make([]AgentLoglinePayload, 0)

	for _, lg := range loglines {
		linePayload := AgentLoglinePayload{}

		// Check if loglines contain ResourceD base64 wire protocol.
		// If so, convert to plain text.
		if strings.HasPrefix(lg, "type:base64") {
			lg = logline.ParseSingle(lg).EncodePlain()
		}

		// Check if each logline is NOT encoded in ResourceD log wire protocol
		if !strings.HasPrefix(lg, "type:base64") && !strings.HasPrefix(lg, "type:plain") {
			linePayload.Created = time.Now().UTC().Unix()
			linePayload.Content = lg
		} else {
			liveLogline := logline.ParseSingle(lg)
			linePayload.Created = liveLogline.Created
			linePayload.Content = liveLogline.Content
		}

		if linePayload.Content != "" {
			toSend.Data.Loglines = append(toSend.Data.Loglines, linePayload)
		}
	}

	return toSend
}

// SendLogToMaster sends log lines to master.
func (b *Base) SendLogToMaster(accessToken, masterURLHost, masterURLPath string, hostData *host.Host, loglines []string, source string) error {
	// Don't do anything if there are no log lines to send.
	if len(loglines) == 0 {
		return nil
	}

	if masterURLPath == "" {
		masterURLPath = "/api/logs"
	}

	data := b.logPayloadForMaster(hostData, loglines, source)

	dataJson, err := json.Marshal(data)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error": err.Error(),
		}).Error("Failed to marshal JSON payload for sending logs to ResourceD Master")

		return err
	}

	url := masterURLHost + masterURLPath

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(dataJson))
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error": err.Error(),
		}).Error("Failed to create request struct for sending logs to ResourceD Master")

		return err
	}

	req.SetBasicAuth(accessToken, "")

	client := pester.New()
	client.MaxRetries = 3
	client.Backoff = pester.ExponentialJitterBackoff
	client.KeepLog = false

	resp, err := client.Do(req)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error":      err.Error(),
			"req.URL":    req.URL.String(),
			"req.Method": req.Method,
		}).Error("Failed to send logs data to ResourceD Master")

		return err
	}

	if resp != nil && resp.Body != nil {
		defer resp.Body.Close()
	}

	return err
}

// SendLogToAgent sends log lines to another agent.
func (b *Base) SendLogToAgent(anotherAgentAddr string, maxRetries int, loglines []string, source string) error {
	if len(loglines) == 0 {
		return nil
	}

	conn, err := net.Dial("tcp", anotherAgentAddr)
	attempts := 0

	for {
		if err == nil {
			break
		}

		if err != nil && attempts > maxRetries {
			return err
		}

		if err != nil {
			attempts = attempts + 1
			time.Sleep(pester.ExponentialJitterBackoff(attempts))
			conn, err = net.Dial("tcp", anotherAgentAddr)
			continue
		}
	}

	if conn != nil {
		defer conn.Close()

		w := bufio.NewWriter(conn)

		for i, lg := range loglines {
			// Check if each logline is NOT encoded in ResourceD log wire protocol
			if !strings.HasPrefix(lg, "type:base64") && !strings.HasPrefix(lg, "type:plain") {
				loglines[i] = logline.LiveLogline{Created: time.Now().UTC().Unix(), Content: lg}.EncodeBase64()
				lg = loglines[i]
			}

			fmt.Fprint(w, lg)
			w.Flush()
		}
	}

	return err
}

func (b *Base) SendLogToSyslog(protocol string, addr string, priority syslog.Priority, tag string, loglines []string, source string) error {
	logHandler, err := syslog.Dial(protocol, addr, priority, tag)
	if err != nil {
		return err
	}
	defer logHandler.Close()

	for _, lg := range loglines {
		logHandler.Write([]byte(lg))
	}

	return nil
}

// WriteToFile writes log lines to local file.
func (b *Base) WriteToFile(targetFile string, loglines []string) error {
	// Check if loglines contain ResourceD base64 wire protocol.
	// If so, convert to plain text.
	for i, lg := range loglines {
		if strings.HasPrefix(lg, "type:base64") || strings.HasPrefix(lg, "type:plain") {
			loglines[i] = logline.ParseSingle(lg).PlainContent()
		}
	}

	fileHandle, err := os.OpenFile(targetFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		return err
	}
	defer fileHandle.Close()

	for _, logline := range loglines {
		if strings.HasSuffix(logline, "\n") {
			fileHandle.WriteString(logline)
		} else {
			fileHandle.WriteString(logline + "\n")
		}
	}

	return nil
}
<commit_msg>parse internal format first before sending log.<commit_after>// Package loggers provides objects that gathers resource data from a host.
package loggers

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log/syslog"
	"net"
	"net/http"
	"os"
	"reflect"
	"regexp"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/hpcloud/tail"
	"github.com/sethgrid/pester"

	resourced_config "github.com/resourced/resourced/config"
	"github.com/resourced/resourced/host"
	"github.com/resourced/resourced/libmap"
	"github.com/resourced/resourced/logline"
)

var loggerConstructors = make(map[string]func() ILogger)

func init() {
	Register("Base", NewBase)
}

// Register makes any logger constructor available by name.
func Register(name string, constructor func() ILogger) {
	if constructor == nil {
		panic("logger: Register logger constructor is nil")
	}
	if _, dup := loggerConstructors[name]; dup {
		panic("logger: Register called twice for logger constructor " + name)
	}
	loggerConstructors[name] = constructor
}

// NewGoStruct instantiates ILogger
func NewGoStruct(name string) (ILogger, error) {
	constructor, ok := loggerConstructors[name]
	if !ok {
		return nil, errors.New("GoStruct is undefined.")
	}

	return constructor(), nil
}

// NewGoStructByConfig instantiates ILogger given Config struct
func NewGoStructByConfig(config resourced_config.Config) (ILogger, error) {
	lgr, err := NewGoStruct(config.GoStruct)
	if err != nil {
		return nil, err
	}

	lgr.SetSource(config.Source)
	lgr.SetBufferSize(config.BufferSize)
	lgr.SetTargets(config.Targets)

	// Populate ILogger fields dynamically
	if len(config.GoStructFields) > 0 {
		for structFieldInString, value := range config.GoStructFields {
			goStructField := reflect.ValueOf(lgr).Elem().FieldByName(structFieldInString)

			if goStructField.IsValid() && goStructField.CanSet() {
				valueOfValue := reflect.ValueOf(value)
				goStructField.Set(valueOfValue)
			}
		}
	}

	return lgr, err
}

// ILogger is generic interface for all loggers.
type ILogger interface {
	SetSource(string)
	GetSource() string

	SetBufferSize(int64)
	GetBufferSize() int64

	SetTargets([]resourced_config.LogTargetConfig)
	GetTargets() []resourced_config.LogTargetConfig

	SetLoglines(string, []string)
	GetLoglines(string) []string
	GetLoglinesLength(string) int
	GetAndResetLoglines(string) []string
	ResetLoglines(string)
	ProcessOutgoingLoglines([]string, []string, []string) []string

	LogErrorAndResetLoglinesIfNeeded(string, error, string)

	SendLogToMaster(string, string, string, *host.Host, []string, string) error

	SendLogToAgent(string, int, []string, string) error

	SendLogToSyslog(string, string, syslog.Priority, string, []string, string) error

	WriteToFile(string, []string) error
}

type ILoggerChannel interface {
	ILogger
	PubSubKey(string) string
	RunBlockingChannel(string, <-chan interface{})
}

type ILoggerFile interface {
	ILogger
	RunBlockingFile(string)
}

func NewBase() ILogger {
	b := &Base{}
	b.Data = libmap.NewTSafeMapStrings(nil)
	b.BufferSize = 1000000

	return b
}

type Base struct {
	Source     string
	BufferSize int64
	Targets    []resourced_config.LogTargetConfig

	Data *libmap.TSafeMapStrings
}

// PubSubKey is used for forwarding live log line from TCP listener to every target's channel.
func (b *Base) PubSubKey(targetEndpoint string) string {
	return b.GetSource() + "->" + targetEndpoint
}

// RunBlockingChannel pulls log line from channel continuously.
func (b *Base) RunBlockingChannel(name string, ch <-chan interface{}) {
	for line := range ch {
		b.Data.Append(name, line.(string))
	}
}

// RunBlockingFile tails the file continuously.
func (b *Base) RunBlockingFile(file string) {
	t, err := tail.TailFile(file, tail.Config{
		Follow:   true,
		Location: &tail.SeekInfo{Offset: 0, Whence: os.SEEK_END},
	})
	if err == nil {
		if !b.Data.Exists(file) {
			b.Data.Set(file, make([]string, 0))
		}

		for line := range t.Lines {
			b.Data.Append(file, line.Text)
		}
	}
}

// SetSource
func (b *Base) SetSource(source string) {
	b.Source = source
}

// GetSource returns the source field.
func (b *Base) GetSource() string {
	return b.Source
}

// SetBufferSize sets BufferSize
func (b *Base) SetBufferSize(bufferSize int64) {
	b.BufferSize = bufferSize
}

// GetBufferSize returns BufferSize
func (b *Base) GetBufferSize() int64 {
	return b.BufferSize
}

// SetTargets sets []LogTargetConfig
func (b *Base) SetTargets(targets []resourced_config.LogTargetConfig) {
	b.Targets = targets
}

// SetLoglines sets loglines.
func (b *Base) SetLoglines(source string, loglines []string) {
	b.Data.Set(source, loglines)
}

// GetLoglines returns loglines.
func (b *Base) GetLoglines(source string) []string {
	return b.Data.Get(source)
}

// GetLoglinesLength returns the count of loglines.
func (b *Base) GetLoglinesLength(source string) int {
	return len(b.Data.Get(source))
}

// GetAndResetLoglines returns loglines.
func (b *Base) GetAndResetLoglines(source string) []string {
	loglines := b.Data.Get(source)
	b.ResetLoglines(source)
	return loglines
}

// ResetLoglines wipes it clean.
func (b *Base) ResetLoglines(source string) {
	b.Data.Reset(source)
}

// GetTargets returns slice of LogTargetConfig.
func (b *Base) GetTargets() []resourced_config.LogTargetConfig {
	return b.Targets
}

func (b *Base) LogErrorAndResetLoglinesIfNeeded(source string, err error, message string) {
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error": err.Error(),
		}).Error(message)

		// Check if we have to prune in-memory log lines.
		if int64(b.GetLoglinesLength(source)) > b.GetBufferSize() {
			b.ResetLoglines(source)
		}
	}
}

// denyLoglines filters out every log line that matches denyList regex.
func (b *Base) denyLoglines(loglines []string, denyList []string) []string {
	newLoglines := make([]string, 0)

	for _, logline := range loglines {
		for _, deny := range denyList {
			if deny == "" {
				newLoglines = append(newLoglines, logline)
				continue
			}

			match, err := regexp.MatchString(deny, logline)
			if err != nil || !match {
				newLoglines = append(newLoglines, logline)
			}
		}
	}

	return newLoglines
}

// allowLoglines filters out every log line that matches allowList regex.
func (b *Base) allowLoglines(loglines []string, allowList []string) []string {
	newLoglines := make([]string, 0)

	for _, logline := range loglines {
		for _, allow := range allowList {
			match, err := regexp.MatchString(allow, logline)
			if err == nil && match {
				newLoglines = append(newLoglines, logline)
			}
		}
	}

	return newLoglines
}

// ProcessOutgoingLoglines before forwarding to targets.
func (b *Base) ProcessOutgoingLoglines(loglines []string, allowList []string, denyList []string) []string {
	if len(allowList) > 0 {
		return b.allowLoglines(loglines, allowList)
	}
	if len(denyList) > 0 {
		return b.denyLoglines(loglines, denyList)
	}

	return loglines
}

// logPayloadForMaster packages the log data before sending to master.
func (b *Base) logPayloadForMaster(hostData *host.Host, loglines []string, source string) AgentLogPayload {
	toSend := AgentLogPayload{}
	toSend.Host.Name = hostData.Name
	toSend.Host.Tags = hostData.Tags
	toSend.Data.Filename = source
	toSend.Data.Loglines = make([]AgentLoglinePayload, 0)

	for _, lg := range loglines {
		linePayload := AgentLoglinePayload{}

		// Check if loglines contain ResourceD base64 wire protocol.
		// If so, convert to plain text.
		if strings.HasPrefix(lg, "type:base64") {
			lg = logline.ParseSingle(lg).EncodePlain()
		}

		// Check if each logline is NOT encoded in ResourceD log wire protocol
		if !strings.HasPrefix(lg, "type:base64") && !strings.HasPrefix(lg, "type:plain") {
			linePayload.Created = time.Now().UTC().Unix()
			linePayload.Content = lg
		} else {
			liveLogline := logline.ParseSingle(lg)
			linePayload.Created = liveLogline.Created
			linePayload.Content = liveLogline.Content
		}

		if linePayload.Content != "" {
			toSend.Data.Loglines = append(toSend.Data.Loglines, linePayload)
		}
	}

	return toSend
}

// SendLogToMaster sends log lines to master.
func (b *Base) SendLogToMaster(accessToken, masterURLHost, masterURLPath string, hostData *host.Host, loglines []string, source string) error {
	// Don't do anything if there are no log lines to send.
	if len(loglines) == 0 {
		return nil
	}

	if masterURLPath == "" {
		masterURLPath = "/api/logs"
	}

	data := b.logPayloadForMaster(hostData, loglines, source)

	dataJson, err := json.Marshal(data)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error": err.Error(),
		}).Error("Failed to marshal JSON payload for sending logs to ResourceD Master")

		return err
	}

	url := masterURLHost + masterURLPath

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(dataJson))
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error": err.Error(),
		}).Error("Failed to create request struct for sending logs to ResourceD Master")

		return err
	}

	req.SetBasicAuth(accessToken, "")

	client := pester.New()
	client.MaxRetries = 3
	client.Backoff = pester.ExponentialJitterBackoff
	client.KeepLog = false

	resp, err := client.Do(req)
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error":      err.Error(),
			"req.URL":    req.URL.String(),
			"req.Method": req.Method,
		}).Error("Failed to send logs data to ResourceD Master")

		return err
	}

	if resp != nil && resp.Body != nil {
		defer resp.Body.Close()
	}

	return err
}

// SendLogToAgent sends log lines to another agent.
func (b *Base) SendLogToAgent(anotherAgentAddr string, maxRetries int, loglines []string, source string) error {
	if len(loglines) == 0 {
		return nil
	}

	conn, err := net.Dial("tcp", anotherAgentAddr)
	attempts := 0

	for {
		if err == nil {
			break
		}

		if err != nil && attempts > maxRetries {
			return err
		}

		if err != nil {
			attempts = attempts + 1
			time.Sleep(pester.ExponentialJitterBackoff(attempts))
			conn, err = net.Dial("tcp", anotherAgentAddr)
			continue
		}
	}

	if conn != nil {
		defer conn.Close()

		w := bufio.NewWriter(conn)

		for i, lg := range loglines {
			// Check if each logline is NOT encoded in ResourceD log wire protocol
			if !strings.HasPrefix(lg, "type:base64") && !strings.HasPrefix(lg, "type:plain") {
				loglines[i] = logline.LiveLogline{Created: time.Now().UTC().Unix(), Content: lg}.EncodeBase64()
				lg = loglines[i]
			}

			fmt.Fprint(w, lg)
			w.Flush()
		}
	}

	return err
}

func (b *Base) SendLogToSyslog(protocol string, addr string, priority syslog.Priority, tag string, loglines []string, source string) error {
	if len(loglines) == 0 {
		return nil
	}

	logHandler, err := syslog.Dial(protocol, addr, priority, tag)
	if err != nil {
		return err
	}
	defer logHandler.Close()

	for _, lg := range loglines {
		logHandler.Write([]byte(logline.ParseSingle(lg).PlainContent()))
	}

	return nil
}

// WriteToFile writes log lines to local file.
func (b *Base) WriteToFile(targetFile string, loglines []string) error {
	// Check if loglines contain ResourceD base64 wire protocol.
	// If so, convert to plain text.
	for i, lg := range loglines {
		if strings.HasPrefix(lg, "type:base64") || strings.HasPrefix(lg, "type:plain") {
			loglines[i] = logline.ParseSingle(lg).PlainContent()
		}
	}

	fileHandle, err := os.OpenFile(targetFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		return err
	}
	defer fileHandle.Close()

	for _, logline := range loglines {
		if strings.HasSuffix(logline, "\n") {
			fileHandle.WriteString(logline)
		} else {
			fileHandle.WriteString(logline + "\n")
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package hpcloud

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
)

/* Server flavours Smallest to Largest */
type Flavor int

const (
	XSmall = Flavor(100) + iota
	Small
	Medium
	Large
	XLarge
	DblXLarge
)

/* Available images */
type ServerImage int

const (
	UbuntuLucid10_04Kernel    = ServerImage(1235)
	UbuntuLucid10_04          = 1236
	UbuntuMaverick10_10Kernel = 1237
	UbuntuMaverick10_10       = 1238
	UbuntuNatty11_04Kernel    = 1239
	UbuntuNatty11_04          = 1240
	UbuntuOneiric11_10        = 5579
	UbuntuPrecise12_04        = 8419
	CentOS5_8Server64         = 54021
	CentOS6_2Server64Kernel   = 1356
	CentOS6_2Server64Ramdisk  = 1357
	CentOS6_2Server64         = 1358
	DebianSqueeze6_0_3Kernel  = 1359
	DebianSqueeze6_0_3Ramdisk = 1360
	DebianSqueeze6_0_3Server  = 1361
	Fedora16Server64          = 16291
	BitNamiDrupal7_14_0       = 22729
	BitNamiWebPack1_2_0       = 22731
	BitNamiDevPack1_0_0       = 4654
	ActiveStateStackatov1_2_6 = 14345
	ActiveStateStackatov2_2_2 = 59297
	ActiveStateStackatov2_2_3 = 60815
	EnterpriseDBPPAS9_1_2     = 9953
	EnterpriseDBPSQL9_1_3     = 9995
)

type Link struct {
	HREF string `json:"href"`
	Rel  string `json:"rel"`
}

/*
  Several embedded types are simply an ID string with a slice of Link
*/
type IDLink struct {
	Name  string `json:"name"`
	ID    string `json:"id"`
	Links []Link `json:"links"`
}

type Flavor_ struct {
	Name  string `json:"name"`
	ID    int64  `json:"id"`
	Links []Link `json:"links"`
}

type Flavors struct {
	F []Flavor_ `json:"flavors"`
}

type Image struct {
	I struct {
		Name     string            `json:"name"`
		ID       string            `json:"id"`
		Links    []Link            `json:"links"`
		Progress int               `json:"progress"`
		Metadata map[string]string `json:"metadata"`
		Status   string            `json:"status"`
		Updated  string            `json:"updated"`
	} `json:"image"`
}

type Images struct {
	I []IDLink `json:"images"`
}

/*
  This type describes the JSON data which should be sent to the create
  server resource.
*/
type Server struct {
	ConfigDrive    bool              `json:"config_drive"`
	FlavorRef      Flavor            `json:"flavorRef"`
	ImageRef       ServerImage       `json:"imageRef"`
	MaxCount       int               `json:"max_count"`
	MinCount       int               `json:"min_count"`
	Name           string            `json:"name"`
	Key            string            `json:"key_name"`
	Personality    string            `json:"personality"`
	UserData       string            `json:"user_data"`
	SecurityGroups []IDLink          `json:"security_groups"`
	Metadata       map[string]string `json:"metadata"`
}

/*
  This type describes the JSON response from a successful CreateServer
  call.
*/
type ServerResponse struct {
	S struct {
		Status         string            `json:"status"`
		Updated        string            `json:"update"`
		HostID         string            `json:"hostId"`
		UserID         string            `json:"user_id"`
		Name           string            `json:"name"`
		Links          []Link            `json:"links"`
		Addresses      interface{}       `json:"addresses"`
		TenantID       string            `json:"tenant_id"`
		Image          IDLink            `json:"image"`
		Created        string            `json:"created"`
		UUID           string            `json:"uuid"`
		AccessIPv4     string            `json:"accessIPv4"`
		AccessIPv6     string            `json:"accessIPv6"`
		KeyName        string            `json:"key_name"`
		AdminPass      string            `json:"adminPass"`
		Flavor         IDLink            `json:"flavor"`
		ConfigDrive    string            `json:"config_drive"`
		ID             int64             `json:"id"`
		SecurityGroups []IDLink          `json:"security_groups"`
		Metadata       map[string]string `json:"metadata"`
	} `json:"server"`
}

func (a Access) CreateServer(s Server) (*ServerResponse, error) {
	b, err := s.MarshalJSON()
	if err != nil {
		return nil, err
	}

	body, err := a.baseComputeRequest("servers", "POST", strings.NewReader(string(b)))
	if err != nil {
		return nil, err
	}
	sr := &ServerResponse{}
	err = json.Unmarshal(body, sr)
	if err != nil {
		return nil, err
	}
	return sr, nil
}

func (a Access) DeleteServer(server_id string) error {
	_, err := a.baseComputeRequest(fmt.Sprintf("servers/%s", server_id), "DELETE", nil)
	if err != nil {
		return err
	}
	return nil
}

func (a Access) RebootServer(server_id string) error {
	s := `{"reboot":{"type":"SOFT"}}`
	_, err := a.baseComputeRequest(fmt.Sprintf("servers/%s/action", server_id), "POST", strings.NewReader(s))
	if err != nil {
		return err
	}
	return nil

}

func (a Access) ListFlavors() (*Flavors, error) {
	body, err := a.baseComputeRequest("flavors", "GET", nil)
	if err != nil {
		return nil, err
	}

	fl := &Flavors{}
	err = json.Unmarshal(body, fl)
	if err != nil {
		return nil, err
	}
	return fl, nil
}

func (a Access) ListImages() (*Images, error) {
	body, err := a.baseComputeRequest("images", "GET", nil)
	if err != nil {
		return nil, err
	}
	im := &Images{}
	err = json.Unmarshal(body, im)
	if err != nil {
		return nil, err
	}
	return im, nil
}

func (a Access) DeleteImage(image_id string) error {
	_, err := a.baseComputeRequest(fmt.Sprintf("images/%s", image_id), "DELETE", nil)
	if err != nil {
		return err
	}
	return nil
}

func (a Access) ListImage(image_id string) (*Image, error) {
	body, err := a.baseComputeRequest(fmt.Sprintf("images/%s", image_id), "GET", nil)
	if err != nil {
		return nil, err
	}
	i := &Image{}
	err = json.Unmarshal(body, i)
	if err != nil {
		return nil, err
	}
	return i, nil
}

func (a Access) baseComputeRequest(url, method string, b io.Reader) ([]byte, error) {
	path := fmt.Sprintf("%s%s/%s", COMPUTE_URL, a.TenantID, url)
	client := &http.Client{}
	req, err := http.NewRequest(method, path, b)
	if err != nil {
		return nil, err
	}
	req.Header.Add("X-Auth-Token", a.A.Token.ID)
	req.Header.Add("Content-type", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	switch resp.StatusCode {
	case http.StatusOK, http.StatusNonAuthoritativeInfo, http.StatusAccepted:
		return body, nil
	case http.StatusNotFound:
		nf := &NotFound{}
		err = json.Unmarshal(body, nf)
		if err != nil {
			return nil, err
		}
		return nil, errors.New(nf.NF.Message)
	default:
		br := &BadRequest{}
		err = json.Unmarshal(body, br)
		if err != nil {
			return nil, err
		}
		return nil, errors.New(br.B.Message)
	}
	panic("Unreachable")
}

func (s Server) MarshalJSON() ([]byte, error) {
	b := bytes.NewBufferString("")
	b.WriteString(`{"server":{`)
	/* The available images are 100-105, x-small to x-large. */
	if s.FlavorRef < 100 || s.FlavorRef > 105 {
		return []byte{},
			errors.New("Flavor Reference refers to a non-existant flavour.")
	} else {
		b.WriteString(fmt.Sprintf(`"flavorRef":%d`, s.FlavorRef))
	}
	if s.ImageRef == 0 {
		return []byte{},
			errors.New("An image name is required.")
	} else {
		b.WriteString(fmt.Sprintf(`,"imageRef":%d`, s.ImageRef))
	}
	if s.Name == "" {
		return []byte{},
			errors.New("A name is required")
	} else {
		b.WriteString(fmt.Sprintf(`,"name":"%s"`, s.Name))
	}

	/* Optional items */
	if s.Key != "" {
		b.WriteString(fmt.Sprintf(`,"key_name":"%s"`, s.Key))
	}
	if s.ConfigDrive {
		b.WriteString(`,"config_drive": true`)
	}
	if s.MinCount > 0 {
		b.WriteString(fmt.Sprintf(`,"min_count":%d`, s.MinCount))
	}
	if s.MaxCount > 0 {
		b.WriteString(fmt.Sprintf(`,"max_count":%d`, s.MaxCount))
	}
	if s.UserData != "" {
		/* user_data needs to be base64'd */
		newb := make([]byte, 0, len(s.UserData))
		base64.StdEncoding.Encode([]byte(s.UserData), newb)
		b.WriteString(fmt.Sprintf(`,"user_data": "%s",`, string(newb)))
	}
	if len(s.Personality) > 255 {
		return []byte{},
			errors.New("Server's personality cannot have >255 bytes.")
	} else if s.Personality != "" {
		b.WriteString(fmt.Sprintf(`,"personality":"%s",`, s.Personality))
	}
	if len(s.Metadata) > 0 {
		fmt.Println(len(s.Metadata))
		b.WriteString(`,"metadata":{`)
		cnt := 0
		for key, value := range s.Metadata {
			b.WriteString(fmt.Sprintf(`"%s": "%s"`, key, value))
			if cnt+1 != len(s.Metadata) {
				b.WriteString(",")
				cnt++
			} else {
				b.WriteString("}")
			}
		}
	}
	if len(s.SecurityGroups) > 0 {
		b.WriteString(`,"security_groups":[`)
		cnt := 0
		for _, sg := range s.SecurityGroups {
			b.WriteString(fmt.Sprintf(`{"name": "%s"}`, sg.Name))
			if cnt+1 != len(s.SecurityGroups) {
				b.WriteString(",")
				cnt++
			} else {
				b.WriteString("]")
			}
		}
	}
	b.WriteString("}}")
	return b.Bytes(), nil
}
<commit_msg>Whitespace.<commit_after>package hpcloud

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
)

/* Server flavours Smallest to Largest */
type Flavor int

const (
	XSmall = Flavor(100) + iota
	Small
	Medium
	Large
	XLarge
	DblXLarge
)

/* Available images */
type ServerImage int

const (
	UbuntuLucid10_04Kernel    = ServerImage(1235)
	UbuntuLucid10_04          = 1236
	UbuntuMaverick10_10Kernel = 1237
	UbuntuMaverick10_10       = 1238
	UbuntuNatty11_04Kernel    = 1239
	UbuntuNatty11_04          = 1240
	UbuntuOneiric11_10        = 5579
	UbuntuPrecise12_04        = 8419
	CentOS5_8Server64         = 54021
	CentOS6_2Server64Kernel   = 1356
	CentOS6_2Server64Ramdisk  = 1357
	CentOS6_2Server64         = 1358
	DebianSqueeze6_0_3Kernel  = 1359
	DebianSqueeze6_0_3Ramdisk = 1360
	DebianSqueeze6_0_3Server  = 1361
	Fedora16Server64          = 16291
	BitNamiDrupal7_14_0       = 22729
	BitNamiWebPack1_2_0       = 22731
	BitNamiDevPack1_0_0       = 4654
	ActiveStateStackatov1_2_6 = 14345
	ActiveStateStackatov2_2_2 = 59297
	ActiveStateStackatov2_2_3 = 60815
	EnterpriseDBPPAS9_1_2     = 9953
	EnterpriseDBPSQL9_1_3     = 9995
)

type Link struct {
	HREF string `json:"href"`
	Rel  string `json:"rel"`
}

/*
  Several embedded types are simply an ID string with a slice of Link
*/
type IDLink struct {
	Name  string `json:"name"`
	ID    string `json:"id"`
	Links []Link `json:"links"`
}

type Flavor_ struct {
	Name  string `json:"name"`
	ID    int64  `json:"id"`
	Links []Link `json:"links"`
}

type Flavors struct {
	F []Flavor_ `json:"flavors"`
}

type Image struct {
	I struct {
		Name     string            `json:"name"`
		ID       string            `json:"id"`
		Links    []Link            `json:"links"`
		Progress int               `json:"progress"`
		Metadata map[string]string `json:"metadata"`
		Status   string            `json:"status"`
		Updated  string            `json:"updated"`
	} `json:"image"`
}

type Images struct {
	I []IDLink `json:"images"`
}

/*
  This type describes the JSON data which should be sent to the create
  server resource.
*/
type Server struct {
	ConfigDrive    bool              `json:"config_drive"`
	FlavorRef      Flavor            `json:"flavorRef"`
	ImageRef       ServerImage       `json:"imageRef"`
	MaxCount       int               `json:"max_count"`
	MinCount       int               `json:"min_count"`
	Name           string            `json:"name"`
	Key            string            `json:"key_name"`
	Personality    string            `json:"personality"`
	UserData       string            `json:"user_data"`
	SecurityGroups []IDLink          `json:"security_groups"`
	Metadata       map[string]string `json:"metadata"`
}

/*
  This type describes the JSON response from a successful CreateServer
  call.
*/
type ServerResponse struct {
	S struct {
		Status         string            `json:"status"`
		Updated        string            `json:"update"`
		HostID         string            `json:"hostId"`
		UserID         string            `json:"user_id"`
		Name           string            `json:"name"`
		Links          []Link            `json:"links"`
		Addresses      interface{}       `json:"addresses"`
		TenantID       string            `json:"tenant_id"`
		Image          IDLink            `json:"image"`
		Created        string            `json:"created"`
		UUID           string            `json:"uuid"`
		AccessIPv4     string            `json:"accessIPv4"`
		AccessIPv6     string            `json:"accessIPv6"`
		KeyName        string            `json:"key_name"`
		AdminPass      string            `json:"adminPass"`
		Flavor         IDLink            `json:"flavor"`
		ConfigDrive    string            `json:"config_drive"`
		ID             int64             `json:"id"`
		SecurityGroups []IDLink          `json:"security_groups"`
		Metadata       map[string]string `json:"metadata"`
	} `json:"server"`
}

func (a Access) CreateServer(s Server) (*ServerResponse, error) {
	b, err := s.MarshalJSON()
	if err != nil {
		return nil, err
	}

	body, err := a.baseComputeRequest("servers", "POST",
		strings.NewReader(string(b)),
	)
	if err != nil {
		return nil, err
	}
	sr := &ServerResponse{}
	err = json.Unmarshal(body, sr)
	if err != nil {
		return nil, err
	}
	return sr, nil
}

func (a Access) DeleteServer(server_id string) error {
	_, err := a.baseComputeRequest(
		fmt.Sprintf("servers/%s", server_id),
		"DELETE", nil,
	)
	if err != nil {
		return err
	}
	return nil
}

func (a Access) RebootServer(server_id string) error {
	s := `{"reboot":{"type":"SOFT"}}`
	_, err := a.baseComputeRequest(
		fmt.Sprintf("servers/%s/action", server_id),
		"POST", strings.NewReader(s),
	)
	if err != nil {
		return err
	}
	return nil

}

func (a Access) ListFlavors() (*Flavors, error) {
	body, err := a.baseComputeRequest("flavors", "GET", nil)
	if err != nil {
		return nil, err
	}

	fl := &Flavors{}
	err = json.Unmarshal(body, fl)
	if err != nil {
		return nil, err
	}
	return fl, nil
}

func (a Access) ListImages() (*Images, error) {
	body, err := a.baseComputeRequest("images", "GET", nil)
	if err != nil {
		return nil, err
	}
	im := &Images{}
	err = json.Unmarshal(body, im)
	if err != nil {
		return nil, err
	}
	return im, nil
}

func (a Access) DeleteImage(image_id string) error {
	_, err := a.baseComputeRequest(
		fmt.Sprintf("images/%s", image_id), "DELETE", nil,
	)
	if err != nil {
		return err
	}
	return nil
}

func (a Access) ListImage(image_id string) (*Image, error) {
	body, err := a.baseComputeRequest(
		fmt.Sprintf("images/%s", image_id), "GET", nil,
	)
	if err != nil {
		return nil, err
	}
	i := &Image{}
	err = json.Unmarshal(body, i)
	if err != nil {
		return nil, err
	}
	return i, nil
}

func (a Access) baseComputeRequest(url, method string, b io.Reader) ([]byte, error) {
	path := fmt.Sprintf("%s%s/%s", COMPUTE_URL, a.TenantID, url)
	client := &http.Client{}
	req, err := http.NewRequest(method, path, b)
	if err != nil {
		return nil, err
	}
	req.Header.Add("X-Auth-Token", a.A.Token.ID)
	req.Header.Add("Content-type", "application/json")
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	switch resp.StatusCode {
	case http.StatusAccepted:
	case http.StatusNonAuthoritativeInfo:
	case http.StatusOK:
		return body, nil
	case http.StatusNotFound:
		nf := &NotFound{}
		err = json.Unmarshal(body, nf)
		if err != nil {
			return nil, err
		}
		return nil, errors.New(nf.NF.Message)
	default:
		br := &BadRequest{}
		err = json.Unmarshal(body, br)
		if err != nil {
			return nil, err
		}
		return nil, errors.New(br.B.Message)
	}
	panic("Unreachable")
}

func (s Server) MarshalJSON() ([]byte, error) {
	b := bytes.NewBufferString("")
	b.WriteString(`{"server":{`)
	/* The available images are 100-105, x-small to x-large. */
	if s.FlavorRef < 100 || s.FlavorRef > 105 {
		return []byte{},
			errors.New("Flavor Reference refers to a non-existant flavour.")
	} else {
		b.WriteString(fmt.Sprintf(`"flavorRef":%d`, s.FlavorRef))
	}
	if s.ImageRef == 0 {
		return []byte{},
			errors.New("An image name is required.")
	} else {
		b.WriteString(fmt.Sprintf(`,"imageRef":%d`, s.ImageRef))
	}
	if s.Name == "" {
		return []byte{},
			errors.New("A name is required")
	} else {
		b.WriteString(fmt.Sprintf(`,"name":"%s"`, s.Name))
	}

	/* Optional items */
	if s.Key != "" {
		b.WriteString(fmt.Sprintf(`,"key_name":"%s"`, s.Key))
	}
	if s.ConfigDrive {
		b.WriteString(`,"config_drive": true`)
	}
	if s.MinCount > 0 {
		b.WriteString(fmt.Sprintf(`,"min_count":%d`, s.MinCount))
	}
	if s.MaxCount > 0 {
		b.WriteString(fmt.Sprintf(`,"max_count":%d`, s.MaxCount))
	}
	if s.UserData != "" {
		/* user_data needs to be base64'd */
		newb := make([]byte, 0, len(s.UserData))
		base64.StdEncoding.Encode([]byte(s.UserData), newb)
		b.WriteString(fmt.Sprintf(`,"user_data": "%s",`, string(newb)))
	}
	if len(s.Personality) > 255 {
		return []byte{},
			errors.New("Server's personality cannot have >255 bytes.")
	} else if s.Personality != "" {
		b.WriteString(fmt.Sprintf(`,"personality":"%s",`, s.Personality))
	}
	if len(s.Metadata) > 0 {
		fmt.Println(len(s.Metadata))
		b.WriteString(`,"metadata":{`)
		cnt := 0
		for key, value := range s.Metadata {
			b.WriteString(fmt.Sprintf(`"%s": "%s"`, key, value))
			if cnt+1 != len(s.Metadata) {
				b.WriteString(",")
				cnt++
			} else {
				b.WriteString("}")
			}
		}
	}
	if len(s.SecurityGroups) > 0 {
		b.WriteString(`,"security_groups":[`)
		cnt := 0
		for _, sg := range s.SecurityGroups {
			b.WriteString(fmt.Sprintf(`{"name": "%s"}`, sg.Name))
			if cnt+1 != len(s.SecurityGroups) {
				b.WriteString(",")
				cnt++
			} else {
				b.WriteString("]")
			}
		}
	}
	b.WriteString("}}")
	return b.Bytes(), nil
}
<|endoftext|>