<commit_before>package data

import (
	"encoding/csv"
	"time"
	"fmt"
	"strconv"
	"io"
)

func CSVParse(file io.Reader) (labels []string, data []Record) {
	labels, data = csvParse(file)
	return
}

func csvParse(file io.Reader) (labels []string, data []Record) {
	reader := csv.NewReader (file)
	tmpdata, err := reader.ReadAll()
	if  err != nil {
		fmt.Println(err)
	}
	fmt.Println(len(tmpdata) - 1)
	//labels = make([]string, 6)
	labels = tmpdata[0]
	data  = make([]Record,  len(tmpdata)-1)
	for i := 1; i<len(tmpdata)-1; i++ {
		data[i-1].Time, _ = time.Parse(ISO, tmpdata[i][0])
		data[i-1].Radiation, err = strconv.ParseFloat(tmpdata[i][1], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Humidity, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Temperature, err = strconv.ParseFloat(tmpdata[i][3], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Wind, err = strconv.ParseFloat(tmpdata[i][4], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Power, err = strconv.ParseFloat(tmpdata[i][5], 64)
		if err != nil {
			data[i-1].Null = true
		}
	}
	fmt.Println(len(data))
	data = fillRecords (data)
	fmt.Println(len(data))
	return
}

func fillRecords (emptyData []Record) (data []Record){
	gradRad, gradHumidity, gradTemp, gradWind := 0.0, 0.0, 0.0, 0.0
	fmt.Println(len(emptyData))
	for i := 0; i<len(emptyData); i++ {
		if emptyData[i].empty && i > 0 {
			emptyData[i].Radiation = emptyData[i-1].Radiation + gradRad
			emptyData[i].Humidity = emptyData[i-1].Humidity + gradHumidity
			emptyData[i].Temperature = emptyData[i-1].Temperature + gradTemp
			emptyData[i].Wind = emptyData[i-1].Wind + gradWind
			emptyData[i].empty = false
		} else {
			if i + 4 < len (emptyData) {
				gradRad = (emptyData[i+4].Radiation - emptyData[i].Radiation)/4
				gradHumidity = (emptyData[i+4].Humidity - emptyData[i].Humidity)/4
				gradTemp = (emptyData[i+4].Temperature - emptyData[i].Temperature)/4
				gradWind = (emptyData[i+4].Wind - emptyData[i].Wind)/4
			} else {
				gradRad = 0
				gradHumidity = 0
				gradTemp = 0
				gradWind = 0
			}
		}
	}
	return emptyData
}
<commit_msg>checking index limits<commit_after>package data

import (
	"encoding/csv"
	"time"
	"fmt"
	"strconv"
	"io"
)

func CSVParse(file io.Reader) (labels []string, data []Record) {
	labels, data = csvParse(file)
	return
}

func csvParse(file io.Reader) (labels []string, data []Record) {
	reader := csv.NewReader (file)
	tmpdata, err := reader.ReadAll()
	if  err != nil {
		fmt.Println(err)
	}
	fmt.Println(len(tmpdata) - 1)
	//labels = make([]string, 6)
	labels = tmpdata[0]
	data  = make([]Record,  len(tmpdata))
	for i := 1; i<len(tmpdata)-1; i++ {
		data[i-1].Time, _ = time.Parse(ISO, tmpdata[i][0])
		data[i-1].Radiation, err = strconv.ParseFloat(tmpdata[i][1], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Humidity, err = strconv.ParseFloat(tmpdata[i][2], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Temperature, err = strconv.ParseFloat(tmpdata[i][3], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Wind, err = strconv.ParseFloat(tmpdata[i][4], 64)
		if err != nil {
			data[i-1].empty = true
		}
		data[i-1].Power, err = strconv.ParseFloat(tmpdata[i][5], 64)
		if err != nil {
			data[i-1].Null = true
		}
	}
	fmt.Println(len(data))
	data = fillRecords (data)
	fmt.Println(len(data))
	return
}

func fillRecords (emptyData []Record) (data []Record){
	gradRad, gradHumidity, gradTemp, gradWind := 0.0, 0.0, 0.0, 0.0
	fmt.Println(len(emptyData))
	for i := 0; i<len(emptyData); i++ {
		if emptyData[i].empty && i > 0 {
			emptyData[i].Radiation = emptyData[i-1].Radiation + gradRad
			emptyData[i].Humidity = emptyData[i-1].Humidity + gradHumidity
			emptyData[i].Temperature = emptyData[i-1].Temperature + gradTemp
			emptyData[i].Wind = emptyData[i-1].Wind + gradWind
			emptyData[i].empty = false
		} else {
			if i + 4 < len (emptyData) {
				gradRad = (emptyData[i+4].Radiation - emptyData[i].Radiation)/4
				gradHumidity = (emptyData[i+4].Humidity - emptyData[i].Humidity)/4
				gradTemp = (emptyData[i+4].Temperature - emptyData[i].Temperature)/4
				gradWind = (emptyData[i+4].Wind - emptyData[i].Wind)/4
			} else {
				gradRad = 0
				gradHumidity = 0
				gradTemp = 0
				gradWind = 0
			}
		}
	}
	return emptyData
}
<|endoftext|>


<commit_before>package models

import (
	"socialapi/request"
	"strings"
	"time"

	"github.com/koding/bongo"
)

// ChannelLink holds the link between two channels
type ChannelLink struct {
	// Id holds the unique id of the link between channels
	Id int64 `json:"id,string"`

	// RootId is the id of the root channel
	RootId int64 `json:"rootId,string"       sql:"NOT NULL"`

	// LeafId is the id of the leaf channel
	LeafId int64 `json:"leafId,string"       sql:"NOT NULL"`

	// CreatedAt holds the creation time of the channel_link
	CreatedAt time.Time `json:"createdAt"    sql:"NOT NULL"`
}

func (c *ChannelLink) validate() error {
	if c.LeafId == 0 {
		return ErrLeafIsNotSet
	}

	if c.RootId == 0 {
		return ErrRootIsNotSet
	}

	return nil
}

func (c *ChannelLink) List(q *request.Query) ([]Channel, error) {
	if c.RootId == 0 {
		return nil, ErrRootIsNotSet
	}

	var leafIds []int64

	bq := &bongo.Query{
		Selector: map[string]interface{}{
			"root_id": c.RootId,
		},
		Pluck: "leaf_id",
	}

	err := c.Some(&leafIds, bq)
	if err != nil {
		return nil, err
	}

	return NewChannel().FetchByIds(leafIds)
}

func (c *ChannelLink) UnLink() error {
	if err := c.validate(); err != nil {
		return err
	}

	// first update the leaf node with it's previous channel type constant
	ch := NewChannel()
	if err := ch.ById(c.LeafId); err != nil {
		if err == bongo.RecordNotFound {
			return ErrChannelNotFound
		}

		return err
	}

	ch.TypeConstant = strings.TrimPrefix(
		ch.TypeConstant,
		ChannelLinkedPrefix,
	)

	if err := ch.Update(); err != nil {
		return err
	}

	// then delete the link between two channels
	bq := &bongo.Query{
		Selector: map[string]interface{}{
			"root_id": c.RootId,
			"leaf_id": c.LeafId,
		},
	}

	if err := c.One(bq); err != nil {
		return err
	}

	return ch.Delete()
}

func (c *ChannelLink) Blacklist() error {
	if c.RootId == 0 {
		return ErrRootIsNotSet
	}

	bq := &bongo.Query{
		Selector: map[string]interface{}{
			"root_id": c.RootId,
		},
	}

	// if channel has some leaves, do not continue
	if c.One(bq) != bongo.RecordNotFound {
		return ErrChannelHasLeaves
	}

	// first update the leaf node with it's previous channel type constant
	ch := NewChannel()
	if err := ch.ById(c.RootId); err != nil {
		if err == bongo.RecordNotFound {
			return ErrChannelNotFound
		}

		return err
	}

	// mark channel as linked
	ch.TypeConstant = ChannelLinkedPrefix + ch.TypeConstant

	if err := ch.Update(); err != nil {
		return err
	}

	return nil
}
<commit_msg>Socialapi: put options into channel link struct<commit_after>package models

import (
	"socialapi/request"
	"strings"
	"time"

	"github.com/koding/bongo"
)

// ChannelLink holds the link between two channels
type ChannelLink struct {
	// Id holds the unique id of the link between channels
	Id int64 `json:"id,string"`

	// RootId is the id of the root channel
	RootId int64 `json:"rootId,string"       sql:"NOT NULL"`

	// LeafId is the id of the leaf channel
	LeafId int64 `json:"leafId,string"       sql:"NOT NULL"`

	// CreatedAt holds the creation time of the channel_link
	CreatedAt time.Time `json:"createdAt"    sql:"NOT NULL"`

	// options for operations

	// DeleteMessages remove the messages of a channel
	DeleteMessages bool `json:"deleteMessages,omitempty" sql:"-"`
}

func (c *ChannelLink) validate() error {
	if c.LeafId == 0 {
		return ErrLeafIsNotSet
	}

	if c.RootId == 0 {
		return ErrRootIsNotSet
	}

	return nil
}

func (c *ChannelLink) List(q *request.Query) ([]Channel, error) {
	if c.RootId == 0 {
		return nil, ErrRootIsNotSet
	}

	var leafIds []int64

	bq := &bongo.Query{
		Selector: map[string]interface{}{
			"root_id": c.RootId,
		},
		Pluck: "leaf_id",
	}

	err := c.Some(&leafIds, bq)
	if err != nil {
		return nil, err
	}

	return NewChannel().FetchByIds(leafIds)
}

func (c *ChannelLink) UnLink() error {
	if err := c.validate(); err != nil {
		return err
	}

	// first update the leaf node with it's previous channel type constant
	ch := NewChannel()
	if err := ch.ById(c.LeafId); err != nil {
		if err == bongo.RecordNotFound {
			return ErrChannelNotFound
		}

		return err
	}

	ch.TypeConstant = strings.TrimPrefix(
		ch.TypeConstant,
		ChannelLinkedPrefix,
	)

	if err := ch.Update(); err != nil {
		return err
	}

	// then delete the link between two channels
	bq := &bongo.Query{
		Selector: map[string]interface{}{
			"root_id": c.RootId,
			"leaf_id": c.LeafId,
		},
	}

	if err := c.One(bq); err != nil {
		return err
	}

	return ch.Delete()
}

func (c *ChannelLink) Blacklist() error {
	if c.RootId == 0 {
		return ErrRootIsNotSet
	}

	bq := &bongo.Query{
		Selector: map[string]interface{}{
			"root_id": c.RootId,
		},
	}

	// if channel has some leaves, do not continue
	if c.One(bq) != bongo.RecordNotFound {
		return ErrChannelHasLeaves
	}

	// first update the leaf node with it's previous channel type constant
	ch := NewChannel()
	if err := ch.ById(c.RootId); err != nil {
		if err == bongo.RecordNotFound {
			return ErrChannelNotFound
		}

		return err
	}

	// mark channel as linked
	ch.TypeConstant = ChannelLinkedPrefix + ch.TypeConstant

	if err := ch.Update(); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// demonstrate for loop with all the possible ways to use it

package main

import ()

func main() {
	// for
}
<commit_msg>language basics: for<commit_after>// demonstrate for loop with all the possible ways to use it
// print 0-10

package main

import (
	"fmt"
)

func main() {

	// default syntax
	// for declare/init; check; inc{}

	for i := 0; i <= 10; i++ {
		fmt.Println(i)
	}

	j := 0
	for ; j <= 10; j++ {
		fmt.Println(j)
	}

	k := 0
	for ; ; k++ {
		if k > 10 {
			break
		}
		fmt.Println(k)
	}

	l := 0
	for {
		if l > 10 {
			break
		}
		fmt.Println(l)
		l++
	}

	// infinite loop, run with caution, use ctrl+c to kill the program to stop
	// for{
	// 	fmt.Println("hello")
	// }

}
<|endoftext|>


<commit_before>package aws

import (
  "fmt"

  "github.com/hashicorp/terraform/helper/schema"
)

// See http://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/billing-getting-started.html#step-2
var billingAccountId = "386209384616"

func dataSourceAwsBillingServiceAccount() *schema.Resource {
  return &schema.Resource{
    Read: dataSourceAwsBillingServiceAccountRead,

    Schema: map[string]*schema.Schema{
      "arn": &schema.Schema{
        Type:     schema.TypeString,
        Computed: true,
      },
    },
  }
}

func dataSourceAwsBillingServiceAccountRead(d *schema.ResourceData, meta interface{}) error {
  d.SetId(billingAccountId)

  d.Set("arn", "arn:aws:iam::"+billingAccountId+":root")

  return nil
}
<commit_msg>Fix formatting<commit_after>package aws

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"
)

// See http://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/billing-getting-started.html#step-2
var billingAccountId = "386209384616"

func dataSourceAwsBillingServiceAccount() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsBillingServiceAccountRead,

		Schema: map[string]*schema.Schema{
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func dataSourceAwsBillingServiceAccountRead(d *schema.ResourceData, meta interface{}) error {
	d.SetId(billingAccountId)

	d.Set("arn", "arn:aws:iam::"+billingAccountId+":root")

	return nil
}
<|endoftext|>


<commit_before>package kodingkite

import (
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"

	kodingconfig "koding/tools/config"

	"github.com/koding/kite"
	kiteconfig "github.com/koding/kite/config"
	"github.com/koding/kite/kontrolclient"
	"github.com/koding/kite/registration"
	"github.com/koding/kite/server"
	"github.com/koding/logging"
)

type KodingKite struct {
	*server.Server
	Kontrol      *kontrolclient.KontrolClient
	Registration *registration.Registration
	KodingConfig *kodingconfig.Config
	registerIP   string
}

// New returns a new kite instance based on for the given Koding configurations
func New(kodingConf *kodingconfig.Config, name, version string) (*KodingKite, error) {
	kiteConf, err := kiteconfig.Get()
	if err != nil {
		return nil, err
	}

	kiteConf.Environment = kodingConf.Environment

	k := kite.New(name, version)
	k.Config = kiteConf

	server := server.New(k)

	kon := kontrolclient.New(k)

	kk := &KodingKite{
		Server:       server,
		Kontrol:      kon,
		Registration: registration.New(kon),
		KodingConfig: kodingConf,
	}

	kk.registerIP, err = getRegisterIP(kodingConf.Environment)
	if err != nil {
		return nil, err
	}

	syslog, err := logging.NewSyslogHandler(name)
	if err != nil {
		log.Fatalf("Cannot connect to syslog: %s", err.Error())
	}

	kk.Log.SetHandler(logging.NewMultiHandler(logging.StderrHandler, syslog))

	return kk, nil
}

func getRegisterIP(environment string) (string, error) {
	var ip string

	switch environment {
	case "production":
		fallthrough
	case "staging":
		// Magical IP address of Openstack Metadata Service
		// http://docs.openstack.org/grizzly/openstack-compute/admin/content/metadata-service.html
		resp, err := http.Get("http://169.254.169.254/latest/meta-data/public-ipv4")
		if err != nil {
			return "", errors.New("cannot get public IP address: " + err.Error())
		}

		defer resp.Body.Close()

		if resp.StatusCode != 200 {
			return "", errors.New("unexpected status code: " + resp.Status)
		}

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", err
		}

		ip = string(body)
	case "vagrant":
		ip = "127.0.0.1"
	default:
		return "", errors.New("I don't know which IP address to register in this environment: " + environment)
	}

	if ip == "" {
		return "", errors.New("no suitable IP address is found")
	}

	return ip, nil
}

func (s *KodingKite) Start() {
	s.Log.Info("Kite has started: %s", s.Kite.Kite())

	registerWithURL := &url.URL{
		Scheme: "ws",
		Host:   s.registerIP + ":" + strconv.Itoa(s.Config.Port),
		Path:   "/",
	}

	connected, err := s.Kontrol.DialForever()
	if err != nil {
		s.Server.Log.Fatal("Cannot dial kontrol: %s", err.Error())
	}
	s.Server.Start()
	go func() {
		<-connected
		s.Registration.RegisterToKontrol(registerWithURL)
	}()
}

func (s *KodingKite) Run() {
	s.Start()
	<-s.Server.CloseNotify()
}

func (s *KodingKite) Close() {
	s.Kontrol.Close()
	s.Server.Close()
}
<commit_msg>kite: move unexported func below exported ones<commit_after>package kodingkite

import (
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"

	kodingconfig "koding/tools/config"

	"github.com/koding/kite"
	kiteconfig "github.com/koding/kite/config"
	"github.com/koding/kite/kontrolclient"
	"github.com/koding/kite/registration"
	"github.com/koding/kite/server"
	"github.com/koding/logging"
)

type KodingKite struct {
	*server.Server
	Kontrol      *kontrolclient.KontrolClient
	Registration *registration.Registration
	KodingConfig *kodingconfig.Config
	registerIP   string
}

// New returns a new kite instance based on for the given Koding configurations
func New(kodingConf *kodingconfig.Config, name, version string) (*KodingKite, error) {
	kiteConf, err := kiteconfig.Get()
	if err != nil {
		return nil, err
	}

	kiteConf.Environment = kodingConf.Environment

	k := kite.New(name, version)
	k.Config = kiteConf

	server := server.New(k)

	kon := kontrolclient.New(k)

	kk := &KodingKite{
		Server:       server,
		Kontrol:      kon,
		Registration: registration.New(kon),
		KodingConfig: kodingConf,
	}

	kk.registerIP, err = getRegisterIP(kodingConf.Environment)
	if err != nil {
		return nil, err
	}

	syslog, err := logging.NewSyslogHandler(name)
	if err != nil {
		log.Fatalf("Cannot connect to syslog: %s", err.Error())
	}

	kk.Log.SetHandler(logging.NewMultiHandler(logging.StderrHandler, syslog))

	return kk, nil
}

func (s *KodingKite) Start() {
	s.Log.Info("Kite has started: %s", s.Kite.Kite())

	registerWithURL := &url.URL{
		Scheme: "ws",
		Host:   s.registerIP + ":" + strconv.Itoa(s.Config.Port),
		Path:   "/",
	}

	connected, err := s.Kontrol.DialForever()
	if err != nil {
		s.Server.Log.Fatal("Cannot dial kontrol: %s", err.Error())
	}
	s.Server.Start()
	go func() {
		<-connected
		s.Registration.RegisterToKontrol(registerWithURL)
	}()
}

func (s *KodingKite) Run() {
	s.Start()
	<-s.Server.CloseNotify()
}

func (s *KodingKite) Close() {
	s.Kontrol.Close()
	s.Server.Close()
}

func getRegisterIP(environment string) (string, error) {
	var ip string

	switch environment {
	case "production":
		fallthrough
	case "staging":
		// Magical IP address of Openstack Metadata Service
		// http://docs.openstack.org/grizzly/openstack-compute/admin/content/metadata-service.html
		resp, err := http.Get("http://169.254.169.254/latest/meta-data/public-ipv4")
		if err != nil {
			return "", errors.New("cannot get public IP address: " + err.Error())
		}

		defer resp.Body.Close()

		if resp.StatusCode != 200 {
			return "", errors.New("unexpected status code: " + resp.Status)
		}

		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", err
		}

		ip = string(body)
	case "vagrant":
		ip = "127.0.0.1"
	default:
		return "", errors.New("I don't know which IP address to register in this environment: " + environment)
	}

	if ip == "" {
		return "", errors.New("no suitable IP address is found")
	}

	return ip, nil
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"time"

	"github.com/koding/bongo"
)

type Interaction struct {
	// unique identifier of the Interaction
	Id int64 `json:"id"`

	// Id of the interacted message
	MessageId int64 `json:"messageId"`

	// Id of the actor
	AccountId int64 `json:"accountId"`

	// Type of the interaction
	Type string `json:"type"`

	// Creation of the interaction
	CreatedAt time.Time `json:"createdAt"`
}

var AllowedInteractions = map[string]struct{}{
	"like":     struct{}{},
	"upvote":   struct{}{},
	"downvote": struct{}{},
}

const (
	Interaction_TYPE_LIKE     = "like"
	Interaction_TYPE_UPVOTE   = "upvote"
	Interaction_TYPE_DONWVOTE = "downvote"
)

func (i *Interaction) GetId() int64 {
	return i.Id
}

func (i *Interaction) TableName() string {
	return "interaction"
}

func (i *Interaction) Self() bongo.Modellable {
	return i
}

func NewInteraction() *Interaction {
	return &Interaction{}
}

func (i *Interaction) Fetch() error {
	return bongo.B.Fetch(i)
}

func (i *Interaction) Create() error {
	return bongo.B.Create(i)
}

func (i *Interaction) Delete() error {
	if err := bongo.B.DB.
		Where("message_id = ? and account_id = ?", i.MessageId, i.AccountId).
		Delete(NewInteraction()).Error; err != nil {
		return err
	}
	return nil
}

func (c *Interaction) List(interactionType string) ([]int64, error) {
	var interactions []int64

	if c.MessageId == 0 {
		return interactions, errors.New("ChannelId is not set")
	}

	if err := bongo.B.DB.Table(c.TableName()).
		Where("message_id = ?", c.MessageId).
		Pluck("account_id", &interactions).
		Error; err != nil {
		return nil, err
	}

	if interactions == nil {
		return make([]int64, 0), nil
	}

	// change this part to use c.m.some

	// selector := map[string]interface{}{
	// 	"message_id": c.MessageId,
	// }

	// pluck := map[string]interface{}{
	// 	"account_id": true,
	// }

	// err := c.m.Some(c, &interactions, selector, nil, pluck)
	// if err != nil && err != gorm.RecordNotFound {
	// 	return nil, err
	// }

	return interactions, nil
}
<commit_msg>Social: add plugins for realtime events<commit_after>package models

import (
	"errors"
	"time"

	"github.com/koding/bongo"
)

type Interaction struct {
	// unique identifier of the Interaction
	Id int64 `json:"id"`

	// Id of the interacted message
	MessageId int64 `json:"messageId"`

	// Id of the actor
	AccountId int64 `json:"accountId"`

	// Type of the interaction
	Type string `json:"type"`

	// Creation of the interaction
	CreatedAt time.Time `json:"createdAt"`
}

var AllowedInteractions = map[string]struct{}{
	"like":     struct{}{},
	"upvote":   struct{}{},
	"downvote": struct{}{},
}

const (
	Interaction_TYPE_LIKE     = "like"
	Interaction_TYPE_UPVOTE   = "upvote"
	Interaction_TYPE_DONWVOTE = "downvote"
)

func (i *Interaction) GetId() int64 {
	return i.Id
}

func (i *Interaction) TableName() string {
	return "interaction"
}

func (i *Interaction) Self() bongo.Modellable {
	return i
}

func NewInteraction() *Interaction {
	return &Interaction{}
}

func (i *Interaction) Fetch() error {
	return bongo.B.Fetch(i)
}

func (i *Interaction) Create() error {
	return bongo.B.Create(i)
}

func (i *Interaction) AfterCreate() {
	bongo.B.AfterCreate(i)
}

func (i *Interaction) AfterUpdate() {
	bongo.B.AfterUpdate(i)
}

func (i *Interaction) AfterDelete() {
	bongo.B.AfterDelete(i)
}

func (i *Interaction) Delete() error {
	if err := bongo.B.DB.
		Where("message_id = ? and account_id = ?", i.MessageId, i.AccountId).
		Delete(NewInteraction()).Error; err != nil {
		return err
	}
	return nil
}

func (c *Interaction) List(interactionType string) ([]int64, error) {
	var interactions []int64

	if c.MessageId == 0 {
		return interactions, errors.New("ChannelId is not set")
	}

	if err := bongo.B.DB.Table(c.TableName()).
		Where("message_id = ?", c.MessageId).
		Pluck("account_id", &interactions).
		Error; err != nil {
		return nil, err
	}

	if interactions == nil {
		return make([]int64, 0), nil
	}

	// change this part to use c.m.some

	// selector := map[string]interface{}{
	// 	"message_id": c.MessageId,
	// }

	// pluck := map[string]interface{}{
	// 	"account_id": true,
	// }

	// err := c.m.Some(c, &interactions, selector, nil, pluck)
	// if err != nil && err != gorm.RecordNotFound {
	// 	return nil, err
	// }

	return interactions, nil
}
<|endoftext|>


<commit_before>package models

import (
	kodingmodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"math/rand"
	"strconv"
	"time"

	. "github.com/smartystreets/goconvey/convey"
	"labix.org/v2/mgo/bson"
)

func CreateChannelWithParticipants() (*Channel, []*Account) {
	account1 := CreateAccountWithTest()
	account2 := CreateAccountWithTest()
	account3 := CreateAccountWithTest()
	accounts := []*Account{account1, account2, account3}

	channel := CreateChannelWithTest(account1.Id)
	AddParticipants(channel.Id, account1.Id, account2.Id, account3.Id)

	return channel, accounts
}

func CreateChannelWithTest(accountId int64) *Channel {
	// create and account instance
	channel := NewChannel()
	channel.CreatorId = accountId

	err := channel.Create()
	So(err, ShouldBeNil)

	return channel
}

func CreateMessage(channelId, accountId int64, typeConstant string) *ChannelMessage {
	cm := NewChannelMessage()

	cm.AccountId = accountId
	// set channel id
	cm.InitialChannelId = channelId
	cm.TypeConstant = typeConstant
	// set body
	cm.Body = "testing message"

	err := cm.Create()
	So(err, ShouldBeNil)

	return cm
}

func CreateTrollMessage(channelId, accountId int64, typeConstant string) *ChannelMessage {
	cm := NewChannelMessage()

	cm.AccountId = accountId
	// set channel id
	cm.InitialChannelId = channelId
	cm.TypeConstant = typeConstant
	// set body
	cm.Body = "testing message"
	cm.MetaBits = Troll

	err := cm.Create()
	So(err, ShouldBeNil)

	return cm
}

func AddParticipants(channelId int64, accountIds ...int64) {

	for _, accountId := range accountIds {
		participant := NewChannelParticipant()
		participant.ChannelId = channelId
		participant.AccountId = accountId

		err := participant.Create()
		So(err, ShouldBeNil)
	}
}

func createAccount() (*Account, error) {
	// create and account instance
	author := NewAccount()

	// create a fake mongo id
	oldId := bson.NewObjectId()
	// assign it to our test user
	author.OldId = oldId.Hex()

	// seed the random data generator
	rand.Seed(time.Now().UnixNano())

	author.Nick = "malitest" + strconv.Itoa(rand.Intn(10e9))

	if err := author.Create(); err != nil {
		return nil, err
	}

	return author, nil
}

func CreateAccountWithTest() *Account {
	account, err := createAccount()
	So(err, ShouldBeNil)
	So(account, ShouldNotBeNil)
	So(account.Id, ShouldNotEqual, 0)
	return account
}

func createChannel(accountId int64) (*Channel, error) {
	// create and account instance
	channel := NewChannel()
	channel.CreatorId = accountId

	if err := channel.Create(); err != nil {
		return nil, err
	}

	return channel, nil
}

func createMessageWithTest() *ChannelMessage {
	cm := NewChannelMessage()

	// init account
	account, err := createAccount()
	So(err, ShouldBeNil)
	So(account, ShouldNotBeNil)
	So(account.Id, ShouldNotEqual, 0)
	// init channel
	channel, err := createChannel(account.Id)
	So(err, ShouldBeNil)
	So(channel, ShouldNotBeNil)

	// set account id
	cm.AccountId = account.Id
	// set channel id
	cm.InitialChannelId = channel.Id
	// set body
	cm.Body = "5five"
	return cm
}

func FetchOrCreateSession(nick string) (*kodingmodels.Session, error) {
	session, err := modelhelper.GetOneSessionForAccount(nick)
	if err == nil {
		return session, nil
	}

	return modelhelper.CreateSessionForAccount(nick)
}

func CreateAccountInBothDbs() (*Account, error) {
	return CreateAccountInBothDbsWithNick(bson.NewObjectId().Hex())
}

func CreateAccountInBothDbsWithNick(nick string) (*Account, error) {
	accId := bson.NewObjectId()
	accHex := nick

	oldAcc := &kodingmodels.Account{
		Id: accId,
		Profile: struct {
			Nickname  string `bson:"nickname" json:"nickname"`
			FirstName string `bson:"firstName" json:"firstName"`
			LastName  string `bson:"lastName" json:"lastName"`
			Hash      string `bson:"hash" json:"hash"`
		}{
			Nickname: accHex,
		},
	}

	err := modelhelper.CreateAccount(oldAcc)
	if err != nil {
		return nil, err
	}

	oldUser := &kodingmodels.User{
		ObjectId:       bson.NewObjectId(),
		Password:       accHex,
		Salt:           accHex,
		Name:           accHex,
		Email:          accHex,
		EmailFrequency: kodingmodels.EmailFrequency{},
	}

	err = modelhelper.CreateUser(oldUser)
	if err != nil {
		return nil, err
	}

	a := NewAccount()
	a.Nick = accHex
	a.OldId = accId.Hex()
	if err := a.Create(); err != nil {
		return nil, err
	}

	return a, nil
}
<commit_msg>Socialapi: use nick variable while setting nick of account<commit_after>package models

import (
	kodingmodels "koding/db/models"
	"koding/db/mongodb/modelhelper"
	"math/rand"
	"strconv"
	"time"

	. "github.com/smartystreets/goconvey/convey"
	"labix.org/v2/mgo/bson"
)

func CreateChannelWithParticipants() (*Channel, []*Account) {
	account1 := CreateAccountWithTest()
	account2 := CreateAccountWithTest()
	account3 := CreateAccountWithTest()
	accounts := []*Account{account1, account2, account3}

	channel := CreateChannelWithTest(account1.Id)
	AddParticipants(channel.Id, account1.Id, account2.Id, account3.Id)

	return channel, accounts
}

func CreateChannelWithTest(accountId int64) *Channel {
	// create and account instance
	channel := NewChannel()
	channel.CreatorId = accountId

	err := channel.Create()
	So(err, ShouldBeNil)

	return channel
}

func CreateMessage(channelId, accountId int64, typeConstant string) *ChannelMessage {
	cm := NewChannelMessage()

	cm.AccountId = accountId
	// set channel id
	cm.InitialChannelId = channelId
	cm.TypeConstant = typeConstant
	// set body
	cm.Body = "testing message"

	err := cm.Create()
	So(err, ShouldBeNil)

	return cm
}

func CreateTrollMessage(channelId, accountId int64, typeConstant string) *ChannelMessage {
	cm := NewChannelMessage()

	cm.AccountId = accountId
	// set channel id
	cm.InitialChannelId = channelId
	cm.TypeConstant = typeConstant
	// set body
	cm.Body = "testing message"
	cm.MetaBits = Troll

	err := cm.Create()
	So(err, ShouldBeNil)

	return cm
}

func AddParticipants(channelId int64, accountIds ...int64) {

	for _, accountId := range accountIds {
		participant := NewChannelParticipant()
		participant.ChannelId = channelId
		participant.AccountId = accountId

		err := participant.Create()
		So(err, ShouldBeNil)
	}
}

func createAccount() (*Account, error) {
	// create and account instance
	author := NewAccount()

	// create a fake mongo id
	oldId := bson.NewObjectId()
	// assign it to our test user
	author.OldId = oldId.Hex()

	// seed the random data generator
	rand.Seed(time.Now().UnixNano())

	author.Nick = "malitest" + strconv.Itoa(rand.Intn(10e9))

	if err := author.Create(); err != nil {
		return nil, err
	}

	return author, nil
}

func CreateAccountWithTest() *Account {
	account, err := createAccount()
	So(err, ShouldBeNil)
	So(account, ShouldNotBeNil)
	So(account.Id, ShouldNotEqual, 0)
	return account
}

func createChannel(accountId int64) (*Channel, error) {
	// create and account instance
	channel := NewChannel()
	channel.CreatorId = accountId

	if err := channel.Create(); err != nil {
		return nil, err
	}

	return channel, nil
}

func createMessageWithTest() *ChannelMessage {
	cm := NewChannelMessage()

	// init account
	account, err := createAccount()
	So(err, ShouldBeNil)
	So(account, ShouldNotBeNil)
	So(account.Id, ShouldNotEqual, 0)
	// init channel
	channel, err := createChannel(account.Id)
	So(err, ShouldBeNil)
	So(channel, ShouldNotBeNil)

	// set account id
	cm.AccountId = account.Id
	// set channel id
	cm.InitialChannelId = channel.Id
	// set body
	cm.Body = "5five"
	return cm
}

func FetchOrCreateSession(nick string) (*kodingmodels.Session, error) {
	session, err := modelhelper.GetOneSessionForAccount(nick)
	if err == nil {
		return session, nil
	}

	return modelhelper.CreateSessionForAccount(nick)
}

func CreateAccountInBothDbs() (*Account, error) {
	return CreateAccountInBothDbsWithNick(bson.NewObjectId().Hex())
}

func CreateAccountInBothDbsWithNick(nick string) (*Account, error) {
	accId := bson.NewObjectId()
	accHex := nick

	oldAcc := &kodingmodels.Account{
		Id: accId,
		Profile: struct {
			Nickname  string `bson:"nickname" json:"nickname"`
			FirstName string `bson:"firstName" json:"firstName"`
			LastName  string `bson:"lastName" json:"lastName"`
			Hash      string `bson:"hash" json:"hash"`
		}{
			Nickname: nick,
		},
	}

	err := modelhelper.CreateAccount(oldAcc)
	if err != nil {
		return nil, err
	}

	oldUser := &kodingmodels.User{
		ObjectId:       bson.NewObjectId(),
		Password:       accHex,
		Salt:           accHex,
		Name:           nick,
		Email:          accHex + "@koding.com",
		EmailFrequency: kodingmodels.EmailFrequency{},
	}

	err = modelhelper.CreateUser(oldUser)
	if err != nil {
		return nil, err
	}

	a := NewAccount()
	a.Nick = nick
	a.OldId = accId.Hex()
	if err := a.Create(); err != nil {
		return nil, err
	}

	return a, nil
}
<|endoftext|>


<commit_before>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package externalaccount

import (
	"bytes"
	"context"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"golang.org/x/oauth2"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"sort"
	"strings"
	"time"
)

type awsSecurityCredentials struct {
	AccessKeyID     string `json:"AccessKeyID"`
	SecretAccessKey string `json:"SecretAccessKey"`
	SecurityToken   string `json:"Token"`
}

// awsRequestSigner is a utility class to sign http requests using a AWS V4 signature.
type awsRequestSigner struct {
	RegionName             string
	AwsSecurityCredentials awsSecurityCredentials
}

// getenv aliases os.Getenv for testing
var getenv = os.Getenv

const (
	// AWS Signature Version 4 signing algorithm identifier.
	awsAlgorithm = "AWS4-HMAC-SHA256"

	// The termination string for the AWS credential scope value as defined in
	// https://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
	awsRequestType = "aws4_request"

	// The AWS authorization header name for the security session token if available.
	awsSecurityTokenHeader = "x-amz-security-token"

	// The AWS authorization header name for the auto-generated date.
	awsDateHeader = "x-amz-date"

	awsTimeFormatLong  = "20060102T150405Z"
	awsTimeFormatShort = "20060102"
)

func getSha256(input []byte) (string, error) {
	hash := sha256.New()
	if _, err := hash.Write(input); err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(nil)), nil
}

func getHmacSha256(key, input []byte) ([]byte, error) {
	hash := hmac.New(sha256.New, key)
	if _, err := hash.Write(input); err != nil {
		return nil, err
	}
	return hash.Sum(nil), nil
}

func cloneRequest(r *http.Request) *http.Request {
	r2 := new(http.Request)
	*r2 = *r
	if r.Header != nil {
		r2.Header = make(http.Header, len(r.Header))

		// Find total number of values.
		headerCount := 0
		for _, headerValues := range r.Header {
			headerCount += len(headerValues)
		}
		copiedHeaders := make([]string, headerCount) // shared backing array for headers' values

		for headerKey, headerValues := range r.Header {
			headerCount = copy(copiedHeaders, headerValues)
			r2.Header[headerKey] = copiedHeaders[:headerCount:headerCount]
			copiedHeaders = copiedHeaders[headerCount:]
		}
	}
	return r2
}

func canonicalPath(req *http.Request) string {
	result := req.URL.EscapedPath()
	if result == "" {
		return "/"
	}
	return path.Clean(result)
}

func canonicalQuery(req *http.Request) string {
	queryValues := req.URL.Query()
	for queryKey := range queryValues {
		sort.Strings(queryValues[queryKey])
	}
	return queryValues.Encode()
}

func canonicalHeaders(req *http.Request) (string, string) {
	// Header keys need to be sorted alphabetically.
	var headers []string
	lowerCaseHeaders := make(http.Header)
	for k, v := range req.Header {
		k := strings.ToLower(k)
		if _, ok := lowerCaseHeaders[k]; ok {
			// include additional values
			lowerCaseHeaders[k] = append(lowerCaseHeaders[k], v...)
		} else {
			headers = append(headers, k)
			lowerCaseHeaders[k] = v
		}
	}
	sort.Strings(headers)

	var fullHeaders bytes.Buffer
	for _, header := range headers {
		headerValue := strings.Join(lowerCaseHeaders[header], ",")
		fullHeaders.WriteString(header)
		fullHeaders.WriteRune(':')
		fullHeaders.WriteString(headerValue)
		fullHeaders.WriteRune('\n')
	}

	return strings.Join(headers, ";"), fullHeaders.String()
}

func requestDataHash(req *http.Request) (string, error) {
	var requestData []byte
	if req.Body != nil {
		requestBody, err := req.GetBody()
		if err != nil {
			return "", err
		}
		defer requestBody.Close()

		requestData, err = ioutil.ReadAll(io.LimitReader(requestBody, 1<<20))
		if err != nil {
			return "", err
		}
	}

	return getSha256(requestData)
}

func requestHost(req *http.Request) string {
	if req.Host != "" {
		return req.Host
	}
	return req.URL.Host
}

func canonicalRequest(req *http.Request, canonicalHeaderColumns, canonicalHeaderData string) (string, error) {
	dataHash, err := requestDataHash(req)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", req.Method, canonicalPath(req), canonicalQuery(req), canonicalHeaderData, canonicalHeaderColumns, dataHash), nil
}

// SignRequest adds the appropriate headers to an http.Request
// or returns an error if something prevented this.
func (rs *awsRequestSigner) SignRequest(req *http.Request) error {
	signedRequest := cloneRequest(req)
	timestamp := now()

	signedRequest.Header.Add("host", requestHost(req))

	if rs.AwsSecurityCredentials.SecurityToken != "" {
		signedRequest.Header.Add(awsSecurityTokenHeader, rs.AwsSecurityCredentials.SecurityToken)
	}

	if signedRequest.Header.Get("date") == "" {
		signedRequest.Header.Add(awsDateHeader, timestamp.Format(awsTimeFormatLong))
	}

	authorizationCode, err := rs.generateAuthentication(signedRequest, timestamp)
	if err != nil {
		return err
	}
	signedRequest.Header.Set("Authorization", authorizationCode)

	req.Header = signedRequest.Header
	return nil
}

func (rs *awsRequestSigner) generateAuthentication(req *http.Request, timestamp time.Time) (string, error) {
	canonicalHeaderColumns, canonicalHeaderData := canonicalHeaders(req)

	dateStamp := timestamp.Format(awsTimeFormatShort)
	serviceName := ""
	if splitHost := strings.Split(requestHost(req), "."); len(splitHost) > 0 {
		serviceName = splitHost[0]
	}

	credentialScope := fmt.Sprintf("%s/%s/%s/%s", dateStamp, rs.RegionName, serviceName, awsRequestType)

	requestString, err := canonicalRequest(req, canonicalHeaderColumns, canonicalHeaderData)
	if err != nil {
		return "", err
	}
	requestHash, err := getSha256([]byte(requestString))
	if err != nil {
		return "", err
	}

	stringToSign := fmt.Sprintf("%s\n%s\n%s\n%s", awsAlgorithm, timestamp.Format(awsTimeFormatLong), credentialScope, requestHash)

	signingKey := []byte("AWS4" + rs.AwsSecurityCredentials.SecretAccessKey)
	for _, signingInput := range []string{
		dateStamp, rs.RegionName, serviceName, awsRequestType, stringToSign,
	} {
		signingKey, err = getHmacSha256(signingKey, []byte(signingInput))
		if err != nil {
			return "", err
		}
	}

	return fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", awsAlgorithm, rs.AwsSecurityCredentials.AccessKeyID, credentialScope, canonicalHeaderColumns, hex.EncodeToString(signingKey)), nil
}

type awsCredentialSource struct {
	EnvironmentID               string
	RegionURL                   string
	RegionalCredVerificationURL string
	CredVerificationURL         string
	TargetResource              string
	requestSigner               *awsRequestSigner
	region                      string
	ctx                         context.Context
	client                      *http.Client
}

type awsRequestHeader struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type awsRequest struct {
	URL     string             `json:"url"`
	Method  string             `json:"method"`
	Headers []awsRequestHeader `json:"headers"`
}

func (cs awsCredentialSource) doRequest(req *http.Request) (*http.Response, error) {
	if cs.client == nil {
		cs.client = oauth2.NewClient(cs.ctx, nil)
	}
	return cs.client.Do(req.WithContext(cs.ctx))
}

func (cs awsCredentialSource) subjectToken() (string, error) {
	if cs.requestSigner == nil {
		awsSecurityCredentials, err := cs.getSecurityCredentials()
		if err != nil {
			return "", err
		}

		if cs.region, err = cs.getRegion(); err != nil {
			return "", err
		}

		cs.requestSigner = &awsRequestSigner{
			RegionName:             cs.region,
			AwsSecurityCredentials: awsSecurityCredentials,
		}
	}

	// Generate the signed request to AWS STS GetCallerIdentity API.
	// Use the required regional endpoint. Otherwise, the request will fail.
	req, err := http.NewRequest("POST", strings.Replace(cs.RegionalCredVerificationURL, "{region}", cs.region, 1), nil)
	if err != nil {
		return "", err
	}
	// The full, canonical resource name of the workload identity pool
	// provider, with or without the HTTPS prefix.
	// Including this header as part of the signature is recommended to
	// ensure data integrity.
	if cs.TargetResource != "" {
		req.Header.Add("x-goog-cloud-target-resource", cs.TargetResource)
	}
	cs.requestSigner.SignRequest(req)

	/*
	   The GCP STS endpoint expects the headers to be formatted as:
	   # [
	   #   {key: 'x-amz-date', value: '...'},
	   #   {key: 'Authorization', value: '...'},
	   #   ...
	   # ]
	   # And then serialized as:
	   # quote(json.dumps({
	   #   url: '...',
	   #   method: 'POST',
	   #   headers: [{key: 'x-amz-date', value: '...'}, ...]
	   # }))
	*/

	awsSignedReq := awsRequest{
		URL:    req.URL.String(),
		Method: "POST",
	}
	for headerKey, headerList := range req.Header {
		for _, headerValue := range headerList {
			awsSignedReq.Headers = append(awsSignedReq.Headers, awsRequestHeader{
				Key:   headerKey,
				Value: headerValue,
			})
		}
	}
	sort.Slice(awsSignedReq.Headers, func(i, j int) bool {
		headerCompare := strings.Compare(awsSignedReq.Headers[i].Key, awsSignedReq.Headers[j].Key)
		if headerCompare == 0 {
			return strings.Compare(awsSignedReq.Headers[i].Value, awsSignedReq.Headers[j].Value) < 0
		}
		return headerCompare < 0
	})

	result, err := json.Marshal(awsSignedReq)
	if err != nil {
		return "", err
	}
	return url.QueryEscape(string(result)), nil
}

func (cs *awsCredentialSource) getRegion() (string, error) {
	if envAwsRegion := getenv("AWS_REGION"); envAwsRegion != "" {
		return envAwsRegion, nil
	} if envAwsRegion := getenv("AWS_DEFAULT_REGION"); envAwsRegion != "" {
		return envAwsRegion, nil
	}

	if cs.RegionURL == "" {
		return "", errors.New("oauth2/google: unable to determine AWS region")
	}

	req, err := http.NewRequest("GET", cs.RegionURL, nil)
	if err != nil {
		return "", err
	}

	resp, err := cs.doRequest(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if err != nil {
		return "", err
	}

	if resp.StatusCode != 200 {
		return "", fmt.Errorf("oauth2/google: unable to retrieve AWS region - %s", string(respBody))
	}

	// This endpoint will return the region in format: us-east-2b.
	// Only the us-east-2 part should be used.
	respBodyEnd := 0
	if len(respBody) > 1 {
		respBodyEnd = len(respBody) - 1
	}
	return string(respBody[:respBodyEnd]), nil
}

func (cs *awsCredentialSource) getSecurityCredentials() (result awsSecurityCredentials, err error) {
	if accessKeyID := getenv("AWS_ACCESS_KEY_ID"); accessKeyID != "" {
		if secretAccessKey := getenv("AWS_SECRET_ACCESS_KEY"); secretAccessKey != "" {
			return awsSecurityCredentials{
				AccessKeyID:     accessKeyID,
				SecretAccessKey: secretAccessKey,
				SecurityToken:   getenv("AWS_SESSION_TOKEN"),
			}, nil
		}
	}

	roleName, err := cs.getMetadataRoleName()
	if err != nil {
		return
	}

	credentials, err := cs.getMetadataSecurityCredentials(roleName)
	if err != nil {
		return
	}

	if credentials.AccessKeyID == "" {
		return result, errors.New("oauth2/google: missing AccessKeyId credential")
	}

	if credentials.SecretAccessKey == "" {
		return result, errors.New("oauth2/google: missing SecretAccessKey credential")
	}

	return credentials, nil
}

func (cs *awsCredentialSource) getMetadataSecurityCredentials(roleName string) (awsSecurityCredentials, error) {
	var result awsSecurityCredentials

	req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", cs.CredVerificationURL, roleName), nil)
	if err != nil {
		return result, err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := cs.doRequest(req)
	if err != nil {
		return result, err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if err != nil {
		return result, err
	}

	if resp.StatusCode != 200 {
		return result, fmt.Errorf("oauth2/google: unable to retrieve AWS security credentials - %s", string(respBody))
	}

	err = json.Unmarshal(respBody, &result)
	return result, err
}

func (cs *awsCredentialSource) getMetadataRoleName() (string, error) {
	if cs.CredVerificationURL == "" {
		return "", errors.New("oauth2/google: unable to determine the AWS metadata server security credentials endpoint")
	}

	req, err := http.NewRequest("GET", cs.CredVerificationURL, nil)
	if err != nil {
		return "", err
	}

	resp, err := cs.doRequest(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if err != nil {
		return "", err
	}

	if resp.StatusCode != 200 {
		return "", fmt.Errorf("oauth2/google: unable to retrieve AWS role name - %s", string(respBody))
	}

	return string(respBody), nil
}
<commit_msg>google: fix syntax error<commit_after>// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package externalaccount

import (
	"bytes"
	"context"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path"
	"sort"
	"strings"
	"time"

	"golang.org/x/oauth2"
)

type awsSecurityCredentials struct {
	AccessKeyID     string `json:"AccessKeyID"`
	SecretAccessKey string `json:"SecretAccessKey"`
	SecurityToken   string `json:"Token"`
}

// awsRequestSigner is a utility class to sign http requests using a AWS V4 signature.
type awsRequestSigner struct {
	RegionName             string
	AwsSecurityCredentials awsSecurityCredentials
}

// getenv aliases os.Getenv for testing
var getenv = os.Getenv

const (
	// AWS Signature Version 4 signing algorithm identifier.
	awsAlgorithm = "AWS4-HMAC-SHA256"

	// The termination string for the AWS credential scope value as defined in
	// https://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
	awsRequestType = "aws4_request"

	// The AWS authorization header name for the security session token if available.
	awsSecurityTokenHeader = "x-amz-security-token"

	// The AWS authorization header name for the auto-generated date.
	awsDateHeader = "x-amz-date"

	awsTimeFormatLong  = "20060102T150405Z"
	awsTimeFormatShort = "20060102"
)

func getSha256(input []byte) (string, error) {
	hash := sha256.New()
	if _, err := hash.Write(input); err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(nil)), nil
}

func getHmacSha256(key, input []byte) ([]byte, error) {
	hash := hmac.New(sha256.New, key)
	if _, err := hash.Write(input); err != nil {
		return nil, err
	}
	return hash.Sum(nil), nil
}

func cloneRequest(r *http.Request) *http.Request {
	r2 := new(http.Request)
	*r2 = *r
	if r.Header != nil {
		r2.Header = make(http.Header, len(r.Header))

		// Find total number of values.
		headerCount := 0
		for _, headerValues := range r.Header {
			headerCount += len(headerValues)
		}
		copiedHeaders := make([]string, headerCount) // shared backing array for headers' values

		for headerKey, headerValues := range r.Header {
			headerCount = copy(copiedHeaders, headerValues)
			r2.Header[headerKey] = copiedHeaders[:headerCount:headerCount]
			copiedHeaders = copiedHeaders[headerCount:]
		}
	}
	return r2
}

func canonicalPath(req *http.Request) string {
	result := req.URL.EscapedPath()
	if result == "" {
		return "/"
	}
	return path.Clean(result)
}

func canonicalQuery(req *http.Request) string {
	queryValues := req.URL.Query()
	for queryKey := range queryValues {
		sort.Strings(queryValues[queryKey])
	}
	return queryValues.Encode()
}

func canonicalHeaders(req *http.Request) (string, string) {
	// Header keys need to be sorted alphabetically.
	var headers []string
	lowerCaseHeaders := make(http.Header)
	for k, v := range req.Header {
		k := strings.ToLower(k)
		if _, ok := lowerCaseHeaders[k]; ok {
			// include additional values
			lowerCaseHeaders[k] = append(lowerCaseHeaders[k], v...)
		} else {
			headers = append(headers, k)
			lowerCaseHeaders[k] = v
		}
	}
	sort.Strings(headers)

	var fullHeaders bytes.Buffer
	for _, header := range headers {
		headerValue := strings.Join(lowerCaseHeaders[header], ",")
		fullHeaders.WriteString(header)
		fullHeaders.WriteRune(':')
		fullHeaders.WriteString(headerValue)
		fullHeaders.WriteRune('\n')
	}

	return strings.Join(headers, ";"), fullHeaders.String()
}

func requestDataHash(req *http.Request) (string, error) {
	var requestData []byte
	if req.Body != nil {
		requestBody, err := req.GetBody()
		if err != nil {
			return "", err
		}
		defer requestBody.Close()

		requestData, err = ioutil.ReadAll(io.LimitReader(requestBody, 1<<20))
		if err != nil {
			return "", err
		}
	}

	return getSha256(requestData)
}

func requestHost(req *http.Request) string {
	if req.Host != "" {
		return req.Host
	}
	return req.URL.Host
}

func canonicalRequest(req *http.Request, canonicalHeaderColumns, canonicalHeaderData string) (string, error) {
	dataHash, err := requestDataHash(req)
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", req.Method, canonicalPath(req), canonicalQuery(req), canonicalHeaderData, canonicalHeaderColumns, dataHash), nil
}

// SignRequest adds the appropriate headers to an http.Request
// or returns an error if something prevented this.
func (rs *awsRequestSigner) SignRequest(req *http.Request) error {
	signedRequest := cloneRequest(req)
	timestamp := now()

	signedRequest.Header.Add("host", requestHost(req))

	if rs.AwsSecurityCredentials.SecurityToken != "" {
		signedRequest.Header.Add(awsSecurityTokenHeader, rs.AwsSecurityCredentials.SecurityToken)
	}

	if signedRequest.Header.Get("date") == "" {
		signedRequest.Header.Add(awsDateHeader, timestamp.Format(awsTimeFormatLong))
	}

	authorizationCode, err := rs.generateAuthentication(signedRequest, timestamp)
	if err != nil {
		return err
	}
	signedRequest.Header.Set("Authorization", authorizationCode)

	req.Header = signedRequest.Header
	return nil
}

func (rs *awsRequestSigner) generateAuthentication(req *http.Request, timestamp time.Time) (string, error) {
	canonicalHeaderColumns, canonicalHeaderData := canonicalHeaders(req)

	dateStamp := timestamp.Format(awsTimeFormatShort)
	serviceName := ""
	if splitHost := strings.Split(requestHost(req), "."); len(splitHost) > 0 {
		serviceName = splitHost[0]
	}

	credentialScope := fmt.Sprintf("%s/%s/%s/%s", dateStamp, rs.RegionName, serviceName, awsRequestType)

	requestString, err := canonicalRequest(req, canonicalHeaderColumns, canonicalHeaderData)
	if err != nil {
		return "", err
	}
	requestHash, err := getSha256([]byte(requestString))
	if err != nil {
		return "", err
	}

	stringToSign := fmt.Sprintf("%s\n%s\n%s\n%s", awsAlgorithm, timestamp.Format(awsTimeFormatLong), credentialScope, requestHash)

	signingKey := []byte("AWS4" + rs.AwsSecurityCredentials.SecretAccessKey)
	for _, signingInput := range []string{
		dateStamp, rs.RegionName, serviceName, awsRequestType, stringToSign,
	} {
		signingKey, err = getHmacSha256(signingKey, []byte(signingInput))
		if err != nil {
			return "", err
		}
	}

	return fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", awsAlgorithm, rs.AwsSecurityCredentials.AccessKeyID, credentialScope, canonicalHeaderColumns, hex.EncodeToString(signingKey)), nil
}

type awsCredentialSource struct {
	EnvironmentID               string
	RegionURL                   string
	RegionalCredVerificationURL string
	CredVerificationURL         string
	TargetResource              string
	requestSigner               *awsRequestSigner
	region                      string
	ctx                         context.Context
	client                      *http.Client
}

type awsRequestHeader struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type awsRequest struct {
	URL     string             `json:"url"`
	Method  string             `json:"method"`
	Headers []awsRequestHeader `json:"headers"`
}

func (cs awsCredentialSource) doRequest(req *http.Request) (*http.Response, error) {
	if cs.client == nil {
		cs.client = oauth2.NewClient(cs.ctx, nil)
	}
	return cs.client.Do(req.WithContext(cs.ctx))
}

func (cs awsCredentialSource) subjectToken() (string, error) {
	if cs.requestSigner == nil {
		awsSecurityCredentials, err := cs.getSecurityCredentials()
		if err != nil {
			return "", err
		}

		if cs.region, err = cs.getRegion(); err != nil {
			return "", err
		}

		cs.requestSigner = &awsRequestSigner{
			RegionName:             cs.region,
			AwsSecurityCredentials: awsSecurityCredentials,
		}
	}

	// Generate the signed request to AWS STS GetCallerIdentity API.
	// Use the required regional endpoint. Otherwise, the request will fail.
	req, err := http.NewRequest("POST", strings.Replace(cs.RegionalCredVerificationURL, "{region}", cs.region, 1), nil)
	if err != nil {
		return "", err
	}
	// The full, canonical resource name of the workload identity pool
	// provider, with or without the HTTPS prefix.
	// Including this header as part of the signature is recommended to
	// ensure data integrity.
	if cs.TargetResource != "" {
		req.Header.Add("x-goog-cloud-target-resource", cs.TargetResource)
	}
	cs.requestSigner.SignRequest(req)

	/*
	   The GCP STS endpoint expects the headers to be formatted as:
	   # [
	   #   {key: 'x-amz-date', value: '...'},
	   #   {key: 'Authorization', value: '...'},
	   #   ...
	   # ]
	   # And then serialized as:
	   # quote(json.dumps({
	   #   url: '...',
	   #   method: 'POST',
	   #   headers: [{key: 'x-amz-date', value: '...'}, ...]
	   # }))
	*/

	awsSignedReq := awsRequest{
		URL:    req.URL.String(),
		Method: "POST",
	}
	for headerKey, headerList := range req.Header {
		for _, headerValue := range headerList {
			awsSignedReq.Headers = append(awsSignedReq.Headers, awsRequestHeader{
				Key:   headerKey,
				Value: headerValue,
			})
		}
	}
	sort.Slice(awsSignedReq.Headers, func(i, j int) bool {
		headerCompare := strings.Compare(awsSignedReq.Headers[i].Key, awsSignedReq.Headers[j].Key)
		if headerCompare == 0 {
			return strings.Compare(awsSignedReq.Headers[i].Value, awsSignedReq.Headers[j].Value) < 0
		}
		return headerCompare < 0
	})

	result, err := json.Marshal(awsSignedReq)
	if err != nil {
		return "", err
	}
	return url.QueryEscape(string(result)), nil
}

func (cs *awsCredentialSource) getRegion() (string, error) {
	if envAwsRegion := getenv("AWS_REGION"); envAwsRegion != "" {
		return envAwsRegion, nil
	}
	if envAwsRegion := getenv("AWS_DEFAULT_REGION"); envAwsRegion != "" {
		return envAwsRegion, nil
	}

	if cs.RegionURL == "" {
		return "", errors.New("oauth2/google: unable to determine AWS region")
	}

	req, err := http.NewRequest("GET", cs.RegionURL, nil)
	if err != nil {
		return "", err
	}

	resp, err := cs.doRequest(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if err != nil {
		return "", err
	}

	if resp.StatusCode != 200 {
		return "", fmt.Errorf("oauth2/google: unable to retrieve AWS region - %s", string(respBody))
	}

	// This endpoint will return the region in format: us-east-2b.
	// Only the us-east-2 part should be used.
	respBodyEnd := 0
	if len(respBody) > 1 {
		respBodyEnd = len(respBody) - 1
	}
	return string(respBody[:respBodyEnd]), nil
}

func (cs *awsCredentialSource) getSecurityCredentials() (result awsSecurityCredentials, err error) {
	if accessKeyID := getenv("AWS_ACCESS_KEY_ID"); accessKeyID != "" {
		if secretAccessKey := getenv("AWS_SECRET_ACCESS_KEY"); secretAccessKey != "" {
			return awsSecurityCredentials{
				AccessKeyID:     accessKeyID,
				SecretAccessKey: secretAccessKey,
				SecurityToken:   getenv("AWS_SESSION_TOKEN"),
			}, nil
		}
	}

	roleName, err := cs.getMetadataRoleName()
	if err != nil {
		return
	}

	credentials, err := cs.getMetadataSecurityCredentials(roleName)
	if err != nil {
		return
	}

	if credentials.AccessKeyID == "" {
		return result, errors.New("oauth2/google: missing AccessKeyId credential")
	}

	if credentials.SecretAccessKey == "" {
		return result, errors.New("oauth2/google: missing SecretAccessKey credential")
	}

	return credentials, nil
}

func (cs *awsCredentialSource) getMetadataSecurityCredentials(roleName string) (awsSecurityCredentials, error) {
	var result awsSecurityCredentials

	req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", cs.CredVerificationURL, roleName), nil)
	if err != nil {
		return result, err
	}
	req.Header.Add("Content-Type", "application/json")

	resp, err := cs.doRequest(req)
	if err != nil {
		return result, err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if err != nil {
		return result, err
	}

	if resp.StatusCode != 200 {
		return result, fmt.Errorf("oauth2/google: unable to retrieve AWS security credentials - %s", string(respBody))
	}

	err = json.Unmarshal(respBody, &result)
	return result, err
}

func (cs *awsCredentialSource) getMetadataRoleName() (string, error) {
	if cs.CredVerificationURL == "" {
		return "", errors.New("oauth2/google: unable to determine the AWS metadata server security credentials endpoint")
	}

	req, err := http.NewRequest("GET", cs.CredVerificationURL, nil)
	if err != nil {
		return "", err
	}

	resp, err := cs.doRequest(req)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	respBody, err := ioutil.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if err != nil {
		return "", err
	}

	if resp.StatusCode != 200 {
		return "", fmt.Errorf("oauth2/google: unable to retrieve AWS role name - %s", string(respBody))
	}

	return string(respBody), nil
}
<|endoftext|>


<commit_before>package sudoku

type SolveTechnique interface {
	Name() string
	Description(*SolveStep) string
	//IMPORTANT: a step should return a step IFF that step is valid AND the step would cause useful work to be done if applied.

	//Find returns as many steps as it can find in the grid for that technique. This helps ensure that when we pick a step,
	//it's more likely to be an "easy" step because there will be more of them at any time.
	Find(*Grid) []*SolveStep
	IsFill() bool
	//How difficult a real human would say this technique is. Generally inversely related to how often a real person would pick it. 0.0 to 1.0.
	Difficulty() float64
}

type basicSolveTechnique struct {
	name       string
	isFill     bool
	difficulty float64
}

func init() {

	//TODO: calculate more realistic weights.

	Techniques = []SolveTechnique{
		hiddenSingleInRow{
			basicSolveTechnique{
				"Necessary In Row",
				true,
				0.0,
			},
		},
		hiddenSingleInCol{
			basicSolveTechnique{
				"Necessary In Col",
				true,
				0.0,
			},
		},
		hiddenSingleInBlock{
			basicSolveTechnique{
				"Necessary In Block",
				true,
				0.0,
			},
		},
		nakedSingleTechnique{
			basicSolveTechnique{
				"Only Legal Number",
				true,
				5.0,
			},
		},
		pointingPairRow{
			basicSolveTechnique{
				"Pointing Pair Row",
				false,
				25.0,
			},
		},
		pointingPairCol{
			basicSolveTechnique{
				"Pointing Pair Col",
				false,
				25.0,
			},
		},
		nakedPairCol{
			basicSolveTechnique{
				"Naked Pair Col",
				false,
				75.0,
			},
		},
		nakedPairRow{
			basicSolveTechnique{
				"Naked Pair Row",
				false,
				75.0,
			},
		},
		nakedPairBlock{
			basicSolveTechnique{
				"Naked Pair Block",
				false,
				85.0,
			},
		},
		nakedTripleCol{
			basicSolveTechnique{
				"Naked Triple Col",
				false,
				125.0,
			},
		},
		nakedTripleRow{
			basicSolveTechnique{
				"Naked Triple Row",
				false,
				125.0,
			},
		},
		nakedTripleBlock{
			basicSolveTechnique{
				"Naked Triple Block",
				false,
				140.0,
			},
		},
		hiddenPairRow{
			basicSolveTechnique{
				"Hidden Pair Row",
				false,
				300.0,
			},
		},
		hiddenPairCol{
			basicSolveTechnique{
				"Hidden Pair Col",
				false,
				300.0,
			},
		},
		hiddenPairBlock{
			basicSolveTechnique{
				"Hidden Pair Block",
				false,
				250.0,
			},
		},
	}
}

func (self basicSolveTechnique) Name() string {
	return self.name
}

func (self basicSolveTechnique) IsFill() bool {
	return self.isFill
}

func (self basicSolveTechnique) Difficulty() float64 {
	return self.difficulty
}

//This is useful both for hidden and naked subset techniques
func subsetIndexes(len int, size int) [][]int {
	//Given size of array to generate subset for, and size of desired subset, returns an array of all subset-indexes to try.
	//Sanity check
	if size > len {
		return nil
	}

	//returns an array of slices of size size that give you all of the subsets of a list of length len
	result := make([][]int, 0)
	counters := make([]int, size)
	for i, _ := range counters {
		counters[i] = i
	}
	for {
		innerResult := make([]int, size)
		for i, counter := range counters {
			innerResult[i] = counter
		}
		result = append(result, innerResult)
		//Now, increment.
		//Start at the end and try to increment each counter one.
		incremented := false
		for i := size - 1; i >= 0; i-- {

			counter := counters[i]
			if counter < len-(size-i) {
				//Found one!
				counters[i]++
				incremented = true
				if i < size-1 {
					//It was an inner counter; need to set all of the higher counters to one above the one to the left.
					base := counters[i] + 1
					for j := i + 1; j < size; j++ {
						counters[j] = base
						base++
					}
				}
				break
			}
		}
		//If we couldn't increment any, there's nothing to do.
		if !incremented {
			break
		}
	}
	return result
}
<commit_msg>Added an orienting comment in hs_techniques.go<commit_after>package sudoku

/*
	This file is where the basic solve technique infrastructure is defined.

	The techniques that Human Solve uses are initalized and stored into human_solves.go Techniques slice here.

	Specific techniques are implemented in hst_*.go files (hst == human solve techniques), where there's
	a separate file for each class of technique.

*/

type SolveTechnique interface {
	Name() string
	Description(*SolveStep) string
	//IMPORTANT: a step should return a step IFF that step is valid AND the step would cause useful work to be done if applied.

	//Find returns as many steps as it can find in the grid for that technique. This helps ensure that when we pick a step,
	//it's more likely to be an "easy" step because there will be more of them at any time.
	Find(*Grid) []*SolveStep
	IsFill() bool
	//How difficult a real human would say this technique is. Generally inversely related to how often a real person would pick it. 0.0 to 1.0.
	Difficulty() float64
}

type basicSolveTechnique struct {
	name       string
	isFill     bool
	difficulty float64
}

func init() {

	//TODO: calculate more realistic weights.

	Techniques = []SolveTechnique{
		hiddenSingleInRow{
			basicSolveTechnique{
				"Necessary In Row",
				true,
				0.0,
			},
		},
		hiddenSingleInCol{
			basicSolveTechnique{
				"Necessary In Col",
				true,
				0.0,
			},
		},
		hiddenSingleInBlock{
			basicSolveTechnique{
				"Necessary In Block",
				true,
				0.0,
			},
		},
		nakedSingleTechnique{
			basicSolveTechnique{
				"Only Legal Number",
				true,
				5.0,
			},
		},
		pointingPairRow{
			basicSolveTechnique{
				"Pointing Pair Row",
				false,
				25.0,
			},
		},
		pointingPairCol{
			basicSolveTechnique{
				"Pointing Pair Col",
				false,
				25.0,
			},
		},
		nakedPairCol{
			basicSolveTechnique{
				"Naked Pair Col",
				false,
				75.0,
			},
		},
		nakedPairRow{
			basicSolveTechnique{
				"Naked Pair Row",
				false,
				75.0,
			},
		},
		nakedPairBlock{
			basicSolveTechnique{
				"Naked Pair Block",
				false,
				85.0,
			},
		},
		nakedTripleCol{
			basicSolveTechnique{
				"Naked Triple Col",
				false,
				125.0,
			},
		},
		nakedTripleRow{
			basicSolveTechnique{
				"Naked Triple Row",
				false,
				125.0,
			},
		},
		nakedTripleBlock{
			basicSolveTechnique{
				"Naked Triple Block",
				false,
				140.0,
			},
		},
		hiddenPairRow{
			basicSolveTechnique{
				"Hidden Pair Row",
				false,
				300.0,
			},
		},
		hiddenPairCol{
			basicSolveTechnique{
				"Hidden Pair Col",
				false,
				300.0,
			},
		},
		hiddenPairBlock{
			basicSolveTechnique{
				"Hidden Pair Block",
				false,
				250.0,
			},
		},
	}
}

func (self basicSolveTechnique) Name() string {
	return self.name
}

func (self basicSolveTechnique) IsFill() bool {
	return self.isFill
}

func (self basicSolveTechnique) Difficulty() float64 {
	return self.difficulty
}

//This is useful both for hidden and naked subset techniques
func subsetIndexes(len int, size int) [][]int {
	//Given size of array to generate subset for, and size of desired subset, returns an array of all subset-indexes to try.
	//Sanity check
	if size > len {
		return nil
	}

	//returns an array of slices of size size that give you all of the subsets of a list of length len
	result := make([][]int, 0)
	counters := make([]int, size)
	for i, _ := range counters {
		counters[i] = i
	}
	for {
		innerResult := make([]int, size)
		for i, counter := range counters {
			innerResult[i] = counter
		}
		result = append(result, innerResult)
		//Now, increment.
		//Start at the end and try to increment each counter one.
		incremented := false
		for i := size - 1; i >= 0; i-- {

			counter := counters[i]
			if counter < len-(size-i) {
				//Found one!
				counters[i]++
				incremented = true
				if i < size-1 {
					//It was an inner counter; need to set all of the higher counters to one above the one to the left.
					base := counters[i] + 1
					for j := i + 1; j < size; j++ {
						counters[j] = base
						base++
					}
				}
				break
			}
		}
		//If we couldn't increment any, there's nothing to do.
		if !incremented {
			break
		}
	}
	return result
}
<|endoftext|>


<commit_before>package translations_test

import (
	"github.com/cloudfoundry/jibber_jabber"

	. "github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
)

var _ = Describe("i18n support and language detection", func() {
	BeforeEach(func() {
		userLocale, err := jibber_jabber.DetectIETF()
		Expect(err).NotTo(HaveOccurred())
		Expect(userLocale).To(Equal("fr-FR"), "This test can only be run when the system's language is set to french")
	})

	It("returns the french translation for cf quota", func() {
		Eventually(Cf("help", "quota")).Should(Say("Montrez l'information de quota"))
	})
})
<commit_msg>Update french test per updated translations<commit_after>package translations_test

import (
	"github.com/cloudfoundry/jibber_jabber"

	. "github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
)

var _ = Describe("i18n support and language detection", func() {
	BeforeEach(func() {
		userLocale, err := jibber_jabber.DetectIETF()
		Expect(err).NotTo(HaveOccurred())
		Expect(userLocale).To(Equal("fr-FR"), "This test can only be run when the system's language is set to french")
	})

	It("returns the french translation for cf quota", func() {
		Eventually(Cf("help", "quota")).Should(Say("Afficher les informations de quota"))
	})
})
<|endoftext|>


<commit_before>package fs

import (
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/gabrielperezs/monad"

	"github.com/gallir/radix.improved/redis"
	"github.com/gallir/smart-relayer/lib"
)

// Server is the thread that listen for clients' connections
type Server struct {
	sync.Mutex
	config   lib.RelayerConfig
	done     chan bool
	exiting  bool
	reseting bool
	listener net.Listener
	C        chan *Msg
	shards   uint32

	desired int
	writers chan *writer
	monad   *monad.Monad

	lastError time.Time
	errors    int64

	s3sess *session.Session
}

var (
	sep     = []byte("\t")
	newLine = []byte("\n")
)

var (
	errBadCmd      = errors.New("ERR bad command")
	errKO          = errors.New("fatal error")
	errSet         = errors.New("ERR - syntax: SET project key [timestamp] value")
	errGet         = errors.New("ERR - syntax: GET project key [timestamp]")
	errChanFull    = errors.New("ERR - The file can't be created")
	errNotFound    = errors.New("KO - Key not found")
	errClosing     = errors.New("ERR - Daemon closing")
	respOK         = redis.NewRespSimple("OK")
	respTrue       = redis.NewResp(1)
	respBadCommand = redis.NewResp(errBadCmd)
	respKO         = redis.NewResp(errKO)
	respBadSet     = redis.NewResp(errSet)
	respBadGet     = redis.NewResp(errGet)
	respChanFull   = redis.NewResp(errChanFull)
	respNotFound   = redis.NewResp(errNotFound)
	respClosing    = redis.NewResp(errClosing)
	commands       map[string]*redis.Resp

	defaultShards                = 64
	defaultMinWriters            = 2
	defaultMaxWriters            = 256
	defaultInterval              = 500 * time.Millisecond
	defaultWriterCooldown        = 15 * time.Second
	defaultWriterThresholdWarmUp = 50.0 // percent
	defaultBuffer                = 20000
	defaultPath                  = "/tmp"
)

func init() {
	commands = map[string]*redis.Resp{
		"PING": respOK,
		"SET":  respOK,
		"GET":  respOK,
	}
}

// New creates a new Redis local server
func New(c lib.RelayerConfig, done chan bool) (*Server, error) {
	srv := &Server{
		done:    done,
		errors:  0,
		writers: make(chan *writer, defaultMaxWriters),
	}

	srv.Reload(&c)

	return srv, nil
}

// Reload the configuration
func (srv *Server) Reload(c *lib.RelayerConfig) (err error) {
	srv.Lock()
	defer srv.Unlock()

	srv.config = *c
	if srv.config.Buffer == 0 {
		srv.config.Buffer = defaultBuffer
	}

	if srv.config.MaxConnections == 0 {
		srv.config.MaxConnections = defaultMaxWriters
	}

	if srv.config.Path == "" {
		srv.config.Path = defaultPath
	}

	// If Shards is 0 we manage it as undefined so we apply the default value
	if srv.config.Shards == 0 {
		srv.config.Shards = defaultShards
	}
	// If the Shards are higher than 0 (could be by default or defined by the config file)
	// In case the shards are defined with a value less than 0 we will manage it as disabled
	// definding srv.shards as 0, look in function srv.path
	if srv.config.Shards > 0 {
		srv.shards = uint32(srv.config.Shards)
	}

	if srv.C == nil {
		srv.C = make(chan *Msg, srv.config.Buffer)
	}

	if cap(srv.C) < srv.config.Buffer {
		log.Printf("FS WARNING: the new buffer size %d is bigger than the real capacity of the channel %d", srv.config.Buffer, cap(srv.C))
		srv.config.Buffer = cap(srv.C)
	}

	awsOpt := session.Options{
		Profile: srv.config.Profile,
		Config: aws.Config{
			Region: &srv.config.Region,
		},
	}

	if sess, err := session.NewSessionWithOptions(awsOpt); err == nil {
		srv.s3sess = sess
	} else {
		log.Printf("FS ERROR: invalid S3 session: %s", err)
		srv.s3sess = nil
	}

	monadCfg := &monad.Config{
		Min:            uint64(defaultMinWriters),
		Max:            uint64(srv.config.MaxConnections),
		Interval:       defaultInterval,
		CoolDownPeriod: defaultWriterCooldown,
		WarmFn: func() bool {
			l := float64(len(srv.C))
			if l == 0 {
				return false
			}

			currPtc := (l / float64(srv.config.Buffer)) * 100

			lib.Debugf("FS: colddown Queue %d/%d (%.0f%%/%.0f%%), Workers %d/%d",
				len(srv.C), cap(srv.C), currPtc, defaultWriterThresholdWarmUp, srv.desired, srv.config.MaxConnections)

			if currPtc > defaultWriterThresholdWarmUp {
				return true
			}
			return false
		},
		DesireFn: func(n uint64) {
			srv.desired = int(n)
			lw := len(srv.writers)

			if lw == srv.desired {
				log.Printf("FS: concurrent writers %d", lw)
				return
			}

			defer func() {
				log.Printf("FS %s clients %d/%d, in the queue %d/%d", srv.config.Listen, len(srv.writers), srv.config.MaxConnections, len(srv.C), srv.config.Buffer)
			}()

			if lw > srv.desired {
				for i := srv.desired; i < lw; i++ {
					w := <-srv.writers
					// exit without blocking
					go w.exit()
				}
				return
			}

			if lw < srv.desired {
				for i := lw; i < srv.desired; i++ {
					srv.writers <- newWriter(srv)
				}
				return
			}
		},
	}

	if srv.monad == nil {
		srv.monad = monad.New(monadCfg)
	} else {
		go srv.monad.Reload(monadCfg)
	}

	log.Printf("FS %s config Writers %d/%d Buffer %d Shard %d",
		srv.config.Listen, defaultMinWriters, srv.config.MaxConnections, srv.config.Buffer, srv.shards)

	return nil
}

// Start accepts incoming connections on the Listener
func (srv *Server) Start() (e error) {
	srv.Lock()
	defer srv.Unlock()

	srv.listener, e = lib.NewListener(srv.config)
	if e != nil {
		return e
	}

	// Serve clients
	go func(l net.Listener) {
		defer srv.listener.Close()
		for {
			netConn, e := l.Accept()
			if e != nil {
				if netErr, ok := e.(net.Error); ok && netErr.Timeout() {
					// Paranoid, ignore timeout errors
					log.Println("FS ERROR: timeout at local listener", srv.config.ListenHost(), e)
					continue
				}
				if srv.exiting {
					log.Println("FS: exiting local listener", srv.config.ListenHost())
					return
				}
				log.Fatalln("FS ERROR: emergency error in local listener", srv.config.ListenHost(), e)
				return
			}
			go srv.handleConnection(netConn)
		}
	}(srv.listener)

	return
}

// Exit closes the listener and send done to main
func (srv *Server) Exit() {
	srv.exiting = true

	if srv.listener != nil {
		srv.listener.Close()
	}

	// Close the channel were we store the active writers
	close(srv.writers)
	for w := range srv.writers {
		w.exit()
	}

	// Close the main channel, all writers will finish
	close(srv.C)

	// finishing the server
	srv.done <- true
}

func (srv *Server) handleConnection(netCon net.Conn) {

	defer netCon.Close()

	reader := redis.NewRespReader(netCon)

	for {

		r := reader.Read()

		if r.IsType(redis.IOErr) {
			if redis.IsTimeout(r) {
				// Paranoid, don't close it just log it
				log.Println("FS ERROR: Local client listen timeout at", srv.config.Listen)
				continue
			}
			// Connection was closed
			return
		}

		req := lib.NewRequest(r, &srv.config)
		if req == nil {
			respBadCommand.WriteTo(netCon)
			continue
		}

		fastResponse, ok := commands[req.Command]
		if !ok {
			respBadCommand.WriteTo(netCon)
			continue
		}

		switch req.Command {
		case "PING":
			fastResponse.WriteTo(netCon)
		case "SET":
			// SET project key [timestamp] value
			if len(req.Items) <= 3 || len(req.Items) > 5 {
				respBadSet.WriteTo(netCon)
				continue
			}

			if err := srv.set(netCon, req.Items); err != nil {
				switch err {
				case errChanFull:
					log.Printf("FS ERROR SET: channel full %d", len(srv.C))
					respChanFull.WriteTo(netCon)
				default:
					log.Printf("FS ERROR SET: %s", err)
					redis.NewResp(err).WriteTo(netCon)
				}
			}
		case "GET":
			// GET project key [timestamp]
			if len(req.Items) <= 2 || len(req.Items) > 4 {
				respBadGet.WriteTo(netCon)
				continue
			}

			if err := srv.get(netCon, req.Items); err != nil {
				switch err {
				case err.(*os.PathError):
					respNotFound.WriteTo(netCon)
				default:
					log.Printf("FS ERROR GET: %s", err)
					redis.NewResp(err).WriteTo(netCon)
				}
			}
		default:
			log.Panicf("FS ERROR: Invalid command: This never should happen, check the cases or the list of valid command")

		}

	}
}

func (srv *Server) set(netCon net.Conn, items []*redis.Resp) (err error) {

	// Get a message struct from the pool
	msg := getMsg(srv)

	// Find the project name
	msg.project, err = items[1].Str()
	if err != nil {
		return err
	}

	// Find the key name of the message
	msg.k, err = items[2].Str()
	if err != nil {
		return err
	}

	if len(items) == 4 {
		// If have 4 items means that the client sent the content without timestamp
		// in this case the message will have the current timestamp
		if b, err := items[3].Bytes(); err == nil {
			msg.b.Write(b)
		} else {
			return err
		}
		// Define the current timestamp
		msg.t = time.Now()
	} else {
		// If have more than 4 items we read timestamp
		if i, err := items[3].Int64(); err == nil {
			msg.t = time.Unix(i, 0)
		} else {
			return err
		}

		if b, err := items[4].Bytes(); err == nil {
			msg.b.Write(b)
		} else {
			return err
		}
	}

	r := fmt.Sprintf("%s/%s", msg.fullpath(), msg.filename())

	// If the server is configured in sync mode we will try to write
	// the file directly and respons to the client with the result
	if srv.config.Mode == "sync" {
		defer putMsg(msg)

		w := &writer{srv: srv}
		if err := w.writeTo(msg); err != nil {
			redis.NewResp(err).WriteTo(netCon)
			return err
		}
		redis.NewResp(r).WriteTo(netCon)
		return nil
	}

	// When the process have to close the chan srv.C can be closed
	// before arrive to this part of the code. For this reason we
	// check if the server is closing and respond an error message
	// to avoild a race condition trying to send a message to closed
	// channel
	if srv.exiting {
		respClosing.WriteTo(netCon)
		return nil
	}

	// Try to send the message to the chan srv.C if is full we response
	// with the chan full message
	select {
	case srv.C <- msg:
		redis.NewResp(r).WriteTo(netCon)
		return nil
	default:
		return errChanFull
	}
}

func (srv *Server) get(netCon net.Conn, items []*redis.Resp) (err error) {
	msg := getMsg(srv)
	defer putMsg(msg)

	// Find the project name
	msg.project, err = items[1].Str()
	if err != nil {
		return err
	}

	// Find the key name
	msg.k, err = items[2].Str()
	if err != nil {
		return err
	}

	// Verify if the 3th item is a int64 value to be converted in time
	if len(items) > 3 {
		if i, err := items[3].Int64(); err == nil {
			msg.t = time.Unix(i, 0)
		} else {
			return err
		}
	}

	var b []byte
	b, err = msg.Bytes()
	if err != nil {
		return err
	}

	redis.NewResp(b).WriteTo(netCon)
	return nil
}
<commit_msg>Message log in Reload event<commit_after>package fs

import (
	"errors"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/gabrielperezs/monad"

	"github.com/gallir/radix.improved/redis"
	"github.com/gallir/smart-relayer/lib"
)

// Server is the thread that listen for clients' connections
type Server struct {
	sync.Mutex
	config   lib.RelayerConfig
	done     chan bool
	exiting  bool
	reseting bool
	listener net.Listener
	C        chan *Msg
	shards   uint32

	desired int
	writers chan *writer
	monad   *monad.Monad

	lastError time.Time
	errors    int64

	s3sess *session.Session
}

var (
	sep     = []byte("\t")
	newLine = []byte("\n")
)

var (
	errBadCmd      = errors.New("ERR bad command")
	errKO          = errors.New("fatal error")
	errSet         = errors.New("ERR - syntax: SET project key [timestamp] value")
	errGet         = errors.New("ERR - syntax: GET project key [timestamp]")
	errChanFull    = errors.New("ERR - The file can't be created")
	errNotFound    = errors.New("KO - Key not found")
	errClosing     = errors.New("ERR - Daemon closing")
	respOK         = redis.NewRespSimple("OK")
	respTrue       = redis.NewResp(1)
	respBadCommand = redis.NewResp(errBadCmd)
	respKO         = redis.NewResp(errKO)
	respBadSet     = redis.NewResp(errSet)
	respBadGet     = redis.NewResp(errGet)
	respChanFull   = redis.NewResp(errChanFull)
	respNotFound   = redis.NewResp(errNotFound)
	respClosing    = redis.NewResp(errClosing)
	commands       map[string]*redis.Resp

	defaultShards                = 64
	defaultMinWriters            = 2
	defaultMaxWriters            = 256
	defaultInterval              = 500 * time.Millisecond
	defaultWriterCooldown        = 15 * time.Second
	defaultWriterThresholdWarmUp = 50.0 // percent
	defaultBuffer                = 20000
	defaultPath                  = "/tmp"
)

func init() {
	commands = map[string]*redis.Resp{
		"PING": respOK,
		"SET":  respOK,
		"GET":  respOK,
	}
}

// New creates a new Redis local server
func New(c lib.RelayerConfig, done chan bool) (*Server, error) {
	srv := &Server{
		done:    done,
		errors:  0,
		writers: make(chan *writer, defaultMaxWriters),
	}

	srv.Reload(&c)

	return srv, nil
}

// Reload the configuration
func (srv *Server) Reload(c *lib.RelayerConfig) (err error) {
	srv.Lock()
	defer srv.Unlock()

	srv.config = *c
	if srv.config.Buffer == 0 {
		srv.config.Buffer = defaultBuffer
	}

	if srv.config.MaxConnections == 0 {
		srv.config.MaxConnections = defaultMaxWriters
	}

	if srv.config.Path == "" {
		srv.config.Path = defaultPath
	}

	// If Shards is 0 we manage it as undefined so we apply the default value
	if srv.config.Shards == 0 {
		srv.config.Shards = defaultShards
	}
	// If the Shards are higher than 0 (could be by default or defined by the config file)
	// In case the shards are defined with a value less than 0 we will manage it as disabled
	// definding srv.shards as 0, look in function srv.path
	if srv.config.Shards > 0 {
		srv.shards = uint32(srv.config.Shards)
	}

	if srv.C == nil {
		srv.C = make(chan *Msg, srv.config.Buffer)
	}

	if cap(srv.C) < srv.config.Buffer {
		log.Printf("FS WARNING: the new buffer size %d is bigger than the real capacity of the channel %d", srv.config.Buffer, cap(srv.C))
		srv.config.Buffer = cap(srv.C)
	}

	awsOpt := session.Options{
		Profile: srv.config.Profile,
		Config: aws.Config{
			Region: &srv.config.Region,
		},
	}

	if sess, err := session.NewSessionWithOptions(awsOpt); err == nil {
		srv.s3sess = sess
	} else {
		log.Printf("FS ERROR: invalid S3 session: %s", err)
		srv.s3sess = nil
	}

	monadCfg := &monad.Config{
		Min:            uint64(defaultMinWriters),
		Max:            uint64(srv.config.MaxConnections),
		Interval:       defaultInterval,
		CoolDownPeriod: defaultWriterCooldown,
		WarmFn: func() bool {
			l := float64(len(srv.C))
			if l == 0 {
				return false
			}

			currPtc := (l / float64(srv.config.Buffer)) * 100

			lib.Debugf("FS: colddown Queue %d/%d (%.0f%%/%.0f%%), Workers %d/%d",
				len(srv.C), cap(srv.C), currPtc, defaultWriterThresholdWarmUp, srv.desired, srv.config.MaxConnections)

			if currPtc > defaultWriterThresholdWarmUp {
				return true
			}
			return false
		},
		DesireFn: func(n uint64) {
			srv.desired = int(n)
			lw := len(srv.writers)

			if lw == srv.desired {
				log.Printf("FS: concurrent writers %d", lw)
				return
			}

			defer func() {
				log.Printf("FS %s clients %d/%d, in the queue %d/%d", srv.config.Listen, len(srv.writers), srv.config.MaxConnections, len(srv.C), srv.config.Buffer)
			}()

			if lw > srv.desired {
				for i := srv.desired; i < lw; i++ {
					w := <-srv.writers
					// exit without blocking
					go w.exit()
				}
				return
			}

			if lw < srv.desired {
				for i := lw; i < srv.desired; i++ {
					srv.writers <- newWriter(srv)
				}
				return
			}
		},
	}

	if srv.monad == nil {
		srv.monad = monad.New(monadCfg)
	} else {
		go srv.monad.Reload(monadCfg)
	}

	log.Printf("FS %s config Writers %d/%d Buffer %d/%d Shard %d",
		srv.config.Listen, defaultMinWriters, srv.config.MaxConnections, len(srv.C), srv.config.Buffer, srv.shards)

	return nil
}

// Start accepts incoming connections on the Listener
func (srv *Server) Start() (e error) {
	srv.Lock()
	defer srv.Unlock()

	srv.listener, e = lib.NewListener(srv.config)
	if e != nil {
		return e
	}

	// Serve clients
	go func(l net.Listener) {
		defer srv.listener.Close()
		for {
			netConn, e := l.Accept()
			if e != nil {
				if netErr, ok := e.(net.Error); ok && netErr.Timeout() {
					// Paranoid, ignore timeout errors
					log.Println("FS ERROR: timeout at local listener", srv.config.ListenHost(), e)
					continue
				}
				if srv.exiting {
					log.Println("FS: exiting local listener", srv.config.ListenHost())
					return
				}
				log.Fatalln("FS ERROR: emergency error in local listener", srv.config.ListenHost(), e)
				return
			}
			go srv.handleConnection(netConn)
		}
	}(srv.listener)

	return
}

// Exit closes the listener and send done to main
func (srv *Server) Exit() {
	srv.exiting = true

	if srv.listener != nil {
		srv.listener.Close()
	}

	// Close the channel were we store the active writers
	close(srv.writers)
	for w := range srv.writers {
		w.exit()
	}

	// Close the main channel, all writers will finish
	close(srv.C)

	// finishing the server
	srv.done <- true
}

func (srv *Server) handleConnection(netCon net.Conn) {

	defer netCon.Close()

	reader := redis.NewRespReader(netCon)

	for {

		r := reader.Read()

		if r.IsType(redis.IOErr) {
			if redis.IsTimeout(r) {
				// Paranoid, don't close it just log it
				log.Println("FS ERROR: Local client listen timeout at", srv.config.Listen)
				continue
			}
			// Connection was closed
			return
		}

		req := lib.NewRequest(r, &srv.config)
		if req == nil {
			respBadCommand.WriteTo(netCon)
			continue
		}

		fastResponse, ok := commands[req.Command]
		if !ok {
			respBadCommand.WriteTo(netCon)
			continue
		}

		switch req.Command {
		case "PING":
			fastResponse.WriteTo(netCon)
		case "SET":
			// SET project key [timestamp] value
			if len(req.Items) <= 3 || len(req.Items) > 5 {
				respBadSet.WriteTo(netCon)
				continue
			}

			if err := srv.set(netCon, req.Items); err != nil {
				switch err {
				case errChanFull:
					log.Printf("FS ERROR SET: channel full %d", len(srv.C))
					respChanFull.WriteTo(netCon)
				default:
					log.Printf("FS ERROR SET: %s", err)
					redis.NewResp(err).WriteTo(netCon)
				}
			}
		case "GET":
			// GET project key [timestamp]
			if len(req.Items) <= 2 || len(req.Items) > 4 {
				respBadGet.WriteTo(netCon)
				continue
			}

			if err := srv.get(netCon, req.Items); err != nil {
				switch err {
				case err.(*os.PathError):
					respNotFound.WriteTo(netCon)
				default:
					log.Printf("FS ERROR GET: %s", err)
					redis.NewResp(err).WriteTo(netCon)
				}
			}
		default:
			log.Panicf("FS ERROR: Invalid command: This never should happen, check the cases or the list of valid command")

		}

	}
}

func (srv *Server) set(netCon net.Conn, items []*redis.Resp) (err error) {

	// Get a message struct from the pool
	msg := getMsg(srv)

	// Find the project name
	msg.project, err = items[1].Str()
	if err != nil {
		return err
	}

	// Find the key name of the message
	msg.k, err = items[2].Str()
	if err != nil {
		return err
	}

	if len(items) == 4 {
		// If have 4 items means that the client sent the content without timestamp
		// in this case the message will have the current timestamp
		if b, err := items[3].Bytes(); err == nil {
			msg.b.Write(b)
		} else {
			return err
		}
		// Define the current timestamp
		msg.t = time.Now()
	} else {
		// If have more than 4 items we read timestamp
		if i, err := items[3].Int64(); err == nil {
			msg.t = time.Unix(i, 0)
		} else {
			return err
		}

		if b, err := items[4].Bytes(); err == nil {
			msg.b.Write(b)
		} else {
			return err
		}
	}

	r := fmt.Sprintf("%s/%s", msg.fullpath(), msg.filename())

	// If the server is configured in sync mode we will try to write
	// the file directly and respons to the client with the result
	if srv.config.Mode == "sync" {
		defer putMsg(msg)

		w := &writer{srv: srv}
		if err := w.writeTo(msg); err != nil {
			redis.NewResp(err).WriteTo(netCon)
			return err
		}
		redis.NewResp(r).WriteTo(netCon)
		return nil
	}

	// When the process have to close the chan srv.C can be closed
	// before arrive to this part of the code. For this reason we
	// check if the server is closing and respond an error message
	// to avoild a race condition trying to send a message to closed
	// channel
	if srv.exiting {
		respClosing.WriteTo(netCon)
		return nil
	}

	// Try to send the message to the chan srv.C if is full we response
	// with the chan full message
	select {
	case srv.C <- msg:
		redis.NewResp(r).WriteTo(netCon)
		return nil
	default:
		return errChanFull
	}
}

func (srv *Server) get(netCon net.Conn, items []*redis.Resp) (err error) {
	msg := getMsg(srv)
	defer putMsg(msg)

	// Find the project name
	msg.project, err = items[1].Str()
	if err != nil {
		return err
	}

	// Find the key name
	msg.k, err = items[2].Str()
	if err != nil {
		return err
	}

	// Verify if the 3th item is a int64 value to be converted in time
	if len(items) > 3 {
		if i, err := items[3].Int64(); err == nil {
			msg.t = time.Unix(i, 0)
		} else {
			return err
		}
	}

	var b []byte
	b, err = msg.Bytes()
	if err != nil {
		return err
	}

	redis.NewResp(b).WriteTo(netCon)
	return nil
}
<|endoftext|>