<commit_before>package status

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"sync"

	"github.com/byuoitav/av-api/base"
	"github.com/byuoitav/av-api/dbo"
	"github.com/byuoitav/configuration-database-microservice/accessors"
	"github.com/byuoitav/event-router-microservice/eventinfrastructure"
)

func GetRoomStatus(building string, roomName string) (base.PublicRoom, error) {

	//get room from database
	room, err := dbo.GetRoomByInfo(building, roomName)
	if err != nil {
		return base.PublicRoom{}, err
	}

	log.Printf("Generating commands...")
	commands, err := generateStatusCommands(room, DEFAULT_MAP)
	if err != nil {
		return base.PublicRoom{}, err
	}

	log.Printf("Running commands...")
	responses, err := runStatusCommands(commands)
	if err != nil {
		return base.PublicRoom{}, err
	}

	log.Printf("Evaluating Responses...")
	roomStatus, err := evaluateResponses(responses)
	if err != nil {
		return base.PublicRoom{}, err
	}

	roomStatus.Building = building
	roomStatus.Room = roomName

	return roomStatus, nil
}

func generateStatusCommands(room accessors.Room, commandMap map[string]StatusEvaluator) ([]StatusCommand, error) {

	var outputs []StatusCommand

	//iterate over each status evaluator
	for _, command := range room.Configuration.Evaluators {

		if strings.HasPrefix(command.EvaluatorKey, FLAG) {

			evaluator := DEFAULT_MAP[command.EvaluatorKey]

			//Idenify relevant devices
			devices, err := evaluator.GetDevices(room)
			if err != nil {
				return []StatusCommand{}, err
			}

			//Generate actions by iterating over the commands of each device
			commands, err := evaluator.GenerateCommands(devices)
			if err != nil {
				return []StatusCommand{}, err
			}

			//log.Printf("Appending commands: %v to action list", commands)
			outputs = append(outputs, commands...)
		}
	}

	//log.Printf("Final command output: %v", outputs)
	return outputs, nil
}

func runStatusCommands(commands []StatusCommand) (outputs []Status, err error) {
	//log.Printf("Commands: %v", commands)
	if len(commands) == 0 {
		err = errors.New("No commands")
		return
	}

	//map device names to commands
	commandMap := make(map[string][]StatusCommand)

	log.Printf("Building device map")
	for _, command := range commands {

		_, present := commandMap[command.Device.Name]
		if !present {
			commandMap[command.Device.Name] = []StatusCommand{command}
			log.Printf("Device %s identified", command.Device.Name)
		} else {
			commandMap[command.Device.Name] = append(commandMap[command.Device.Name], command)
		}

	}

	//make a channel with the same number of 'slots' as devices
	log.Printf("Creating channel")
	channel := make(chan Status, len(commandMap))
	var group sync.WaitGroup

	for device, deviceCommands := range commandMap {

		//spin up new go routine
		log.Printf("Starting new goroutine for device %s", device)
		group.Add(1)
		go issueCommands(deviceCommands, channel, &group)
	}

	log.Printf("Waiting for WaitGroup")
	group.Wait()
	log.Printf("done waiting")

	log.Printf("closing channel...")
	close(channel)

	for output := range channel {
		if output.ErrorMessage != nil {
			log.Printf("Error querying status with destination: %s", output.DestinationDevice.Device.Name)
			event := eventinfrastructure.Event{Event: "Status Retrieval",
				Success:  false,
				Building: output.DestinationDevice.Device.Building.Name,
				Room:     output.DestinationDevice.Device.Room.Name,
				Device:   output.DestinationDevice.Device.Name,
			}
			base.Publish(event)
		}
		log.Printf("Appending results of %s to output", output.DestinationDevice.Device.Name)
		outputs = append(outputs, output)
	}
	return
}

//builds a Status object corresponding to a device and writes it to the channel
func issueCommands(commands []StatusCommand, channel chan Status, control *sync.WaitGroup) {

	//add task to waitgroup

	//final output
	output := Status{DestinationDevice: commands[0].DestinationDevice}
	statuses := make(map[string]interface{})

	//iterate over list of StatusCommands
	//TODO:make sure devices can handle rapid-fire API requests
	for _, command := range commands {

		//build url
		url := command.Action.Microservice + command.Action.Endpoint.Path
		for formal, actual := range command.Parameters {
			toReplace := ":" + formal
			if !strings.Contains(url, toReplace) {
				errorMessage := "Could not find parameter " + toReplace + " issuing the command " + command.Action.Name
				output.ErrorMessage = &errorMessage
				log.Printf(errorMessage)
			} else {
				url = strings.Replace(url, toReplace, actual, -1)
			}
		}

		log.Printf("Sending requqest to %s", url)
		response, err := http.Get(url)
		if err != nil {
			errorMessage := err.Error()
			output.ErrorMessage = &errorMessage
			log.Printf("Error getting response from %s", command.Device.Name)
			continue
		}
		defer response.Body.Close()

		body, err := ioutil.ReadAll(response.Body)
		if err != nil {
			errorMessage := err.Error()
			output.ErrorMessage = &errorMessage
			log.Printf("Error reading response from %s", command.Device.Name)
			continue
		}
		log.Printf("Microservice returned: %s", body)

		var status map[string]interface{}
		err = json.Unmarshal(body, &status)
		if err != nil {
			errorMessage := err.Error()
			output.ErrorMessage = &errorMessage
			log.Printf("Error unmarshalling response from %s", command.Device.Name)
			continue
		}

		log.Printf("Copying data into output")
		for device, object := range status {
			statuses[device] = object
			log.Printf("%s maps to %v", device, object)
		}
	}

	//set the map of statuses to output
	output.Status = statuses
	//write output to channel
	log.Printf("writing output to channel")
	for key, value := range output.Status {
		log.Printf("%s maps to %v", key, value)
	}

	channel <- output
	log.Printf("done acquiring status for %s", output.DestinationDevice.Device.Name)
	control.Done()
}

func evaluateResponses(responses []Status) (base.PublicRoom, error) {

	var AudioDevices []base.AudioDevice
	var Displays []base.Display

	for _, device := range responses {

		log.Printf("Populating struct for device %s", device.DestinationDevice.Device.Name)

		log.Printf("Ranging over status")
		for key, value := range device.Status {
			log.Printf("Found status: %s with response %v", key, value)
		}

		if device.DestinationDevice.AudioDevice {

			var audioDevice base.AudioDevice

			//fixME make this look like the way we get power and input

			muted, ok := device.Status["muted"]
			mutedBool, ok := muted.(bool)
			if ok {
				audioDevice.Muted = &mutedBool
			}

			volume, ok := device.Status["volume"]
			volumeInt, ok := volume.(int)
			if ok {
				audioDevice.Volume = &volumeInt
			}

			power, ok := device.Status["power"]
			powerString, ok := power.(string)
			if ok {
				audioDevice.Power = powerString
			}

			input, ok := device.Status["input"]
			inputString, ok := input.(string)
			if ok {
				audioDevice.Input = inputString
			}

			audioDevice.Name = device.DestinationDevice.Device.Name

			log.Printf("Appending device: %s to AudioDevice array", audioDevice.Name)
			AudioDevices = append(AudioDevices, audioDevice)
		}

		if device.DestinationDevice.Display {
			var display base.Display

			blanked, ok := device.Status["blanked"]
			blankedBool, ok := blanked.(bool)
			if ok {
				display.Blanked = &blankedBool
			}

			power, ok := device.Status["power"]
			powerString, ok := power.(string)
			if ok {
				display.Power = powerString
			}

			input, ok := device.Status["input"]
			inputString, ok := input.(string)
			if ok {
				display.Input = inputString
			}

			display.Name = device.DestinationDevice.Device.Name

			log.Printf("Appending device: %s to Dispaly array", display.Name)
			Displays = append(Displays, display)

		}

	}

	return base.PublicRoom{Displays: Displays, AudioDevices: AudioDevices}, nil
}
<commit_msg>a little cleaner<commit_after>package status

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"sync"

	"github.com/byuoitav/av-api/base"
	"github.com/byuoitav/av-api/dbo"
	"github.com/byuoitav/configuration-database-microservice/accessors"
	"github.com/byuoitav/event-router-microservice/eventinfrastructure"
)

func GetRoomStatus(building string, roomName string) (base.PublicRoom, error) {

	room, err := dbo.GetRoomByInfo(building, roomName)
	if err != nil {
		return base.PublicRoom{}, err
	}

	commands, err := generateStatusCommands(room, DEFAULT_MAP)
	if err != nil {
		return base.PublicRoom{}, err
	}

	responses, err := runStatusCommands(commands)
	if err != nil {
		return base.PublicRoom{}, err
	}

	roomStatus, err := evaluateResponses(responses)
	if err != nil {
		return base.PublicRoom{}, err
	}

	roomStatus.Building = building
	roomStatus.Room = roomName

	return roomStatus, nil
}

func generateStatusCommands(room accessors.Room, commandMap map[string]StatusEvaluator) ([]StatusCommand, error) {

	log.Printf("Generating commands...")

	var output []StatusCommand

	for _, possibleEvaluator := range room.Configuration.Evaluators {

		if strings.HasPrefix(possibleEvaluator.EvaluatorKey, FLAG) {

			currentEvaluator := DEFAULT_MAP[possibleEvaluator.EvaluatorKey]

			devices, err := currentEvaluator.GetDevices(room)
			if err != nil {
				return []StatusCommand{}, err
			}

			commands, err := currentEvaluator.GenerateCommands(devices)
			if err != nil {
				return []StatusCommand{}, err
			}

			output = append(output, commands...)
		}
	}

	return output, nil
}

func runStatusCommands(commands []StatusCommand) (outputs []Status, err error) {

	log.Printf("Running commands...")

	if len(commands) == 0 {
		err = errors.New("No commands")
		return
	}

	//map device names to commands
	commandMap := make(map[string][]StatusCommand)

	log.Printf("Building device map")
	for _, command := range commands {

		_, present := commandMap[command.Device.Name]
		if !present {
			commandMap[command.Device.Name] = []StatusCommand{command}
			log.Printf("Device %s identified", command.Device.Name)
		} else {
			commandMap[command.Device.Name] = append(commandMap[command.Device.Name], command)
		}

	}

	log.Printf("Creating channel")
	channel := make(chan Status, len(commandMap))
	var group sync.WaitGroup

	for device, deviceCommands := range commandMap {
		group.Add(1)
		go issueCommands(deviceCommands, channel, &group)
	}

	log.Printf("Waiting for WaitGroup")
	group.Wait()
	log.Printf("done waiting")

	log.Printf("closing channel...")
	close(channel)

	for output := range channel {
		if output.ErrorMessage != nil {
			log.Printf("Error querying status with destination: %s", output.DestinationDevice.Device.Name)
			event := eventinfrastructure.Event{Event: "Status Retrieval",
				Success:  false,
				Building: output.DestinationDevice.Device.Building.Name,
				Room:     output.DestinationDevice.Device.Room.Name,
				Device:   output.DestinationDevice.Device.Name,
			}
			base.Publish(event)
		}
		log.Printf("Appending results of %s to output", output.DestinationDevice.Device.Name)
		outputs = append(outputs, output)
	}
	return
}

//builds a Status object corresponding to a device and writes it to the channel
func issueCommands(commands []StatusCommand, channel chan Status, control *sync.WaitGroup) {

	//add task to waitgroup

	//final output
	output := Status{DestinationDevice: commands[0].DestinationDevice}
	statuses := make(map[string]interface{})

	//iterate over list of StatusCommands
	//TODO:make sure devices can handle rapid-fire API requests
	for _, command := range commands {

		//build url
		url := command.Action.Microservice + command.Action.Endpoint.Path
		for formal, actual := range command.Parameters {
			toReplace := ":" + formal
			if !strings.Contains(url, toReplace) {
				errorMessage := "Could not find parameter " + toReplace + " issuing the command " + command.Action.Name
				output.ErrorMessage = &errorMessage
				log.Printf(errorMessage)
			} else {
				url = strings.Replace(url, toReplace, actual, -1)
			}
		}

		log.Printf("Sending requqest to %s", url)
		response, err := http.Get(url)
		if err != nil {
			errorMessage := err.Error()
			output.ErrorMessage = &errorMessage
			log.Printf("Error getting response from %s", command.Device.Name)
			continue
		}
		defer response.Body.Close()

		body, err := ioutil.ReadAll(response.Body)
		if err != nil {
			errorMessage := err.Error()
			output.ErrorMessage = &errorMessage
			log.Printf("Error reading response from %s", command.Device.Name)
			continue
		}
		log.Printf("Microservice returned: %s", body)

		var status map[string]interface{}
		err = json.Unmarshal(body, &status)
		if err != nil {
			errorMessage := err.Error()
			output.ErrorMessage = &errorMessage
			log.Printf("Error unmarshalling response from %s", command.Device.Name)
			continue
		}

		log.Printf("Copying data into output")
		for device, object := range status {
			statuses[device] = object
			log.Printf("%s maps to %v", device, object)
		}
	}

	//set the map of statuses to output
	output.Status = statuses
	//write output to channel
	log.Printf("writing output to channel")
	for key, value := range output.Status {
		log.Printf("%s maps to %v", key, value)
	}

	channel <- output
	log.Printf("done acquiring status for %s", output.DestinationDevice.Device.Name)
	control.Done()
}

func evaluateResponses(responses []Status) (base.PublicRoom, error) {

	log.Printf("Evaluating responses...")

	var AudioDevices []base.AudioDevice
	var Displays []base.Display

	for _, device := range responses {

		if device.DestinationDevice.AudioDevice {

			log.Printf("Adding audio device: %s", device.DestinationDevice.Device.Name)

			var audioDevice base.AudioDevice

			muted, ok := device.Status["muted"]
			mutedBool, ok := muted.(bool)
			if ok {
				audioDevice.Muted = &mutedBool
			}

			volume, ok := device.Status["volume"]
			volumeInt, ok := volume.(int)
			if ok {
				audioDevice.Volume = &volumeInt
			}

			power, ok := device.Status["power"]
			powerString, ok := power.(string)
			if ok {
				audioDevice.Power = powerString
			}

			input, ok := device.Status["input"]
			inputString, ok := input.(string)
			if ok {
				audioDevice.Input = inputString
			}

			audioDevice.Name = device.DestinationDevice.Device.Name

			AudioDevices = append(AudioDevices, audioDevice)
		}

		if device.DestinationDevice.Display {

			log.Printf("Adding display: %s", device.DestinationDevice.Device.Name)

			var display base.Display

			blanked, ok := device.Status["blanked"]
			blankedBool, ok := blanked.(bool)
			if ok {
				display.Blanked = &blankedBool
			}

			power, ok := device.Status["power"]
			powerString, ok := power.(string)
			if ok {
				display.Power = powerString
			}

			input, ok := device.Status["input"]
			inputString, ok := input.(string)
			if ok {
				display.Input = inputString
			}

			display.Name = device.DestinationDevice.Device.Name

			log.Printf("Appending device: %s to Dispaly array", display.Name)
			Displays = append(Displays, display)

		}

	}

	return base.PublicRoom{Displays: Displays, AudioDevices: AudioDevices}, nil
}
<|endoftext|>


<commit_before>package kafkaadmin

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	kafka "github.com/packetloop/go-kafkaesque"
)

func TestAccKafkaAdminTopicCreate(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreventPostDestroyRefresh: true,
		PreCheck:                  func() { testAccPreCheck(t) },
		Providers:                 testAccProviders,
		CheckDestroy:              testCheckTopicDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckKafkaTopicCreate,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckTopicExists("kafka_topic.foo"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "name", "mytopic"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "partitions", "2"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "replication_factor", "3"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "cleanup_policy", "compact"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "retention_ms", "-1"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "segment_bytes", "1073741824"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "segment_ms", "604800000"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "retention_bytes", "-1"),
				),
			},
		},
	})
}

const testAccCheckKafkaTopicCreate = `
resource "kafka_topic" "foo" {
  name = "mytopic"
  partitions = 2
  replication_factor = 3
}
`

func TestAccKafkaAdminTopicCreateWithConfig(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreventPostDestroyRefresh: true,
		PreCheck:                  func() { testAccPreCheck(t) },
		Providers:                 testAccProviders,
		CheckDestroy:              testCheckTopicDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckKafkaTopicCreateConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckTopicExists("kafka_topic.foobar"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "name", "mytopicconfig"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "partitions", "2"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "replication_factor", "3"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "cleanup_policy", "delete"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "retention_ms", "300000"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "segment_bytes", "10737418"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "segment_ms", "600000"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "retention_bytes", "100000"),
				),
			},
		},
	})
}

const testAccCheckKafkaTopicCreateConfig = `
resource "kafka_topic" "foobar" {
  name = "mytopicconfig"
  partitions = 2
  replication_factor = 3
  retention_ms = 300000
  cleanup_policy = "delete"
  segment_bytes = 10737418
  min_insync_replicas = 2
  retention_bytes = 100000
  segment_ms = 600000
}
`

func testAccCheckTopicExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		client := testAccProvider.Meta().(*Conn).sclient
		return TopicExistsHelper(s, client)
	}
}

func testCheckTopicDestroy(state *terraform.State) error {
	for _, res := range state.RootModule().Resources {
		if res.Type != "kafka_topic" {
			continue
		}

		topicName := res.Primary.ID

		client := testAccProvider.Meta().(*Conn).sclient
		_, err := client.GetTopic(topicName)
		// If err is not nil, it means topic still exist.
		if err == nil {
			return fmt.Errorf("ERROR DESTROY TOPIC '%s': %v", topicName, err)
		}
		return nil
	}
	return nil
}

func TopicExistsHelper(s *terraform.State, client *kafka.Client) error {
	for _, r := range s.RootModule().Resources {
		id := r.Primary.ID

		// If topic exist, returns error nil.
		if _, err := client.GetTopic(id); err != nil {
			return fmt.Errorf("ERROR TOPIC '%s' DOES NOT EXIST: %v", id, err)
		}
	}
	return nil
}
<commit_msg>mayh-9612 - Add test case<commit_after>package kafkaadmin

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	kafka "github.com/packetloop/go-kafkaesque"
)

func TestAccKafkaAdminTopicCreate(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreventPostDestroyRefresh: true,
		PreCheck:                  func() { testAccPreCheck(t) },
		Providers:                 testAccProviders,
		CheckDestroy:              testCheckTopicDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckKafkaTopicCreate,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckTopicExists("kafka_topic.foo"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "name", "mytopic"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "partitions", "2"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "replication_factor", "3"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "cleanup_policy", "compact"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "retention_ms", "-1"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "segment_bytes", "1073741824"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "segment_ms", "604800000"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "retention_bytes", "-1"),
				),
			},
		},
	})
}

const testAccCheckKafkaTopicCreate = `
resource "kafka_topic" "foo" {
  name = "mytopic"
  partitions = 2
  replication_factor = 3
}
`

func TestAccKafkaAdminTopicCreateWithConfig(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreventPostDestroyRefresh: true,
		PreCheck:                  func() { testAccPreCheck(t) },
		Providers:                 testAccProviders,
		CheckDestroy:              testCheckTopicDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckKafkaTopicCreateConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckTopicExists("kafka_topic.foobar"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "name", "mytopicconfig"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "partitions", "2"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "replication_factor", "3"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "cleanup_policy", "delete"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "retention_ms", "300000"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "segment_bytes", "10737418"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "segment_ms", "600000"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foobar", "retention_bytes", "100000"),
				),
			},
		},
	})
}

const testAccCheckKafkaTopicCreateConfig = `
resource "kafka_topic" "foobar" {
  name = "mytopicconfig"
  partitions = 2
  replication_factor = 3
  retention_ms = 300000
  cleanup_policy = "delete"
  segment_bytes = 10737418
  min_insync_replicas = 2
  retention_bytes = 100000
  segment_ms = 600000
}
`

func TestAccKafkaAdminTopicUpdate(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreventPostDestroyRefresh: true,
		PreCheck:                  func() { testAccPreCheck(t) },
		Providers:                 testAccProviders,
		CheckDestroy:              testCheckTopicDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckKafkaTopicCreate,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckTopicExists("kafka_topic.foo"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "name", "mytopic"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "partitions", "2"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "replication_factor", "3"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "retention_ms", "-1"),
				),
			},
			{
				Config: testAccCheckKafkaTopicUpdate,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckTopicExists("kafka_topic.foo"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "name", "mytopic"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "partitions", "2"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "replication_factor", "3"),
					resource.TestCheckResourceAttr(
						"kafka_topic.foo", "retention_ms", "100000"),
				),
			},
		},
	})
}

const testAccCheckKafkaTopicUpdate = `
resource "kafka_topic" "foo" {
  name = "mytopic"
  partitions = 2
  replication_factor = 3
  retention_ms = 100000
}
`

func testAccCheckTopicExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		client := testAccProvider.Meta().(*Conn).sclient
		return TopicExistsHelper(s, client)
	}
}

func testCheckTopicDestroy(state *terraform.State) error {
	for _, res := range state.RootModule().Resources {
		if res.Type != "kafka_topic" {
			continue
		}

		topicName := res.Primary.ID

		client := testAccProvider.Meta().(*Conn).sclient
		_, err := client.GetTopic(topicName)
		// If err is not nil, it means topic still exist.
		if err == nil {
			return fmt.Errorf("ERROR DESTROY TOPIC '%s': %v", topicName, err)
		}
		return nil
	}
	return nil
}

func TopicExistsHelper(s *terraform.State, client *kafka.Client) error {
	for _, r := range s.RootModule().Resources {
		id := r.Primary.ID

		// If topic exist, returns error nil.
		if _, err := client.GetTopic(id); err != nil {
			return fmt.Errorf("ERROR TOPIC '%s' DOES NOT EXIST: %v", id, err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package controller

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/avct/uasurfer"
	"github.com/goadesign/goa"
	influxClient "github.com/influxdata/influxdb/client/v2"
	"github.com/pkg/errors"
	"github.com/snowplow/referer-parser/go"
	"gitlab.com/remp/remp/Beam/go/cmd/tracker/app"
	"gitlab.com/remp/remp/Beam/go/model"
)

// TrackController implements the track resource.
type TrackController struct {
	*goa.Controller
	EventProducer   sarama.AsyncProducer
	PropertyStorage model.PropertyStorage
}

// Event represents Influx event structure
type Event struct {
	Action   string                 `json:"action"`
	Category string                 `json:"category"`
	Fields   map[string]interface{} `json:"fields"`
	Value    float64                `json:"value"`
}

// NewTrackController creates a track controller.
func NewTrackController(service *goa.Service, ep sarama.AsyncProducer, ps model.PropertyStorage) *TrackController {
	return &TrackController{
		Controller:      service.NewController("TrackController"),
		EventProducer:   ep,
		PropertyStorage: ps,
	}
}

// Commerce runs the commerce action.
func (c *TrackController) Commerce(ctx *app.CommerceTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"step": ctx.Payload.Step,
	}
	if ctx.Payload.RempCommerceID != nil {
		tags["remp_commerce_id"] = *ctx.Payload.RempCommerceID
	}

	values := map[string]interface{}{}

	if ctx.Payload.Article != nil {
		at, av := articleValues(ctx.Payload.Article)
		for key, tag := range at {
			tags[key] = tag
		}
		for key, val := range av {
			values[key] = val
		}
	}

	switch ctx.Payload.Step {
	case "checkout":
		values["funnel_id"] = ctx.Payload.Checkout.FunnelID
	case "payment":
		if ctx.Payload.Payment.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Payment.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Payment.ProductIds, ",")
		values["revenue"] = ctx.Payload.Payment.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Payment.TransactionID
		tags["currency"] = ctx.Payload.Payment.Revenue.Currency
	case "purchase":
		if ctx.Payload.Purchase.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Purchase.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Purchase.ProductIds, ",")
		values["revenue"] = ctx.Payload.Purchase.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Purchase.TransactionID
		tags["currency"] = ctx.Payload.Purchase.Revenue.Currency
	case "refund":
		if ctx.Payload.Refund.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Refund.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Refund.ProductIds, ",")
		values["revenue"] = ctx.Payload.Refund.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Refund.TransactionID
		tags["currency"] = ctx.Payload.Refund.Revenue.Currency
	default:
		return fmt.Errorf("unhandled commerce step: %s", ctx.Payload.Step)
	}

	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, model.TableCommerce, tags, values); err != nil {
		return err
	}

	topic := fmt.Sprintf("%s_%s", "commerce", ctx.Payload.Step)
	value, err := json.Marshal(ctx.Payload)
	if err != nil {
		return errors.Wrap(err, "unable to marshal payload for kafka")
	}
	c.pushPublic(topic, value)

	return ctx.Accepted()
}

// Event runs the event action.
func (c *TrackController) Event(ctx *app.EventTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"category": ctx.Payload.Category,
		"action":   ctx.Payload.Action,
	}
	if ctx.Payload.RempEventID != nil {
		tags["remp_event_id"] = *ctx.Payload.RempEventID
	}
	fields := map[string]interface{}{}
	if ctx.Payload.Value != nil {
		fields["value"] = *ctx.Payload.Value
	}
	for key, val := range ctx.Payload.Tags {
		tags[key] = val
	}
	for key, val := range ctx.Payload.Fields {
		fields[key] = val
	}
	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, model.TableEvents, tags, fields); err != nil {
		return err
	}

	// push public

	topic := fmt.Sprintf("%s_%s", ctx.Payload.Category, ctx.Payload.Action)
	value, err := json.Marshal(ctx.Payload)
	if err != nil {
		return errors.Wrap(err, "unable to marshal payload for kafka")
	}
	c.pushPublic(topic, value)

	return ctx.Accepted()
}

// Pageview runs the pageview action.
func (c *TrackController) Pageview(ctx *app.PageviewTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"category": model.CategoryPageview,
	}
	values := map[string]interface{}{}

	var measurement string
	switch ctx.Payload.Action {
	case model.ActionPageviewLoad:
		tags["action"] = model.ActionPageviewLoad
		measurement = model.TablePageviews
	case model.ActionPageviewTimespent:
		tags["action"] = model.ActionPageviewTimespent
		measurement = model.TableTimespent
		if ctx.Payload.Timespent != nil {
			values["timespent"] = ctx.Payload.Timespent.Seconds
			tags["unload"] = "0"
			if ctx.Payload.Timespent.Unload != nil && *ctx.Payload.Timespent.Unload {
				tags["unload"] = "1"
			}
		}
	default:
		return ctx.BadRequest(fmt.Errorf("incorrect pageview action [%s]", ctx.Payload.Action))
	}

	if ctx.Payload.Article != nil {
		tags[model.FlagArticle] = "1"
		at, av := articleValues(ctx.Payload.Article)
		for key, tag := range at {
			tags[key] = tag
		}
		for key, val := range av {
			values[key] = val
		}
	} else {
		tags[model.FlagArticle] = "0"
	}

	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, measurement, tags, values); err != nil {
		return err
	}
	return ctx.Accepted()
}

func articleValues(article *app.Article) (map[string]string, map[string]interface{}) {
	tags := map[string]string{
		"article_id": article.ID,
	}
	values := map[string]interface{}{}
	if article.AuthorID != nil {
		tags["author_id"] = *article.AuthorID
	}
	if article.Category != nil {
		tags["category"] = *article.Category
	}
	if article.Locked != nil {
		if *article.Locked {
			tags["locked"] = "1"
		} else {
			tags["locked"] = "0"
		}
	}
	for key, variant := range article.Variants {
		tags[fmt.Sprintf("%s_variant", key)] = variant
	}
	if article.Tags != nil {
		values["tags"] = strings.Join(article.Tags, ",")
	}
	return tags, values
}

// pushInternal pushes new event to the InfluxDB.
func (c *TrackController) pushInternal(system *app.System, user *app.User,
	measurement string, tags map[string]string, fields map[string]interface{}) error {
	fields["token"] = system.PropertyToken

	if user != nil {
		if user.IPAddress != nil {
			fields["ip"] = *user.IPAddress
		}
		if user.URL != nil {
			fields["url"] = *user.URL
		}
		if user.UserAgent != nil {
			fields["user_agent"] = *user.UserAgent

			ua := uasurfer.Parse(*user.UserAgent)
			fields["derived_device"] = strings.TrimPrefix(ua.DeviceType.String(), "Device")
			fields["derived_os"] = strings.TrimPrefix(ua.OS.Name.String(), "OS")
			fields["derived_os_version"] = fmt.Sprintf("%d.%d", ua.OS.Version.Major, ua.OS.Version.Minor)
			fields["derived_platform"] = strings.TrimPrefix(ua.OS.Platform.String(), "Platform")
			fields["derived_browser"] = strings.TrimPrefix(ua.Browser.Name.String(), "Browser")
			fields["derived_browser_version"] = fmt.Sprintf("%d.%d", ua.Browser.Version.Major, ua.Browser.Version.Minor)
		}

		if user.Referer != nil {
			fields["referer"] = *user.Referer
			parsedRef := refererparser.Parse(*user.Referer)
			if user.URL != nil {
				parsedRef.SetCurrent(*user.URL)
			}
			tags["derived_referer_medium"] = parsedRef.Medium
			tags["derived_referer_source"] = parsedRef.Referer
		} else {
			tags["derived_referer_medium"] = "direct"
		}

		if tags["derived_referer_medium"] == "unknown" {
			tags["derived_referer_medium"] = "external"
			if user.URL != nil {
				tags["derived_referer_source"] = *user.URL
			}
		}

		if user.Adblock != nil {
			if *user.Adblock {
				tags["adblock"] = "1"
			} else {
				tags["adblock"] = "0"
			}
		}
		if user.WindowHeight != nil {
			fields["window_height"] = *user.WindowHeight
		}
		if user.WindowWidth != nil {
			fields["window_width"] = *user.WindowWidth
		}
		if user.Cookies != nil {
			if *user.Cookies {
				tags["cookies"] = "1"
			} else {
				tags["cookies"] = "0"
			}
		}
		if user.Websockets != nil {
			if *user.Websockets {
				tags["websockets"] = "1"
			} else {
				tags["websockets"] = "0"
			}
		}
		if user.ID != nil {
			tags["user_id"] = *user.ID
			tags["signed_in"] = "1"
		} else {
			tags["signed_in"] = "0"
		}
		if user.BrowserID != nil {
			tags["browser_id"] = *user.BrowserID
		}
		if user.RempSessionID != nil {
			tags["remp_session_id"] = *user.RempSessionID
		}
		if user.RempPageviewID != nil {
			tags["remp_pageview_id"] = *user.RempPageviewID
		}
		if user.Subscriber != nil {
			if *user.Subscriber {
				tags["subscriber"] = "1"
			} else {
				tags["subscriber"] = "0"
			}
		}

		if user.Source != nil {
			if user.Source.Social != nil {
				tags["social"] = *user.Source.Social
			}
			if user.Source.Ref != nil {
				tags["ref_source"] = *user.Source.Ref
			}
			if user.Source.UtmSource != nil {
				tags["utm_source"] = *user.Source.UtmSource
			}
			if user.Source.UtmMedium != nil {
				tags["utm_medium"] = *user.Source.UtmMedium
			}
			if user.Source.UtmCampaign != nil {
				tags["utm_campaign"] = *user.Source.UtmCampaign
			}
			if user.Source.UtmContent != nil {
				tags["utm_content"] = *user.Source.UtmContent
			}
			if user.Source.BannerVariant != nil {
				tags["banner_variant"] = *user.Source.BannerVariant
			}
		}
	} else {
		tags["signed_in"] = "0"
	}

	p, err := influxClient.NewPoint(measurement, tags, fields, system.Time)
	if err != nil {
		return err
	}
	c.EventProducer.Input() <- &sarama.ProducerMessage{
		Topic: "beam_events",
		Value: sarama.StringEncoder(p.String()),
	}
	return nil
}

func (c *TrackController) pushPublic(topic string, value []byte) {
	c.EventProducer.Input() <- &sarama.ProducerMessage{
		Topic: topic,
		Value: sarama.ByteEncoder(value),
	}
}
<commit_msg>Do not check if referer medium is unknown if no referer is given<commit_after>package controller

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/avct/uasurfer"
	"github.com/goadesign/goa"
	influxClient "github.com/influxdata/influxdb/client/v2"
	"github.com/pkg/errors"
	"github.com/snowplow/referer-parser/go"
	"gitlab.com/remp/remp/Beam/go/cmd/tracker/app"
	"gitlab.com/remp/remp/Beam/go/model"
)

// TrackController implements the track resource.
type TrackController struct {
	*goa.Controller
	EventProducer   sarama.AsyncProducer
	PropertyStorage model.PropertyStorage
}

// Event represents Influx event structure
type Event struct {
	Action   string                 `json:"action"`
	Category string                 `json:"category"`
	Fields   map[string]interface{} `json:"fields"`
	Value    float64                `json:"value"`
}

// NewTrackController creates a track controller.
func NewTrackController(service *goa.Service, ep sarama.AsyncProducer, ps model.PropertyStorage) *TrackController {
	return &TrackController{
		Controller:      service.NewController("TrackController"),
		EventProducer:   ep,
		PropertyStorage: ps,
	}
}

// Commerce runs the commerce action.
func (c *TrackController) Commerce(ctx *app.CommerceTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"step": ctx.Payload.Step,
	}
	if ctx.Payload.RempCommerceID != nil {
		tags["remp_commerce_id"] = *ctx.Payload.RempCommerceID
	}

	values := map[string]interface{}{}

	if ctx.Payload.Article != nil {
		at, av := articleValues(ctx.Payload.Article)
		for key, tag := range at {
			tags[key] = tag
		}
		for key, val := range av {
			values[key] = val
		}
	}

	switch ctx.Payload.Step {
	case "checkout":
		values["funnel_id"] = ctx.Payload.Checkout.FunnelID
	case "payment":
		if ctx.Payload.Payment.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Payment.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Payment.ProductIds, ",")
		values["revenue"] = ctx.Payload.Payment.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Payment.TransactionID
		tags["currency"] = ctx.Payload.Payment.Revenue.Currency
	case "purchase":
		if ctx.Payload.Purchase.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Purchase.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Purchase.ProductIds, ",")
		values["revenue"] = ctx.Payload.Purchase.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Purchase.TransactionID
		tags["currency"] = ctx.Payload.Purchase.Revenue.Currency
	case "refund":
		if ctx.Payload.Refund.FunnelID != nil {
			values["funnel_id"] = *ctx.Payload.Refund.FunnelID
		}
		values["product_ids"] = strings.Join(ctx.Payload.Refund.ProductIds, ",")
		values["revenue"] = ctx.Payload.Refund.Revenue.Amount
		values["transaction_id"] = ctx.Payload.Refund.TransactionID
		tags["currency"] = ctx.Payload.Refund.Revenue.Currency
	default:
		return fmt.Errorf("unhandled commerce step: %s", ctx.Payload.Step)
	}

	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, model.TableCommerce, tags, values); err != nil {
		return err
	}

	topic := fmt.Sprintf("%s_%s", "commerce", ctx.Payload.Step)
	value, err := json.Marshal(ctx.Payload)
	if err != nil {
		return errors.Wrap(err, "unable to marshal payload for kafka")
	}
	c.pushPublic(topic, value)

	return ctx.Accepted()
}

// Event runs the event action.
func (c *TrackController) Event(ctx *app.EventTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"category": ctx.Payload.Category,
		"action":   ctx.Payload.Action,
	}
	if ctx.Payload.RempEventID != nil {
		tags["remp_event_id"] = *ctx.Payload.RempEventID
	}
	fields := map[string]interface{}{}
	if ctx.Payload.Value != nil {
		fields["value"] = *ctx.Payload.Value
	}
	for key, val := range ctx.Payload.Tags {
		tags[key] = val
	}
	for key, val := range ctx.Payload.Fields {
		fields[key] = val
	}
	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, model.TableEvents, tags, fields); err != nil {
		return err
	}

	// push public

	topic := fmt.Sprintf("%s_%s", ctx.Payload.Category, ctx.Payload.Action)
	value, err := json.Marshal(ctx.Payload)
	if err != nil {
		return errors.Wrap(err, "unable to marshal payload for kafka")
	}
	c.pushPublic(topic, value)

	return ctx.Accepted()
}

// Pageview runs the pageview action.
func (c *TrackController) Pageview(ctx *app.PageviewTrackContext) error {
	_, ok, err := c.PropertyStorage.Get(ctx.Payload.System.PropertyToken.String())
	if err != nil {
		return err
	}
	if !ok {
		return ctx.NotFound()
	}

	tags := map[string]string{
		"category": model.CategoryPageview,
	}
	values := map[string]interface{}{}

	var measurement string
	switch ctx.Payload.Action {
	case model.ActionPageviewLoad:
		tags["action"] = model.ActionPageviewLoad
		measurement = model.TablePageviews
	case model.ActionPageviewTimespent:
		tags["action"] = model.ActionPageviewTimespent
		measurement = model.TableTimespent
		if ctx.Payload.Timespent != nil {
			values["timespent"] = ctx.Payload.Timespent.Seconds
			tags["unload"] = "0"
			if ctx.Payload.Timespent.Unload != nil && *ctx.Payload.Timespent.Unload {
				tags["unload"] = "1"
			}
		}
	default:
		return ctx.BadRequest(fmt.Errorf("incorrect pageview action [%s]", ctx.Payload.Action))
	}

	if ctx.Payload.Article != nil {
		tags[model.FlagArticle] = "1"
		at, av := articleValues(ctx.Payload.Article)
		for key, tag := range at {
			tags[key] = tag
		}
		for key, val := range av {
			values[key] = val
		}
	} else {
		tags[model.FlagArticle] = "0"
	}

	if err := c.pushInternal(ctx.Payload.System, ctx.Payload.User, measurement, tags, values); err != nil {
		return err
	}
	return ctx.Accepted()
}

func articleValues(article *app.Article) (map[string]string, map[string]interface{}) {
	tags := map[string]string{
		"article_id": article.ID,
	}
	values := map[string]interface{}{}
	if article.AuthorID != nil {
		tags["author_id"] = *article.AuthorID
	}
	if article.Category != nil {
		tags["category"] = *article.Category
	}
	if article.Locked != nil {
		if *article.Locked {
			tags["locked"] = "1"
		} else {
			tags["locked"] = "0"
		}
	}
	for key, variant := range article.Variants {
		tags[fmt.Sprintf("%s_variant", key)] = variant
	}
	if article.Tags != nil {
		values["tags"] = strings.Join(article.Tags, ",")
	}
	return tags, values
}

// pushInternal pushes new event to the InfluxDB.
func (c *TrackController) pushInternal(system *app.System, user *app.User,
	measurement string, tags map[string]string, fields map[string]interface{}) error {
	fields["token"] = system.PropertyToken

	if user != nil {
		if user.IPAddress != nil {
			fields["ip"] = *user.IPAddress
		}
		if user.URL != nil {
			fields["url"] = *user.URL
		}
		if user.UserAgent != nil {
			fields["user_agent"] = *user.UserAgent

			ua := uasurfer.Parse(*user.UserAgent)
			fields["derived_device"] = strings.TrimPrefix(ua.DeviceType.String(), "Device")
			fields["derived_os"] = strings.TrimPrefix(ua.OS.Name.String(), "OS")
			fields["derived_os_version"] = fmt.Sprintf("%d.%d", ua.OS.Version.Major, ua.OS.Version.Minor)
			fields["derived_platform"] = strings.TrimPrefix(ua.OS.Platform.String(), "Platform")
			fields["derived_browser"] = strings.TrimPrefix(ua.Browser.Name.String(), "Browser")
			fields["derived_browser_version"] = fmt.Sprintf("%d.%d", ua.Browser.Version.Major, ua.Browser.Version.Minor)
		}

		if user.Referer != nil {
			fields["referer"] = *user.Referer
			parsedRef := refererparser.Parse(*user.Referer)
			if user.URL != nil {
				parsedRef.SetCurrent(*user.URL)
			}
			tags["derived_referer_medium"] = parsedRef.Medium
			tags["derived_referer_source"] = parsedRef.Referer

			if tags["derived_referer_medium"] == "unknown" {
				tags["derived_referer_medium"] = "external"
				if user.URL != nil {
					tags["derived_referer_source"] = *user.URL
				}
			}

		} else {
			tags["derived_referer_medium"] = "direct"
		}

		if user.Adblock != nil {
			if *user.Adblock {
				tags["adblock"] = "1"
			} else {
				tags["adblock"] = "0"
			}
		}
		if user.WindowHeight != nil {
			fields["window_height"] = *user.WindowHeight
		}
		if user.WindowWidth != nil {
			fields["window_width"] = *user.WindowWidth
		}
		if user.Cookies != nil {
			if *user.Cookies {
				tags["cookies"] = "1"
			} else {
				tags["cookies"] = "0"
			}
		}
		if user.Websockets != nil {
			if *user.Websockets {
				tags["websockets"] = "1"
			} else {
				tags["websockets"] = "0"
			}
		}
		if user.ID != nil {
			tags["user_id"] = *user.ID
			tags["signed_in"] = "1"
		} else {
			tags["signed_in"] = "0"
		}
		if user.BrowserID != nil {
			tags["browser_id"] = *user.BrowserID
		}
		if user.RempSessionID != nil {
			tags["remp_session_id"] = *user.RempSessionID
		}
		if user.RempPageviewID != nil {
			tags["remp_pageview_id"] = *user.RempPageviewID
		}
		if user.Subscriber != nil {
			if *user.Subscriber {
				tags["subscriber"] = "1"
			} else {
				tags["subscriber"] = "0"
			}
		}

		if user.Source != nil {
			if user.Source.Social != nil {
				tags["social"] = *user.Source.Social
			}
			if user.Source.Ref != nil {
				tags["ref_source"] = *user.Source.Ref
			}
			if user.Source.UtmSource != nil {
				tags["utm_source"] = *user.Source.UtmSource
			}
			if user.Source.UtmMedium != nil {
				tags["utm_medium"] = *user.Source.UtmMedium
			}
			if user.Source.UtmCampaign != nil {
				tags["utm_campaign"] = *user.Source.UtmCampaign
			}
			if user.Source.UtmContent != nil {
				tags["utm_content"] = *user.Source.UtmContent
			}
			if user.Source.BannerVariant != nil {
				tags["banner_variant"] = *user.Source.BannerVariant
			}
		}
	} else {
		tags["signed_in"] = "0"
	}

	p, err := influxClient.NewPoint(measurement, tags, fields, system.Time)
	if err != nil {
		return err
	}
	c.EventProducer.Input() <- &sarama.ProducerMessage{
		Topic: "beam_events",
		Value: sarama.StringEncoder(p.String()),
	}
	return nil
}

func (c *TrackController) pushPublic(topic string, value []byte) {
	c.EventProducer.Input() <- &sarama.ProducerMessage{
		Topic: topic,
		Value: sarama.ByteEncoder(value),
	}
}
<|endoftext|>


<commit_before>package filesys

import (
	"context"
	"fmt"
	"os"
	"path"
	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"path/filepath"
	"time"
)

type Dir struct {
	Path string
	wfs  *WFS
}

var _ = fs.Node(&Dir{})
var _ = fs.NodeCreater(&Dir{})
var _ = fs.NodeMkdirer(&Dir{})
var _ = fs.NodeStringLookuper(&Dir{})
var _ = fs.HandleReadDirAller(&Dir{})
var _ = fs.NodeRemover(&Dir{})
var _ = fs.NodeRenamer(&Dir{})

func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {

	if dir.Path == "/" {
		attr.Valid = time.Second
		attr.Mode = os.ModeDir | 0777
		return nil
	}

	item := dir.wfs.listDirectoryEntriesCache.Get(dir.Path)
	if item != nil && !item.Expired() {
		entry := item.Value().(*filer_pb.Entry)

		attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
		attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
		attr.Gid = entry.Attributes.Gid
		attr.Uid = entry.Attributes.Uid

		return nil
	}

	parent, name := filepath.Split(dir.Path)

	var attributes *filer_pb.FuseAttributes

	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.GetEntryAttributesRequest{
			Name:      name,
			ParentDir: parent,
		}

		glog.V(1).Infof("read dir %s attr: %v", dir.Path, request)
		resp, err := client.GetEntryAttributes(context, request)
		if err != nil {
			glog.V(0).Infof("read dir %s attr %v: %v", dir.Path, request, err)
			return err
		}

		attributes = resp.Attributes

		return nil
	})

	if err != nil {
		return err
	}

	// glog.V(1).Infof("dir %s: %v", dir.Path, attributes)
	// glog.V(1).Infof("dir %s permission: %v", dir.Path, os.FileMode(attributes.FileMode))

	attr.Mode = os.FileMode(attributes.FileMode) | os.ModeDir
	if dir.Path == "/" && attributes.FileMode == 0 {
		attr.Valid = time.Second
	}

	attr.Mtime = time.Unix(attributes.Mtime, 0)
	attr.Ctime = time.Unix(attributes.Crtime, 0)
	attr.Gid = attributes.Gid
	attr.Uid = attributes.Uid

	return nil
}

func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk) *File {
	return &File{
		Name: name,
		dir:  dir,
		wfs:  dir.wfs,
		// attributes: &filer_pb.FuseAttributes{},
		Chunks: chunks,
	}
}

func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {

	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.CreateEntryRequest{
			Directory: dir.Path,
			Entry: &filer_pb.Entry{
				Name:        req.Name,
				IsDirectory: req.Mode&os.ModeDir > 0,
				Attributes: &filer_pb.FuseAttributes{
					Mtime:    time.Now().Unix(),
					Crtime:   time.Now().Unix(),
					FileMode: uint32(req.Mode),
					Uid:      req.Uid,
					Gid:      req.Gid,
				},
			},
		}

		glog.V(1).Infof("create: %v", request)
		if _, err := client.CreateEntry(ctx, request); err != nil {
			return fmt.Errorf("create file: %v", err)
		}

		return nil
	})

	if err == nil {
		file := dir.newFile(req.Name, nil)
		file.isOpen = true
		return file, dir.wfs.AcquireHandle(file, req.Uid, req.Gid), nil
	}

	return nil, nil, err
}

func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {

	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.CreateEntryRequest{
			Directory: dir.Path,
			Entry: &filer_pb.Entry{
				Name:        req.Name,
				IsDirectory: true,
				Attributes: &filer_pb.FuseAttributes{
					Mtime:    time.Now().Unix(),
					Crtime:   time.Now().Unix(),
					FileMode: uint32(req.Mode),
					Uid:      req.Uid,
					Gid:      req.Gid,
				},
			},
		}

		glog.V(1).Infof("mkdir: %v", request)
		if _, err := client.CreateEntry(ctx, request); err != nil {
			glog.V(0).Infof("mkdir %v: %v", request, err)
			return fmt.Errorf("make dir: %v", err)
		}

		return nil
	})

	if err == nil {
		node := &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs}
		return node, nil
	}

	return nil, err
}

func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err error) {

	var entry *filer_pb.Entry
	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.LookupDirectoryEntryRequest{
			Directory: dir.Path,
			Name:      name,
		}

		glog.V(4).Infof("lookup directory entry: %v", request)
		resp, err := client.LookupDirectoryEntry(ctx, request)
		if err != nil {
			return err
		}

		entry = resp.Entry

		return nil
	})

	if entry != nil {
		if entry.IsDirectory {
			node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs}
		} else {
			node = dir.newFile(name, entry.Chunks)
		}
		return node, nil
	}

	return nil, fuse.ENOENT
}

func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {

	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.ListEntriesRequest{
			Directory: dir.Path,
		}

		glog.V(4).Infof("read directory: %v", request)
		resp, err := client.ListEntries(ctx, request)
		if err != nil {
			return err
		}

		for _, entry := range resp.Entries {
			if entry.IsDirectory {
				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
				ret = append(ret, dirent)
			} else {
				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
				ret = append(ret, dirent)
			}
			dir.wfs.listDirectoryEntriesCache.Set(dir.Path+"/"+entry.Name, entry, 300*time.Millisecond)
		}

		return nil
	})

	return ret, err
}

func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {

	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.DeleteEntryRequest{
			Directory:    dir.Path,
			Name:         req.Name,
			IsDirectory:  req.Dir,
			IsDeleteData: true,
		}

		glog.V(1).Infof("remove directory entry: %v", request)
		_, err := client.DeleteEntry(ctx, request)
		if err != nil {
			return err
		}

		return nil
	})

}

func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {

	newDir := newDirectory.(*Dir)

	var entry *filer_pb.Entry
	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		// find existing entry
		{
			request := &filer_pb.LookupDirectoryEntryRequest{
				Directory: dir.Path,
				Name:      req.OldName,
			}

			glog.V(4).Infof("find existing directory entry: %v", request)
			resp, err := client.LookupDirectoryEntry(ctx, request)
			if err != nil {
				return err
			}

			entry = resp.Entry

			if entry.IsDirectory {
				// do not support moving directory
				return fuse.ENOTSUP
			}

			glog.V(4).Infof("found existing directory entry resp: %+v", resp)

		}

		// add to new directory
		{
			request := &filer_pb.CreateEntryRequest{
				Directory: newDir.Path,
				Entry: &filer_pb.Entry{
					Name:        req.NewName,
					IsDirectory: false,
					Attributes:  entry.Attributes,
					Chunks:      entry.Chunks,
				},
			}

			glog.V(1).Infof("create new entry: %v", request)
			if _, err := client.CreateEntry(ctx, request); err != nil {
				return fmt.Errorf("create new entry: %v", err)
			}
		}

		// delete old entry
		{
			request := &filer_pb.DeleteEntryRequest{
				Directory:    dir.Path,
				Name:         req.OldName,
				IsDirectory:  false,
				IsDeleteData: false,
			}

			glog.V(1).Infof("remove old entry: %v", request)
			_, err := client.DeleteEntry(ctx, request)
			if err != nil {
				return err
			}

		}

		return nil
	})

	return err
}
<commit_msg>better error log<commit_after>package filesys

import (
	"context"
	"os"
	"path"
	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"path/filepath"
	"time"
)

type Dir struct {
	Path string
	wfs  *WFS
}

var _ = fs.Node(&Dir{})
var _ = fs.NodeCreater(&Dir{})
var _ = fs.NodeMkdirer(&Dir{})
var _ = fs.NodeStringLookuper(&Dir{})
var _ = fs.HandleReadDirAller(&Dir{})
var _ = fs.NodeRemover(&Dir{})
var _ = fs.NodeRenamer(&Dir{})

func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {

	if dir.Path == "/" {
		attr.Valid = time.Second
		attr.Mode = os.ModeDir | 0777
		return nil
	}

	item := dir.wfs.listDirectoryEntriesCache.Get(dir.Path)
	if item != nil && !item.Expired() {
		entry := item.Value().(*filer_pb.Entry)

		attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
		attr.Ctime = time.Unix(entry.Attributes.Crtime, 0)
		attr.Gid = entry.Attributes.Gid
		attr.Uid = entry.Attributes.Uid

		return nil
	}

	parent, name := filepath.Split(dir.Path)

	var attributes *filer_pb.FuseAttributes

	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.GetEntryAttributesRequest{
			Name:      name,
			ParentDir: parent,
		}

		glog.V(1).Infof("read dir %s attr: %v", dir.Path, request)
		resp, err := client.GetEntryAttributes(context, request)
		if err != nil {
			glog.V(0).Infof("read dir %s attr %v: %v", dir.Path, request, err)
			return err
		}

		attributes = resp.Attributes

		return nil
	})

	if err != nil {
		return err
	}

	// glog.V(1).Infof("dir %s: %v", dir.Path, attributes)
	// glog.V(1).Infof("dir %s permission: %v", dir.Path, os.FileMode(attributes.FileMode))

	attr.Mode = os.FileMode(attributes.FileMode) | os.ModeDir
	if dir.Path == "/" && attributes.FileMode == 0 {
		attr.Valid = time.Second
	}

	attr.Mtime = time.Unix(attributes.Mtime, 0)
	attr.Ctime = time.Unix(attributes.Crtime, 0)
	attr.Gid = attributes.Gid
	attr.Uid = attributes.Uid

	return nil
}

func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk) *File {
	return &File{
		Name: name,
		dir:  dir,
		wfs:  dir.wfs,
		// attributes: &filer_pb.FuseAttributes{},
		Chunks: chunks,
	}
}

func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {

	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.CreateEntryRequest{
			Directory: dir.Path,
			Entry: &filer_pb.Entry{
				Name:        req.Name,
				IsDirectory: req.Mode&os.ModeDir > 0,
				Attributes: &filer_pb.FuseAttributes{
					Mtime:    time.Now().Unix(),
					Crtime:   time.Now().Unix(),
					FileMode: uint32(req.Mode),
					Uid:      req.Uid,
					Gid:      req.Gid,
				},
			},
		}

		glog.V(1).Infof("create: %v", request)
		if _, err := client.CreateEntry(ctx, request); err != nil {
			glog.V(0).Infof("create %s/%s: %v", dir.Path, req.Name, err)
			return fuse.EIO
		}

		return nil
	})

	if err == nil {
		file := dir.newFile(req.Name, nil)
		file.isOpen = true
		return file, dir.wfs.AcquireHandle(file, req.Uid, req.Gid), nil
	}

	return nil, nil, err
}

func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {

	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.CreateEntryRequest{
			Directory: dir.Path,
			Entry: &filer_pb.Entry{
				Name:        req.Name,
				IsDirectory: true,
				Attributes: &filer_pb.FuseAttributes{
					Mtime:    time.Now().Unix(),
					Crtime:   time.Now().Unix(),
					FileMode: uint32(req.Mode),
					Uid:      req.Uid,
					Gid:      req.Gid,
				},
			},
		}

		glog.V(1).Infof("mkdir: %v", request)
		if _, err := client.CreateEntry(ctx, request); err != nil {
			glog.V(0).Infof("mkdir %s/%s: %v", dir.Path, req.Name, err)
			return fuse.EIO
		}

		return nil
	})

	if err == nil {
		node := &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs}
		return node, nil
	}

	return nil, err
}

func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err error) {

	var entry *filer_pb.Entry
	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.LookupDirectoryEntryRequest{
			Directory: dir.Path,
			Name:      name,
		}

		glog.V(4).Infof("lookup directory entry: %v", request)
		resp, err := client.LookupDirectoryEntry(ctx, request)
		if err != nil {
			// glog.V(0).Infof("lookup %s/%s: %v", dir.Path, name, err)
			return fuse.ENOENT
		}

		entry = resp.Entry

		return nil
	})

	if entry != nil {
		if entry.IsDirectory {
			node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs}
		} else {
			node = dir.newFile(name, entry.Chunks)
		}
		return node, nil
	}

	return nil, fuse.ENOENT
}

func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) {

	err = dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.ListEntriesRequest{
			Directory: dir.Path,
		}

		glog.V(4).Infof("read directory: %v", request)
		resp, err := client.ListEntries(ctx, request)
		if err != nil {
			glog.V(0).Infof("list %s: %v", dir.Path, err)
			return fuse.EIO
		}

		for _, entry := range resp.Entries {
			if entry.IsDirectory {
				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_Dir}
				ret = append(ret, dirent)
			} else {
				dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File}
				ret = append(ret, dirent)
			}
			dir.wfs.listDirectoryEntriesCache.Set(dir.Path+"/"+entry.Name, entry, 300*time.Millisecond)
		}

		return nil
	})

	return ret, err
}

func (dir *Dir) Remove(ctx context.Context, req *fuse.RemoveRequest) error {

	return dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.DeleteEntryRequest{
			Directory:    dir.Path,
			Name:         req.Name,
			IsDirectory:  req.Dir,
			IsDeleteData: true,
		}

		glog.V(1).Infof("remove directory entry: %v", request)
		_, err := client.DeleteEntry(ctx, request)
		if err != nil {
			glog.V(0).Infof("remove %s/%s: %v", dir.Path, req.Name, err)
			return fuse.EIO
		}

		return nil
	})

}

func (dir *Dir) Rename(ctx context.Context, req *fuse.RenameRequest, newDirectory fs.Node) error {

	newDir := newDirectory.(*Dir)

	var entry *filer_pb.Entry
	err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		// find existing entry
		{
			request := &filer_pb.LookupDirectoryEntryRequest{
				Directory: dir.Path,
				Name:      req.OldName,
			}

			glog.V(4).Infof("find existing directory entry: %v", request)
			resp, err := client.LookupDirectoryEntry(ctx, request)
			if err != nil {
				glog.V(0).Infof("renaming find %s/%s: %v", dir.Path, req.OldName, err)
				return fuse.ENOENT
			}

			entry = resp.Entry

			if entry.IsDirectory {
				// do not support moving directory
				return fuse.ENOTSUP
			}

			glog.V(4).Infof("found existing directory entry resp: %+v", resp)

		}

		// add to new directory
		{
			request := &filer_pb.CreateEntryRequest{
				Directory: newDir.Path,
				Entry: &filer_pb.Entry{
					Name:        req.NewName,
					IsDirectory: false,
					Attributes:  entry.Attributes,
					Chunks:      entry.Chunks,
				},
			}

			glog.V(1).Infof("create new entry: %v", request)
			if _, err := client.CreateEntry(ctx, request); err != nil {
				glog.V(0).Infof("renaming create %s/%s: %v", newDir.Path, req.NewName, err)
				return fuse.EIO
			}
		}

		// delete old entry
		{
			request := &filer_pb.DeleteEntryRequest{
				Directory:    dir.Path,
				Name:         req.OldName,
				IsDirectory:  false,
				IsDeleteData: false,
			}

			glog.V(1).Infof("remove old entry: %v", request)
			_, err := client.DeleteEntry(ctx, request)
			if err != nil {
				glog.V(0).Infof("renaming delete %s/%s: %v", dir.Path, req.OldName, err)
				return fuse.EIO
			}

		}

		return nil
	})

	return err
}
<|endoftext|>


<commit_before>package storage

import (
	"encoding/json"
	"github.com/huacnlee/gobackup/config"
	"github.com/huacnlee/gobackup/helper"
	"github.com/huacnlee/gobackup/logger"
	"io/ioutil"
	"os"
	"path"
	"time"
)

type PackageList []Package

type Package struct {
	FileKey   string    `json:"file_key"`
	CreatedAt time.Time `json:"created_at"`
}

var (
	cyclerPath = path.Join(config.HomeDir, ".gobackup/cycler")
)

type Cycler struct {
	packages PackageList
	isLoaded bool
}

func (c *Cycler) add(fileKey string) {
	c.packages = append(c.packages, Package{
		FileKey:   fileKey,
		CreatedAt: time.Now(),
	})
}

func (c *Cycler) shiftByKeep(keep int) (first *Package) {
	total := len(c.packages)
	if total <= keep {
		return nil
	}

	first, c.packages = &c.packages[0], c.packages[1:]
	return
}

func (c *Cycler) run(model string, fileKey string, keep int, deletePackage func(fileKey string) error) {
	cyclerFileName := path.Join(cyclerPath, model + ".json")

	c.load(cyclerFileName)
	c.add(fileKey)
	defer c.save(cyclerFileName)

	if keep == 0 {
		return
	}

	for {
		pkg := c.shiftByKeep(keep)
		if pkg == nil {
			break
		}

		err := deletePackage(pkg.FileKey)
		if err != nil {
			logger.Warn("remove failed: ", err)
		}
	}
}

func (c *Cycler) load(cyclerFileName string) {
	if !helper.IsExistsPath(cyclerPath) {
		helper.Exec("mkdir", "-p", cyclerPath)
	}

	if !helper.IsExistsPath(cyclerFileName) {
		helper.Exec("touch", cyclerFileName)
	}

	f, err := ioutil.ReadFile(cyclerFileName)
	if err != nil {
		logger.Error("Load cycler.json failed:", err)
		return
	}
	err = json.Unmarshal(f, &c.packages)
	if err != nil {
		logger.Error("Unmarshal cycler.json failed:", err)
	}
	c.isLoaded = true
}

func (c *Cycler) save(cyclerFileName string) {
	if !c.isLoaded {
		logger.Warn("Skip save cycler.json because it not loaded")
		return
	}

	data, err := json.Marshal(&c.packages)
	if err != nil {
		logger.Error("Marshal packages to cycler.json failed: ", err)
		return
	}

	err = ioutil.WriteFile(cyclerFileName, data, os.ModePerm)
	if err != nil {
		logger.Error("Save cycler.json failed: ", err)
		return
	}
}
<commit_msg>use helper.mkdirp<commit_after>package storage

import (
	"encoding/json"
	"github.com/huacnlee/gobackup/config"
	"github.com/huacnlee/gobackup/helper"
	"github.com/huacnlee/gobackup/logger"
	"io/ioutil"
	"os"
	"path"
	"time"
)

type PackageList []Package

type Package struct {
	FileKey   string    `json:"file_key"`
	CreatedAt time.Time `json:"created_at"`
}

var (
	cyclerPath = path.Join(config.HomeDir, ".gobackup/cycler")
)

type Cycler struct {
	packages PackageList
	isLoaded bool
}

func (c *Cycler) add(fileKey string) {
	c.packages = append(c.packages, Package{
		FileKey:   fileKey,
		CreatedAt: time.Now(),
	})
}

func (c *Cycler) shiftByKeep(keep int) (first *Package) {
	total := len(c.packages)
	if total <= keep {
		return nil
	}

	first, c.packages = &c.packages[0], c.packages[1:]
	return
}

func (c *Cycler) run(model string, fileKey string, keep int, deletePackage func(fileKey string) error) {
	cyclerFileName := path.Join(cyclerPath, model + ".json")

	c.load(cyclerFileName)
	c.add(fileKey)
	defer c.save(cyclerFileName)

	if keep == 0 {
		return
	}

	for {
		pkg := c.shiftByKeep(keep)
		if pkg == nil {
			break
		}

		err := deletePackage(pkg.FileKey)
		if err != nil {
			logger.Warn("remove failed: ", err)
		}
	}
}

func (c *Cycler) load(cyclerFileName string) {
  helper.MkdirP(cyclerPath)

	if !helper.IsExistsPath(cyclerFileName) {
		helper.Exec("touch", cyclerFileName)
	}

	f, err := ioutil.ReadFile(cyclerFileName)
	if err != nil {
		logger.Error("Load cycler.json failed:", err)
		return
	}
	err = json.Unmarshal(f, &c.packages)
	if err != nil {
		logger.Error("Unmarshal cycler.json failed:", err)
	}
	c.isLoaded = true
}

func (c *Cycler) save(cyclerFileName string) {
	if !c.isLoaded {
		logger.Warn("Skip save cycler.json because it not loaded")
		return
	}

	data, err := json.Marshal(&c.packages)
	if err != nil {
		logger.Error("Marshal packages to cycler.json failed: ", err)
		return
	}

	err = ioutil.WriteFile(cyclerFileName, data, os.ModePerm)
	if err != nil {
		logger.Error("Save cycler.json failed: ", err)
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"code.google.com/p/plotinum/plotutil"
	"code.google.com/p/plotinum/vg"
)

type Record struct {
	Date   string
	Nins   int64
	Ndel   int64
	Commit string
	Author string
	File   string
}

type Nloc struct {
	Date time.Time
	Nloc int64
}

func parsedate(s string) (time.Time, error) {
	var err error

	a := strings.Split(s, "-")

	if len(a) != 3 {
		return time.Now(), errors.New("time format syntax error, expected YYYY-MM-DD")
	}

	var year, month, day uint64
	year, err = strconv.ParseUint(a[0], 10, 32)
	if err != nil {
		return time.Now(), err
	}

	month, err = strconv.ParseUint(a[1], 10, 32)
	if err != nil {
		return time.Now(), err
	}

	day, err = strconv.ParseUint(a[2], 10, 32)
	if err != nil {
		return time.Now(), err
	}

	return time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.Local), nil
}

func reader_setup(inpath string) (*csv.Reader, error) {
	file, err := os.Open(inpath)
	if err != nil {
		return nil, err
	}
	reader := csv.NewReader(file)

	return reader, nil
}

func record_get(infile *csv.Reader) (*Record, error) {
	line, err := infile.Read()
	if err == io.EOF {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}

	if len(line) < 6 {
		return nil, errors.New("short line")
	}

	var r Record

	r.Date = line[0]
	r.Nins, err = strconv.ParseInt(line[1], 0, 64)
	if err != nil {
		return nil, err
	}
	r.Ndel, err = strconv.ParseInt(line[2], 0, 64)
	if err != nil {
		return nil, err
	}
	r.Commit = line[3]
	r.Author = line[4]
	r.File = line[5]

	return &r, nil
}

func chart_draw_pchg(path string, nloc []Nloc, span int) error {
	var v plotter.Values

	for i := span; i < len(nloc); i += span {
		start := float64(nloc[i-span].Nloc)
		cur := float64(nloc[i].Nloc)

		var pinc float64
		if start < cur {
			pinc = 100 - (start/cur)*100.0
		} else {
			pinc = 100 - (cur/start)*100.0
		}

		v = append(v, pinc)
	}

	w := vg.Points(2)

	bc, err := plotter.NewBarChart(v, w)
	if err != nil {
		return err
	}

	bc.LineStyle.Width = vg.Length(0)
	bc.Color = plotutil.Color(0)
	bc.Offset = -w

	p, err := plot.New()
	if err != nil {
		return err
	}

	p.Add(bc)

	return p.Save(10, 5, path)
}

func chart_draw_nloc(path string, nloc []Nloc) error {
	pts := make(plotter.XYs, len(nloc))
	for i := 0; i < len(nloc); i++ {
		pt := &pts[i]
		pt.X = float64(i)
		pt.Y = float64(nloc[i].Nloc)
	}

	lc, err := plotter.NewLine(pts)
	if err != nil {
		return err
	}
	lc.LineStyle.Width = vg.Points(1)

	p, err := plot.New()
	if err != nil {
		return err
	}

	p.Add(lc)

	return p.Save(10, 5, path)
}

func main() {
	var inpath string
	var nlocfile string
	var pcntfile string
	var pcntspan int

	flag.StringVar(&inpath, "infile", "", "Input path to .csv")
	flag.StringVar(&nlocfile, "nloc", "", "Output path to num lines of code over time chart (png, pdf, svg, etc)")
	flag.StringVar(&pcntfile, "pcnt", "", "Output path to %change over time chart (png, pdf, svg, etc)")
	flag.IntVar(&pcntspan, "pspan", 7, "Number of days per data point in the %change chart")
	flag.Parse()

	if inpath == "" {
		panic(errors.New("infile required"))
	}

	infile, err := reader_setup(inpath)
	if err != nil {
		panic(err)
	}

	var totnloc int64 = 0
	var curdate time.Time
	var newdate time.Time
	var curnloc int64 = 0
	var nloc []Nloc
	first := true
	for {
		r, err := record_get(infile)
		if err != nil {
			panic(err)
		}
		if r == nil {
			break
		}

		if first { // first record, set start date
			curdate, err = parsedate(r.Date)
			if err != nil {
				panic(err)
			}
			first = false
		}

		newdate, err = parsedate(r.Date)
		if err != nil {
			panic(err)
		}

		if newdate.Before(curdate) {
			panic(errors.New(fmt.Sprintf("date %s before %s", newdate.String(), curdate.String())))
		}

		if newdate.Equal(curdate) {
			curnloc += r.Nins - r.Ndel // accumulate to have one data point per day
			continue
		}

		for curdate.Before(newdate) { // days without data are flat
			n := Nloc{curdate, totnloc}
			nloc = append(nloc, n)
			curdate = curdate.AddDate(0, 0, 1)
		}

		totnloc += curnloc
		curnloc = r.Nins - r.Ndel
		curdate = newdate
	}

	if curnloc != 0 {
		totnloc += curnloc

		n := Nloc{curdate, totnloc}
		nloc = append(nloc, n)

		curnloc = 0
	}

	fmt.Printf("Total Days: %d\n", len(nloc))
	fmt.Printf("Total NLOC %v %v through %v\n", totnloc, nloc[0].Date, nloc[len(nloc)-1].Date)

	if nlocfile != "" {
		err = chart_draw_nloc(nlocfile, nloc)
		if err != nil {
			panic(err)
		}
	}

	if pcntfile != "" {
		err = chart_draw_pchg(pcntfile, nloc, pcntspan)
		if err != nil {
			panic(err)
		}
	}
}
<commit_msg>fix percent change over time calculations<commit_after>package main

import (
	"encoding/csv"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"code.google.com/p/plotinum/plotutil"
	"code.google.com/p/plotinum/vg"
)

type Record struct {
	Date   string
	Nins   int64
	Ndel   int64
	Commit string
	Author string
	File   string
}

type Nloc struct {
	Date time.Time
	Nloc int64
}

func parsedate(s string) (time.Time, error) {
	var err error

	a := strings.Split(s, "-")

	if len(a) != 3 {
		return time.Now(), errors.New("time format syntax error, expected YYYY-MM-DD")
	}

	var year, month, day uint64
	year, err = strconv.ParseUint(a[0], 10, 32)
	if err != nil {
		return time.Now(), err
	}

	month, err = strconv.ParseUint(a[1], 10, 32)
	if err != nil {
		return time.Now(), err
	}

	day, err = strconv.ParseUint(a[2], 10, 32)
	if err != nil {
		return time.Now(), err
	}

	return time.Date(int(year), time.Month(month), int(day), 0, 0, 0, 0, time.Local), nil
}

func reader_setup(inpath string) (*csv.Reader, error) {
	file, err := os.Open(inpath)
	if err != nil {
		return nil, err
	}
	reader := csv.NewReader(file)

	return reader, nil
}

func record_get(infile *csv.Reader) (*Record, error) {
	line, err := infile.Read()
	if err == io.EOF {
		return nil, nil
	}
	if err != nil {
		return nil, err
	}

	if len(line) < 6 {
		return nil, errors.New("short line")
	}

	var r Record

	r.Date = line[0]
	r.Nins, err = strconv.ParseInt(line[1], 0, 64)
	if err != nil {
		return nil, err
	}
	r.Ndel, err = strconv.ParseInt(line[2], 0, 64)
	if err != nil {
		return nil, err
	}
	r.Commit = line[3]
	r.Author = line[4]
	r.File = line[5]

	return &r, nil
}

func chart_draw_pchg(path string, nloc []Nloc, span int) error {
	var v plotter.Values

	st := time.Unix(0, 0)
	et := st.AddDate(0, 0, span)

	sn := float64(nloc[0].Nloc)

	var now time.Time
	var pinc float64
	for i := 0; i < len(nloc); i++ {
		now = nloc[i].Date
		if now.Before(et) {
			continue
		}

		cn := float64(nloc[i].Nloc)

		if sn < cn {
			pinc = 100 - (sn / cn) * 100.0
		} else if (sn > cn) {
			pinc = 100 - (cn / sn) * 100.0
		} else {
			pinc = 0.0
		}

		v = append(v, pinc)

		sn = cn
		st = now
		et = st.AddDate(0, 0, span)
	}

	w := vg.Points(2)

	bc, err := plotter.NewBarChart(v, w)
	if err != nil {
		return err
	}

	bc.LineStyle.Width = vg.Length(0)
	bc.Color = plotutil.Color(0)
	bc.Offset = -w

	p, err := plot.New()
	if err != nil {
		return err
	}

	p.Add(bc)

	return p.Save(10, 5, path)
}

func chart_draw_nloc(path string, nloc []Nloc) error {
	pts := make(plotter.XYs, len(nloc))
	for i := 0; i < len(nloc); i++ {
		pt := &pts[i]
		pt.X = float64(i)
		pt.Y = float64(nloc[i].Nloc)
	}

	lc, err := plotter.NewLine(pts)
	if err != nil {
		return err
	}
	lc.LineStyle.Width = vg.Points(1)

	p, err := plot.New()
	if err != nil {
		return err
	}

	p.Add(lc)

	return p.Save(10, 5, path)
}

func main() {
	var inpath string
	var nlocfile string
	var pcntfile string
	var pcntspan int

	flag.StringVar(&inpath, "infile", "", "Input path to .csv")
	flag.StringVar(&nlocfile, "nloc", "", "Output path to num lines of code over time chart (png, pdf, svg, etc)")
	flag.StringVar(&pcntfile, "pcnt", "", "Output path to %change over time chart (png, pdf, svg, etc)")
	flag.IntVar(&pcntspan, "pspan", 7, "Number of days per data point in the %change chart")
	flag.Parse()

	if inpath == "" {
		panic(errors.New("infile required"))
	}

	infile, err := reader_setup(inpath)
	if err != nil {
		panic(err)
	}

	var totnloc int64 = 0
	var curdate time.Time
	var newdate time.Time
	var curnloc int64 = 0
	var nloc []Nloc
	first := true
	for {
		r, err := record_get(infile)
		if err != nil {
			panic(err)
		}
		if r == nil {
			break
		}

		if first { // first record, set start date
			curdate, err = parsedate(r.Date)
			if err != nil {
				panic(err)
			}
			first = false
		}

		newdate, err = parsedate(r.Date)
		if err != nil {
			panic(err)
		}

		if newdate.Before(curdate) {
			panic(errors.New(fmt.Sprintf("date %s before %s", newdate.String(), curdate.String())))
		}

		if newdate.Equal(curdate) {
			curnloc += r.Nins - r.Ndel // accumulate to have one data point per day
			continue
		}

		for curdate.Before(newdate) { // days without data are flat
			n := Nloc{curdate, totnloc}
			nloc = append(nloc, n)
			curdate = curdate.AddDate(0, 0, 1)
		}

		totnloc += curnloc
		curnloc = r.Nins - r.Ndel
		curdate = newdate
	}

	if curnloc != 0 {
		totnloc += curnloc

		n := Nloc{curdate, totnloc}
		nloc = append(nloc, n)

		curnloc = 0
	}

	fmt.Printf("Total Days: %d\n", len(nloc))
	fmt.Printf("Total NLOC %v %v through %v\n", totnloc, nloc[0].Date, nloc[len(nloc)-1].Date)

	if nlocfile != "" {
		err = chart_draw_nloc(nlocfile, nloc)
		if err != nil {
			panic(err)
		}
	}

	if pcntfile != "" {
		err = chart_draw_pchg(pcntfile, nloc, pcntspan)
		if err != nil {
			panic(err)
		}
	}
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"os"

	"github.com/chrislusf/seaweedfs/weed/storage/backend"
	"github.com/chrislusf/seaweedfs/weed/storage/idx"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	. "github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uint64, e error) {
	var indexSize int64
	if indexSize, e = verifyIndexFileIntegrity(indexFile); e != nil {
		return 0, fmt.Errorf("verifyIndexFileIntegrity %s failed: %v", indexFile.Name(), e)
	}
	if indexSize == 0 {
		return 0, nil
	}
	var lastIdxEntry []byte
	if lastIdxEntry, e = readIndexEntryAtOffset(indexFile, indexSize-NeedleMapEntrySize); e != nil {
		return 0, fmt.Errorf("readLastIndexEntry %s failed: %v", indexFile.Name(), e)
	}
	key, offset, size := idx.IdxFileEntry(lastIdxEntry)
	if offset.IsZero() {
		return 0, nil
	}
	if size < 0 {
		size = -size
	}
	if lastAppendAtNs, e = verifyNeedleIntegrity(v.DataBackend, v.Version(), offset.ToAcutalOffset(), key, size); e != nil {
		return lastAppendAtNs, fmt.Errorf("verifyNeedleIntegrity %s failed: %v", indexFile.Name(), e)
	}
	return
}

func verifyIndexFileIntegrity(indexFile *os.File) (indexSize int64, err error) {
	if indexSize, err = util.GetFileSize(indexFile); err == nil {
		if indexSize%NeedleMapEntrySize != 0 {
			err = fmt.Errorf("index file's size is %d bytes, maybe corrupted", indexSize)
		}
	}
	return
}

func readIndexEntryAtOffset(indexFile *os.File, offset int64) (bytes []byte, err error) {
	if offset < 0 {
		err = fmt.Errorf("offset %d for index file is invalid", offset)
		return
	}
	bytes = make([]byte, NeedleMapEntrySize)
	_, err = indexFile.ReadAt(bytes, offset)
	return
}

func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version, offset int64, key NeedleId, size Size) (lastAppendAtNs uint64, err error) {
	n := new(needle.Needle)
	if err = n.ReadData(datFile, offset, size, v); err != nil {
		return n.AppendAtNs, fmt.Errorf("read data [%d,%d) : %v", offset, offset+int64(size), err)
	}
	if n.Id != key {
		return n.AppendAtNs, fmt.Errorf("index key %#x does not match needle's Id %#x", key, n.Id)
	}
	return n.AppendAtNs, err
}
<commit_msg>volume: validate volume correctness if last entry is a deletion<commit_after>package storage

import (
	"fmt"
	"os"

	"github.com/chrislusf/seaweedfs/weed/storage/backend"
	"github.com/chrislusf/seaweedfs/weed/storage/idx"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	. "github.com/chrislusf/seaweedfs/weed/storage/types"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uint64, e error) {
	var indexSize int64
	if indexSize, e = verifyIndexFileIntegrity(indexFile); e != nil {
		return 0, fmt.Errorf("verifyIndexFileIntegrity %s failed: %v", indexFile.Name(), e)
	}
	if indexSize == 0 {
		return 0, nil
	}
	var lastIdxEntry []byte
	if lastIdxEntry, e = readIndexEntryAtOffset(indexFile, indexSize-NeedleMapEntrySize); e != nil {
		return 0, fmt.Errorf("readLastIndexEntry %s failed: %v", indexFile.Name(), e)
	}
	key, offset, size := idx.IdxFileEntry(lastIdxEntry)
	if offset.IsZero() {
		return 0, nil
	}
	if size < 0 {
		// read the deletion entry
		if lastAppendAtNs, e = verifyDeletedNeedleIntegrity(v.DataBackend, v.Version(), key); e != nil {
			return lastAppendAtNs, fmt.Errorf("verifyNeedleIntegrity %s failed: %v", indexFile.Name(), e)
		}
	} else {
		if lastAppendAtNs, e = verifyNeedleIntegrity(v.DataBackend, v.Version(), offset.ToAcutalOffset(), key, size); e != nil {
			return lastAppendAtNs, fmt.Errorf("verifyNeedleIntegrity %s failed: %v", indexFile.Name(), e)
		}
	}
	return
}

func verifyIndexFileIntegrity(indexFile *os.File) (indexSize int64, err error) {
	if indexSize, err = util.GetFileSize(indexFile); err == nil {
		if indexSize%NeedleMapEntrySize != 0 {
			err = fmt.Errorf("index file's size is %d bytes, maybe corrupted", indexSize)
		}
	}
	return
}

func readIndexEntryAtOffset(indexFile *os.File, offset int64) (bytes []byte, err error) {
	if offset < 0 {
		err = fmt.Errorf("offset %d for index file is invalid", offset)
		return
	}
	bytes = make([]byte, NeedleMapEntrySize)
	_, err = indexFile.ReadAt(bytes, offset)
	return
}

func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version, offset int64, key NeedleId, size Size) (lastAppendAtNs uint64, err error) {
	n := new(needle.Needle)
	if err = n.ReadData(datFile, offset, size, v); err != nil {
		return n.AppendAtNs, fmt.Errorf("read data [%d,%d) : %v", offset, offset+int64(size), err)
	}
	if n.Id != key {
		return n.AppendAtNs, fmt.Errorf("index key %#x does not match needle's Id %#x", key, n.Id)
	}
	return n.AppendAtNs, err
}

func verifyDeletedNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version, key NeedleId) (lastAppendAtNs uint64, err error) {
	n := new(needle.Needle)
	size := n.DiskSize(v)
	var fileSize int64
	fileSize, _, err = datFile.GetStat()
	if err != nil {
		return 0, fmt.Errorf("GetStat: %v", err)
	}
	if err = n.ReadData(datFile, fileSize-size, Size(0), v); err != nil {
		return n.AppendAtNs, fmt.Errorf("read data [%d,%d) : %v", fileSize-size, size, err)
	}
	if n.Id != key {
		return n.AppendAtNs, fmt.Errorf("index key %#x does not match needle's Id %#x", key, n.Id)
	}
	return n.AppendAtNs, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"time"
)

var out_connections int

/*
Forwards the TCP to either TCP or HTTP,
but sometimes pauses output to test timeout recovery.
Combine with Ctrl-C to test reconnects.
*/
func main() {
	out_connections = 0
	not_paused := true
	ticker := time.NewTicker(8 * time.Second).C
	go func() {
		for _ = range ticker {
			not_paused = !not_paused
			fmt.Printf("out_connections: %d\n", out_connections)
		}
	}()

	go Timeout_HTTP(&not_paused)
	go Timeout_TCP(&not_paused)
	go Redirect_once()
	go Redirect_loop()
	go Flood_HTTP()
	go Flood_TCP()
	time.Sleep(time.Hour)
}

func read_pause_TCP(send chan<- []byte, not_paused, stop *bool) {
	addr, err := net.ResolveTCPAddr("tcp", "153.44.253.27:5631")
	CheckErr(err, "Resolve kystverket address")
	conn, err := net.DialTCP("tcp", nil, addr)
	CheckErr(err, "Connect to kystverket")
	defer conn.Close() // FIXME can fail
	buf := make([]byte, 4096)
	for !*stop {
		n, err := conn.Read(buf)
		CheckErr(err, "read tcp")
		if *not_paused && len(send) < cap(send) {
			content := make([]byte, n)
			copy(content, buf[:n])
			send <- content
			fmt.Println(string(buf[0:n]))
		}
	}
}

func Timeout_HTTP(not_paused *bool) {
	read := make(chan []byte, 200)
	h := func(w http.ResponseWriter, _ *http.Request) {
		// I guess the caller closes the connection...out_connections++
		defer func() { out_connections-- }()
		stop := false
		defer func() { stop = true }()
		go read_pause_TCP(read, not_paused, &stop)
		//w.Header().Set("Content-Type", "text/plain; charset=utf-8") // normal header
		w.Header().Set("Transfer-Encoding", "chunked")
		w.Header().Set("Server", "test_timeout")
		w.WriteHeader(http.StatusOK)
		//go func() {
		for s := range read {
			_, err := w.Write(s)
			if err != nil {
				fmt.Printf("write to HTTP error: %s\n", err)
				return
			}
			w.(http.Flusher).Flush()
		}
		//}()
	}
	s := http.Server{
		Addr:    "127.0.0.1:12340",
		Handler: http.HandlerFunc(h),
	}
	err := s.ListenAndServe()
	CheckErr(err, "listen to HTTP")
}

func Timeout_TCP(not_paused *bool) {
	a, err := net.ResolveTCPAddr("tcp", "127.0.0.1:12341")
	CheckErr(err, "resolve TCP address")
	l, err := net.ListenTCP("tcp", a)
	CheckErr(err, "listen for TCP")
	defer closeAndCheck(l, "timeout_TCP server")
	read := make(chan []byte, 200)
	for {
		c, err := l.AcceptTCP()
		CheckErr(err, "accept TCP connection")
		go func() {
			defer closeAndCheck(c, "timeout_TCP connection")
			out_connections++
			defer func() { out_connections-- }()
			stop := false
			defer func() { stop = true }()
			go read_pause_TCP(read, not_paused, &stop)
			for s := range read {
				_, err := c.Write(s)
				if err != nil {
					fmt.Printf("write to TCP error: %s\n", err)
					break
				}
			}
		}()
	}
}

func Redirect_once() {
	h := http.RedirectHandler("http://127.0.0.1:12340", http.StatusMovedPermanently)
	s := http.Server{
		Addr:    "127.0.0.1:12342",
		Handler: h,
	}
	err := s.ListenAndServe()
	CheckErr(err, "listen to HTTP")
}

func Redirect_loop() {
	h := http.RedirectHandler("http://127.0.0.1:12343", http.StatusMovedPermanently)
	s := http.Server{
		Addr:    "127.0.0.1:12343",
		Handler: h,
	}
	err := s.ListenAndServe()
	CheckErr(err, "listen to HTTP")
}

const floodPacket = "!BSVDM,2,1,6,A,59NSF?02;Ic4DiPoP00i0Nt>0t@E8L5<0000001@:H@964Q60;lPASQDh000,0*11\r\n!BSVDM,2,2,6,A,00000000000,2*3B\r\n"

func Flood_HTTP() {
	h := func(w http.ResponseWriter, _ *http.Request) {
		// I guess the caller closes the connection...
		out_connections++
		defer func() { out_connections-- }()
		//w.Header().Set("Content-Type", "text/plain; charset=utf-8") // normal header
		w.Header().Set("Transfer-Encoding", "chunked")
		w.Header().Set("Server", "test_timeout")
		w.WriteHeader(http.StatusOK)
		//go func() {
		for {
			_, err := w.Write([]byte(floodPacket))
			if err != nil {
				fmt.Printf("write to HTTP error: %s\n", err)
				return
			}
			w.(http.Flusher).Flush()
		}
		//}()
	}
	s := http.Server{
		Addr:    "127.0.0.1:12344",
		Handler: http.HandlerFunc(h),
	}
	err := s.ListenAndServe()
	CheckErr(err, "listen to HTTP")
}

func Flood_TCP() {
	a, err := net.ResolveTCPAddr("tcp", "127.0.0.1:12345")
	CheckErr(err, "resolve TCP address")
	l, err := net.ListenTCP("tcp", a)
	CheckErr(err, "listen for TCP")
	defer closeAndCheck(l, "flood_tcp server")
	for {
		c, err := l.AcceptTCP()
		CheckErr(err, "accept TCP connection")
		go func() {
			out_connections++
			defer closeAndCheck(c, "flood_tcp connection")
			defer func() { out_connections-- }()
			for {
				_, err := c.Write([]byte(floodPacket))
				if err != nil {
					fmt.Printf("write to TCP error: %s\n", err)
					break
				}
			}
		}()
	}
}

func closeAndCheck(c io.Closer, name string) {
	err := c.Close()
	if err != nil {
		log.Fatalf("error when closing %s: %s", name, err.Error())
	}
}

func CheckErr(err error, msg string) {
	if err != nil {
		log.Fatalf("Failed to %s: %s\n", msg, err.Error())
	}
}

func ErrIf(cond bool, msg string) {
	if cond {
		log.Fatalln(msg)
	}
}
<commit_msg>Fix a missing line break<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"time"
)

var out_connections int

/*
Forwards the TCP to either TCP or HTTP,
but sometimes pauses output to test timeout recovery.
Combine with Ctrl-C to test reconnects.
*/
func main() {
	out_connections = 0
	not_paused := true
	ticker := time.NewTicker(8 * time.Second).C
	go func() {
		for _ = range ticker {
			not_paused = !not_paused
			fmt.Printf("out_connections: %d\n", out_connections)
		}
	}()

	go Timeout_HTTP(&not_paused)
	go Timeout_TCP(&not_paused)
	go Redirect_once()
	go Redirect_loop()
	go Flood_HTTP()
	go Flood_TCP()
	time.Sleep(time.Hour)
}

func read_pause_TCP(send chan<- []byte, not_paused, stop *bool) {
	addr, err := net.ResolveTCPAddr("tcp", "153.44.253.27:5631")
	CheckErr(err, "Resolve kystverket address")
	conn, err := net.DialTCP("tcp", nil, addr)
	CheckErr(err, "Connect to kystverket")
	defer conn.Close() // FIXME can fail
	buf := make([]byte, 4096)
	for !*stop {
		n, err := conn.Read(buf)
		CheckErr(err, "read tcp")
		if *not_paused && len(send) < cap(send) {
			content := make([]byte, n)
			copy(content, buf[:n])
			send <- content
			fmt.Println(string(buf[0:n]))
		}
	}
}

func Timeout_HTTP(not_paused *bool) {
	read := make(chan []byte, 200)
	h := func(w http.ResponseWriter, _ *http.Request) {
		// I guess the caller closes the connection...
		out_connections++
		defer func() { out_connections-- }()
		stop := false
		defer func() { stop = true }()
		go read_pause_TCP(read, not_paused, &stop)
		//w.Header().Set("Content-Type", "text/plain; charset=utf-8") // normal header
		w.Header().Set("Transfer-Encoding", "chunked")
		w.Header().Set("Server", "test_timeout")
		w.WriteHeader(http.StatusOK)
		//go func() {
		for s := range read {
			_, err := w.Write(s)
			if err != nil {
				fmt.Printf("write to HTTP error: %s\n", err)
				return
			}
			w.(http.Flusher).Flush()
		}
		//}()
	}
	s := http.Server{
		Addr:    "127.0.0.1:12340",
		Handler: http.HandlerFunc(h),
	}
	err := s.ListenAndServe()
	CheckErr(err, "listen to HTTP")
}

func Timeout_TCP(not_paused *bool) {
	a, err := net.ResolveTCPAddr("tcp", "127.0.0.1:12341")
	CheckErr(err, "resolve TCP address")
	l, err := net.ListenTCP("tcp", a)
	CheckErr(err, "listen for TCP")
	defer closeAndCheck(l, "timeout_TCP server")
	read := make(chan []byte, 200)
	for {
		c, err := l.AcceptTCP()
		CheckErr(err, "accept TCP connection")
		go func() {
			defer closeAndCheck(c, "timeout_TCP connection")
			out_connections++
			defer func() { out_connections-- }()
			stop := false
			defer func() { stop = true }()
			go read_pause_TCP(read, not_paused, &stop)
			for s := range read {
				_, err := c.Write(s)
				if err != nil {
					fmt.Printf("write to TCP error: %s\n", err)
					break
				}
			}
		}()
	}
}

func Redirect_once() {
	h := http.RedirectHandler("http://127.0.0.1:12340", http.StatusMovedPermanently)
	s := http.Server{
		Addr:    "127.0.0.1:12342",
		Handler: h,
	}
	err := s.ListenAndServe()
	CheckErr(err, "listen to HTTP")
}

func Redirect_loop() {
	h := http.RedirectHandler("http://127.0.0.1:12343", http.StatusMovedPermanently)
	s := http.Server{
		Addr:    "127.0.0.1:12343",
		Handler: h,
	}
	err := s.ListenAndServe()
	CheckErr(err, "listen to HTTP")
}

const floodPacket = "!BSVDM,2,1,6,A,59NSF?02;Ic4DiPoP00i0Nt>0t@E8L5<0000001@:H@964Q60;lPASQDh000,0*11\r\n!BSVDM,2,2,6,A,00000000000,2*3B\r\n"

func Flood_HTTP() {
	h := func(w http.ResponseWriter, _ *http.Request) {
		// I guess the caller closes the connection...
		out_connections++
		defer func() { out_connections-- }()
		//w.Header().Set("Content-Type", "text/plain; charset=utf-8") // normal header
		w.Header().Set("Transfer-Encoding", "chunked")
		w.Header().Set("Server", "test_timeout")
		w.WriteHeader(http.StatusOK)
		//go func() {
		for {
			_, err := w.Write([]byte(floodPacket))
			if err != nil {
				fmt.Printf("write to HTTP error: %s\n", err)
				return
			}
			w.(http.Flusher).Flush()
		}
		//}()
	}
	s := http.Server{
		Addr:    "127.0.0.1:12344",
		Handler: http.HandlerFunc(h),
	}
	err := s.ListenAndServe()
	CheckErr(err, "listen to HTTP")
}

func Flood_TCP() {
	a, err := net.ResolveTCPAddr("tcp", "127.0.0.1:12345")
	CheckErr(err, "resolve TCP address")
	l, err := net.ListenTCP("tcp", a)
	CheckErr(err, "listen for TCP")
	defer closeAndCheck(l, "flood_tcp server")
	for {
		c, err := l.AcceptTCP()
		CheckErr(err, "accept TCP connection")
		go func() {
			out_connections++
			defer closeAndCheck(c, "flood_tcp connection")
			defer func() { out_connections-- }()
			for {
				_, err := c.Write([]byte(floodPacket))
				if err != nil {
					fmt.Printf("write to TCP error: %s\n", err)
					break
				}
			}
		}()
	}
}

func closeAndCheck(c io.Closer, name string) {
	err := c.Close()
	if err != nil {
		log.Fatalf("error when closing %s: %s", name, err.Error())
	}
}

func CheckErr(err error, msg string) {
	if err != nil {
		log.Fatalf("Failed to %s: %s\n", msg, err.Error())
	}
}

func ErrIf(cond bool, msg string) {
	if cond {
		log.Fatalln(msg)
	}
}
<|endoftext|>


<commit_before>package bpmon

import (
	"fmt"
	"reflect"
	"testing"
	"time"
)

func TestStringToWeekday(t *testing.T) {
	tests := []struct {
		str         string
		day         time.Weekday
		errExpected bool
	}{
		{str: "monday", day: time.Monday, errExpected: false},
		{str: "Tuesday", day: time.Tuesday, errExpected: false},
		{str: "Wednesday", day: time.Wednesday, errExpected: false},
		{str: "Thursday", day: time.Thursday, errExpected: false},
		{str: "FRIDAY", day: time.Friday, errExpected: false},
		{str: "Saturday", day: time.Saturday, errExpected: false},
		{str: "Sunday", day: time.Sunday, errExpected: false},
		{str: "Casual-Friday", errExpected: true},
	}

	for _, test := range tests {
		day, err := toWeekday(test.str)
		if err == nil && test.errExpected == true {
			t.Errorf("Error expected for '%s' but test succeeded", test.str)
		} else if err != nil && test.errExpected == false {
			t.Errorf("No error expected for '%s' but test failed: %s", test.str, err.Error())
		} else if err == nil && test.errExpected == false {
			if day != test.day {
				t.Errorf("Result not as expected for '%s': Should be '%v', is '%v'", test.str, test.day, day)
			}
		}
	}
}

func ParseTime(str string) time.Time {
	format := "15:04:05.000"
	t, err := time.Parse(format, str)
	if err != nil {
		panic(fmt.Sprintf("Time in test malformed, is '%s', must match '%s', error is: %s", str, format, err.Error()))
	}
	return t
}

func TestStringsToAvailabilityTime(t *testing.T) {
	tests := []struct {
		str         []string
		at          AvailabilityTime
		errExpected bool
	}{
		{
			str: []string{"09:00:00-12:00:00"},
			at: AvailabilityTime{
				TimeRanges: []TimeRange{
					{Start: ParseTime("09:00:00.000"), End: ParseTime("12:00:00.000")},
				},
				AllDay: false,
			},
			errExpected: false,
		},
		{
			str:         []string{},
			errExpected: true,
		},
		{
			str:         []string{"12:00:00"},
			errExpected: true,
		},
		{
			str:         []string{"foo-bar"},
			errExpected: true,
		},
		//{
		//	str: []string{"ALLDAY", "09:00:00-12:00:00"},
		//	at: AvailabilityTime{
		//		AllDay: true,
		//	},
		//	errExpected: false,
		//},
	}

	for _, test := range tests {
		at, err := toAvailabilityTime(test.str)
		if err == nil && test.errExpected == true {
			t.Errorf("Error expected for '%s' but test succeeded", test.str)
		} else if err != nil && test.errExpected == false {
			t.Errorf("No error expected for '%s' but test failed: %s", test.str, err.Error())
		} else if err == nil && test.errExpected == false {
			eq := reflect.DeepEqual(at, test.at)
			if !eq {
				t.Errorf("Results do not match for %v: '%v' vs. '%v'", test.str, at, test.at)
			}
		}
	}
}

func ParseDate(str string) time.Time {
	format := "Mon 2006/01/02 15:04:05.000"
	t, err := time.Parse(format, str)
	if err != nil {
		panic(fmt.Sprintf("Time in test malformed, is '%s', must match '%s', error is: %s", str, format, err.Error()))
	}
	return t
}

func TestContains(t *testing.T) {
	a := Availability{
		time.Monday: AvailabilityTime{
			TimeRanges: []TimeRange{
				{Start: ParseTime("09:00:00.000"), End: ParseTime("12:00:00.000")},
			},
			AllDay: false,
		},
		time.Friday: AvailabilityTime{
			AllDay: true,
		},
	}

	tests := []struct {
		inAvalilability bool
		timestamp       time.Time
	}{
		{
			timestamp:       ParseDate("Mon 2017/03/20 08:00:00.000"),
			inAvalilability: false,
		},
		{
			timestamp:       ParseDate("Mon 2017/03/20 09:00:00.001"),
			inAvalilability: true,
		},
		{
			timestamp:       ParseDate("Fri 2017/03/17 09:00:00.001"),
			inAvalilability: true,
		},
	}

	for _, test := range tests {
		contained := a.Contains(test.timestamp)
		if !contained && test.inAvalilability {
			t.Errorf("Time %v is not in avalability but should", test.timestamp)
		}
		if contained && !test.inAvalilability {
			t.Errorf("Time %v is in avalability but should not", test.timestamp)
		}

	}

}
<commit_msg>added availability test<commit_after>package bpmon

import (
	"fmt"
	"reflect"
	"testing"
	"time"
)

func TestStringToWeekday(t *testing.T) {
	tests := []struct {
		str         string
		day         time.Weekday
		errExpected bool
	}{
		{str: "monday", day: time.Monday, errExpected: false},
		{str: "Tuesday", day: time.Tuesday, errExpected: false},
		{str: "Wednesday", day: time.Wednesday, errExpected: false},
		{str: "Thursday", day: time.Thursday, errExpected: false},
		{str: "FRIDAY", day: time.Friday, errExpected: false},
		{str: "Saturday", day: time.Saturday, errExpected: false},
		{str: "Sunday", day: time.Sunday, errExpected: false},
		{str: "Casual-Friday", errExpected: true},
	}

	for _, test := range tests {
		day, err := toWeekday(test.str)
		if err == nil && test.errExpected == true {
			t.Errorf("Error expected for '%s' but test succeeded", test.str)
		} else if err != nil && test.errExpected == false {
			t.Errorf("No error expected for '%s' but test failed: %s", test.str, err.Error())
		} else if err == nil && test.errExpected == false {
			if day != test.day {
				t.Errorf("Result not as expected for '%s': Should be '%v', is '%v'", test.str, test.day, day)
			}
		}
	}
}

func ParseTime(str string) time.Time {
	format := "15:04:05.000"
	t, err := time.Parse(format, str)
	if err != nil {
		panic(fmt.Sprintf("Time in test malformed, is '%s', must match '%s', error is: %s", str, format, err.Error()))
	}
	return t
}

func TestStringsToAvailabilityTime(t *testing.T) {
	tests := []struct {
		str         []string
		at          AvailabilityTime
		errExpected bool
	}{
		{
			str: []string{"09:00:00-12:00:00"},
			at: AvailabilityTime{
				TimeRanges: []TimeRange{
					{Start: ParseTime("09:00:00.000"), End: ParseTime("12:00:00.000")},
				},
				AllDay: false,
			},
			errExpected: false,
		},
		{
			str:         []string{},
			errExpected: true,
		},
		{
			str:         []string{"12:00:00"},
			errExpected: true,
		},
		{
			str:         []string{"foo-bar"},
			errExpected: true,
		},
		{
			str:         []string{"13:00:00-bar"},
			errExpected: true,
		},
		//{
		//	str: []string{"ALLDAY", "09:00:00-12:00:00"},
		//	at: AvailabilityTime{
		//		AllDay: true,
		//	},
		//	errExpected: false,
		//},
	}

	for _, test := range tests {
		at, err := toAvailabilityTime(test.str)
		if err == nil && test.errExpected == true {
			t.Errorf("Error expected for '%s' but test succeeded", test.str)
		} else if err != nil && test.errExpected == false {
			t.Errorf("No error expected for '%s' but test failed: %s", test.str, err.Error())
		} else if err == nil && test.errExpected == false {
			eq := reflect.DeepEqual(at, test.at)
			if !eq {
				t.Errorf("Results do not match for %v: '%v' vs. '%v'", test.str, at, test.at)
			}
		}
	}
}

func ParseDate(str string) time.Time {
	format := "Mon 2006/01/02 15:04:05.000"
	t, err := time.Parse(format, str)
	if err != nil {
		panic(fmt.Sprintf("Time in test malformed, is '%s', must match '%s', error is: %s", str, format, err.Error()))
	}
	return t
}

func TestContains(t *testing.T) {
	a := Availability{
		time.Monday: AvailabilityTime{
			TimeRanges: []TimeRange{
				{Start: ParseTime("09:00:00.000"), End: ParseTime("12:00:00.000")},
			},
			AllDay: false,
		},
		time.Friday: AvailabilityTime{
			AllDay: true,
		},
	}

	tests := []struct {
		inAvalilability bool
		timestamp       time.Time
	}{
		{
			timestamp:       ParseDate("Mon 2017/03/20 08:00:00.000"),
			inAvalilability: false,
		},
		{
			timestamp:       ParseDate("Mon 2017/03/20 09:00:00.001"),
			inAvalilability: true,
		},
		{
			timestamp:       ParseDate("Fri 2017/03/17 09:00:00.001"),
			inAvalilability: true,
		},
	}

	for _, test := range tests {
		contained := a.Contains(test.timestamp)
		if !contained && test.inAvalilability {
			t.Errorf("Time %v is not in avalability but should", test.timestamp)
		}
		if contained && !test.inAvalilability {
			t.Errorf("Time %v is in avalability but should not", test.timestamp)
		}

	}

}
<|endoftext|>