<commit_before>package sdp

import (
	"bufio"
	"strconv"
	"strings"

	"github.com/pkg/errors"
)

type attributeStatus struct {
	seen          bool
	value         string
	allowMultiple bool
}

// Detect if the current attribute is ok to be read (detect out of order errors)
// or if it has already been set
func attributeValid(statuses []*attributeStatus, attribute string) (err error) {
	attrFound := false
	for _, v := range statuses {
		if attrFound && v.seen {
			return errors.Errorf("Attribute %s was found, but later attribute %s has already been set", attribute, v.value)
		}

		if v.value == attribute {
			if v.seen && !v.allowMultiple {
				return errors.Errorf("Attribute %s was attempted to be set twice: %v", attribute, v.value)
			}
			attrFound = true
			v.seen = true
		}
	}
	return nil
}

func nextLine(scanner *bufio.Scanner) (key, value string, scanStatus bool, err error) {
	if scanStatus = scanner.Scan(); !scanStatus {
		return key, value, scanStatus, scanner.Err()
	}

	if len(scanner.Text()) < 3 {
		return key, value, scanStatus, errors.Errorf("line is not long enough to contain both a key and value: %s", scanner.Text())
	} else if scanner.Text()[1] != '=' {
		return key, value, scanStatus, errors.Errorf("line is not a proper key value pair, second character is not `=`: %s", scanner.Text())
	}

	return string(scanner.Text()[0]), scanner.Text()[2:], scanStatus, err
}

// Unmarshal populates a SessionDescription from a raw string
//
// Some lines in each description are REQUIRED and some are OPTIONAL,
// but all MUST appear in exactly the order given here (the fixed order
// greatly enhances error detection and allows for a simple parser).
// OPTIONAL items are marked with a "*".
// v=  (protocol version)
// o=  (originator and session identifier)
// s=  (session name)
// i=* (session information)
// u=* (URI of description)
// e=* (email address)
// p=* (phone number)
// c=* (connection information -- not required if included in all media)
// b=* (zero or more bandwidth information lines)
// t=* (One or more time descriptions)
// r=* (One or more repeat descriptions)
// z=* (time zone adjustments)
// k=* (encryption key)
// a=* (zero or more session attribute lines)
// Zero or more media descriptions
// https://tools.ietf.org/html/rfc4566#section-5
func (s *SessionDescription) Unmarshal(raw string) error {
	earlyEndErr := errors.Errorf("session description ended before all required values were found")
	orderedSessionAttributes := []*attributeStatus{
		&attributeStatus{value: "v"},
		&attributeStatus{value: "o"},
		&attributeStatus{value: "s"},
		&attributeStatus{value: "i"},
		&attributeStatus{value: "u"},
		&attributeStatus{value: "e"},
		&attributeStatus{value: "p"},
		&attributeStatus{value: "c"},
		&attributeStatus{value: "b", allowMultiple: true},
		&attributeStatus{value: "t", allowMultiple: true},
		&attributeStatus{value: "r", allowMultiple: true},
		&attributeStatus{value: "z", allowMultiple: true},
		&attributeStatus{value: "k", allowMultiple: true},
		&attributeStatus{value: "a", allowMultiple: true},
		&attributeStatus{value: "m", allowMultiple: true},
	}

	s.Reset()
	scanner := bufio.NewScanner(strings.NewReader(raw))

	// v=
	key, value, scanStatus, err := nextLine(scanner)
	if err != nil {
		return err
	} else if !scanStatus {
		return earlyEndErr
	} else if key != "v" {
		return errors.Errorf("v (protocol version) was expected, but not found")
	} else if s.ProtocolVersion, err = strconv.Atoi(value); err != nil {
		return errors.Errorf("Failed to take protocol version to int")
	}

	// o=
	key, value, scanStatus, err = nextLine(scanner)
	if err != nil {
		return err
	} else if !scanStatus {
		return earlyEndErr
	} else if key != "o" {
		return errors.Errorf("o (originator and session identifier) was expected, but not found")
	}
	s.Origin = value

	key, value, scanStatus, err = nextLine(scanner)
	if err != nil {
		return err
	} else if !scanStatus {
		return earlyEndErr
	} else if key != "s" {
		return errors.Errorf("o (session name) was expected, but not found")
	}
	s.SessionName = value

	for {
		key, value, scanStatus, err = nextLine(scanner)
		if err != nil || !scanStatus {
			return err
		}

		if err := attributeValid(orderedSessionAttributes, key); err != nil {
			return err
		}

		switch key {
		case "i":
			s.SessionInformation = value
		case "u":
			s.URI = value
		case "e":
			s.EmailAddress = value
		case "p":
			s.PhoneNumber = value
		case "c":
			s.ConnectionData = value
		case "b":
			s.Bandwidth = append(s.Bandwidth, value)
		case "t":
			s.Timing = append(s.Timing, value)
		case "r":
			s.RepeatTimes = append(s.RepeatTimes, value)
		case "z":
			s.TimeZones = append(s.TimeZones, value)
		case "k":
			s.EncryptionKeys = append(s.EncryptionKeys, value)
		case "a":
			s.Attributes = append(s.Attributes, value)
		case "m":
			return s.unmarshalMedias(scanner, value)
		default:
			return errors.Errorf("Invalid session attribute: %s", key)
		}
	}
}

func (s *SessionDescription) unmarshalMedias(scanner *bufio.Scanner, firstMediaName string) (err error) {
	currentMedia := &MediaDescription{MediaName: firstMediaName}
	orderedMediaAttributes := []*attributeStatus{
		&attributeStatus{value: "i"},
		&attributeStatus{value: "c"},
		&attributeStatus{value: "b", allowMultiple: true},
		&attributeStatus{value: "k", allowMultiple: true},
		&attributeStatus{value: "a", allowMultiple: true},
	}
	resetMediaAttributes := func() {
		for _, v := range orderedMediaAttributes {
			v.seen = false
		}
	}

	for {
		key, value, scanStatus, err := nextLine(scanner)
		if err != nil || !scanStatus { // This handles EOF, finish current MediaDescription
			s.MediaDescriptions = append(s.MediaDescriptions, currentMedia)
			return err
		}

		if err := attributeValid(orderedMediaAttributes, key); err != nil {
			return err
		}

		switch key {
		case "m":
			s.MediaDescriptions = append(s.MediaDescriptions, currentMedia)
			resetMediaAttributes()
			currentMedia = &MediaDescription{MediaName: value}
		case "i":
			currentMedia.MediaInformation = value
		case "c":
			currentMedia.ConnectionData = value
		case "b":
			currentMedia.Bandwidth = append(currentMedia.Bandwidth, value)
		case "k":
			currentMedia.EncryptionKeys = append(currentMedia.EncryptionKeys, value)
		case "a":
			currentMedia.Attributes = append(currentMedia.Attributes, value)
		default:
			return errors.Errorf("Invalid media attribute: %s", key)
		}
	}
}
<commit_msg>Fix gocyclo warnings<commit_after>package sdp

import (
	"bufio"
	"strconv"
	"strings"

	"github.com/pkg/errors"
)

type attributeStatus struct {
	seen          bool
	value         string
	allowMultiple bool
}

// Detect if the current attribute is ok to be read (detect out of order errors)
// or if it has already been set
func attributeValid(statuses []*attributeStatus, attribute string) (err error) {
	attrFound := false
	for _, v := range statuses {
		if attrFound && v.seen {
			return errors.Errorf("Attribute %s was found, but later attribute %s has already been set", attribute, v.value)
		}

		if v.value == attribute {
			if v.seen && !v.allowMultiple {
				return errors.Errorf("Attribute %s was attempted to be set twice: %v", attribute, v.value)
			}
			attrFound = true
			v.seen = true
		}
	}
	return nil
}

func nextLine(scanner *bufio.Scanner) (key, value string, scanStatus bool, err error) {
	if scanStatus = scanner.Scan(); !scanStatus {
		return key, value, scanStatus, scanner.Err()
	}

	if len(scanner.Text()) < 3 {
		return key, value, scanStatus, errors.Errorf("line is not long enough to contain both a key and value: %s", scanner.Text())
	} else if scanner.Text()[1] != '=' {
		return key, value, scanStatus, errors.Errorf("line is not a proper key value pair, second character is not `=`: %s", scanner.Text())
	}

	return string(scanner.Text()[0]), scanner.Text()[2:], scanStatus, err
}

// Unmarshal populates a SessionDescription from a raw string
//
// Some lines in each description are REQUIRED and some are OPTIONAL,
// but all MUST appear in exactly the order given here (the fixed order
// greatly enhances error detection and allows for a simple parser).
// OPTIONAL items are marked with a "*".
// v=  (protocol version)
// o=  (originator and session identifier)
// s=  (session name)
// i=* (session information)
// u=* (URI of description)
// e=* (email address)
// p=* (phone number)
// c=* (connection information -- not required if included in all media)
// b=* (zero or more bandwidth information lines)
// t=* (One or more time descriptions)
// r=* (One or more repeat descriptions)
// z=* (time zone adjustments)
// k=* (encryption key)
// a=* (zero or more session attribute lines)
// Zero or more media descriptions
// https://tools.ietf.org/html/rfc4566#section-5
func (s *SessionDescription) Unmarshal(raw string) error {
	earlyEndErr := errors.Errorf("session description ended before all required values were found")

	s.Reset()
	scanner := bufio.NewScanner(strings.NewReader(raw))

	// v=
	key, value, scanStatus, err := nextLine(scanner)
	if err != nil {
		return err
	} else if !scanStatus {
		return earlyEndErr
	} else if key != "v" {
		return errors.Errorf("v (protocol version) was expected, but not found")
	} else if s.ProtocolVersion, err = strconv.Atoi(value); err != nil {
		return errors.Errorf("Failed to take protocol version to int")
	}

	// o=
	key, value, scanStatus, err = nextLine(scanner)
	if err != nil {
		return err
	} else if !scanStatus {
		return earlyEndErr
	} else if key != "o" {
		return errors.Errorf("o (originator and session identifier) was expected, but not found")
	}
	s.Origin = value

	key, value, scanStatus, err = nextLine(scanner)
	if err != nil {
		return err
	} else if !scanStatus {
		return earlyEndErr
	} else if key != "s" {
		return errors.Errorf("o (session name) was expected, but not found")
	}
	s.SessionName = value

	return s.unmarshalOptionalAttributes(scanner)
}

func (s *SessionDescription) unmarshalOptionalAttributes(scanner *bufio.Scanner) error {
	orderedSessionAttributes := []*attributeStatus{
		&attributeStatus{value: "v"},
		&attributeStatus{value: "o"},
		&attributeStatus{value: "s"},
		&attributeStatus{value: "i"},
		&attributeStatus{value: "u"},
		&attributeStatus{value: "e"},
		&attributeStatus{value: "p"},
		&attributeStatus{value: "c"},
		&attributeStatus{value: "b", allowMultiple: true},
		&attributeStatus{value: "t", allowMultiple: true},
		&attributeStatus{value: "r", allowMultiple: true},
		&attributeStatus{value: "z", allowMultiple: true},
		&attributeStatus{value: "k", allowMultiple: true},
		&attributeStatus{value: "a", allowMultiple: true},
		&attributeStatus{value: "m", allowMultiple: true},
	}

	for {
		key, value, scanStatus, err := nextLine(scanner)
		if err != nil || !scanStatus {
			return err
		}

		if err := attributeValid(orderedSessionAttributes, key); err != nil {
			return err
		}

		switch key {
		case "i":
			s.SessionInformation = value
		case "u":
			s.URI = value
		case "e":
			s.EmailAddress = value
		case "p":
			s.PhoneNumber = value
		case "c":
			s.ConnectionData = value
		case "b":
			s.Bandwidth = append(s.Bandwidth, value)
		case "t":
			s.Timing = append(s.Timing, value)
		case "r":
			s.RepeatTimes = append(s.RepeatTimes, value)
		case "z":
			s.TimeZones = append(s.TimeZones, value)
		case "k":
			s.EncryptionKeys = append(s.EncryptionKeys, value)
		case "a":
			s.Attributes = append(s.Attributes, value)
		case "m":
			return s.unmarshalMedias(scanner, value)
		default:
			return errors.Errorf("Invalid session attribute: %s", key)
		}
	}
}

func (s *SessionDescription) unmarshalMedias(scanner *bufio.Scanner, firstMediaName string) (err error) {
	currentMedia := &MediaDescription{MediaName: firstMediaName}
	orderedMediaAttributes := []*attributeStatus{
		&attributeStatus{value: "i"},
		&attributeStatus{value: "c"},
		&attributeStatus{value: "b", allowMultiple: true},
		&attributeStatus{value: "k", allowMultiple: true},
		&attributeStatus{value: "a", allowMultiple: true},
	}
	resetMediaAttributes := func() {
		for _, v := range orderedMediaAttributes {
			v.seen = false
		}
	}

	for {
		key, value, scanStatus, err := nextLine(scanner)
		if err != nil || !scanStatus { // This handles EOF, finish current MediaDescription
			s.MediaDescriptions = append(s.MediaDescriptions, currentMedia)
			return err
		}

		if err := attributeValid(orderedMediaAttributes, key); err != nil {
			return err
		}

		switch key {
		case "m":
			s.MediaDescriptions = append(s.MediaDescriptions, currentMedia)
			resetMediaAttributes()
			currentMedia = &MediaDescription{MediaName: value}
		case "i":
			currentMedia.MediaInformation = value
		case "c":
			currentMedia.ConnectionData = value
		case "b":
			currentMedia.Bandwidth = append(currentMedia.Bandwidth, value)
		case "k":
			currentMedia.EncryptionKeys = append(currentMedia.EncryptionKeys, value)
		case "a":
			currentMedia.Attributes = append(currentMedia.Attributes, value)
		default:
			return errors.Errorf("Invalid media attribute: %s", key)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/alexjohnj/flasher/tbutils"
	"github.com/codegangsta/cli"
	"github.com/nsf/termbox-go"
	"log"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Name = "flasher"
	app.Version = "0.3.0-DEV"
	app.Author = "Alex Jackson"
	app.Email = "alex@alexj.org"

	app.Commands = []cli.Command{
		{
			Name:      "flash",
			ShortName: "f",
			Usage:     "flasher flash [flashcard-file.json]",
			Action:    cliFlash,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "no-shuffle, n",
					Usage: "Presents flashcards in the order they are written in the source JSON file.",
				},
			},
		},
		{
			Name:      "info",
			ShortName: "i",
			Usage:     "flasher info [flashcard-file.json]",
			Action:    cliInfo,
		},
	}
	app.Run(os.Args)
}

func cliFlash(c *cli.Context) {
	// Load flashcards

	if len(c.Args()) != 1 {
		log.Printf("Incorrect usage\n")
		cli.ShowCommandHelp(c, "flash")
		os.Exit(1)
	}

	flashcardStack := new(cardStack)
	err := flashcardStack.loadFlashcardStack(c.Args()[0])

	if err != nil {
		log.Fatal(err)
	}

	if !c.Bool("no-shuffle") {
		flashcardStack.shuffle()
	}

	// Init termbox
	err = termbox.Init()
	if err != nil {
		panic(err)
	}
	defer termbox.Close()
	drawAll(flashcardStack)

	//Main Run loop
mainloop:
	for {
		switch event := termbox.PollEvent(); event.Type {
		case termbox.EventKey:
			switch event.Key {
			case termbox.KeyEsc, termbox.KeyCtrlC:
				break mainloop
			case termbox.KeyEnter, termbox.KeyArrowRight:
				flashcardStack.advanceStack()
				drawAll(flashcardStack)
			case termbox.KeyBackspace2, termbox.KeyArrowLeft:
				flashcardStack.revertStack()
				drawAll(flashcardStack)
			}
		case termbox.EventResize:
			drawAll(flashcardStack)
		}
		drawAll(flashcardStack)
	}
}

func cliInfo(c *cli.Context) {
	if len(c.Args()) != 1 {
		log.Printf("Incorrect usage\n")
		cli.ShowCommandHelp(c, "info")
		os.Exit(1)
	}

	flashcardStack := new(cardStack)
	err := flashcardStack.loadFlashcardStack(c.Args()[0])

	if err != nil {
		log.Fatalf("%s is an invalid file: %s", c.Args()[0], err.Error())
	}

	fmt.Printf("Deck Name: %s\nAuthor: %s\nNumber of Cards: %d\n", flashcardStack.Title, flashcardStack.Author, len(flashcardStack.Flashcards))
}

func drawAll(stack *cardStack) {
	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
	w, h := termbox.Size()

	// Draw termbox border
	termbox.SetCell(0, 0, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(w-1, 0, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(0, h-1, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(w-1, h-1, '+', termbox.ColorDefault, termbox.ColorDefault)

	for x := 1; x < w-1; x++ {
		termbox.SetCell(x, 0, '-', termbox.ColorDefault, termbox.ColorDefault)
		termbox.SetCell(x, h-1, '-', termbox.ColorDefault, termbox.ColorDefault)
	}

	// Draw the Stack's title
	titleXCoord := tbutils.CalculateXCenterCoord(stack.Title)
	tbutils.DrawText(titleXCoord, 1, stack.Title)

	// Draw the current card
	currentQuestion := stack.getCurrentFlashcard()
	currentQuestion.drawQuestion()
	if stack.ShowAnswer {
		currentQuestion.drawAnswer()
		// Draw the Q/A divider
		for x := 0; x < w; x++ {
			termbox.SetCell(x, (3 * h / 8), '-', termbox.ColorBlue, termbox.ColorDefault)
		}
	}

	// Draw the current index
	indexStr := fmt.Sprintf("(%d/%d)", stack.StackIndex+1, len(stack.Flashcards))
	indexXCoord, indexYCoord := tbutils.CalculateXCenterCoord(indexStr), h-1
	tbutils.DrawText(indexXCoord, indexYCoord, indexStr)

	// Write out the back buffer
	termbox.Flush()
}
<commit_msg>Added end of stack message<commit_after>package main

import (
	"fmt"
	"github.com/alexjohnj/flasher/tbutils"
	"github.com/codegangsta/cli"
	"github.com/nsf/termbox-go"
	"log"
	"os"
)

func main() {
	app := cli.NewApp()
	app.Name = "flasher"
	app.Version = "0.3.0-DEV"
	app.Author = "Alex Jackson"
	app.Email = "alex@alexj.org"

	app.Commands = []cli.Command{
		{
			Name:      "flash",
			ShortName: "f",
			Usage:     "flasher flash [flashcard-file.json]",
			Action:    cliFlash,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "no-shuffle, n",
					Usage: "Presents flashcards in the order they are written in the source JSON file.",
				},
			},
		},
		{
			Name:      "info",
			ShortName: "i",
			Usage:     "flasher info [flashcard-file.json]",
			Action:    cliInfo,
		},
	}
	app.Run(os.Args)
}

func cliFlash(c *cli.Context) {
	// Load flashcards

	if len(c.Args()) != 1 {
		log.Printf("Incorrect usage\n")
		cli.ShowCommandHelp(c, "flash")
		os.Exit(1)
	}

	flashcardStack := new(cardStack)
	err := flashcardStack.loadFlashcardStack(c.Args()[0])

	if err != nil {
		log.Fatal(err)
	}

	if !c.Bool("no-shuffle") {
		flashcardStack.shuffle()
	}

	// Init termbox
	err = termbox.Init()
	if err != nil {
		panic(err)
	}
	defer termbox.Close()
	drawAll(flashcardStack)

	//Main Run loop
mainloop:
	for {
		switch event := termbox.PollEvent(); event.Type {
		case termbox.EventKey:
			switch event.Key {
			case termbox.KeyEsc, termbox.KeyCtrlC:
				break mainloop
			case termbox.KeyEnter, termbox.KeyArrowRight:
				flashcardStack.advanceStack()
				drawAll(flashcardStack)
			case termbox.KeyBackspace2, termbox.KeyArrowLeft:
				flashcardStack.revertStack()
				drawAll(flashcardStack)
			}
		case termbox.EventResize:
			drawAll(flashcardStack)
		}
		drawAll(flashcardStack)
	}
}

func cliInfo(c *cli.Context) {
	if len(c.Args()) != 1 {
		log.Printf("Incorrect usage\n")
		cli.ShowCommandHelp(c, "info")
		os.Exit(1)
	}

	flashcardStack := new(cardStack)
	err := flashcardStack.loadFlashcardStack(c.Args()[0])

	if err != nil {
		log.Fatalf("%s is an invalid file: %s", c.Args()[0], err.Error())
	}

	fmt.Printf("Deck Name: %s\nAuthor: %s\nNumber of Cards: %d\n", flashcardStack.Title, flashcardStack.Author, len(flashcardStack.Flashcards))
}

func drawAll(stack *cardStack) {
	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
	w, h := termbox.Size()
	endOfStack := stack.StackIndex+1 == len(stack.Flashcards) && stack.ShowAnswer

	// Draw termbox border
	termbox.SetCell(0, 0, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(w-1, 0, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(0, h-1, '+', termbox.ColorDefault, termbox.ColorDefault)
	termbox.SetCell(w-1, h-1, '+', termbox.ColorDefault, termbox.ColorDefault)

	for x := 1; x < w-1; x++ {
		termbox.SetCell(x, 0, '-', termbox.ColorDefault, termbox.ColorDefault)
		termbox.SetCell(x, h-1, '-', termbox.ColorDefault, termbox.ColorDefault)
	}

	// Draw the Stack's title
	titleXCoord := tbutils.CalculateXCenterCoord(stack.Title)
	tbutils.DrawText(titleXCoord, 1, stack.Title)

	// Draw the current card
	currentQuestion := stack.getCurrentFlashcard()
	currentQuestion.drawQuestion()
	if stack.ShowAnswer {
		currentQuestion.drawAnswer()
		// Draw the Q/A divider
		for x := 0; x < w; x++ {
			termbox.SetCell(x, (3 * h / 8), '-', termbox.ColorBlue, termbox.ColorDefault)
		}
	}

	// Draw end of stack message
	if endOfStack {
		endOfStackMessageLine1 := "End of Stack..."
		endOfStackMessageLine2 := "(q)Quit (r)Restart (x)Reshuffle & Restart."
		eosXCoord1, eosYCoord1 := tbutils.CalculateXCenterCoord(endOfStackMessageLine1), (3 * (h / 4))
		eosXCoord2, eosYCoord2 := tbutils.CalculateXCenterCoord(endOfStackMessageLine2), (3*h/4)+1
		tbutils.DrawText(eosXCoord1, eosYCoord1, endOfStackMessageLine1)
		tbutils.DrawText(eosXCoord2, eosYCoord2, endOfStackMessageLine2)
	}

	// Draw the current index
	indexStr := fmt.Sprintf("(%d/%d)", stack.StackIndex+1, len(stack.Flashcards))
	indexXCoord, indexYCoord := tbutils.CalculateXCenterCoord(indexStr), h-1
	tbutils.DrawText(indexXCoord, indexYCoord, indexStr)

	// Write out the back buffer
	termbox.Flush()
}
<|endoftext|>


<commit_before>package flv

import (
	"bytes"
	"fmt"
	"github.com/metachord/amf.go/amf0"
	"io"
	"os"
)

type Header struct {
	Version uint16
	Body    []byte
}

type Frame interface {
	WriteFrame(io.Writer) error
	GetStream() uint32
	GetDts() uint32
	String() string
}

type CFrame struct {
	Stream      uint32
	Dts         uint32
	Type        TagType
	Flavor      Flavor
	Position    int64
	Body        []byte
	PrevTagSize uint32
}

type VideoFrame struct {
	CFrame
	CodecId VideoCodec
	Width   uint16
	Height  uint16
}

type AudioFrame struct {
	CFrame
	CodecId  AudioCodec
	Rate     uint32
	BitSize  AudioSize
	Channels AudioType
}

type MetaFrame struct {
	CFrame
}

func (f CFrame) WriteFrame(w io.Writer) error {
	bl := uint32(len(f.Body))
	var err error
	err = writeType(w, f.Type)
	if err != nil {
		return err
	}
	err = writeBodyLength(w, bl)
	err = writeDts(w, f.Dts)
	err = writeStream(w, f.Stream)
	err = writeBody(w, f.Body)
	prevTagSize := bl + uint32(TAG_HEADER_LENGTH)
	err = writePrevTagSize(w, prevTagSize)
	return nil
}

func (f CFrame) GetStream() uint32 {
	return f.Stream
}
func (f CFrame) GetDts() uint32 {
	return f.Dts
}

func writeType(w io.Writer, t TagType) error {
	_, err := w.Write([]byte{byte(t)})
	return err
}

func writeBodyLength(w io.Writer, bl uint32) error {
	_, err := w.Write([]byte{byte(bl >> 16), byte((bl >> 8) & 0xFF), byte(bl & 0xFF)})
	return err
}

func writeDts(w io.Writer, dts uint32) error {
	_, err := w.Write([]byte{byte((dts >> 16) & 0xFF), byte((dts >> 8) & 0xFF), byte(dts & 0xFF)})
	if err != nil {
		return err
	}
	_, err = w.Write([]byte{byte((dts >> 24) & 0xFF)})
	return err
}

func writeStream(w io.Writer, stream uint32) error {
	_, err := w.Write([]byte{byte(stream >> 16), byte((stream >> 8) & 0xFF), byte(stream & 0xFF)})
	return err
}

func writeBody(w io.Writer, body []byte) error {
	_, err := w.Write(body)
	return err
}

func writePrevTagSize(w io.Writer, prevTagSize uint32) error {
	_, err := w.Write([]byte{byte((prevTagSize >> 24) & 0xFF), byte((prevTagSize >> 16) & 0xFF), byte((prevTagSize >> 8) & 0xFF), byte(prevTagSize & 0xFF)})
	return err
}

func (f VideoFrame) String() string {
	s := ""
	switch f.Flavor {
	case KEYFRAME:
		s = fmt.Sprintf("%10d\t%d\t%d\t%s\t%s\t{%dx%d,%d bytes} keyframe", f.CFrame.Stream, f.CFrame.Dts, f.CFrame.Position, f.CFrame.Type, f.CodecId, f.Width, f.Height, len(f.CFrame.Body))
	default:
		s = fmt.Sprintf("%10d\t%d\t%d\t%s\t%s\t{%dx%d,%d bytes}", f.CFrame.Stream, f.CFrame.Dts, f.CFrame.Position, f.CFrame.Type, f.CodecId, f.Width, f.Height, len(f.CFrame.Body))
	}
	return s
}

func (f AudioFrame) String() string {
	return fmt.Sprintf("%10d\t%d\t%d\t%s\t%s\t{%d,%s,%s}", f.CFrame.Stream, f.CFrame.Dts, f.CFrame.Position, f.CFrame.Type, f.CodecId, f.Rate, f.BitSize, f.Channels)
}

func (f MetaFrame) String() string {
	buf := bytes.NewReader(f.CFrame.Body)
	dec := amf0.NewDecoder(buf)
	evName, err := dec.Decode()
	mds := ""
	if err == nil {
		switch evName {
		case amf0.StringType("onMetaData"):
			md, err := dec.Decode()
			if err == nil {

				var ea map[amf0.StringType]interface{}
				switch md := md.(type) {
				case *amf0.EcmaArrayType:
					ea = *md
				case *amf0.ObjectType:
					ea = *md
				}
				for k, v := range ea {
					mds += fmt.Sprintf("%v=%+v;", k, v)
				}
			}
		}
	}

	return fmt.Sprintf("%10d\t%d\t%d\t%s\t%s", f.CFrame.Stream, f.CFrame.Dts, f.CFrame.Position, f.CFrame.Type, mds)
}

type FlvReader struct {
	InFile *os.File
	width  uint16
	height uint16
}

func NewReader(inFile *os.File) *FlvReader {
	return &FlvReader{
		InFile: inFile,
		width:  0,
		height: 0,
	}
}

type FlvWriter struct {
	OutFile *os.File
}

func NewWriter(outFile *os.File) *FlvWriter {
	return &FlvWriter{
		OutFile: outFile,
	}
}

func (frReader *FlvReader) ReadHeader() (*Header, error) {
	header := make([]byte, HEADER_LENGTH+4)
	_, err := frReader.InFile.Read(header)
	if err != nil {
		return nil, err
	}

	sig := header[0:3]
	if bytes.Compare(sig, []byte(SIG)) != 0 {
		return nil, fmt.Errorf("bad file format")
	}
	version := (uint16(header[3]) << 8) | (uint16(header[4]) << 0)
	//skip := header[4:5]
	//offset := header[5:9]
	//next_id := header[9:13]

	return &Header{Version: version, Body: header}, nil
}

func (frWriter *FlvWriter) WriteHeader(header *Header) error {
	_, err := frWriter.OutFile.Write(header.Body)
	if err != nil {
		return err
	}
	return nil
}

func (frReader *FlvReader) ReadFrame() (fr Frame, e error) {

	var n int
	var err error

	curPos, _ := frReader.InFile.Seek(0, os.SEEK_CUR)

	tagHeaderB := make([]byte, TAG_HEADER_LENGTH)
	n, err = frReader.InFile.Read(tagHeaderB)
	if n == 0 {
		return nil, nil
	}
	if TagSize(n) != TAG_HEADER_LENGTH {
		return nil, fmt.Errorf("bad tag length: %d", n)
	}
	if err != nil {
		return nil, err
	}

	tagType := TagType(tagHeaderB[0])
	bodyLen := (uint32(tagHeaderB[1]) << 16) | (uint32(tagHeaderB[2]) << 8) | (uint32(tagHeaderB[3]) << 0)
	ts := (uint32(tagHeaderB[4]) << 16) | (uint32(tagHeaderB[5]) << 8) | (uint32(tagHeaderB[6]) << 0)
	tsExt := uint32(tagHeaderB[7])
	stream := (uint32(tagHeaderB[8]) << 16) | (uint32(tagHeaderB[9]) << 8) | (uint32(tagHeaderB[10]) << 0)

	var dts uint32
	dts = (tsExt << 24) | ts

	bodyBuf := make([]byte, bodyLen)
	n, err = frReader.InFile.Read(bodyBuf)
	if err != nil {
		return nil, err
	}

	prevTagSizeB := make([]byte, PREV_TAG_SIZE_LENGTH)
	n, err = frReader.InFile.Read(prevTagSizeB)
	if err != nil {
		return nil, err
	}
	prevTagSize := (uint32(prevTagSizeB[0]) << 24) | (uint32(prevTagSizeB[1]) << 16) | (uint32(prevTagSizeB[2]) << 8) | (uint32(prevTagSizeB[3]) << 0)

	pFrame := CFrame{
		Stream:      stream,
		Dts:         dts,
		Type:        tagType,
		Position:    curPos,
		Body:        bodyBuf,
		PrevTagSize: prevTagSize,
	}

	var resFrame Frame

	switch tagType {
	case TAG_TYPE_META:
		pFrame.Flavor = METADATA
		resFrame = MetaFrame{pFrame}
	case TAG_TYPE_VIDEO:
		vft := VideoFrameType(uint8(bodyBuf[0]) >> 4)
		codecId := VideoCodec(uint8(bodyBuf[0]) & 0x0F)
		switch vft {
		case VIDEO_FRAME_TYPE_KEYFRAME:
			pFrame.Flavor = KEYFRAME
			switch codecId {
			case VIDEO_CODEC_ON2VP6:
				hHelper := (uint16(bodyBuf[1]) >> 4) & 0x0F
				wHelper := uint16(bodyBuf[1]) & 0x0F
				w := uint16(bodyBuf[5])
				h := uint16(bodyBuf[6])

				frReader.width = w*16 - wHelper
				frReader.height = h*16 - hHelper
			}
		default:
			pFrame.Flavor = FRAME
		}
		resFrame = VideoFrame{CFrame: pFrame, CodecId: codecId, Width: frReader.width, Height: frReader.height}
	case TAG_TYPE_AUDIO:
		pFrame.Flavor = FRAME
		codecId := AudioCodec(uint8(bodyBuf[0]) >> 4)
		rate := audioRate(AudioRate((uint8(bodyBuf[0]) >> 2) & 0x03))
		bitSize := AudioSize((uint8(bodyBuf[0]) >> 1) & 0x01)
		channels := AudioType(uint8(bodyBuf[0]) & 0x01)
		resFrame = AudioFrame{CFrame: pFrame, CodecId: codecId, Rate: rate, BitSize: bitSize, Channels: channels}
	}

	return resFrame, nil
}

func audioRate(ar AudioRate) uint32 {
	var ret uint32
	switch ar {
	case AUDIO_RATE_5_5:
		ret = 5500
	case AUDIO_RATE_11:
		ret = 11000
	case AUDIO_RATE_22:
		ret = 22000
	case AUDIO_RATE_44:
		ret = 44000
	default:
		ret = 0
	}
	return ret
}

func (frWriter *FlvWriter) WriteFrame(fr Frame) (e error) {
	return fr.WriteFrame(frWriter.OutFile)
}
<commit_msg>Handle frames with empty body<commit_after>package flv

import (
	"bytes"
	"fmt"
	"github.com/metachord/amf.go/amf0"
	"io"
	"os"
)

type Header struct {
	Version uint16
	Body    []byte
}

type Frame interface {
	WriteFrame(io.Writer) error
	GetStream() uint32
	GetDts() uint32
	String() string
}

type CFrame struct {
	Stream      uint32
	Dts         uint32
	Type        TagType
	Flavor      Flavor
	Position    int64
	Body        []byte
	PrevTagSize uint32
}

type VideoFrame struct {
	CFrame
	CodecId VideoCodec
	Width   uint16
	Height  uint16
}

type AudioFrame struct {
	CFrame
	CodecId  AudioCodec
	Rate     uint32
	BitSize  AudioSize
	Channels AudioType
}

type MetaFrame struct {
	CFrame
}

func (f CFrame) WriteFrame(w io.Writer) error {
	bl := uint32(len(f.Body))
	var err error
	err = writeType(w, f.Type)
	if err != nil {
		return err
	}
	err = writeBodyLength(w, bl)
	err = writeDts(w, f.Dts)
	err = writeStream(w, f.Stream)
	err = writeBody(w, f.Body)
	prevTagSize := bl + uint32(TAG_HEADER_LENGTH)
	err = writePrevTagSize(w, prevTagSize)
	return nil
}

func (f CFrame) GetStream() uint32 {
	return f.Stream
}
func (f CFrame) GetDts() uint32 {
	return f.Dts
}

func writeType(w io.Writer, t TagType) error {
	_, err := w.Write([]byte{byte(t)})
	return err
}

func writeBodyLength(w io.Writer, bl uint32) error {
	_, err := w.Write([]byte{byte(bl >> 16), byte((bl >> 8) & 0xFF), byte(bl & 0xFF)})
	return err
}

func writeDts(w io.Writer, dts uint32) error {
	_, err := w.Write([]byte{byte((dts >> 16) & 0xFF), byte((dts >> 8) & 0xFF), byte(dts & 0xFF)})
	if err != nil {
		return err
	}
	_, err = w.Write([]byte{byte((dts >> 24) & 0xFF)})
	return err
}

func writeStream(w io.Writer, stream uint32) error {
	_, err := w.Write([]byte{byte(stream >> 16), byte((stream >> 8) & 0xFF), byte(stream & 0xFF)})
	return err
}

func writeBody(w io.Writer, body []byte) error {
	_, err := w.Write(body)
	return err
}

func writePrevTagSize(w io.Writer, prevTagSize uint32) error {
	_, err := w.Write([]byte{byte((prevTagSize >> 24) & 0xFF), byte((prevTagSize >> 16) & 0xFF), byte((prevTagSize >> 8) & 0xFF), byte(prevTagSize & 0xFF)})
	return err
}

func (f VideoFrame) String() string {
	s := ""
	switch f.Flavor {
	case KEYFRAME:
		s = fmt.Sprintf("%10d\t%d\t%d\t%s\t%s\t{%dx%d,%d bytes} keyframe", f.CFrame.Stream, f.CFrame.Dts, f.CFrame.Position, f.CFrame.Type, f.CodecId, f.Width, f.Height, len(f.CFrame.Body))
	default:
		s = fmt.Sprintf("%10d\t%d\t%d\t%s\t%s\t{%dx%d,%d bytes}", f.CFrame.Stream, f.CFrame.Dts, f.CFrame.Position, f.CFrame.Type, f.CodecId, f.Width, f.Height, len(f.CFrame.Body))
	}
	return s
}

func (f AudioFrame) String() string {
	return fmt.Sprintf("%10d\t%d\t%d\t%s\t%s\t{%d,%s,%s}", f.CFrame.Stream, f.CFrame.Dts, f.CFrame.Position, f.CFrame.Type, f.CodecId, f.Rate, f.BitSize, f.Channels)
}

func (f MetaFrame) String() string {
	buf := bytes.NewReader(f.CFrame.Body)
	dec := amf0.NewDecoder(buf)
	evName, err := dec.Decode()
	mds := ""
	if err == nil {
		switch evName {
		case amf0.StringType("onMetaData"):
			md, err := dec.Decode()
			if err == nil {

				var ea map[amf0.StringType]interface{}
				switch md := md.(type) {
				case *amf0.EcmaArrayType:
					ea = *md
				case *amf0.ObjectType:
					ea = *md
				}
				for k, v := range ea {
					mds += fmt.Sprintf("%v=%+v;", k, v)
				}
			}
		}
	}

	return fmt.Sprintf("%10d\t%d\t%d\t%s\t%s", f.CFrame.Stream, f.CFrame.Dts, f.CFrame.Position, f.CFrame.Type, mds)
}

type FlvReader struct {
	InFile *os.File
	width  uint16
	height uint16
}

func NewReader(inFile *os.File) *FlvReader {
	return &FlvReader{
		InFile: inFile,
		width:  0,
		height: 0,
	}
}

type FlvWriter struct {
	OutFile *os.File
}

func NewWriter(outFile *os.File) *FlvWriter {
	return &FlvWriter{
		OutFile: outFile,
	}
}

func (frReader *FlvReader) ReadHeader() (*Header, error) {
	header := make([]byte, HEADER_LENGTH+4)
	_, err := frReader.InFile.Read(header)
	if err != nil {
		return nil, err
	}

	sig := header[0:3]
	if bytes.Compare(sig, []byte(SIG)) != 0 {
		return nil, fmt.Errorf("bad file format")
	}
	version := (uint16(header[3]) << 8) | (uint16(header[4]) << 0)
	//skip := header[4:5]
	//offset := header[5:9]
	//next_id := header[9:13]

	return &Header{Version: version, Body: header}, nil
}

func (frWriter *FlvWriter) WriteHeader(header *Header) error {
	_, err := frWriter.OutFile.Write(header.Body)
	if err != nil {
		return err
	}
	return nil
}

func (frReader *FlvReader) ReadFrame() (fr Frame, e error) {

	var n int
	var err error

	curPos, _ := frReader.InFile.Seek(0, os.SEEK_CUR)

	tagHeaderB := make([]byte, TAG_HEADER_LENGTH)
	n, err = frReader.InFile.Read(tagHeaderB)
	if n == 0 {
		return nil, nil
	}
	if TagSize(n) != TAG_HEADER_LENGTH {
		return nil, fmt.Errorf("bad tag length: %d", n)
	}
	if err != nil {
		return nil, err
	}

	tagType := TagType(tagHeaderB[0])
	bodyLen := (uint32(tagHeaderB[1]) << 16) | (uint32(tagHeaderB[2]) << 8) | (uint32(tagHeaderB[3]) << 0)
	ts := (uint32(tagHeaderB[4]) << 16) | (uint32(tagHeaderB[5]) << 8) | (uint32(tagHeaderB[6]) << 0)
	tsExt := uint32(tagHeaderB[7])
	stream := (uint32(tagHeaderB[8]) << 16) | (uint32(tagHeaderB[9]) << 8) | (uint32(tagHeaderB[10]) << 0)

	var dts uint32
	dts = (tsExt << 24) | ts

	bodyBuf := make([]byte, bodyLen)
	n, err = frReader.InFile.Read(bodyBuf)
	if err != nil {
		return nil, err
	}

	prevTagSizeB := make([]byte, PREV_TAG_SIZE_LENGTH)
	n, err = frReader.InFile.Read(prevTagSizeB)
	if err != nil {
		return nil, err
	}
	prevTagSize := (uint32(prevTagSizeB[0]) << 24) | (uint32(prevTagSizeB[1]) << 16) | (uint32(prevTagSizeB[2]) << 8) | (uint32(prevTagSizeB[3]) << 0)

	pFrame := CFrame{
		Stream:      stream,
		Dts:         dts,
		Type:        tagType,
		Position:    curPos,
		Body:        bodyBuf,
		PrevTagSize: prevTagSize,
	}

	var resFrame Frame

	switch tagType {
	case TAG_TYPE_META:
		pFrame.Flavor = METADATA
		resFrame = MetaFrame{CFrame: pFrame}
	case TAG_TYPE_VIDEO:
		if len(bodyBuf) > 0 {
			vft := VideoFrameType(uint8(bodyBuf[0]) >> 4)
			codecId := VideoCodec(uint8(bodyBuf[0]) & 0x0F)
			switch vft {
			case VIDEO_FRAME_TYPE_KEYFRAME:
				pFrame.Flavor = KEYFRAME
				switch codecId {
				case VIDEO_CODEC_ON2VP6:
					hHelper := (uint16(bodyBuf[1]) >> 4) & 0x0F
					wHelper := uint16(bodyBuf[1]) & 0x0F
					w := uint16(bodyBuf[5])
					h := uint16(bodyBuf[6])

					frReader.width = w*16 - wHelper
					frReader.height = h*16 - hHelper
				}
			default:
				pFrame.Flavor = FRAME
			}
			resFrame = VideoFrame{CFrame: pFrame, CodecId: codecId, Width: frReader.width, Height: frReader.height}
		} else {
			resFrame = VideoFrame{CFrame: pFrame, CodecId: VIDEO_CODEC_UNDEFINED, Width: frReader.width, Height: frReader.height}
		}
	case TAG_TYPE_AUDIO:
		pFrame.Flavor = FRAME
		if len(bodyBuf) > 0 {
			codecId := AudioCodec(uint8(bodyBuf[0]) >> 4)
			rate := audioRate(AudioRate((uint8(bodyBuf[0]) >> 2) & 0x03))
			bitSize := AudioSize((uint8(bodyBuf[0]) >> 1) & 0x01)
			channels := AudioType(uint8(bodyBuf[0]) & 0x01)
			resFrame = AudioFrame{CFrame: pFrame, CodecId: codecId, Rate: rate, BitSize: bitSize, Channels: channels}
		} else {
			resFrame = AudioFrame{CFrame: pFrame, CodecId: AUDIO_CODEC_UNDEFINED, Rate: audioRate(AUDIO_RATE_UNDEFINED), BitSize: AUDIO_SIZE_UNDEFINED, Channels: AUDIO_TYPE_UNDEFINED}
		}
	}

	return resFrame, nil
}

func audioRate(ar AudioRate) uint32 {
	var ret uint32
	switch ar {
	case AUDIO_RATE_5_5:
		ret = 5500
	case AUDIO_RATE_11:
		ret = 11000
	case AUDIO_RATE_22:
		ret = 22000
	case AUDIO_RATE_44:
		ret = 44000
	default:
		ret = 0
	}
	return ret
}

func (frWriter *FlvWriter) WriteFrame(fr Frame) (e error) {
	return fr.WriteFrame(frWriter.OutFile)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package listener listens to Kubernetes for policy updates.
package listener

import (
	"errors"
	"fmt"
	"net/http"
	"os"

	"github.com/romana/core/common"
	"github.com/romana/core/common/log/trace"
	"github.com/romana/core/tenant"
	log "github.com/romana/rlog"
	"k8s.io/client-go/1.5/pkg/apis/extensions/v1beta1"

	"k8s.io/client-go/1.5/kubernetes"
	"k8s.io/client-go/1.5/tools/cache"
	"k8s.io/client-go/1.5/tools/clientcmd"
)

type networkPolicyAction int

const (
	networkPolicyActionDelete networkPolicyAction = iota
	networkPolicyActionAdd
	networkPolicyActionModify
)

const (
	HttpGetParamWatch           = "watch=true"
	HttpGetParamResourceVersion = "resourceVersion"
)

// KubeListener is a Service that listens to updates
// from Kubernetes by connecting to the endpoints specified
// and consuming chunked JSON documents. The endpoints are
// constructed from kubeURL and the following paths:
// 1. namespaceNotificationPath for namespace additions/deletions
// 2. policyNotificationPathPrefix + <namespace name> + policyNotificationPathPostfix
//    for policy additions/deletions.
type KubeListener struct {
	config                        common.ServiceConfig
	restClient                    *common.RestClient
	kubeURL                       string
	namespaceNotificationPath     string
	policyNotificationPathPrefix  string
	policyNotificationPathPostfix string
	segmentLabelName              string
	tenantLabelName               string
	lastEventPerNamespace         map[string]uint64
	namespaceBufferSize           uint64

	kubeClient *kubernetes.Clientset
	Watchers   map[string]cache.ListerWatcher
}

// Routes returns various routes used in the service.
func (l *KubeListener) Routes() common.Routes {
	routes := common.Routes{}
	return routes
}

// Name implements method of Service interface.
func (l *KubeListener) Name() string {
	return "kubernetesListener"
}

func (l *KubeListener) CreateSchema(overwrite bool) error {
	return nil
}

// SetConfig implements SetConfig function of the Service interface.
func (l *KubeListener) SetConfig(config common.ServiceConfig) error {
	confString := "/etc/romana/romana.conf.yml:kubernetesListener:config:"
	log.Trace(trace.Inside, confString, config)

	m := config.ServiceSpecific
	if kl, ok := m["kubernetes_url"]; !ok || kl == "" {
		return fmt.Errorf("%s%s", confString, "kubernetes_url required in config.")
	}
	l.kubeURL = m["kubernetes_url"].(string)

	if nnp, ok := m["namespace_notification_path"]; !ok || nnp == "" {
		return fmt.Errorf("%s%s", confString, "namespace_notification_path required in config.")
	}
	l.namespaceNotificationPath = m["namespace_notification_path"].(string)

	if pnppre, ok := m["policy_notification_path_prefix"]; !ok || pnppre == "" {
		return fmt.Errorf("%s%s", confString, "policy_notification_path_prefix required in config.")
	}
	l.policyNotificationPathPrefix = m["policy_notification_path_prefix"].(string)

	if pnppost, ok := m["policy_notification_path_prefix"]; !ok || pnppost == "" {
		return fmt.Errorf("%s%s", confString, "policy_notification_path_postfix required in config.")
	}
	l.policyNotificationPathPostfix = m["policy_notification_path_postfix"].(string)

	if sln, ok := m["segment_label_name"]; !ok || sln == "" {
		return fmt.Errorf("%s%s", confString, "segment_label_name required in config.")
	}
	l.segmentLabelName = m["segment_label_name"].(string)

	if tln, ok := m["tenant_label_name"]; !ok || tln == "" {
		return fmt.Errorf("%s%s", confString, "tenant_label_name required in config.")
	}
	l.tenantLabelName = m["tenant_label_name"].(string)

	l.namespaceBufferSize = 1000

	if kc, ok := m["kubernetes_config"]; !ok || kc == "" {
		// Default kubernetes config location on ubuntu
		// TODO: this should not be hard coded, other
		//       distributions may have other user names.
		m["kubernetes_config"] = "/home/ubuntu/.kube/config"
	}

	// TODO, this loads kubernetes config from flags provided in main
	// should be loading from path provided by romana-root. Stas.
	kubeClientConfig, err := clientcmd.BuildConfigFromFlags("", m["kubernetes_config"].(string))
	if err != nil {
		return errors.New(fmt.Sprintf("Failed to load kubernetes kubeClientConfig %s", err))
	}
	clientset, err := kubernetes.NewForConfig(kubeClientConfig)
	if err != nil {
		return fmt.Errorf("Failed to make kubernetes client %s", err)
	}
	l.kubeClient = clientset

	return nil
}

// TODO there should be a better way to introduce translator
// then global variable like this one.
var PTranslator Translator

// Run configures and runs listener service.
func Run(rootServiceURL string, cred *common.Credential) (*common.RestServiceInfo, error) {
	clientConfig := common.GetDefaultRestClientConfig(rootServiceURL)
	clientConfig.Credential = cred
	client, err := common.NewRestClient(clientConfig)
	if err != nil {
		return nil, err
	}
	KubeListener := &KubeListener{}
	KubeListener.restClient = client

	config, err := client.GetServiceConfig(KubeListener.Name())
	if err != nil {
		return nil, err
	}
	return common.InitializeService(KubeListener, *config, cred)
}

// getOrAddSegment finds a segment (based on segment selector).
// If not found, it adds one.
func (l *KubeListener) getOrAddSegment(namespace string, kubeSegmentName string) (*tenant.Segment, error) {
	ten := &tenant.Tenant{}
	ten.Name = namespace
	// TODO this should be changed to find EXACTLY one after deletion functionality is implemented
	err := l.restClient.Find(ten, common.FindLast)
	if err != nil {
		return nil, err
	}

	seg := &tenant.Segment{}
	seg.Name = kubeSegmentName
	seg.TenantID = ten.ID
	err = l.restClient.Find(seg, common.FindExactlyOne)
	if err == nil {
		return seg, nil
	}

	switch err := err.(type) {
	case common.HttpError:
		if err.StatusCode == http.StatusNotFound {
			// Not found, so let's create a segment.
			segreq := tenant.Segment{Name: kubeSegmentName, TenantID: ten.ID}
			segURL, err2 := l.restClient.GetServiceUrl("tenant")
			if err2 != nil {
				return nil, err2
			}
			segURL = fmt.Sprintf("%s/tenants/%d/segments", segURL, ten.ID)
			err2 = l.restClient.Post(segURL, segreq, seg)
			if err2 == nil {
				// Successful creation.
				return seg, nil
			}
			// Creation of non-existing segment gave an error.
			switch err2 := err2.(type) {
			case common.HttpError:
				// Maybe someone else just created a segment between the original
				// lookup and now?
				if err2.StatusCode == http.StatusConflict {
					switch details := err2.Details.(type) {
					case tenant.Segment:
						// We expect the existing segment to be returned in the details field.
						return &details, nil
					default:
						// This is unexpected...
						return nil, err
					}
				}
				// Any other HTTP error other than a Conflict here - return it.
				return nil, err2
			default:
				// Any other error - return it
				return nil, err2
			}
		}
		// Any other HTTP error other than a Not found here - return it
		return nil, err
	default:
		// Any other error - return it
		return nil, err
	}
}

// resolveTenantByName retrieves tenant information from romana.
func (l *KubeListener) resolveTenantByName(tenantName string) (*tenant.Tenant, error) {
	t := &tenant.Tenant{Name: tenantName}
	err := l.restClient.Find(t, common.FindLast)
	if err != nil {
		return t, err
	}
	return t, nil
}

// deleteNetworkPolicyByID deletes the policy specified by the policyID
func (l *KubeListener) deleteNetworkPolicyByID(policyID uint64) error {
	policyURL, err := l.restClient.GetServiceUrl("policy")
	if err != nil {
		return err
	}
	policyURL = fmt.Sprintf("%s/policies/%d", policyURL, policyID)
	log.Debugf("deleteNetworkPolicyByID: Deleting policy %d", policyID)
	deletedPolicy := common.Policy{}
	err = l.restClient.Delete(policyURL, nil, &deletedPolicy)
	if err != nil {
		switch err := err.(type) {
		default:
			return err
		case common.HttpError:
			if err.StatusCode == http.StatusNotFound {
				log.Warnf("deleteNetworkPolicyByID: Policy %d not found, ignoring", policyID)
				return nil
			} else {
				return err
			}
		}
	}
	log.Debugf("deleteNetworkPolicyByID: Deleted policy %s", deletedPolicy)
	return err
}

// addNetworkPolicy adds the policy to the policy service.
func (l *KubeListener) addNetworkPolicy(policy common.Policy) error {
	policyURL, err := l.restClient.GetServiceUrl("policy")
	if err != nil {
		return err
	}
	policyURL = fmt.Sprintf("%s/policies", policyURL)
	log.Debugf("Applying policy %s", policy)
	err = l.restClient.Post(policyURL, policy, &policy)
	if err != nil {
		return err
	}
	return nil
}

// deleteNetworkPolicy deletes the policy matching provided policy on whatever
// fields are provided.
func (l *KubeListener) deleteNetworkPolicy(policy common.Policy) error {
	err := l.restClient.Find(&policy, common.FindExactlyOne)
	if err != nil {
		switch err := err.(type) {
		default:
			return err
		case common.HttpError:
			if err.StatusCode == http.StatusNotFound {
				log.Warnf("deleteNetworkPolicy: Policy not found %s, ignoring", kubePolicy.Name)
				return nil
			} else {
				return err
			}
		}
	}
	return l.deleteNetworkPolicyByID(policy.ID)
}

func (l *KubeListener) Initialize(client *common.RestClient) error {
	l.restClient = client

	// TODO, find a better place to initialize
	// the translator. Stas.
	PTranslator.Init(l.restClient, l.segmentLabelName, l.tenantLabelName)
	tc := PTranslator.GetClient()
	if tc == nil {
		log.Critical("Failed to initialize rest client for policy translator.")
		os.Exit(255)
	}

	l.lastEventPerNamespace = make(map[string]uint64)
	log.Infof("%s: Starting server", l.Name())
	nsURL, err := common.CleanURL(fmt.Sprintf("%s/%s/?%s", l.kubeURL, l.namespaceNotificationPath, HttpGetParamWatch))
	if err != nil {
		return err
	}
	log.Infof("Starting to listen on %s", nsURL)
	done := make(chan struct{})
	eventc, err := l.nsWatch(done, nsURL)
	if err != nil {
		log.Critical("Namespace watcher failed to start", err)
		os.Exit(255)
	}

	// events := l.conductor(nsEvents, done)
	l.process(eventc, done)

	ProduceNewPolicyEvents(eventc, done, l)

	log.Info("All routines started")
	return nil
}

// CreateSchema is placeholder for now.
func CreateSchema(rootServiceURL string, overwrite bool) error {
	return nil
}
<commit_msg>go: fix go build issues.<commit_after>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package listener listens to Kubernetes for policy updates.
package listener

import (
	"errors"
	"fmt"
	"net/http"
	"os"

	"github.com/romana/core/common"
	"github.com/romana/core/common/log/trace"
	"github.com/romana/core/tenant"
	log "github.com/romana/rlog"

	"k8s.io/client-go/1.5/kubernetes"
	"k8s.io/client-go/1.5/tools/cache"
	"k8s.io/client-go/1.5/tools/clientcmd"
)

type networkPolicyAction int

const (
	networkPolicyActionDelete networkPolicyAction = iota
	networkPolicyActionAdd
	networkPolicyActionModify
)

const (
	HttpGetParamWatch           = "watch=true"
	HttpGetParamResourceVersion = "resourceVersion"
)

// KubeListener is a Service that listens to updates
// from Kubernetes by connecting to the endpoints specified
// and consuming chunked JSON documents. The endpoints are
// constructed from kubeURL and the following paths:
// 1. namespaceNotificationPath for namespace additions/deletions
// 2. policyNotificationPathPrefix + <namespace name> + policyNotificationPathPostfix
//    for policy additions/deletions.
type KubeListener struct {
	config                        common.ServiceConfig
	restClient                    *common.RestClient
	kubeURL                       string
	namespaceNotificationPath     string
	policyNotificationPathPrefix  string
	policyNotificationPathPostfix string
	segmentLabelName              string
	tenantLabelName               string
	lastEventPerNamespace         map[string]uint64
	namespaceBufferSize           uint64

	kubeClient *kubernetes.Clientset
	Watchers   map[string]cache.ListerWatcher
}

// Routes returns various routes used in the service.
func (l *KubeListener) Routes() common.Routes {
	routes := common.Routes{}
	return routes
}

// Name implements method of Service interface.
func (l *KubeListener) Name() string {
	return "kubernetesListener"
}

func (l *KubeListener) CreateSchema(overwrite bool) error {
	return nil
}

// SetConfig implements SetConfig function of the Service interface.
func (l *KubeListener) SetConfig(config common.ServiceConfig) error {
	confString := "/etc/romana/romana.conf.yml:kubernetesListener:config:"
	log.Trace(trace.Inside, confString, config)

	m := config.ServiceSpecific
	if kl, ok := m["kubernetes_url"]; !ok || kl == "" {
		return fmt.Errorf("%s%s", confString, "kubernetes_url required in config.")
	}
	l.kubeURL = m["kubernetes_url"].(string)

	if nnp, ok := m["namespace_notification_path"]; !ok || nnp == "" {
		return fmt.Errorf("%s%s", confString, "namespace_notification_path required in config.")
	}
	l.namespaceNotificationPath = m["namespace_notification_path"].(string)

	if pnppre, ok := m["policy_notification_path_prefix"]; !ok || pnppre == "" {
		return fmt.Errorf("%s%s", confString, "policy_notification_path_prefix required in config.")
	}
	l.policyNotificationPathPrefix = m["policy_notification_path_prefix"].(string)

	if pnppost, ok := m["policy_notification_path_prefix"]; !ok || pnppost == "" {
		return fmt.Errorf("%s%s", confString, "policy_notification_path_postfix required in config.")
	}
	l.policyNotificationPathPostfix = m["policy_notification_path_postfix"].(string)

	if sln, ok := m["segment_label_name"]; !ok || sln == "" {
		return fmt.Errorf("%s%s", confString, "segment_label_name required in config.")
	}
	l.segmentLabelName = m["segment_label_name"].(string)

	if tln, ok := m["tenant_label_name"]; !ok || tln == "" {
		return fmt.Errorf("%s%s", confString, "tenant_label_name required in config.")
	}
	l.tenantLabelName = m["tenant_label_name"].(string)

	l.namespaceBufferSize = 1000

	if kc, ok := m["kubernetes_config"]; !ok || kc == "" {
		// Default kubernetes config location on ubuntu
		// TODO: this should not be hard coded, other
		//       distributions may have other user names.
		m["kubernetes_config"] = "/home/ubuntu/.kube/config"
	}

	// TODO, this loads kubernetes config from flags provided in main
	// should be loading from path provided by romana-root. Stas.
	kubeClientConfig, err := clientcmd.BuildConfigFromFlags("", m["kubernetes_config"].(string))
	if err != nil {
		return errors.New(fmt.Sprintf("Failed to load kubernetes kubeClientConfig %s", err))
	}
	clientset, err := kubernetes.NewForConfig(kubeClientConfig)
	if err != nil {
		return fmt.Errorf("Failed to make kubernetes client %s", err)
	}
	l.kubeClient = clientset

	return nil
}

// TODO there should be a better way to introduce translator
// then global variable like this one.
var PTranslator Translator

// Run configures and runs listener service.
func Run(rootServiceURL string, cred *common.Credential) (*common.RestServiceInfo, error) {
	clientConfig := common.GetDefaultRestClientConfig(rootServiceURL)
	clientConfig.Credential = cred
	client, err := common.NewRestClient(clientConfig)
	if err != nil {
		return nil, err
	}
	KubeListener := &KubeListener{}
	KubeListener.restClient = client

	config, err := client.GetServiceConfig(KubeListener.Name())
	if err != nil {
		return nil, err
	}
	return common.InitializeService(KubeListener, *config, cred)
}

// getOrAddSegment finds a segment (based on segment selector).
// If not found, it adds one.
func (l *KubeListener) getOrAddSegment(namespace string, kubeSegmentName string) (*tenant.Segment, error) {
	ten := &tenant.Tenant{}
	ten.Name = namespace
	// TODO this should be changed to find EXACTLY one after deletion functionality is implemented
	err := l.restClient.Find(ten, common.FindLast)
	if err != nil {
		return nil, err
	}

	seg := &tenant.Segment{}
	seg.Name = kubeSegmentName
	seg.TenantID = ten.ID
	err = l.restClient.Find(seg, common.FindExactlyOne)
	if err == nil {
		return seg, nil
	}

	switch err := err.(type) {
	case common.HttpError:
		if err.StatusCode == http.StatusNotFound {
			// Not found, so let's create a segment.
			segreq := tenant.Segment{Name: kubeSegmentName, TenantID: ten.ID}
			segURL, err2 := l.restClient.GetServiceUrl("tenant")
			if err2 != nil {
				return nil, err2
			}
			segURL = fmt.Sprintf("%s/tenants/%d/segments", segURL, ten.ID)
			err2 = l.restClient.Post(segURL, segreq, seg)
			if err2 == nil {
				// Successful creation.
				return seg, nil
			}
			// Creation of non-existing segment gave an error.
			switch err2 := err2.(type) {
			case common.HttpError:
				// Maybe someone else just created a segment between the original
				// lookup and now?
				if err2.StatusCode == http.StatusConflict {
					switch details := err2.Details.(type) {
					case tenant.Segment:
						// We expect the existing segment to be returned in the details field.
						return &details, nil
					default:
						// This is unexpected...
						return nil, err
					}
				}
				// Any other HTTP error other than a Conflict here - return it.
				return nil, err2
			default:
				// Any other error - return it
				return nil, err2
			}
		}
		// Any other HTTP error other than a Not found here - return it
		return nil, err
	default:
		// Any other error - return it
		return nil, err
	}
}

// resolveTenantByName retrieves tenant information from romana.
func (l *KubeListener) resolveTenantByName(tenantName string) (*tenant.Tenant, error) {
	t := &tenant.Tenant{Name: tenantName}
	err := l.restClient.Find(t, common.FindLast)
	if err != nil {
		return t, err
	}
	return t, nil
}

// deleteNetworkPolicyByID deletes the policy specified by the policyID
func (l *KubeListener) deleteNetworkPolicyByID(policyID uint64) error {
	policyURL, err := l.restClient.GetServiceUrl("policy")
	if err != nil {
		return err
	}
	policyURL = fmt.Sprintf("%s/policies/%d", policyURL, policyID)
	log.Debugf("deleteNetworkPolicyByID: Deleting policy %d", policyID)
	deletedPolicy := common.Policy{}
	err = l.restClient.Delete(policyURL, nil, &deletedPolicy)
	if err != nil {
		switch err := err.(type) {
		default:
			return err
		case common.HttpError:
			if err.StatusCode == http.StatusNotFound {
				log.Warnf("deleteNetworkPolicyByID: Policy %d not found, ignoring", policyID)
				return nil
			} else {
				return err
			}
		}
	}
	log.Debugf("deleteNetworkPolicyByID: Deleted policy %s", deletedPolicy)
	return err
}

// addNetworkPolicy adds the policy to the policy service.
func (l *KubeListener) addNetworkPolicy(policy common.Policy) error {
	policyURL, err := l.restClient.GetServiceUrl("policy")
	if err != nil {
		return err
	}
	policyURL = fmt.Sprintf("%s/policies", policyURL)
	log.Debugf("Applying policy %s", policy)
	err = l.restClient.Post(policyURL, policy, &policy)
	if err != nil {
		return err
	}
	return nil
}

// deleteNetworkPolicy deletes the policy matching provided policy on whatever
// fields are provided.
func (l *KubeListener) deleteNetworkPolicy(policy common.Policy) error {
	err := l.restClient.Find(&policy, common.FindExactlyOne)
	if err != nil {
		switch err := err.(type) {
		default:
			return err
		case common.HttpError:
			if err.StatusCode == http.StatusNotFound {
				log.Warnf("deleteNetworkPolicy: Policy not found %s, ignoring", policy.Name)
				return nil
			} else {
				return err
			}
		}
	}
	return l.deleteNetworkPolicyByID(policy.ID)
}

func (l *KubeListener) Initialize(client *common.RestClient) error {
	l.restClient = client

	// TODO, find a better place to initialize
	// the translator. Stas.
	PTranslator.Init(l.restClient, l.segmentLabelName, l.tenantLabelName)
	tc := PTranslator.GetClient()
	if tc == nil {
		log.Critical("Failed to initialize rest client for policy translator.")
		os.Exit(255)
	}

	l.lastEventPerNamespace = make(map[string]uint64)
	log.Infof("%s: Starting server", l.Name())
	nsURL, err := common.CleanURL(fmt.Sprintf("%s/%s/?%s", l.kubeURL, l.namespaceNotificationPath, HttpGetParamWatch))
	if err != nil {
		return err
	}
	log.Infof("Starting to listen on %s", nsURL)
	done := make(chan struct{})
	eventc, err := l.nsWatch(done, nsURL)
	if err != nil {
		log.Critical("Namespace watcher failed to start", err)
		os.Exit(255)
	}

	// events := l.conductor(nsEvents, done)
	l.process(eventc, done)

	ProduceNewPolicyEvents(eventc, done, l)

	log.Info("All routines started")
	return nil
}

// CreateSchema is placeholder for now.
func CreateSchema(rootServiceURL string, overwrite bool) error {
	return nil
}
<|endoftext|>


<commit_before>/* Utility functions and methods. Should probably absorbe what's in "common.go"
 * right now. */

/*
 * Copyright (c) 2013-2016, Jeremy Bingham (<jeremy@goiardi.gl>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
Package util contains various utility functions that are useful across all of goiardi.
*/
package util

import (
	"fmt"
	"github.com/ctdk/goiardi/config"
	"net/http"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

// NoDBConfigured is an error for when no database has been configured for use,
// yet an SQL function is being called.
var NoDBConfigured = &gerror{msg: "no db configured, but you tried to use one", status: http.StatusInternalServerError}

// GoiardiObj is an interface for helping goiardi/chef objects, like cookbooks,
// roles, etc., be able to easily make URLs and be identified by name.
type GoiardiObj interface {
	GetName() string
	URLType() string
}

type gerror struct {
	msg    string
	status int
}

// Gerror is an error type that includes an http status code (defaults to
// http.BadRequest).
type Gerror interface {
	String() string
	Error() string
	Status() int
	SetStatus(int)
}

// New makes a new Gerror. Usually you want Errorf.
func New(text string) Gerror {
	return &gerror{msg: text,
		status: http.StatusBadRequest,
	}
}

// Errorf creates a new Gerror, with a formatted error string.
func Errorf(format string, a ...interface{}) Gerror {
	return New(fmt.Sprintf(format, a...))
}

// CastErr will easily cast a different kind of error to a Gerror.
func CastErr(err error) Gerror {
	return Errorf(err.Error())
}

// Error returns the Gerror error message.
func (e *gerror) Error() string {
	return e.msg
}

func (e *gerror) String() string {
	return e.msg
}

// Set the Gerror HTTP status code.
func (e *gerror) SetStatus(s int) {
	e.status = s
}

// Returns the Gerror's HTTP status code.
func (e *gerror) Status() int {
	return e.status
}

// ObjURL crafts a URL for an object.
func ObjURL(obj GoiardiObj) string {
	baseURL := config.ServerBaseURL()
	fullURL := fmt.Sprintf("%s/%s/%s", baseURL, obj.URLType(), obj.GetName())
	return fullURL
}

// CustomObjURL crafts a URL for a Goiardi object with additional path elements.
func CustomObjURL(obj GoiardiObj, path string) string {
	chkPath(&path)
	return fmt.Sprintf("%s%s", ObjURL(obj), path)
}

// CustomURL crafts a URL from the provided path, without providing an object.
func CustomURL(path string) string {
	chkPath(&path)
	return fmt.Sprintf("%s%s", config.ServerBaseURL(), path)
}

func chkPath(p *string) {
	if (*p)[0] != '/' {
		*p = fmt.Sprintf("/%s", *p)
	}
}

// FlattenObj flattens an object and expand its keys into a map[string]string so
// it's suitable for indexing, either with solr (eventually) or with the whipped
// up replacement for local mode. Objects fed into this function *must* have the
// "json" tag set for their struct members.
func FlattenObj(obj interface{}) map[string]interface{} {
	expanded := make(map[string]interface{})
	s := reflect.ValueOf(obj).Elem()
	for i := 0; i < s.NumField(); i++ {
		if !s.Field(i).CanInterface() {
			continue
		}
		v := s.Field(i).Interface()
		key := s.Type().Field(i).Tag.Get("json")
		var mergeKey string
		if key == "automatic" || key == "normal" || key == "default" || key == "override" || key == "raw_data" {
			mergeKey = ""
		} else {
			mergeKey = key
		}
		subExpand := DeepMerge(mergeKey, v)
		/* Now merge the returned map */
		for k, u := range subExpand {
			expanded[k] = u
		}
	}
	return expanded
}

// MapifyObject turns an object into a map[string]interface{}. Useful for when
// you have a slice of objects that you need to trim, mutilate, fold, etc.
// before returning them as JSON.
func MapifyObject(obj interface{}) map[string]interface{} {
	mapified := make(map[string]interface{})
	s := reflect.ValueOf(obj).Elem()
	for i := 0; i < s.NumField(); i++ {
		if !s.Field(i).CanInterface() {
			continue
		}
		v := s.Field(i).Interface()
		key := s.Type().Field(i).Tag.Get("json")
		mapified[key] = v
	}
	return mapified
}

// Indexify prepares a flattened object for indexing by turning it into a sorted
// slice of strings formatted like "key:value".
func Indexify(flattened map[string]interface{}) []string {
	var readyToIndex []string
	// keep values in the index down to a reasonable size
	maxValLen := 150
	for k, v := range flattened {
		switch v := v.(type) {
		case string:
			//v = IndexEscapeStr(v)
			v = TrimStringMax(v, maxValLen)
			line := fmt.Sprintf("%s:%s", k, v)
			readyToIndex = append(readyToIndex, line)
		case []string:
			for _, w := range v {
				//w = IndexEscapeStr(w)
				w = TrimStringMax(w, maxValLen)
				line := fmt.Sprintf("%s:%s", k, w)
				readyToIndex = append(readyToIndex, line)
			}
		default:
			err := fmt.Errorf("We should never have been able to reach this state. Key %s had a value %v of type %T", k, v, v)
			panic(err)
		}
	}
	sort.Strings(readyToIndex)
	return readyToIndex
}

// IndexEscapeStr escapes values to index in the database, so characters that
// need to be escaped for Solr are properly found when using the trie or
// postgres based searches.
func IndexEscapeStr(s string) string {
	s = strings.Replace(s, "[", "\\[", -1)
	s = strings.Replace(s, "]", "\\]", -1)
	s = strings.Replace(s, "::", "\\:\\:", -1)
	return s
}

// DeepMerge merges disparate data structures into a flat hash.
func DeepMerge(key string, source interface{}) map[string]interface{} {
	merger := make(map[string]interface{})
	var sep string
	if config.Config.DotSearch {
		sep = "."
	} else {
		sep = "_"
	}
	switch v := source.(type) {
	case map[string]interface{}:
		/* We also need to get things like
		 * "default_attributes:key" indexed. */
		topLev := make([]string, len(v))
		n := 0
		for k, u := range v {
			if key != "" && !config.Config.UsePostgreSQL {
				topLev[n] = k
				n++
			}
			var nkey string
			if key == "" {
				nkey = k
			} else {
				nkey = fmt.Sprintf("%s%s%s", key, sep, k)
			}
			nm := DeepMerge(nkey, u)
			for j, q := range nm {
				merger[j] = q
			}
		}
		if key != "" && !config.Config.UsePostgreSQL {
			merger[key] = topLev
		}
	case map[string]string:
		/* We also need to get things like
		 * "default_attributes:key" indexed. */
		topLev := make([]string, len(v))
		n := 0
		for k, u := range v {
			if key != "" && !config.Config.UsePostgreSQL {
				topLev[n] = k
				n++
			}
			var nkey string
			if key == "" {
				nkey = k
			} else {
				nkey = fmt.Sprintf("%s%s%s", key, sep, k)
			}
			merger[nkey] = u
		}
		if key != "" && !config.Config.UsePostgreSQL {
			merger[key] = topLev
		}

	case []interface{}:
		km := make([]string, len(v))
		for i, w := range v {
			km[i] = stringify(w)
		}
		merger[key] = km
	case []string:
		km := make([]string, len(v))
		for i, w := range v {
			km[i] = stringify(w)
		}
		merger[key] = km
		/* If this is the run list, break recipes and roles out
		 * into their own separate indexes as well. */
		if key == "run_list" {
			roleMatch := regexp.MustCompile(`^(recipe|role)\[(.*)\]`)
			var roles []string
			var recipes []string
			for _, w := range v {
				rItem := roleMatch.FindStringSubmatch(stringify(w))
				if rItem != nil {
					rType := rItem[1]
					rThing := rItem[2]
					if rType == "role" {
						roles = append(roles, rThing)
					} else if rType == "recipe" {
						recipes = append(recipes, rThing)
					}
				}
			}
			if len(roles) > 0 {
				merger["role"] = roles
			}
			if len(recipes) > 0 {
				merger["recipe"] = recipes
			}
		}
	default:
		merger[key] = stringify(v)
	}
	return merger
}

func stringify(source interface{}) string {
	switch s := source.(type) {
	case string:
		return s
	case uint8, uint16, uint32, uint64:
		n := reflect.ValueOf(s).Uint()
		str := strconv.FormatUint(n, 10)
		return str
	case int8, int16, int32, int64:
		n := reflect.ValueOf(s).Int()
		str := strconv.FormatInt(n, 10)
		return str
	case float32, float64:
		n := reflect.ValueOf(s).Float()
		str := strconv.FormatFloat(n, 'f', -1, 64)
		return str
	case bool:
		str := strconv.FormatBool(s)
		return str
	default:
		/* Just send back whatever %v gives */
		str := fmt.Sprintf("%v", s)
		return str
	}
}

// PgSearchKey removes characters from search term fields that make the ltree
// data type unhappy. This leads to the postgres-based search being, perhaps,
// somewhat less precise than the solr (or ersatz solr) based search, but at the
// same time one that's less resource demanding and covers almost all known use
// cases. Potential bug: Postgres considers some, but not all, unicode letters
// as being alphanumeric; i.e. golang and postgres both consider 'ü' to be a
// letter, but golang accepts 'ሀ' as a letter while postgres does not. This is
// reasonably unlikely to be an issue, but if you're using lots of non-European
// characters in your attributes this could be a problem. We're accepting more
// than raw ASCII alnum however because it's better behavior and because
// Postgres does accept at least some other alphabets as being alphanumeric.
func PgSearchKey(key string) string {
	re := regexp.MustCompile(`[^\pL\pN_\.]`)
	bs := regexp.MustCompile(`_{2,}`)
	ps := regexp.MustCompile(`\.{2,}`) // repeated . will cause trouble too
	return pgKeyReplace(key, re, bs, ps)
}

// PgSearchQueryKey is very similar to PgSearchKey, except that it preserves the
// Solr wildcard charactes '*' and '?' in the queries.
func PgSearchQueryKey(key string) string {
	re := regexp.MustCompile(`[^\pL\pN_\.\*\?]`)
	bs := regexp.MustCompile(`_{2,}`)
	ps := regexp.MustCompile(`\.{2,}`)
	return pgKeyReplace(key, re, bs, ps)
}

func pgKeyReplace(key string, re, bs, ps *regexp.Regexp) string {
	k := re.ReplaceAllString(key, "_")
	k = bs.ReplaceAllString(k, "_")
	k = ps.ReplaceAllString(k, ".")
	k = strings.Trim(k, "_")
	// on the off hand chance we get leading or trailing dots
	k = strings.Trim(k, ".")
	// finally, if converting search query syntax, convert all _ to '.'.
	// This may need to be revisited in more detail if we find ourselves
	// needing more finesse with escaping underscores.
	if config.Config.ConvertSearch {
		k = strings.Replace(k, "_", ".", -1)
		k = ps.ReplaceAllString(k, ".")
	}
	return k
}
<commit_msg>remove duplicate strings from arrays being indexed<commit_after>/* Utility functions and methods. Should probably absorbe what's in "common.go"
 * right now. */

/*
 * Copyright (c) 2013-2016, Jeremy Bingham (<jeremy@goiardi.gl>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
Package util contains various utility functions that are useful across all of goiardi.
*/
package util

import (
	"fmt"
	"github.com/ctdk/goiardi/config"
	"net/http"
	"reflect"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

// NoDBConfigured is an error for when no database has been configured for use,
// yet an SQL function is being called.
var NoDBConfigured = &gerror{msg: "no db configured, but you tried to use one", status: http.StatusInternalServerError}

// GoiardiObj is an interface for helping goiardi/chef objects, like cookbooks,
// roles, etc., be able to easily make URLs and be identified by name.
type GoiardiObj interface {
	GetName() string
	URLType() string
}

type gerror struct {
	msg    string
	status int
}

// Gerror is an error type that includes an http status code (defaults to
// http.BadRequest).
type Gerror interface {
	String() string
	Error() string
	Status() int
	SetStatus(int)
}

// New makes a new Gerror. Usually you want Errorf.
func New(text string) Gerror {
	return &gerror{msg: text,
		status: http.StatusBadRequest,
	}
}

// Errorf creates a new Gerror, with a formatted error string.
func Errorf(format string, a ...interface{}) Gerror {
	return New(fmt.Sprintf(format, a...))
}

// CastErr will easily cast a different kind of error to a Gerror.
func CastErr(err error) Gerror {
	return Errorf(err.Error())
}

// Error returns the Gerror error message.
func (e *gerror) Error() string {
	return e.msg
}

func (e *gerror) String() string {
	return e.msg
}

// Set the Gerror HTTP status code.
func (e *gerror) SetStatus(s int) {
	e.status = s
}

// Returns the Gerror's HTTP status code.
func (e *gerror) Status() int {
	return e.status
}

// ObjURL crafts a URL for an object.
func ObjURL(obj GoiardiObj) string {
	baseURL := config.ServerBaseURL()
	fullURL := fmt.Sprintf("%s/%s/%s", baseURL, obj.URLType(), obj.GetName())
	return fullURL
}

// CustomObjURL crafts a URL for a Goiardi object with additional path elements.
func CustomObjURL(obj GoiardiObj, path string) string {
	chkPath(&path)
	return fmt.Sprintf("%s%s", ObjURL(obj), path)
}

// CustomURL crafts a URL from the provided path, without providing an object.
func CustomURL(path string) string {
	chkPath(&path)
	return fmt.Sprintf("%s%s", config.ServerBaseURL(), path)
}

func chkPath(p *string) {
	if (*p)[0] != '/' {
		*p = fmt.Sprintf("/%s", *p)
	}
}

// FlattenObj flattens an object and expand its keys into a map[string]string so
// it's suitable for indexing, either with solr (eventually) or with the whipped
// up replacement for local mode. Objects fed into this function *must* have the
// "json" tag set for their struct members.
func FlattenObj(obj interface{}) map[string]interface{} {
	expanded := make(map[string]interface{})
	s := reflect.ValueOf(obj).Elem()
	for i := 0; i < s.NumField(); i++ {
		if !s.Field(i).CanInterface() {
			continue
		}
		v := s.Field(i).Interface()
		key := s.Type().Field(i).Tag.Get("json")
		var mergeKey string
		if key == "automatic" || key == "normal" || key == "default" || key == "override" || key == "raw_data" {
			mergeKey = ""
		} else {
			mergeKey = key
		}
		subExpand := DeepMerge(mergeKey, v)
		/* Now merge the returned map */
		for k, u := range subExpand {
			expanded[k] = u
		}
	}
	return expanded
}

// MapifyObject turns an object into a map[string]interface{}. Useful for when
// you have a slice of objects that you need to trim, mutilate, fold, etc.
// before returning them as JSON.
func MapifyObject(obj interface{}) map[string]interface{} {
	mapified := make(map[string]interface{})
	s := reflect.ValueOf(obj).Elem()
	for i := 0; i < s.NumField(); i++ {
		if !s.Field(i).CanInterface() {
			continue
		}
		v := s.Field(i).Interface()
		key := s.Type().Field(i).Tag.Get("json")
		mapified[key] = v
	}
	return mapified
}

// Indexify prepares a flattened object for indexing by turning it into a sorted
// slice of strings formatted like "key:value".
func Indexify(flattened map[string]interface{}) []string {
	var readyToIndex []string
	// keep values in the index down to a reasonable size
	maxValLen := 150
	for k, v := range flattened {
		switch v := v.(type) {
		case string:
			//v = IndexEscapeStr(v)
			v = TrimStringMax(v, maxValLen)
			line := fmt.Sprintf("%s:%s", k, v)
			readyToIndex = append(readyToIndex, line)
		case []string:
			sort.Strings(v)
			v = RemoveDupStrings(v)
			for _, w := range v {
				//w = IndexEscapeStr(w)
				w = TrimStringMax(w, maxValLen)
				line := fmt.Sprintf("%s:%s", k, w)
				readyToIndex = append(readyToIndex, line)
			}
		default:
			err := fmt.Errorf("We should never have been able to reach this state. Key %s had a value %v of type %T", k, v, v)
			panic(err)
		}
	}
	sort.Strings(readyToIndex)
	return readyToIndex
}

// IndexEscapeStr escapes values to index in the database, so characters that
// need to be escaped for Solr are properly found when using the trie or
// postgres based searches.
func IndexEscapeStr(s string) string {
	s = strings.Replace(s, "[", "\\[", -1)
	s = strings.Replace(s, "]", "\\]", -1)
	s = strings.Replace(s, "::", "\\:\\:", -1)
	return s
}

// DeepMerge merges disparate data structures into a flat hash.
func DeepMerge(key string, source interface{}) map[string]interface{} {
	merger := make(map[string]interface{})
	var sep string
	if config.Config.DotSearch {
		sep = "."
	} else {
		sep = "_"
	}
	switch v := source.(type) {
	case map[string]interface{}:
		/* We also need to get things like
		 * "default_attributes:key" indexed. */
		topLev := make([]string, len(v))
		n := 0
		for k, u := range v {
			if key != "" && !config.Config.UsePostgreSQL {
				topLev[n] = k
				n++
			}
			var nkey string
			if key == "" {
				nkey = k
			} else {
				nkey = fmt.Sprintf("%s%s%s", key, sep, k)
			}
			nm := DeepMerge(nkey, u)
			for j, q := range nm {
				merger[j] = q
			}
		}
		if key != "" && !config.Config.UsePostgreSQL {
			merger[key] = topLev
		}
	case map[string]string:
		/* We also need to get things like
		 * "default_attributes:key" indexed. */
		topLev := make([]string, len(v))
		n := 0
		for k, u := range v {
			if key != "" && !config.Config.UsePostgreSQL {
				topLev[n] = k
				n++
			}
			var nkey string
			if key == "" {
				nkey = k
			} else {
				nkey = fmt.Sprintf("%s%s%s", key, sep, k)
			}
			merger[nkey] = u
		}
		if key != "" && !config.Config.UsePostgreSQL {
			merger[key] = topLev
		}

	case []interface{}:
		km := make([]string, len(v))
		for i, w := range v {
			km[i] = stringify(w)
		}
		merger[key] = km
	case []string:
		km := make([]string, len(v))
		for i, w := range v {
			km[i] = stringify(w)
		}
		merger[key] = km
		/* If this is the run list, break recipes and roles out
		 * into their own separate indexes as well. */
		if key == "run_list" {
			roleMatch := regexp.MustCompile(`^(recipe|role)\[(.*)\]`)
			var roles []string
			var recipes []string
			for _, w := range v {
				rItem := roleMatch.FindStringSubmatch(stringify(w))
				if rItem != nil {
					rType := rItem[1]
					rThing := rItem[2]
					if rType == "role" {
						roles = append(roles, rThing)
					} else if rType == "recipe" {
						recipes = append(recipes, rThing)
					}
				}
			}
			if len(roles) > 0 {
				merger["role"] = roles
			}
			if len(recipes) > 0 {
				merger["recipe"] = recipes
			}
		}
	default:
		merger[key] = stringify(v)
	}
	return merger
}

func stringify(source interface{}) string {
	switch s := source.(type) {
	case string:
		return s
	case uint8, uint16, uint32, uint64:
		n := reflect.ValueOf(s).Uint()
		str := strconv.FormatUint(n, 10)
		return str
	case int8, int16, int32, int64:
		n := reflect.ValueOf(s).Int()
		str := strconv.FormatInt(n, 10)
		return str
	case float32, float64:
		n := reflect.ValueOf(s).Float()
		str := strconv.FormatFloat(n, 'f', -1, 64)
		return str
	case bool:
		str := strconv.FormatBool(s)
		return str
	default:
		/* Just send back whatever %v gives */
		str := fmt.Sprintf("%v", s)
		return str
	}
}

// PgSearchKey removes characters from search term fields that make the ltree
// data type unhappy. This leads to the postgres-based search being, perhaps,
// somewhat less precise than the solr (or ersatz solr) based search, but at the
// same time one that's less resource demanding and covers almost all known use
// cases. Potential bug: Postgres considers some, but not all, unicode letters
// as being alphanumeric; i.e. golang and postgres both consider 'ü' to be a
// letter, but golang accepts 'ሀ' as a letter while postgres does not. This is
// reasonably unlikely to be an issue, but if you're using lots of non-European
// characters in your attributes this could be a problem. We're accepting more
// than raw ASCII alnum however because it's better behavior and because
// Postgres does accept at least some other alphabets as being alphanumeric.
func PgSearchKey(key string) string {
	re := regexp.MustCompile(`[^\pL\pN_\.]`)
	bs := regexp.MustCompile(`_{2,}`)
	ps := regexp.MustCompile(`\.{2,}`) // repeated . will cause trouble too
	return pgKeyReplace(key, re, bs, ps)
}

// PgSearchQueryKey is very similar to PgSearchKey, except that it preserves the
// Solr wildcard charactes '*' and '?' in the queries.
func PgSearchQueryKey(key string) string {
	re := regexp.MustCompile(`[^\pL\pN_\.\*\?]`)
	bs := regexp.MustCompile(`_{2,}`)
	ps := regexp.MustCompile(`\.{2,}`)
	return pgKeyReplace(key, re, bs, ps)
}

func pgKeyReplace(key string, re, bs, ps *regexp.Regexp) string {
	k := re.ReplaceAllString(key, "_")
	k = bs.ReplaceAllString(k, "_")
	k = ps.ReplaceAllString(k, ".")
	k = strings.Trim(k, "_")
	// on the off hand chance we get leading or trailing dots
	k = strings.Trim(k, ".")
	// finally, if converting search query syntax, convert all _ to '.'.
	// This may need to be revisited in more detail if we find ourselves
	// needing more finesse with escaping underscores.
	if config.Config.ConvertSearch {
		k = strings.Replace(k, "_", ".", -1)
		k = ps.ReplaceAllString(k, ".")
	}
	return k
}
<|endoftext|>


<commit_before>package util

import (
	"bytes"
	"fmt"
	pbcom "github.com/CSUNetSec/netsec-protobufs/common"
	"net"
)

func GetIP(a *pbcom.IPAddressWrapper) []byte {
	if a.Ipv4 != nil {
		return a.Ipv4
	} else if a.Ipv6 != nil {
		return a.Ipv6
	}
	return nil
}

func IpToRadixkey(b []byte, mask uint8) string {
	var (
		ip     net.IP = b
		buffer bytes.Buffer
	)
	if len(b) == 0 || len(ip) == 0 { // a misparsed ip probably.
		return ""
	}

	if ip.To4() != nil {
		if mask > 32 { //misparsed?
			return ""
		}
		ip = ip.Mask(net.CIDRMask(int(mask), 32)).To4()
	} else {
		if mask > 128 { //misparsed?
			return ""
		}
		ip = ip.Mask(net.CIDRMask(int(mask), 128)).To16()
	}

	for i := 0; i < len(ip); i++ {
		fmt.Fprintf(&buffer, "%08b", ip[i])
	}
	return buffer.String()
}

//help functions for storing IPs in the radix tree
func IpToRadixkey2(b []byte, mask uint8) string {
	var buffer bytes.Buffer
	for i := 0; i < len(b) && i < int(mask); i++ {
		buffer.WriteString(fmt.Sprintf("%08b", b[i]))
	}
	if int(mask) > len(buffer.String()) {
		return buffer.String()
	}
	return buffer.String()[:mask]
}
<commit_msg>implementing radix tree lookup and insert in protoparse for ip masks<commit_after>package util

import (
	"bytes"
	"fmt"
	pbcom "github.com/CSUNetSec/netsec-protobufs/common"
	radix "github.com/armon/go-radix"
	"net"
)

func GetIP(a *pbcom.IPAddressWrapper) []byte {
	if a.Ipv4 != nil {
		return a.Ipv4
	} else if a.Ipv6 != nil {
		return a.Ipv6
	}
	return nil
}

func IpToRadixkey(b []byte, mask uint8) string {
	var (
		ip     net.IP = b
		buffer bytes.Buffer
	)
	if len(b) == 0 || len(ip) == 0 { // a misparsed ip probably.
		return ""
	}

	if ip.To4() != nil {
		if mask > 32 { //misparsed?
			return ""
		}
		ip = ip.Mask(net.CIDRMask(int(mask), 32)).To4()
	} else {
		if mask > 128 { //misparsed?
			return ""
		}
		ip = ip.Mask(net.CIDRMask(int(mask), 128)).To16()
	}

	for i := 0; i < len(ip); i++ {
		fmt.Fprintf(&buffer, "%08b", ip[i])
	}
	return buffer.String()
}

//PrefixTree holds a radix tree which clients
//can insert ips and masks in , and  also lookup
//for their existence.
type PrefixTree struct {
	rt *radix.Tree
}

func NewPrefixTree() PrefixTree {
	return PrefixTree{
		rt: radix.New(),
	}
}

func (pt PrefixTree) Add(ip net.IP, mask uint8) {
	keystr := IpToRadixkey(ip, mask)
	pt.rt.Insert(keystr, true)
}

func (pt PrefixTree) ContainsIpMask(ip net.IP, mask uint8) bool {
	keystr := IpToRadixkey(ip, mask)
	if _, _, found := pt.rt.LongestPrefix(keystr); found {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"fmt"
	"hash/fnv"
	"io"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/gcsproxy"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

// A thin wrapper around an object proxy for an object in GCS, implementing the
// interfaces necessary for the fusefs package.
//
// TODO(jacobsa): After becoming comfortable with the representation of dir and
// its concurrency protection, audit this file and make sure it is up to par.
type file struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	bucket gcs.Bucket

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// A proxy for the file contents in GCS.
	//
	// INVARIANT: objectProxy.CheckInvariants() doesn't panic
	objectProxy *gcsproxy.ObjectProxy // PT_GUARDED_BY(mu)
}

// Make sure file implements the interfaces we think it does.
var (
	_ fusefs.Node          = &file{}
	_ fusefs.NodeGetattrer = &file{}

	_ fusefs.Handle         = &file{}
	_ fusefs.HandleFlusher  = &file{}
	_ fusefs.HandleReader   = &file{}
	_ fusefs.HandleReleaser = &file{}
	_ fusefs.HandleWriter   = &file{}
)

func newFile(
	logger *log.Logger,
	bucket gcs.Bucket,
	objectName string,
	remoteSize uint64) *file {
	f := &file{
		logger:     logger,
		bucket:     bucket,
		objectName: objectName,
		remoteSize: remoteSize,
	}

	f.mu = syncutil.NewInvariantMutex(func() { f.checkInvariants() })

	return f
}

func (f *file) checkInvariants() {
	if f.tempFileDirty && f.tempFile == nil {
		panic("Expected !tempFileDirty when tempFile == nil.")
	}

	if f.tempFile != nil && f.remoteSize != 0 {
		panic("Expected remoteSize == 0 when tempFile != nil.")
	}
}

func generateInodeNumber(objectName string) uint64 {
	h := fnv.New64()
	if _, err := io.WriteString(h, objectName); err != nil {
		panic(err)
	}

	return h.Sum64()
}

// TODO(jacobsa): Share code with Getattr below.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Attr() fuse.Attr {
	f.mu.RLock()
	defer f.mu.RUnlock()

	f.logger.Printf("Attr: [%s]/%s", f.bucket.Name(), f.objectName)

	// Find the current size.
	size := f.remoteSize
	if f.tempFile != nil {
		// See if the scenario in the TODO below can ever even come to pass. If so,
		// file a bug with the fuse package to see what we're intended to do with
		// errors.
		panic("Received an Attr call with a temp file active.")

		fi, err := f.tempFile.Stat()
		if err != nil {
			// TODO(jacobsa): What do we do about this? The fuse package gives us no
			// opportunity for returning an error here.
			panic("tempFile.Stat: " + err.Error())
		}

		size = uint64(fi.Size())
	}

	return fuse.Attr{
		Inode: generateInodeNumber(f.objectName),
		Mode:  0700,
		Size:  size,
	}
}

// LOCKS_EXCLUDED(f.mu)
func (f *file) Getattr(
	ctx context.Context,
	req *fuse.GetattrRequest,
	resp *fuse.GetattrResponse) (err error) {
	f.mu.RLock()
	defer f.mu.RUnlock()

	f.logger.Printf("Getattr: [%s]/%s", f.bucket.Name(), f.objectName)

	// Find the current size.
	size := f.remoteSize
	if f.tempFile != nil {
		var fi os.FileInfo
		if fi, err = f.tempFile.Stat(); err != nil {
			err = fmt.Errorf("tempFile.Stat: %v", err)
			return
		}

		size = uint64(fi.Size())
	}

	resp.Attr = fuse.Attr{
		Inode: generateInodeNumber(f.objectName),
		Mode:  0700,
		Size:  size,

		// TODO(jacobsa): Do something more useful here.
		Mtime: time.Now(),
	}

	return
}

// If the file contents have not yet been fetched to a temporary file, fetch
// them.
//
// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *file) ensureTempFile(ctx context.Context) error {
	// Do we already have a file?
	if f.tempFile != nil {
		return nil
	}

	// Create a temporary file.
	tempFile, err := ioutil.TempFile("", "gcsfuse")
	if err != nil {
		return fmt.Errorf("ioutil.TempFile: %v", err)
	}

	// Create a reader for the object.
	readCloser, err := f.bucket.NewReader(ctx, f.objectName)
	if err != nil {
		return fmt.Errorf("bucket.NewReader: %v", err)
	}

	defer readCloser.Close()

	// Copy the object contents into the file.
	if _, err := io.Copy(tempFile, readCloser); err != nil {
		return fmt.Errorf("io.Copy: %v", err)
	}

	// Save the file for later.
	f.tempFile = tempFile

	// remoteSize is no longer authoritative.
	f.remoteSize = 0

	return nil
}

// Throw away the local temporary file, if any.
//
// TODO(jacobsa): There are a few bugs here.
//
// 1. This is called when a file descriptor is closed (actually the last clone
// of a file descriptor? -- test this), not when the last file descriptor
// referring to an inode is closed. We don't want to throw away the temp file
// just yet. Add tests for this.
//
// 2. When we do throw out the temp file (probably when the inode is being
// forgotten?, we need to write it back if it's dirty.
//
// Add tests for all of these bugs before fixing them.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Release(
	ctx context.Context,
	req *fuse.ReleaseRequest) (err error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	f.logger.Printf("Release: [%s]/%s", f.bucket.Name(), f.objectName)

	// Is there a file to close?
	if f.tempFile == nil {
		return nil
	}

	// Stat it, for use below.
	fileInfo, err := f.tempFile.Stat()
	if err != nil {
		err = fmt.Errorf("Statting temp file: %v", err)
		return
	}

	// Close it, after grabbing its path.
	path := f.tempFile.Name()
	if err = f.tempFile.Close(); err != nil {
		err = fmt.Errorf("Closing temp file: %v", err)
		return
	}

	// Attempt to delete it.
	if err = os.Remove(path); err != nil {
		err = fmt.Errorf("Deleting temp file: %v", err)
		return
	}

	f.tempFile = nil
	f.tempFileDirty = false
	f.remoteSize = uint64(fileInfo.Size())

	return nil
}

// Ensure that the local temporary file is initialized, then read from it.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Read(
	ctx context.Context,
	req *fuse.ReadRequest,
	resp *fuse.ReadResponse) error {
	f.mu.Lock()
	defer f.mu.Unlock()

	f.logger.Printf("Read: [%s]/%s", f.bucket.Name(), f.objectName)

	// Ensure the temp file is present.
	if err := f.ensureTempFile(ctx); err != nil {
		return err
	}

	// Allocate a response buffer.
	resp.Data = make([]byte, req.Size)

	// Read the data.
	n, err := f.tempFile.ReadAt(resp.Data, req.Offset)
	resp.Data = resp.Data[:n]

	// Special case: read(2) doesn't return EOF errors.
	if err == io.EOF {
		err = nil
	}

	return err
}

// Ensure that the local temporary file is initialized, then write to it.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Write(
	ctx context.Context,
	req *fuse.WriteRequest,
	resp *fuse.WriteResponse) (err error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	f.logger.Printf("Write: [%s]/%s", f.bucket.Name(), f.objectName)

	// Ensure the temp file is present. If it's not, grab the current contents
	// from GCS.
	if err = f.ensureTempFile(ctx); err != nil {
		err = fmt.Errorf("ensureTempFile: %v", err)
		return
	}

	// Mark us dirty.
	f.tempFileDirty = true

	// Write to the temp file.
	resp.Size, err = f.tempFile.WriteAt(req.Data, req.Offset)

	return
}

// Put the temporary file back in the bucket if it's dirty.
//
// TODO(jacobsa): This probably isn't the write place to do this. ext2 doesn't
// appear to do anything at all for i_op->flush, for example, and the fuse
// documentation pretty much says as much (http://goo.gl/KkBJM3 "Filesystems
// shouldn't assume that flush will always be called after some writes, or that
// if will be called at all"). Instead:
//
//  1. We should definitely do it on fsync, because the user asked.
//  2. We should definitely do it when the kernel is forgetting the inode,
//     because we won't get another chance.
//  3. Maybe we should do it after some timeout after the file is closed (the
//     file handle is released).
//
// Avoid doing #3 for now, because the kernel may already forget the inode
// after some timeout after it is unused, to avoid data loss due to power loss.
// Do #3 only if it becomes clear that #2 is not sufficient for real users.
//
// I wonder if we also need to do it when destroying. Will the kernel send us
// forgets before destroying? Again, add only if needed.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Flush(
	ctx context.Context,
	req *fuse.FlushRequest) (err error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	f.logger.Printf("Flush: [%s]/%s", f.bucket.Name(), f.objectName)

	// Is there anything interesting for us to do?
	if !f.tempFileDirty {
		return
	}

	// Flush the temp file to GCS.
	createReq := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: f.objectName,
		},
		Contents: f.tempFile,
	}

	if _, err = f.bucket.CreateObject(ctx, createReq); err != nil {
		err = fmt.Errorf("bucket.CreateObject: %v", err)
		return
	}

	// We are no longer dirty.
	//
	// TODO(jacobsa): Add a test for this. Cause a flush to happen, then
	// overwrite object contents out of band, then make sure they don't restore
	// next time flush happens.
	f.tempFileDirty = false

	return
}
<commit_msg>Fixed up newFile.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"fmt"
	"hash/fnv"
	"io"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/gcsproxy"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

// A thin wrapper around an object proxy for an object in GCS, implementing the
// interfaces necessary for the fusefs package.
//
// TODO(jacobsa): After becoming comfortable with the representation of dir and
// its concurrency protection, audit this file and make sure it is up to par.
type file struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// A proxy for the file contents in GCS.
	//
	// INVARIANT: objectProxy.CheckInvariants() doesn't panic
	objectProxy *gcsproxy.ObjectProxy // PT_GUARDED_BY(mu)
}

// Make sure file implements the interfaces we think it does.
var (
	_ fusefs.Node          = &file{}
	_ fusefs.NodeGetattrer = &file{}

	_ fusefs.Handle         = &file{}
	_ fusefs.HandleFlusher  = &file{}
	_ fusefs.HandleReader   = &file{}
	_ fusefs.HandleReleaser = &file{}
	_ fusefs.HandleWriter   = &file{}
)

func newFile(
	logger *log.Logger,
	bucket gcs.Bucket,
	remoteObject *storage.Object) (f *file, err error) {
	// Create the struct.
	f = &file{
		logger: logger,
	}

	// Set up the object proxy.
	f.objectProxy, err = gcsproxy.NewObjectProxy(bucket, remoteObject.Name)
	if err != nil {
		err = fmt.Errorf("NewObjectProxy: %v", err)
		return
	}

	// Set up the mutex.
	f.mu = syncutil.NewInvariantMutex(f.checkInvariants)

	return
}

func (f *file) checkInvariants() {
	if f.tempFileDirty && f.tempFile == nil {
		panic("Expected !tempFileDirty when tempFile == nil.")
	}

	if f.tempFile != nil && f.remoteSize != 0 {
		panic("Expected remoteSize == 0 when tempFile != nil.")
	}
}

func generateInodeNumber(objectName string) uint64 {
	h := fnv.New64()
	if _, err := io.WriteString(h, objectName); err != nil {
		panic(err)
	}

	return h.Sum64()
}

// TODO(jacobsa): Share code with Getattr below.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Attr() fuse.Attr {
	f.mu.RLock()
	defer f.mu.RUnlock()

	f.logger.Printf("Attr: [%s]/%s", f.bucket.Name(), f.objectName)

	// Find the current size.
	size := f.remoteSize
	if f.tempFile != nil {
		// See if the scenario in the TODO below can ever even come to pass. If so,
		// file a bug with the fuse package to see what we're intended to do with
		// errors.
		panic("Received an Attr call with a temp file active.")

		fi, err := f.tempFile.Stat()
		if err != nil {
			// TODO(jacobsa): What do we do about this? The fuse package gives us no
			// opportunity for returning an error here.
			panic("tempFile.Stat: " + err.Error())
		}

		size = uint64(fi.Size())
	}

	return fuse.Attr{
		Inode: generateInodeNumber(f.objectName),
		Mode:  0700,
		Size:  size,
	}
}

// LOCKS_EXCLUDED(f.mu)
func (f *file) Getattr(
	ctx context.Context,
	req *fuse.GetattrRequest,
	resp *fuse.GetattrResponse) (err error) {
	f.mu.RLock()
	defer f.mu.RUnlock()

	f.logger.Printf("Getattr: [%s]/%s", f.bucket.Name(), f.objectName)

	// Find the current size.
	size := f.remoteSize
	if f.tempFile != nil {
		var fi os.FileInfo
		if fi, err = f.tempFile.Stat(); err != nil {
			err = fmt.Errorf("tempFile.Stat: %v", err)
			return
		}

		size = uint64(fi.Size())
	}

	resp.Attr = fuse.Attr{
		Inode: generateInodeNumber(f.objectName),
		Mode:  0700,
		Size:  size,

		// TODO(jacobsa): Do something more useful here.
		Mtime: time.Now(),
	}

	return
}

// If the file contents have not yet been fetched to a temporary file, fetch
// them.
//
// EXCLUSIVE_LOCKS_REQUIRED(f.mu)
func (f *file) ensureTempFile(ctx context.Context) error {
	// Do we already have a file?
	if f.tempFile != nil {
		return nil
	}

	// Create a temporary file.
	tempFile, err := ioutil.TempFile("", "gcsfuse")
	if err != nil {
		return fmt.Errorf("ioutil.TempFile: %v", err)
	}

	// Create a reader for the object.
	readCloser, err := f.bucket.NewReader(ctx, f.objectName)
	if err != nil {
		return fmt.Errorf("bucket.NewReader: %v", err)
	}

	defer readCloser.Close()

	// Copy the object contents into the file.
	if _, err := io.Copy(tempFile, readCloser); err != nil {
		return fmt.Errorf("io.Copy: %v", err)
	}

	// Save the file for later.
	f.tempFile = tempFile

	// remoteSize is no longer authoritative.
	f.remoteSize = 0

	return nil
}

// Throw away the local temporary file, if any.
//
// TODO(jacobsa): There are a few bugs here.
//
// 1. This is called when a file descriptor is closed (actually the last clone
// of a file descriptor? -- test this), not when the last file descriptor
// referring to an inode is closed. We don't want to throw away the temp file
// just yet. Add tests for this.
//
// 2. When we do throw out the temp file (probably when the inode is being
// forgotten?, we need to write it back if it's dirty.
//
// Add tests for all of these bugs before fixing them.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Release(
	ctx context.Context,
	req *fuse.ReleaseRequest) (err error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	f.logger.Printf("Release: [%s]/%s", f.bucket.Name(), f.objectName)

	// Is there a file to close?
	if f.tempFile == nil {
		return nil
	}

	// Stat it, for use below.
	fileInfo, err := f.tempFile.Stat()
	if err != nil {
		err = fmt.Errorf("Statting temp file: %v", err)
		return
	}

	// Close it, after grabbing its path.
	path := f.tempFile.Name()
	if err = f.tempFile.Close(); err != nil {
		err = fmt.Errorf("Closing temp file: %v", err)
		return
	}

	// Attempt to delete it.
	if err = os.Remove(path); err != nil {
		err = fmt.Errorf("Deleting temp file: %v", err)
		return
	}

	f.tempFile = nil
	f.tempFileDirty = false
	f.remoteSize = uint64(fileInfo.Size())

	return nil
}

// Ensure that the local temporary file is initialized, then read from it.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Read(
	ctx context.Context,
	req *fuse.ReadRequest,
	resp *fuse.ReadResponse) error {
	f.mu.Lock()
	defer f.mu.Unlock()

	f.logger.Printf("Read: [%s]/%s", f.bucket.Name(), f.objectName)

	// Ensure the temp file is present.
	if err := f.ensureTempFile(ctx); err != nil {
		return err
	}

	// Allocate a response buffer.
	resp.Data = make([]byte, req.Size)

	// Read the data.
	n, err := f.tempFile.ReadAt(resp.Data, req.Offset)
	resp.Data = resp.Data[:n]

	// Special case: read(2) doesn't return EOF errors.
	if err == io.EOF {
		err = nil
	}

	return err
}

// Ensure that the local temporary file is initialized, then write to it.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Write(
	ctx context.Context,
	req *fuse.WriteRequest,
	resp *fuse.WriteResponse) (err error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	f.logger.Printf("Write: [%s]/%s", f.bucket.Name(), f.objectName)

	// Ensure the temp file is present. If it's not, grab the current contents
	// from GCS.
	if err = f.ensureTempFile(ctx); err != nil {
		err = fmt.Errorf("ensureTempFile: %v", err)
		return
	}

	// Mark us dirty.
	f.tempFileDirty = true

	// Write to the temp file.
	resp.Size, err = f.tempFile.WriteAt(req.Data, req.Offset)

	return
}

// Put the temporary file back in the bucket if it's dirty.
//
// TODO(jacobsa): This probably isn't the write place to do this. ext2 doesn't
// appear to do anything at all for i_op->flush, for example, and the fuse
// documentation pretty much says as much (http://goo.gl/KkBJM3 "Filesystems
// shouldn't assume that flush will always be called after some writes, or that
// if will be called at all"). Instead:
//
//  1. We should definitely do it on fsync, because the user asked.
//  2. We should definitely do it when the kernel is forgetting the inode,
//     because we won't get another chance.
//  3. Maybe we should do it after some timeout after the file is closed (the
//     file handle is released).
//
// Avoid doing #3 for now, because the kernel may already forget the inode
// after some timeout after it is unused, to avoid data loss due to power loss.
// Do #3 only if it becomes clear that #2 is not sufficient for real users.
//
// I wonder if we also need to do it when destroying. Will the kernel send us
// forgets before destroying? Again, add only if needed.
//
// LOCKS_EXCLUDED(f.mu)
func (f *file) Flush(
	ctx context.Context,
	req *fuse.FlushRequest) (err error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	f.logger.Printf("Flush: [%s]/%s", f.bucket.Name(), f.objectName)

	// Is there anything interesting for us to do?
	if !f.tempFileDirty {
		return
	}

	// Flush the temp file to GCS.
	createReq := &gcs.CreateObjectRequest{
		Attrs: storage.ObjectAttrs{
			Name: f.objectName,
		},
		Contents: f.tempFile,
	}

	if _, err = f.bucket.CreateObject(ctx, createReq); err != nil {
		err = fmt.Errorf("bucket.CreateObject: %v", err)
		return
	}

	// We are no longer dirty.
	//
	// TODO(jacobsa): Add a test for this. Cause a flush to happen, then
	// overwrite object contents out of band, then make sure they don't restore
	// next time flush happens.
	f.tempFileDirty = false

	return
}
<|endoftext|>