<commit_before>package lzma

import (
	"bufio"
	"errors"
	"io"
)

// states defines the overall state count
const states = 12

// bufferLen is the value for internal buffering of the decoder.
var bufferLen = 64 * (1 << 10)

// Decoder is able to read a LZMA byte stream and to read the plain text.
type Decoder struct {
	properties Properties
	packedLen  uint64
	total      uint64
	dict       *decoderDict
	state      uint32
	posBitMask uint32
	rd         *rangeDecoder
	match      [states << maxPosBits]prob
	rep        [states]prob
	repG0      [states]prob
	repG1      [states]prob
	repG2      [states]prob
	repG0Long  [states << maxPosBits]prob
}

// Properties returns a set of properties.
func (d *Decoder) Properties() Properties {
	return d.properties
}

// getUint64LE converts the uint64 value stored as little endian to an uint64
// value.
func getUint64LE(b []byte) uint64 {
	x := uint64(b[7]) << 56
	x |= uint64(b[6]) << 48
	x |= uint64(b[5]) << 40
	x |= uint64(b[4]) << 32
	x |= uint64(b[3]) << 24
	x |= uint64(b[2]) << 16
	x |= uint64(b[1]) << 8
	x |= uint64(b[0])
	return x
}

// readUint64LE reads a uint64 little-endian integer from reader.
func readUint64LE(r io.Reader) (x uint64, err error) {
	b := make([]byte, 8)
	if _, err = io.ReadFull(r, b); err != nil {
		return 0, err
	}
	x = getUint64LE(b)
	return x, nil
}

func initProbSlice(p []prob) {
	for i := range p {
		p[i] = probInit
	}
}

// NewDecoder creates an LZMA decoder. It reads the classic, original LZMA
// format. Note that LZMA2 uses a different header format. It satisfies the
// io.Reader interface.
func NewDecoder(r io.Reader) (d *Decoder, err error) {
	f := bufio.NewReader(r)
	properties, err := readProperties(f)
	if err != nil {
		return nil, err
	}
	historyLen := int(properties.DictLen)
	if historyLen < 0 {
		return nil, errors.New(
			"LZMA property DictLen exceeds maximum int value")
	}
	d = &Decoder{
		properties: *properties,
	}
	if d.packedLen, err = readUint64LE(f); err != nil {
		return nil, err
	}
	if d.dict, err = newDecoderDict(bufferLen, historyLen); err != nil {
		return nil, err
	}
	d.posBitMask = (uint32(1) << uint(d.properties.PB)) - 1
	if d.rd, err = newRangeDecoder(f); err != nil {
		return nil, err
	}
	initProbSlice(d.match[:])
	initProbSlice(d.rep[:])
	initProbSlice(d.repG0[:])
	initProbSlice(d.repG1[:])
	initProbSlice(d.repG2[:])
	initProbSlice(d.repG0Long[:])
	return d, nil
}

// Reads reads data from the decoder stream.
//
// The function fill put as much data in the buffer as it is available. The
// function might block and is not reentrant.
//
// The end of the LZMA stream is indicated by EOF. There might be other errors
// returned. The decoder will not be able to recover from an error returned.
func (d *Decoder) Read(p []byte) (n int, err error) {
	for n < len(p) {
		var k int
		k, err = d.dict.Read(p)
		if err != nil {
			return
		}
		n += k
		if n == len(p) {
			return
		}
		if err = d.fill(len(p) - n); err != nil {
			return
		}
	}
	return
}

func (d *Decoder) fill(n int) error {
	panic("TODO")
}

func (d *Decoder) updateStateLiteral() {
	switch {
	case d.state < 4:
		d.state = 0
		return
	case d.state < 10:
		d.state -= 3
		return
	}
	d.state -= 6
	return
}

func (d *Decoder) updateStateMatch() {
	if d.state < 7 {
		d.state = 7
		return
	}
	d.state = 10
	return
}

func (d *Decoder) updateStateRep() {
	if d.state < 7 {
		d.state = 8
	}
	d.state = 11
}

func (d *Decoder) updateStateShortRep() {
	if d.state < 7 {
		d.state = 9
	}
	d.state = 11
}

func (d *Decoder) decodeOp() (op operation, err error) {
	posState := uint32(d.total) & d.posBitMask
	state2 := (d.state << maxPosBits) | posState

	b, err := d.match[state2].Decode(d.rd)
	if err != nil {
		return nil, err
	}
	if b == 0 {
		// literal
		panic("TODO")
	}
	b, err = d.rep[d.state].Decode(d.rd)
	if err != nil {
		return nil, err
	}
	if b == 0 {
		// simple match
		panic("TODO")
	}
	b, err = d.repG0[d.state].Decode(d.rd)
	if b == 0 {
		// rep0
		panic("TODO")
	}
	b, err = d.repG1[d.state].Decode(d.rd)
	if b == 0 {
		// rep match 1
		panic("TODO")
	}
	b, err = d.repG2[d.state].Decode(d.rd)
	if b == 0 {
		// rep match 2
		panic("TODO")
	}
	// rep match 3
	panic("TODO")
}
<commit_msg>lzma: some changes on the decoder<commit_after>package lzma

import (
	"bufio"
	"errors"
	"io"
)

// states defines the overall state count
const states = 12

// bufferLen is the value for internal buffering of the decoder.
var bufferLen = 64 * (1 << 10)

// Decoder is able to read a LZMA byte stream and to read the plain text.
type Decoder struct {
	properties         Properties
	packedLen          uint64
	unpackedLen        uint64
	unpackedLenDefined bool
	total              uint64
	dict               *decoderDict
	state              uint32
	posBitMask         uint32
	rd                 *rangeDecoder
	isMatch            [states << maxPosBits]prob
	isRep              [states]prob
	isRepG0            [states]prob
	isRepG1            [states]prob
	isRepG2            [states]prob
	isRepG0Long        [states << maxPosBits]prob
	rep                [4]uint32
	litDecoder         *literalCodec
	lengthDecoder      *lengthCodec
}

// NewDecoder creates an LZMA decoder. It reads the classic, original LZMA
// format. Note that LZMA2 uses a different header format.
func NewDecoder(r io.Reader) (d *Decoder, err error) {
	f := bufio.NewReader(r)
	properties, err := readProperties(f)
	if err != nil {
		return nil, err
	}
	historyLen := int(properties.DictLen)
	if historyLen < 0 {
		return nil, errors.New(
			"LZMA property DictLen exceeds maximum int value")
	}
	d = &Decoder{
		properties: *properties,
	}
	if d.packedLen, err = readUint64LE(f); err != nil {
		return nil, err
	}
	if d.dict, err = newDecoderDict(bufferLen, historyLen); err != nil {
		return nil, err
	}
	d.posBitMask = (uint32(1) << uint(d.properties.PB)) - 1
	if d.rd, err = newRangeDecoder(f); err != nil {
		return nil, err
	}
	initProbSlice(d.isMatch[:])
	initProbSlice(d.isRep[:])
	initProbSlice(d.isRepG0[:])
	initProbSlice(d.isRepG1[:])
	initProbSlice(d.isRepG2[:])
	initProbSlice(d.isRepG0Long[:])
	d.litDecoder = newLiteralCodec(d.properties.LC, d.properties.LP)
	d.lengthDecoder = newLengthCodec()
	return d, nil
}

// Properties returns a set of properties.
func (d *Decoder) Properties() Properties {
	return d.properties
}

// getUint64LE converts the uint64 value stored as little endian to an uint64
// value.
func getUint64LE(b []byte) uint64 {
	x := uint64(b[7]) << 56
	x |= uint64(b[6]) << 48
	x |= uint64(b[5]) << 40
	x |= uint64(b[4]) << 32
	x |= uint64(b[3]) << 24
	x |= uint64(b[2]) << 16
	x |= uint64(b[1]) << 8
	x |= uint64(b[0])
	return x
}

// readUint64LE reads a uint64 little-endian integer from reader.
func readUint64LE(r io.Reader) (x uint64, err error) {
	b := make([]byte, 8)
	if _, err = io.ReadFull(r, b); err != nil {
		return 0, err
	}
	x = getUint64LE(b)
	return x, nil
}

// initProbSlice initializes a slice of probabilities.
func initProbSlice(p []prob) {
	for i := range p {
		p[i] = probInit
	}
}

// Reads reads data from the decoder stream.
//
// The function fill put as much data in the buffer as it is available. The
// function might block and is not reentrant.
//
// The end of the LZMA stream is indicated by EOF. There might be other errors
// returned. The decoder will not be able to recover from an error returned.
func (d *Decoder) Read(p []byte) (n int, err error) {
	for n < len(p) {
		var k int
		k, err = d.dict.Read(p)
		if err != nil {
			return
		}
		n += k
		if n == len(p) {
			return
		}
		if err = d.fill(len(p) - n); err != nil {
			return
		}
	}
	return
}

// fill puts at lest the requested number of bytes into the decoder dictionary.
func (d *Decoder) fill(n int) error {
	panic("TODO")
}

// updateStateLiteral updates the state for a literal.
func (d *Decoder) updateStateLiteral() {
	switch {
	case d.state < 4:
		d.state = 0
		return
	case d.state < 10:
		d.state -= 3
		return
	}
	d.state -= 6
	return
}

// updateStateMatch updates the state for a match.
func (d *Decoder) updateStateMatch() {
	if d.state < 7 {
		d.state = 7
		return
	}
	d.state = 10
	return
}

// updateStateRep updates the state for a repetition.
func (d *Decoder) updateStateRep() {
	if d.state < 7 {
		d.state = 8
	}
	d.state = 11
}

// updateStateShortRep updates the state for a short repetition.
func (d *Decoder) updateStateShortRep() {
	if d.state < 7 {
		d.state = 9
	}
	d.state = 11
}

// decodeLiteral decodes a literal.
func (d *Decoder) decodeLiteral() (op operation, err error) {
	prevByte := d.dict.getByte(1)
	lp, lc := uint(d.properties.LP), uint(d.properties.LC)
	litState := ((uint32(d.total) & ((1 << lp) - 1)) << lc) |
		(uint32(prevByte) >> (8 - lc))
	match := d.dict.getByte(int(d.rep[0]) + 1)
	s, err := d.litDecoder.Decode(d.rd, d.state, match, litState)
	if err != nil {
		return nil, err
	}
	return lit{s}, nil
}

// applyOp applies an operation.
func (d *Decoder) applyOp(op operation) error {
	total := d.total + uint64(op.Len())
	if d.unpackedLenDefined && d.total > d.unpackedLen {
		return errors.New("lzma: overflow of specified unpacked len")
	}
	d.total = total
	return op.applyDecoderDict(d.dict)
}

// decodeOp decodes an operation.
func (d *Decoder) decodeOp() (op operation, err error) {
	posState := uint32(d.total) & d.posBitMask
	state2 := (d.state << maxPosBits) | posState

	b, err := d.isMatch[state2].Decode(d.rd)
	if err != nil {
		return nil, err
	}
	if b == 0 {
		// literal
		op, err := d.decodeLiteral()
		if err != nil {
			return nil, err
		}
		d.updateStateLiteral()
		return op, nil
	}
	b, err = d.isRep[d.state].Decode(d.rd)
	if err != nil {
		return nil, err
	}
	if b == 0 {
		// simple match
		d.rep[3], d.rep[2], d.rep[1] = d.rep[2], d.rep[1], d.rep[0]
		d.updateStateMatch()
		// n, err := d.lengthDecoder.Decode(d.rd, posState)
		if err != nil {
			return nil, err
		}
		panic("TODO")
	}
	b, err = d.isRepG0[d.state].Decode(d.rd)
	if b == 0 {
		// rep0
		panic("TODO")
	}
	b, err = d.isRepG1[d.state].Decode(d.rd)
	if b == 0 {
		// rep match 1
		panic("TODO")
	}
	b, err = d.isRepG2[d.state].Decode(d.rd)
	if b == 0 {
		// rep match 2
		panic("TODO")
	}
	// rep match 3
	panic("TODO")
}
<|endoftext|>


<commit_before>package core

import (
	"log"
	"testing"
)

func TestNewArkClient(t *testing.T) {
	arkapi := NewArkClient(nil)

	if arkapi == nil {
		t.Error("Error creating client")
	}
	log.Println(t.Name(), "Success")
}

func TestSwitchPeer(t *testing.T) {
	arkapi := NewArkClient(nil)

	if arkapi == nil {
		t.Error("Error creating client")
	}

	arkapi = arkapi.SwitchPeer()
	log.Println(arkapi.GetActivePeer())
	log.Println(t.Name(), "Success")
}

func TestSwitchNetwork(t *testing.T) {
	arkapi := NewArkClient(nil)

	for i := 0; i < 5; i++ {
		arkapi = arkapi.SetActiveConfiguration(MAINNET)
		log.Println("Selected: ", arkapi.GetActivePeer())
	}

	arkapi = arkapi.SwitchPeer()
	log.Println(arkapi.GetActivePeer())
	log.Println(t.Name(), "Success")
}

func TestNewPeerArkApiClient(t *testing.T) {
	arkapi := NewArkClient(nil)

	log.Println(arkapi.GetActivePeer())
	peer := EnvironmentParams.Network.PeerList[3]
	arkapi1 := NewArkClientFromPeer(peer)
	log.Println("New arkapi for peer", arkapi1.GetActivePeer())

	log.Println(arkapi1.GetActivePeer())
	log.Println(t.Name(), "Success")
}

func TestGetRandomXPeers(t *testing.T) {
	arkapi := NewArkClient(nil)

	peers := arkapi.GetRandomXPeers(5)

	for _, el := range peers {
		log.Println(el)
	}
}
<commit_msg>testing devnet peer selection<commit_after>package core

import (
	"log"
	"testing"
)

func TestNewArkClient(t *testing.T) {
	arkapi := NewArkClient(nil)

	if arkapi == nil {
		t.Error("Error creating client")
	}
	log.Println(t.Name(), "Success")
}

func TestSwitchPeer(t *testing.T) {
	arkapi := NewArkClient(nil)

	if arkapi == nil {
		t.Error("Error creating client")
	}

	arkapi = arkapi.SwitchPeer()
	log.Println(arkapi.GetActivePeer())
	log.Println(t.Name(), "Success")
}

func TestSwitchNetwork(t *testing.T) {
	arkapi := NewArkClient(nil)

	for i := 0; i < 5; i++ {
		arkapi = arkapi.SetActiveConfiguration(MAINNET)
		log.Println("Selected: ", arkapi.GetActivePeer())
	}

	arkapi = arkapi.SwitchPeer()
	log.Println(arkapi.GetActivePeer())
	log.Println(t.Name(), "Success")
}

func TestNewPeerArkApiClient(t *testing.T) {
	arkapi := NewArkClient(nil)

	log.Println(arkapi.GetActivePeer())
	peer := EnvironmentParams.Network.PeerList[3]
	arkapi1 := NewArkClientFromPeer(peer)
	log.Println("New arkapi for peer", arkapi1.GetActivePeer())

	log.Println(arkapi1.GetActivePeer())
	log.Println(t.Name(), "Success")
}

func TestGetRandomXPeers(t *testing.T) {
	arkapi := NewArkClient(nil)
	arkapi = arkapi.SetActiveConfiguration(DEVNET)

	peers := arkapi.GetRandomXPeers(5)

	for _, el := range peers {
		log.Println(el)
	}
}
<|endoftext|>


<commit_before>package marshal

import (
	"encoding/json"
	"errors"
	"reflect"
	"strings"

	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/parquet"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/types"
)

//ss is []string
func MarshalJSON(ss []interface{}, schemaHandler *schema.SchemaHandler) (tb *map[string]*layout.Table, err error) {
	defer func() {
		if r := recover(); r != nil {
			switch x := r.(type) {
			case string:
				err = errors.New(x)
			case error:
				err = x
			default:
				err = errors.New("unknown error")
			}
		}
	}()

	res := make(map[string]*layout.Table)
	pathMap := schemaHandler.PathMap
	nodeBuf := NewNodeBuf(1)

	for i := 0; i < len(schemaHandler.SchemaElements); i++ {
		schema := schemaHandler.SchemaElements[i]
		pathStr := schemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res[pathStr] = layout.NewEmptyTable()
			res[pathStr].Path = common.StrToPath(pathStr)
			res[pathStr].MaxDefinitionLevel, _ = schemaHandler.MaxDefinitionLevel(res[pathStr].Path)
			res[pathStr].MaxRepetitionLevel, _ = schemaHandler.MaxRepetitionLevel(res[pathStr].Path)
			res[pathStr].RepetitionType = schema.GetRepetitionType()
			res[pathStr].Schema = schemaHandler.SchemaElements[schemaHandler.MapIndex[pathStr]]
			res[pathStr].Info = schemaHandler.Infos[i]
		}
	}

	stack := make([]*Node, 0, 100)
	for i := 0; i < len(ss); i++ {
		stack = stack[:0]
		nodeBuf.Reset()

		node := nodeBuf.GetNode()
		var ui interface{}

		// `useNumber`causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.
		d := json.NewDecoder(strings.NewReader(ss[i].(string)))
		d.UseNumber()
		d.Decode(&ui)

		node.Val = reflect.ValueOf(ui)
		node.PathMap = pathMap

		stack = append(stack, node)

		for len(stack) > 0 {
			ln := len(stack)
			node = stack[ln-1]
			stack = stack[:ln-1]

			tk := node.Val.Type().Kind()

			pathStr := node.PathMap.Path

			schemaIndex, ok := schemaHandler.MapIndex[pathStr]
			//no schema item will be ignored
			if !ok {
				continue
			}

			schema := schemaHandler.SchemaElements[schemaIndex]

			if tk == reflect.Map {
				keys := node.Val.MapKeys()

				if schema.GetConvertedType() == parquet.ConvertedType_MAP { //real map
					pathStr = pathStr + common.PAR_GO_PATH_DELIMITER + "Key_value"
					if len(keys) <= 0 {
						for key, table := range res {
							if strings.HasPrefix(key, node.PathMap.Path) &&
								(len(key) == len(node.PathMap.Path) || key[len(node.PathMap.Path)] == common.PAR_GO_PATH_DELIMITER[0]) {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}

					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))
					for j := len(keys) - 1; j >= 0; j-- {
						key := keys[j]
						value := node.Val.MapIndex(key).Elem()

						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["Key_value"].Children["Key"]
						newNode.Val = key
						newNode.DL = node.DL + 1
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						stack = append(stack, newNode)

						newNode = nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["Key_value"].Children["Value"]
						newNode.Val = value
						newNode.DL = node.DL + 1
						newPathStr := newNode.PathMap.Path // check again
						newSchemaIndex := schemaHandler.MapIndex[newPathStr]
						newSchema := schemaHandler.SchemaElements[newSchemaIndex]
						if newSchema.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL { //map value only be :optional or required
							newNode.DL++
						}

						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						stack = append(stack, newNode)
					}

				} else { //struct
					keysMap := make(map[string]int)
					for j := 0; j < len(keys); j++ {
						//ExName to InName
						keysMap[common.StringToVariableName(keys[j].String())] = j
					}
					for key, _ := range node.PathMap.Children {
						ki, ok := keysMap[key]

						if ok && node.Val.MapIndex(keys[ki]).Elem().IsValid() {
							newNode := nodeBuf.GetNode()
							newNode.PathMap = node.PathMap.Children[key]
							newNode.Val = node.Val.MapIndex(keys[ki]).Elem()
							newNode.RL = node.RL
							newNode.DL = node.DL
							newPathStr := newNode.PathMap.Path
							newSchemaIndex := schemaHandler.MapIndex[newPathStr]
							newSchema := schemaHandler.SchemaElements[newSchemaIndex]
							if newSchema.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL {
								newNode.DL++
							}
							stack = append(stack, newNode)

						} else {
							newPathStr := node.PathMap.Children[key].Path
							for path, table := range res {
								if strings.HasPrefix(path, newPathStr) &&
									(len(path) == len(newPathStr) || path[len(newPathStr)] == common.PAR_GO_PATH_DELIMITER[0]) {

									table.Values = append(table.Values, nil)
									table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
									table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
								}
							}
						}
					}
				}

			} else if tk == reflect.Slice {
				ln := node.Val.Len()

				if schema.GetConvertedType() == parquet.ConvertedType_LIST { // real LIST
					pathStr = pathStr + common.PAR_GO_PATH_DELIMITER + "List" + common.PAR_GO_PATH_DELIMITER + "Element"
					if ln <= 0 {
						for key, table := range res {
							if strings.HasPrefix(key, node.PathMap.Path) &&
								(len(key) == len(node.PathMap.Path) || key[len(node.PathMap.Path)] == common.PAR_GO_PATH_DELIMITER[0]) {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}
					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))

					for j := ln - 1; j >= 0; j-- {
						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["List"].Children["Element"]
						newNode.Val = node.Val.Index(j).Elem()
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						newNode.DL = node.DL + 1

						newPathStr := newNode.PathMap.Path
						newSchemaIndex := schemaHandler.MapIndex[newPathStr]
						newSchema := schemaHandler.SchemaElements[newSchemaIndex]
						if newSchema.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL { //element of LIST can only be optional or required
							newNode.DL++
						}

						stack = append(stack, newNode)
					}

				} else { //Repeated
					if ln <= 0 {
						for key, table := range res {
							if strings.HasPrefix(key, node.PathMap.Path) &&
								(len(key) == len(node.PathMap.Path) || key[len(node.PathMap.Path)] == common.PAR_GO_PATH_DELIMITER[0]) {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}
					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))

					for j := ln - 1; j >= 0; j-- {
						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap
						newNode.Val = node.Val.Index(j).Elem()
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						newNode.DL = node.DL + 1
						stack = append(stack, newNode)
					}
				}

			} else {
				table := res[node.PathMap.Path]
				pT, cT := schema.Type, schema.ConvertedType
				val := types.JSONTypeToParquetType(node.Val, pT, cT, int(schema.GetTypeLength()), int(schema.GetScale()))

				table.Values = append(table.Values, val)
				table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
				table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
			}
		}
	}

	return &res, nil

}
<commit_msg>support string or []byte for JSON objects<commit_after>package marshal

import (
	"bytes"
	"encoding/json"
	"errors"
	"reflect"
	"strings"

	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/parquet"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/types"
)

//ss is []string
func MarshalJSON(ss []interface{}, schemaHandler *schema.SchemaHandler) (tb *map[string]*layout.Table, err error) {
	defer func() {
		if r := recover(); r != nil {
			switch x := r.(type) {
			case string:
				err = errors.New(x)
			case error:
				err = x
			default:
				err = errors.New("unknown error")
			}
		}
	}()

	res := make(map[string]*layout.Table)
	pathMap := schemaHandler.PathMap
	nodeBuf := NewNodeBuf(1)

	for i := 0; i < len(schemaHandler.SchemaElements); i++ {
		schema := schemaHandler.SchemaElements[i]
		pathStr := schemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res[pathStr] = layout.NewEmptyTable()
			res[pathStr].Path = common.StrToPath(pathStr)
			res[pathStr].MaxDefinitionLevel, _ = schemaHandler.MaxDefinitionLevel(res[pathStr].Path)
			res[pathStr].MaxRepetitionLevel, _ = schemaHandler.MaxRepetitionLevel(res[pathStr].Path)
			res[pathStr].RepetitionType = schema.GetRepetitionType()
			res[pathStr].Schema = schemaHandler.SchemaElements[schemaHandler.MapIndex[pathStr]]
			res[pathStr].Info = schemaHandler.Infos[i]
		}
	}

	stack := make([]*Node, 0, 100)
	for i := 0; i < len(ss); i++ {
		stack = stack[:0]
		nodeBuf.Reset()

		node := nodeBuf.GetNode()
		var ui interface{}

		var d *json.Decoder

		switch t := ss[i].(type) {
		case string:
			d = json.NewDecoder(strings.NewReader(t))
		case []byte:
			d = json.NewDecoder(bytes.NewReader(t))
		}
		// `useNumber`causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.
		d.UseNumber()
		d.Decode(&ui)

		node.Val = reflect.ValueOf(ui)
		node.PathMap = pathMap

		stack = append(stack, node)

		for len(stack) > 0 {
			ln := len(stack)
			node = stack[ln-1]
			stack = stack[:ln-1]

			tk := node.Val.Type().Kind()

			pathStr := node.PathMap.Path

			schemaIndex, ok := schemaHandler.MapIndex[pathStr]
			//no schema item will be ignored
			if !ok {
				continue
			}

			schema := schemaHandler.SchemaElements[schemaIndex]

			if tk == reflect.Map {
				keys := node.Val.MapKeys()

				if schema.GetConvertedType() == parquet.ConvertedType_MAP { //real map
					pathStr = pathStr + common.PAR_GO_PATH_DELIMITER + "Key_value"
					if len(keys) <= 0 {
						for key, table := range res {
							if strings.HasPrefix(key, node.PathMap.Path) &&
								(len(key) == len(node.PathMap.Path) || key[len(node.PathMap.Path)] == common.PAR_GO_PATH_DELIMITER[0]) {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}

					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))
					for j := len(keys) - 1; j >= 0; j-- {
						key := keys[j]
						value := node.Val.MapIndex(key).Elem()

						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["Key_value"].Children["Key"]
						newNode.Val = key
						newNode.DL = node.DL + 1
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						stack = append(stack, newNode)

						newNode = nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["Key_value"].Children["Value"]
						newNode.Val = value
						newNode.DL = node.DL + 1
						newPathStr := newNode.PathMap.Path // check again
						newSchemaIndex := schemaHandler.MapIndex[newPathStr]
						newSchema := schemaHandler.SchemaElements[newSchemaIndex]
						if newSchema.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL { //map value only be :optional or required
							newNode.DL++
						}

						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						stack = append(stack, newNode)
					}

				} else { //struct
					keysMap := make(map[string]int)
					for j := 0; j < len(keys); j++ {
						//ExName to InName
						keysMap[common.StringToVariableName(keys[j].String())] = j
					}
					for key, _ := range node.PathMap.Children {
						ki, ok := keysMap[key]

						if ok && node.Val.MapIndex(keys[ki]).Elem().IsValid() {
							newNode := nodeBuf.GetNode()
							newNode.PathMap = node.PathMap.Children[key]
							newNode.Val = node.Val.MapIndex(keys[ki]).Elem()
							newNode.RL = node.RL
							newNode.DL = node.DL
							newPathStr := newNode.PathMap.Path
							newSchemaIndex := schemaHandler.MapIndex[newPathStr]
							newSchema := schemaHandler.SchemaElements[newSchemaIndex]
							if newSchema.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL {
								newNode.DL++
							}
							stack = append(stack, newNode)

						} else {
							newPathStr := node.PathMap.Children[key].Path
							for path, table := range res {
								if strings.HasPrefix(path, newPathStr) &&
									(len(path) == len(newPathStr) || path[len(newPathStr)] == common.PAR_GO_PATH_DELIMITER[0]) {

									table.Values = append(table.Values, nil)
									table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
									table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
								}
							}
						}
					}
				}

			} else if tk == reflect.Slice {
				ln := node.Val.Len()

				if schema.GetConvertedType() == parquet.ConvertedType_LIST { // real LIST
					pathStr = pathStr + common.PAR_GO_PATH_DELIMITER + "List" + common.PAR_GO_PATH_DELIMITER + "Element"
					if ln <= 0 {
						for key, table := range res {
							if strings.HasPrefix(key, node.PathMap.Path) &&
								(len(key) == len(node.PathMap.Path) || key[len(node.PathMap.Path)] == common.PAR_GO_PATH_DELIMITER[0]) {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}
					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))

					for j := ln - 1; j >= 0; j-- {
						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap.Children["List"].Children["Element"]
						newNode.Val = node.Val.Index(j).Elem()
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						newNode.DL = node.DL + 1

						newPathStr := newNode.PathMap.Path
						newSchemaIndex := schemaHandler.MapIndex[newPathStr]
						newSchema := schemaHandler.SchemaElements[newSchemaIndex]
						if newSchema.GetRepetitionType() == parquet.FieldRepetitionType_OPTIONAL { //element of LIST can only be optional or required
							newNode.DL++
						}

						stack = append(stack, newNode)
					}

				} else { //Repeated
					if ln <= 0 {
						for key, table := range res {
							if strings.HasPrefix(key, node.PathMap.Path) &&
								(len(key) == len(node.PathMap.Path) || key[len(node.PathMap.Path)] == common.PAR_GO_PATH_DELIMITER[0]) {
								table.Values = append(table.Values, nil)
								table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
								table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
							}
						}
					}
					rlNow, _ := schemaHandler.MaxRepetitionLevel(common.StrToPath(pathStr))

					for j := ln - 1; j >= 0; j-- {
						newNode := nodeBuf.GetNode()
						newNode.PathMap = node.PathMap
						newNode.Val = node.Val.Index(j).Elem()
						if j == 0 {
							newNode.RL = node.RL
						} else {
							newNode.RL = rlNow
						}
						newNode.DL = node.DL + 1
						stack = append(stack, newNode)
					}
				}

			} else {
				table := res[node.PathMap.Path]
				pT, cT := schema.Type, schema.ConvertedType
				val := types.JSONTypeToParquetType(node.Val, pT, cT, int(schema.GetTypeLength()), int(schema.GetScale()))

				table.Values = append(table.Values, val)
				table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
				table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
			}
		}
	}

	return &res, nil

}
<|endoftext|>


<commit_before>package mmatcher_test

import (
	"testing"

	. "github.com/oklasoft/mmatcher"
)

func TestPair(t *testing.T) {
	a := NewPair("A1", "B1")
	b := NewPair("A2", "B2")
	c := NewPair("B1", "A1")
	if a.Eql(b) {
		t.Error(a, "not expected to equal", b)
	}
	if !a.Eql(c) {
		t.Error(a, "expected to equal", c)
	}
}

func TestAddRemove(t *testing.T) {
	m := NewMatchSet()
	if 0 != m.NumPairs() {
		t.Error("A new set should be empty, but got", m.NumPairs())
	}

	m.AddPair(NewPair("A1", "B1"))
	if 1 != m.NumPairs() {
		t.Error("Length expected to be 1, but got", m.NumPairs())
	}

	m.AddPair(NewPair("A1", "B2"))
	if 2 != m.NumPairs() {
		t.Error("Length expected to be 2, but got", m.NumPairs())
	}
	m.AddPair(NewPair("A1", "B3"))
	m.AddPair(NewPair("A2", "B2"))
	m.AddPair(NewPair("A2", "B3"))
	m.AddPair(NewPair("A3", "B1"))
	m.AddPair(NewPair("A3", "B2"))

	if 7 != m.NumPairs() {
		t.Error("Length expected to be 7, but got", m.NumPairs())
	}

	m.RemovePair(NewPair("A2", "B2"))
	if 6 != m.NumPairs() {
		t.Error("Afer Removing 1, length should be 6, but was", m.NumPairs(), "in", m)
	}
	m.RemovePair(NewPair("A1", "B1"))
	if 5 != m.NumPairs() {
		t.Error("Afer Removing 2, length should be 5, but was", m.NumPairs(), "in", m)
	}
	m.RemovePair(NewPair("A3", "B2"))
	if 4 != m.NumPairs() {
		t.Error("Afer Removing 3, length should be 4, but was", m.NumPairs(), "in", m)
	}
	m.RemovePair(NewPair("X3", "X2"))
	if 4 != m.NumPairs() {
		t.Error("After removing a non existent the size should still be 4", m.NumPairs(), "in", m)
	}
	m.RemovePair(NewPair("A1", "X2"))
	if 4 != m.NumPairs() {
		t.Error("After removing a non existent the size should still be 4", m.NumPairs(), "in", m)
	}
	m.RemovePair(NewPair("B3", "A2"))
	if 3 != m.NumPairs() {
		t.Error("After removing a pair in oppopsite order there should be 3, but there were", m.NumPairs(), "in", m)
	}
	m.Purge("A1")
	if 1 != m.NumPairs() {
		t.Error("After a purge of A3, we should have only 1 left, but was", m.NumPairs(), "in", m)
	}
}
<commit_msg>Add test for getting optimized match set<commit_after>package mmatcher_test

import (
	"testing"

	. "github.com/oklasoft/mmatcher"
)

func TestPair(t *testing.T) {
	a := NewPair("A1", "B1")
	b := NewPair("A2", "B2")
	c := NewPair("B1", "A1")
	if a.Eql(b) {
		t.Error(a, "not expected to equal", b)
	}
	if !a.Eql(c) {
		t.Error(a, "expected to equal", c)
	}
}

func TestAddRemove(t *testing.T) {
	m := NewMatchSet()
	if 0 != m.NumPairs() {
		t.Error("A new set should be empty, but got", m.NumPairs())
	}

	m.AddPair(NewPair("A1", "B1"))
	if 1 != m.NumPairs() {
		t.Error("Length expected to be 1, but got", m.NumPairs())
	}

	m.AddPair(NewPair("A1", "B2"))
	if 2 != m.NumPairs() {
		t.Error("Length expected to be 2, but got", m.NumPairs())
	}
	m.AddPair(NewPair("A1", "B3"))
	m.AddPair(NewPair("A2", "B2"))
	m.AddPair(NewPair("A2", "B3"))
	m.AddPair(NewPair("A3", "B1"))
	m.AddPair(NewPair("A3", "B2"))

	if 7 != m.NumPairs() {
		t.Error("Length expected to be 7, but got", m.NumPairs())
	}

	m.RemovePair(NewPair("A2", "B2"))
	if 6 != m.NumPairs() {
		t.Error("Afer Removing 1, length should be 6, but was", m.NumPairs(), "in", m)
	}
	m.RemovePair(NewPair("A1", "B1"))
	if 5 != m.NumPairs() {
		t.Error("Afer Removing 2, length should be 5, but was", m.NumPairs(), "in", m)
	}
	m.RemovePair(NewPair("A3", "B2"))
	if 4 != m.NumPairs() {
		t.Error("Afer Removing 3, length should be 4, but was", m.NumPairs(), "in", m)
	}
	m.RemovePair(NewPair("X3", "X2"))
	if 4 != m.NumPairs() {
		t.Error("After removing a non existent the size should still be 4", m.NumPairs(), "in", m)
	}
	m.RemovePair(NewPair("A1", "X2"))
	if 4 != m.NumPairs() {
		t.Error("After removing a non existent the size should still be 4", m.NumPairs(), "in", m)
	}
	m.RemovePair(NewPair("B3", "A2"))
	if 3 != m.NumPairs() {
		t.Error("After removing a pair in oppopsite order there should be 3, but there were", m.NumPairs(), "in", m)
	}
	m.Purge("A1")
	if 1 != m.NumPairs() {
		t.Error("After a purge of A3, we should have only 1 left, but was", m.NumPairs(), "in", m)
	}
}

func TestQuantityOptimize(t *testing.T) {
	m := NewMatchSet()
	m.AddPair(NewPair("A1", "B1"))
	m.AddPair(NewPair("A1", "B2"))
	m.AddPair(NewPair("A1", "B3"))
	m.AddPair(NewPair("A2", "B2"))
	m.AddPair(NewPair("A2", "B3"))
	m.AddPair(NewPair("A3", "B1"))
	m.AddPair(NewPair("A3", "B2"))

	o := m.QuantityOptimized()
	if 3 != o.NumPairs() {
		t.Error("After optimizing we should have 3, but we had", o.NumPairs(), "in", o)
	}
	if 7 != m.NumPairs() {
		t.Error("After making the optimized set, the original should be the same size still", m)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bpf

/*
#cgo CFLAGS: -I../../bpf/include
#include <stdint.h>
#include <linux/unistd.h>
#include <linux/bpf.h>
#include <sys/resource.h>
#include <time.h>

static __u64 ptr_to_u64(const void *ptr)
{
	return (__u64) (unsigned long) ptr;
}

void create_bpf_create_map(enum bpf_map_type map_type, int key_size, int value_size,
			   int max_entries, unsigned long flags, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->map_type = map_type;
	ptr_bpf_attr->key_size = key_size;
	ptr_bpf_attr->value_size = value_size;
	ptr_bpf_attr->max_entries = max_entries;
	ptr_bpf_attr->map_flags = flags;
}

void create_bpf_update_elem(int fd, const void *key, const void *value,
			    unsigned long long flags, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->map_fd = fd;
	ptr_bpf_attr->key = ptr_to_u64(key);
	ptr_bpf_attr->value = ptr_to_u64(value);
	ptr_bpf_attr->flags = flags;
}

void create_bpf_lookup_elem(int fd, const void *key, void *value, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->map_fd = fd;
	ptr_bpf_attr->key = ptr_to_u64(key);
	ptr_bpf_attr->value = ptr_to_u64(value);
}

void create_bpf_delete_elem(int fd, const void *key, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->map_fd = fd;
	ptr_bpf_attr->key = ptr_to_u64(key);
}

void create_bpf_get_next_key(int fd, const void *key, void *next_key, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->map_fd = fd;
	ptr_bpf_attr->key = ptr_to_u64(key);
	ptr_bpf_attr->next_key = ptr_to_u64(next_key);
}

void create_bpf_obj_pin(int fd, const char *pathname, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->pathname = ptr_to_u64(pathname);
	ptr_bpf_attr->bpf_fd = fd;
}

void create_bpf_obj_get(const char *pathname, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->pathname = ptr_to_u64(pathname);
}
*/
import "C"

import (
	"fmt"
	"math"
	"os"
	"path/filepath"
	"unsafe"

	"golang.org/x/sys/unix"
)

const (
	BPF_MAP_TYPE_HASH     = C.BPF_MAP_TYPE_HASH
	BPF_MAP_TYPE_LPM_TRIE = C.BPF_MAP_TYPE_LPM_TRIE

	BPF_F_NO_PREALLOC = C.BPF_F_NO_PREALLOC
)

// CreateMap creates a Map of type mapType, with key size keySize, a value size of
// valueSize and the maximum amount of entries of maxEntries.
// mapType should be one of the bpf_map_type in "uapi/linux/bpf.h"
func CreateMap(mapType int, keySize, valueSize, maxEntries, flags uint32) (int, error) {
	uba := C.union_bpf_attr{}
	C.create_bpf_create_map(
		C.enum_bpf_map_type(mapType),
		C.int(keySize),
		C.int(valueSize),
		C.int(maxEntries),
		C.ulong(flags),
		unsafe.Pointer(&uba),
	)
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_MAP_CREATE,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 {
		return int(ret), nil
	}
	return 0, fmt.Errorf("Unable to create map: %s", err)
}

// UpdateElement updates the map in fd with the given value in the given key.
// The flags can have the following values (if you include "uapi/linux/bpf.h"):
// C.BPF_ANY to create new element or update existing;
// C.BPF_NOEXIST to create new element if it didn't exist;
// C.BPF_EXIST to update existing element.
func UpdateElement(fd int, key, value unsafe.Pointer, flags uint64) error {
	uba := C.union_bpf_attr{}
	C.create_bpf_update_elem(
		C.int(fd),
		key,
		value,
		C.ulonglong(flags),
		unsafe.Pointer(&uba),
	)
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_MAP_UPDATE_ELEM,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 || err != 0 {
		return fmt.Errorf("Unable to update element: %s", err)
	}

	return nil
}

// LookupElement looks up for the map value stored in fd with the given key. The value
// is stored in the value unsafe.Pointer.
func LookupElement(fd int, key, value unsafe.Pointer) error {
	uba := C.union_bpf_attr{}
	C.create_bpf_lookup_elem(
		C.int(fd),
		key,
		value,
		unsafe.Pointer(&uba),
	)
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_MAP_LOOKUP_ELEM,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 || err != 0 {
		return fmt.Errorf("Unable to lookup element: %s", err)
	}

	return nil
}

// DeleteElement deletes the map element with the given key.
func DeleteElement(fd int, key unsafe.Pointer) error {
	uba := C.union_bpf_attr{}
	C.create_bpf_delete_elem(
		C.int(fd),
		key,
		unsafe.Pointer(&uba),
	)
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_MAP_DELETE_ELEM,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 || err != 0 {
		return fmt.Errorf("Unable to delete element: %s", err)
	}

	return nil
}

// GetNextKey stores, in nextKey, the next key after the key of the map in fd.
func GetNextKey(fd int, key, nextKey unsafe.Pointer) error {
	uba := C.union_bpf_attr{}
	C.create_bpf_get_next_key(
		C.int(fd),
		key,
		nextKey,
		unsafe.Pointer(&uba),
	)
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_MAP_GET_NEXT_KEY,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 || err != 0 {
		return fmt.Errorf("Unable to get next key: %s", err)
	}

	return nil
}

// ObjPin stores the map's fd in pathname.
func ObjPin(fd int, pathname string) error {
	pathStr := C.CString(pathname)
	uba := C.union_bpf_attr{}
	C.create_bpf_obj_pin(C.int(fd), pathStr, unsafe.Pointer(&uba))
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_OBJ_PIN,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 || err != 0 {
		return fmt.Errorf("Unable to pin object: %s", err)
	}

	return nil
}

// ObjGet reads the pathname and returns the map's fd read.
func ObjGet(pathname string) (int, error) {
	pathStr := C.CString(pathname)
	uba := C.union_bpf_attr{}
	C.create_bpf_obj_get(pathStr, unsafe.Pointer(&uba))

	fd, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_OBJ_GET,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if fd == 0 || err != 0 {
		return 0, fmt.Errorf("Unable to get object %s: %s", pathname, err)
	}

	return int(fd), nil
}

// ObjClose closes the map's fd.
func ObjClose(fd int) error {
	if fd > 0 {
		return unix.Close(fd)
	}
	return nil
}

func OpenOrCreateMap(path string, mapType int, keySize, valueSize, maxEntries, flags uint32) (int, bool, error) {
	var fd int

	isNewMap := false

	rl := unix.Rlimit{
		Cur: math.MaxUint64,
		Max: math.MaxUint64,
	}

	err := unix.Setrlimit(C.RLIMIT_MEMLOCK, &rl)
	if err != nil {
		return 0, isNewMap, fmt.Errorf("Unable to increase rlimit: %s", err)
	}

	if _, err = os.Stat(path); os.IsNotExist(err) {
		mapDir := filepath.Dir(path)
		if _, err = os.Stat(mapDir); os.IsNotExist(err) {
			if err = os.MkdirAll(mapDir, 0755); err != nil {
				return 0, isNewMap, fmt.Errorf("Unable create map base directory: %s", err)
			}
		}

		fd, err = CreateMap(
			mapType,
			keySize,
			valueSize,
			maxEntries,
			flags,
		)

		defer func() {
			if err != nil {
				// In case of error, we need to close
				// this fd since it was open by CreateMap
				ObjClose(fd)
			}
		}()

		isNewMap = true

		if err != nil {
			return 0, isNewMap, err
		}

		err = ObjPin(fd, path)
		if err != nil {
			return 0, isNewMap, err
		}

		return fd, isNewMap, nil
	}

	fd, err = ObjGet(path)
	return fd, isNewMap, err
}

// GetMtime returns monotonic time that can be used to compare
// values with ktime_get_ns() BPF helper, e.g. needed to check
// the timeout in sec for BPF entries. We return the raw nsec,
// although that is not quite usable for comparison. Go has
// runtime.nanotime() but doesn't expose it as API.
func GetMtime() (uint64, error) {
	var ts unix.Timespec

	err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &ts)
	if err != nil {
		return 0, fmt.Errorf("Unable get time: %s", err)
	}

	return uint64(unix.TimespecToNsec(ts)), nil
}
<commit_msg>pkg/bpf: use RLIMIT_MEMLOCK from x/sys/unix<commit_after>// Copyright 2016-2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bpf

/*
#cgo CFLAGS: -I../../bpf/include
#include <stdint.h>
#include <linux/unistd.h>
#include <linux/bpf.h>

static __u64 ptr_to_u64(const void *ptr)
{
	return (__u64) (unsigned long) ptr;
}

void create_bpf_create_map(enum bpf_map_type map_type, int key_size, int value_size,
			   int max_entries, unsigned long flags, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->map_type = map_type;
	ptr_bpf_attr->key_size = key_size;
	ptr_bpf_attr->value_size = value_size;
	ptr_bpf_attr->max_entries = max_entries;
	ptr_bpf_attr->map_flags = flags;
}

void create_bpf_update_elem(int fd, const void *key, const void *value,
			    unsigned long long flags, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->map_fd = fd;
	ptr_bpf_attr->key = ptr_to_u64(key);
	ptr_bpf_attr->value = ptr_to_u64(value);
	ptr_bpf_attr->flags = flags;
}

void create_bpf_lookup_elem(int fd, const void *key, void *value, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->map_fd = fd;
	ptr_bpf_attr->key = ptr_to_u64(key);
	ptr_bpf_attr->value = ptr_to_u64(value);
}

void create_bpf_delete_elem(int fd, const void *key, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->map_fd = fd;
	ptr_bpf_attr->key = ptr_to_u64(key);
}

void create_bpf_get_next_key(int fd, const void *key, void *next_key, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->map_fd = fd;
	ptr_bpf_attr->key = ptr_to_u64(key);
	ptr_bpf_attr->next_key = ptr_to_u64(next_key);
}

void create_bpf_obj_pin(int fd, const char *pathname, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->pathname = ptr_to_u64(pathname);
	ptr_bpf_attr->bpf_fd = fd;
}

void create_bpf_obj_get(const char *pathname, void *attr)
{
	union bpf_attr* ptr_bpf_attr;
	ptr_bpf_attr = (union bpf_attr*)attr;
	ptr_bpf_attr->pathname = ptr_to_u64(pathname);
}
*/
import "C"

import (
	"fmt"
	"math"
	"os"
	"path/filepath"
	"unsafe"

	"golang.org/x/sys/unix"
)

const (
	BPF_MAP_TYPE_HASH     = C.BPF_MAP_TYPE_HASH
	BPF_MAP_TYPE_LPM_TRIE = C.BPF_MAP_TYPE_LPM_TRIE

	BPF_F_NO_PREALLOC = C.BPF_F_NO_PREALLOC
)

// CreateMap creates a Map of type mapType, with key size keySize, a value size of
// valueSize and the maximum amount of entries of maxEntries.
// mapType should be one of the bpf_map_type in "uapi/linux/bpf.h"
func CreateMap(mapType int, keySize, valueSize, maxEntries, flags uint32) (int, error) {
	uba := C.union_bpf_attr{}
	C.create_bpf_create_map(
		C.enum_bpf_map_type(mapType),
		C.int(keySize),
		C.int(valueSize),
		C.int(maxEntries),
		C.ulong(flags),
		unsafe.Pointer(&uba),
	)
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_MAP_CREATE,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 {
		return int(ret), nil
	}
	return 0, fmt.Errorf("Unable to create map: %s", err)
}

// UpdateElement updates the map in fd with the given value in the given key.
// The flags can have the following values (if you include "uapi/linux/bpf.h"):
// C.BPF_ANY to create new element or update existing;
// C.BPF_NOEXIST to create new element if it didn't exist;
// C.BPF_EXIST to update existing element.
func UpdateElement(fd int, key, value unsafe.Pointer, flags uint64) error {
	uba := C.union_bpf_attr{}
	C.create_bpf_update_elem(
		C.int(fd),
		key,
		value,
		C.ulonglong(flags),
		unsafe.Pointer(&uba),
	)
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_MAP_UPDATE_ELEM,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 || err != 0 {
		return fmt.Errorf("Unable to update element: %s", err)
	}

	return nil
}

// LookupElement looks up for the map value stored in fd with the given key. The value
// is stored in the value unsafe.Pointer.
func LookupElement(fd int, key, value unsafe.Pointer) error {
	uba := C.union_bpf_attr{}
	C.create_bpf_lookup_elem(
		C.int(fd),
		key,
		value,
		unsafe.Pointer(&uba),
	)
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_MAP_LOOKUP_ELEM,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 || err != 0 {
		return fmt.Errorf("Unable to lookup element: %s", err)
	}

	return nil
}

// DeleteElement deletes the map element with the given key.
func DeleteElement(fd int, key unsafe.Pointer) error {
	uba := C.union_bpf_attr{}
	C.create_bpf_delete_elem(
		C.int(fd),
		key,
		unsafe.Pointer(&uba),
	)
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_MAP_DELETE_ELEM,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 || err != 0 {
		return fmt.Errorf("Unable to delete element: %s", err)
	}

	return nil
}

// GetNextKey stores, in nextKey, the next key after the key of the map in fd.
func GetNextKey(fd int, key, nextKey unsafe.Pointer) error {
	uba := C.union_bpf_attr{}
	C.create_bpf_get_next_key(
		C.int(fd),
		key,
		nextKey,
		unsafe.Pointer(&uba),
	)
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_MAP_GET_NEXT_KEY,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 || err != 0 {
		return fmt.Errorf("Unable to get next key: %s", err)
	}

	return nil
}

// ObjPin stores the map's fd in pathname.
func ObjPin(fd int, pathname string) error {
	pathStr := C.CString(pathname)
	uba := C.union_bpf_attr{}
	C.create_bpf_obj_pin(C.int(fd), pathStr, unsafe.Pointer(&uba))
	ret, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_OBJ_PIN,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if ret != 0 || err != 0 {
		return fmt.Errorf("Unable to pin object: %s", err)
	}

	return nil
}

// ObjGet reads the pathname and returns the map's fd read.
func ObjGet(pathname string) (int, error) {
	pathStr := C.CString(pathname)
	uba := C.union_bpf_attr{}
	C.create_bpf_obj_get(pathStr, unsafe.Pointer(&uba))

	fd, _, err := unix.Syscall(
		C.__NR_bpf,
		C.BPF_OBJ_GET,
		uintptr(unsafe.Pointer(&uba)),
		unsafe.Sizeof(uba),
	)

	if fd == 0 || err != 0 {
		return 0, fmt.Errorf("Unable to get object %s: %s", pathname, err)
	}

	return int(fd), nil
}

// ObjClose closes the map's fd.
func ObjClose(fd int) error {
	if fd > 0 {
		return unix.Close(fd)
	}
	return nil
}

func OpenOrCreateMap(path string, mapType int, keySize, valueSize, maxEntries, flags uint32) (int, bool, error) {
	var fd int

	isNewMap := false

	rl := unix.Rlimit{
		Cur: math.MaxUint64,
		Max: math.MaxUint64,
	}

	err := unix.Setrlimit(unix.RLIMIT_MEMLOCK, &rl)
	if err != nil {
		return 0, isNewMap, fmt.Errorf("Unable to increase rlimit: %s", err)
	}

	if _, err = os.Stat(path); os.IsNotExist(err) {
		mapDir := filepath.Dir(path)
		if _, err = os.Stat(mapDir); os.IsNotExist(err) {
			if err = os.MkdirAll(mapDir, 0755); err != nil {
				return 0, isNewMap, fmt.Errorf("Unable create map base directory: %s", err)
			}
		}

		fd, err = CreateMap(
			mapType,
			keySize,
			valueSize,
			maxEntries,
			flags,
		)

		defer func() {
			if err != nil {
				// In case of error, we need to close
				// this fd since it was open by CreateMap
				ObjClose(fd)
			}
		}()

		isNewMap = true

		if err != nil {
			return 0, isNewMap, err
		}

		err = ObjPin(fd, path)
		if err != nil {
			return 0, isNewMap, err
		}

		return fd, isNewMap, nil
	}

	fd, err = ObjGet(path)
	return fd, isNewMap, err
}

// GetMtime returns monotonic time that can be used to compare
// values with ktime_get_ns() BPF helper, e.g. needed to check
// the timeout in sec for BPF entries. We return the raw nsec,
// although that is not quite usable for comparison. Go has
// runtime.nanotime() but doesn't expose it as API.
func GetMtime() (uint64, error) {
	var ts unix.Timespec

	err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &ts)
	if err != nil {
		return 0, fmt.Errorf("Unable get time: %s", err)
	}

	return uint64(unix.TimespecToNsec(ts)), nil
}
<|endoftext|>


<commit_before>// Package cli implements a generic interactive line editor.
package cli

import (
	"io"
	"os"
	"sync"
	"syscall"

	"src.elv.sh/pkg/cli/term"
	"src.elv.sh/pkg/cli/tk"
	"src.elv.sh/pkg/sys"
)

// App represents a CLI app.
type App interface {
	// ReadCode requests the App to read code from the terminal by running an
	// event loop. This function is not re-entrant.
	ReadCode() (string, error)

	// MutateState mutates the state of the app.
	MutateState(f func(*State))
	// CopyState returns a copy of the a state.
	CopyState() State

	// PushAddon pushes a widget to the addon stack.
	PushAddon(w tk.Widget)
	// PopAddon pops the last widget from the addon stack. If the widget
	// implements interface{ Close(accept bool) }, the Close method is called
	// first. This method does nothing if the addon stack is empty.
	PopAddon(accept bool)

	// ActiveWidget returns the currently active widget. If the addon stack is
	// non-empty, it returns the last addon. Otherwise it returns the main code
	// area widget.
	ActiveWidget() tk.Widget
	// FocusedWidget returns the currently focused widget. It is searched like
	// ActiveWidget, but skips widgets that implement interface{ Focus() bool }
	// and return false when .Focus() is called.
	FocusedWidget() tk.Widget

	// CommitEOF causes the main loop to exit with EOF. If this method is called
	// when an event is being handled, the main loop will exit after the handler
	// returns.
	CommitEOF()
	// CommitCode causes the main loop to exit with the current code content. If
	// this method is called when an event is being handled, the main loop will
	// exit after the handler returns.
	CommitCode()

	// Redraw requests a redraw. It never blocks and can be called regardless of
	// whether the App is active or not.
	Redraw()
	// RedrawFull requests a full redraw. It never blocks and can be called
	// regardless of whether the App is active or not.
	RedrawFull()
	// Notify adds a note and requests a redraw.
	Notify(note string)
}

type app struct {
	loop    *loop
	reqRead chan struct{}

	TTY               TTY
	MaxHeight         func() int
	RPromptPersistent func() bool
	BeforeReadline    []func()
	AfterReadline     []func(string)
	Highlighter       Highlighter
	Prompt            Prompt
	RPrompt           Prompt
	GlobalBindings    tk.Bindings

	StateMutex sync.RWMutex
	State      State

	codeArea tk.CodeArea
}

// State represents mutable state of an App.
type State struct {
	// Notes that have been added since the last redraw.
	Notes []string
	// The addon stack. All widgets are shown under the codearea widget. The
	// last widget handles terminal events.
	Addons []tk.Widget
}

// NewApp creates a new App from the given specification.
func NewApp(spec AppSpec) App {
	lp := newLoop()
	a := app{
		loop:              lp,
		TTY:               spec.TTY,
		MaxHeight:         spec.MaxHeight,
		RPromptPersistent: spec.RPromptPersistent,
		BeforeReadline:    spec.BeforeReadline,
		AfterReadline:     spec.AfterReadline,
		Highlighter:       spec.Highlighter,
		Prompt:            spec.Prompt,
		RPrompt:           spec.RPrompt,
		GlobalBindings:    spec.GlobalBindings,
		State:             spec.State,
	}
	if a.TTY == nil {
		a.TTY = NewTTY(os.Stdin, os.Stderr)
	}
	if a.MaxHeight == nil {
		a.MaxHeight = func() int { return -1 }
	}
	if a.RPromptPersistent == nil {
		a.RPromptPersistent = func() bool { return false }
	}
	if a.Highlighter == nil {
		a.Highlighter = dummyHighlighter{}
	}
	if a.Prompt == nil {
		a.Prompt = NewConstPrompt(nil)
	}
	if a.RPrompt == nil {
		a.RPrompt = NewConstPrompt(nil)
	}
	if a.GlobalBindings == nil {
		a.GlobalBindings = tk.DummyBindings{}
	}
	lp.HandleCb(a.handle)
	lp.RedrawCb(a.redraw)

	a.codeArea = tk.NewCodeArea(tk.CodeAreaSpec{
		Bindings:      spec.CodeAreaBindings,
		Highlighter:   a.Highlighter.Get,
		Prompt:        a.Prompt.Get,
		RPrompt:       a.RPrompt.Get,
		Abbreviations: spec.Abbreviations,
		QuotePaste:    spec.QuotePaste,
		OnSubmit:      a.CommitCode,
		State:         spec.CodeAreaState,

		SmallWordAbbreviations: spec.SmallWordAbbreviations,
	})

	return &a
}

func (a *app) MutateState(f func(*State)) {
	a.StateMutex.Lock()
	defer a.StateMutex.Unlock()
	f(&a.State)
}

func (a *app) CopyState() State {
	a.StateMutex.RLock()
	defer a.StateMutex.RUnlock()
	return State{
		append([]string(nil), a.State.Notes...),
		append([]tk.Widget(nil), a.State.Addons...),
	}
}

func (a *app) CodeArea() tk.CodeArea {
	return a.codeArea
}

type closer interface {
	Close(bool)
}

func (a *app) PushAddon(w tk.Widget) {
	a.StateMutex.Lock()
	defer a.StateMutex.Unlock()
	a.State.Addons = append(a.State.Addons, w)
}

func (a *app) PopAddon(accept bool) {
	a.StateMutex.Lock()
	defer a.StateMutex.Unlock()
	if len(a.State.Addons) == 0 {
		return
	}
	if c, ok := a.State.Addons[len(a.State.Addons)-1].(closer); ok {
		c.Close(accept)
	}
	a.State.Addons = a.State.Addons[:len(a.State.Addons)-1]
}

func (a *app) ActiveWidget() tk.Widget {
	a.StateMutex.Lock()
	defer a.StateMutex.Unlock()
	if len(a.State.Addons) > 0 {
		return a.State.Addons[len(a.State.Addons)-1]
	}
	return a.codeArea
}

func (a *app) FocusedWidget() tk.Widget {
	a.StateMutex.Lock()
	defer a.StateMutex.Unlock()
	addons := a.State.Addons
	for i := len(addons) - 1; i >= 0; i-- {
		if hasFocus(addons[i]) {
			return addons[i]
		}
	}
	return a.codeArea
}

func (a *app) resetAllStates() {
	a.MutateState(func(s *State) { *s = State{} })
	a.codeArea.MutateState(
		func(s *tk.CodeAreaState) { *s = tk.CodeAreaState{} })
}

func (a *app) handle(e event) {
	switch e := e.(type) {
	case os.Signal:
		switch e {
		case syscall.SIGHUP:
			a.loop.Return("", io.EOF)
		case syscall.SIGINT:
			a.resetAllStates()
			a.triggerPrompts(true)
		case sys.SIGWINCH:
			a.RedrawFull()
		}
	case term.Event:
		target := a.ActiveWidget()
		handled := target.Handle(e)
		if !handled {
			a.GlobalBindings.Handle(target, e)
		}
		if !a.loop.HasReturned() {
			a.triggerPrompts(false)
			a.reqRead <- struct{}{}
		}
	}
}

func (a *app) triggerPrompts(force bool) {
	a.Prompt.Trigger(force)
	a.RPrompt.Trigger(force)
}

func (a *app) redraw(flag redrawFlag) {
	// Get the dimensions available.
	height, width := a.TTY.Size()
	if maxHeight := a.MaxHeight(); maxHeight > 0 && maxHeight < height {
		height = maxHeight
	}

	var notes []string
	var addons []tk.Widget
	a.MutateState(func(s *State) {
		notes = s.Notes
		s.Notes = nil
		addons = append([]tk.Widget(nil), s.Addons...)
	})

	bufNotes := renderNotes(notes, width)
	isFinalRedraw := flag&finalRedraw != 0
	if isFinalRedraw {
		hideRPrompt := !a.RPromptPersistent()
		if hideRPrompt {
			a.codeArea.MutateState(func(s *tk.CodeAreaState) { s.HideRPrompt = true })
		}
		bufMain := renderApp(a.codeArea, nil /* addon */, width, height)
		if hideRPrompt {
			a.codeArea.MutateState(func(s *tk.CodeAreaState) { s.HideRPrompt = false })
		}
		// Insert a newline after the buffer and position the cursor there.
		bufMain.Extend(term.NewBuffer(width), true)

		a.TTY.UpdateBuffer(bufNotes, bufMain, flag&fullRedraw != 0)
		a.TTY.ResetBuffer()
	} else {
		bufMain := renderApp(a.codeArea, addons, width, height)
		a.TTY.UpdateBuffer(bufNotes, bufMain, flag&fullRedraw != 0)
	}
}

// Renders notes. This does not respect height so that overflow notes end up in
// the scrollback buffer.
func renderNotes(notes []string, width int) *term.Buffer {
	if len(notes) == 0 {
		return nil
	}
	bb := term.NewBufferBuilder(width)
	for i, note := range notes {
		if i > 0 {
			bb.Newline()
		}
		bb.Write(note)
	}
	return bb.Buffer()
}

// Renders the codearea, and uses the rest of the height for the listing.
func renderApp(codeArea tk.Renderer, addons []tk.Widget, width, height int) *term.Buffer {
	buf := codeArea.Render(width, height)
	for _, w := range addons {
		if len(buf.Lines) >= height {
			break
		}
		bufListing := w.Render(width, height-len(buf.Lines))
		buf.Extend(bufListing, hasFocus(w))
	}
	return buf
}

func hasFocus(w interface{}) bool {
	if f, ok := w.(interface{ Focus() bool }); ok {
		return f.Focus()
	}
	return true
}

func (a *app) ReadCode() (string, error) {
	for _, f := range a.BeforeReadline {
		f()
	}
	defer func() {
		content := a.codeArea.CopyState().Buffer.Content
		for _, f := range a.AfterReadline {
			f(content)
		}
		a.resetAllStates()
	}()

	restore, err := a.TTY.Setup()
	if err != nil {
		return "", err
	}
	defer restore()

	var wg sync.WaitGroup
	defer wg.Wait()

	// Relay input events.
	a.reqRead = make(chan struct{}, 1)
	a.reqRead <- struct{}{}
	defer close(a.reqRead)
	defer a.TTY.CloseReader()
	wg.Add(1)
	go func() {
		defer wg.Done()
		for range a.reqRead {
			event, err := a.TTY.ReadEvent()
			if err == nil {
				a.loop.Input(event)
			} else if err == term.ErrStopped {
				return
			} else if term.IsReadErrorRecoverable(err) {
				a.loop.Input(term.NonfatalErrorEvent{Err: err})
			} else {
				a.loop.Input(term.FatalErrorEvent{Err: err})
				return
			}
		}
	}()

	// Relay signals.
	sigCh := a.TTY.NotifySignals()
	defer a.TTY.StopSignals()
	wg.Add(1)
	go func() {
		for sig := range sigCh {
			a.loop.Input(sig)
		}
		wg.Done()
	}()

	// Relay late updates from prompt, rprompt and highlighter.
	stopRelayLateUpdates := make(chan struct{})
	defer close(stopRelayLateUpdates)
	relayLateUpdates := func(ch <-chan struct{}) {
		if ch == nil {
			return
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				select {
				case <-ch:
					a.Redraw()
				case <-stopRelayLateUpdates:
					return
				}
			}
		}()
	}

	relayLateUpdates(a.Prompt.LateUpdates())
	relayLateUpdates(a.RPrompt.LateUpdates())
	relayLateUpdates(a.Highlighter.LateUpdates())

	// Trigger an initial prompt update.
	a.triggerPrompts(true)

	return a.loop.Run()
}

func (a *app) Redraw() {
	a.loop.Redraw(false)
}

func (a *app) RedrawFull() {
	a.loop.Redraw(true)
}

func (a *app) CommitEOF() {
	a.loop.Return("", io.EOF)
}

func (a *app) CommitCode() {
	code := a.codeArea.CopyState().Buffer.Content
	a.loop.Return(code, nil)
}

func (a *app) Notify(note string) {
	a.MutateState(func(s *State) { s.Notes = append(s.Notes, note) })
	a.Redraw()
}
<commit_msg>pkg/cli: Remove (*app).CodeArea.<commit_after>// Package cli implements a generic interactive line editor.
package cli

import (
	"io"
	"os"
	"sync"
	"syscall"

	"src.elv.sh/pkg/cli/term"
	"src.elv.sh/pkg/cli/tk"
	"src.elv.sh/pkg/sys"
)

// App represents a CLI app.
type App interface {
	// ReadCode requests the App to read code from the terminal by running an
	// event loop. This function is not re-entrant.
	ReadCode() (string, error)

	// MutateState mutates the state of the app.
	MutateState(f func(*State))
	// CopyState returns a copy of the a state.
	CopyState() State

	// PushAddon pushes a widget to the addon stack.
	PushAddon(w tk.Widget)
	// PopAddon pops the last widget from the addon stack. If the widget
	// implements interface{ Close(accept bool) }, the Close method is called
	// first. This method does nothing if the addon stack is empty.
	PopAddon(accept bool)

	// ActiveWidget returns the currently active widget. If the addon stack is
	// non-empty, it returns the last addon. Otherwise it returns the main code
	// area widget.
	ActiveWidget() tk.Widget
	// FocusedWidget returns the currently focused widget. It is searched like
	// ActiveWidget, but skips widgets that implement interface{ Focus() bool }
	// and return false when .Focus() is called.
	FocusedWidget() tk.Widget

	// CommitEOF causes the main loop to exit with EOF. If this method is called
	// when an event is being handled, the main loop will exit after the handler
	// returns.
	CommitEOF()
	// CommitCode causes the main loop to exit with the current code content. If
	// this method is called when an event is being handled, the main loop will
	// exit after the handler returns.
	CommitCode()

	// Redraw requests a redraw. It never blocks and can be called regardless of
	// whether the App is active or not.
	Redraw()
	// RedrawFull requests a full redraw. It never blocks and can be called
	// regardless of whether the App is active or not.
	RedrawFull()
	// Notify adds a note and requests a redraw.
	Notify(note string)
}

type app struct {
	loop    *loop
	reqRead chan struct{}

	TTY               TTY
	MaxHeight         func() int
	RPromptPersistent func() bool
	BeforeReadline    []func()
	AfterReadline     []func(string)
	Highlighter       Highlighter
	Prompt            Prompt
	RPrompt           Prompt
	GlobalBindings    tk.Bindings

	StateMutex sync.RWMutex
	State      State

	codeArea tk.CodeArea
}

// State represents mutable state of an App.
type State struct {
	// Notes that have been added since the last redraw.
	Notes []string
	// The addon stack. All widgets are shown under the codearea widget. The
	// last widget handles terminal events.
	Addons []tk.Widget
}

// NewApp creates a new App from the given specification.
func NewApp(spec AppSpec) App {
	lp := newLoop()
	a := app{
		loop:              lp,
		TTY:               spec.TTY,
		MaxHeight:         spec.MaxHeight,
		RPromptPersistent: spec.RPromptPersistent,
		BeforeReadline:    spec.BeforeReadline,
		AfterReadline:     spec.AfterReadline,
		Highlighter:       spec.Highlighter,
		Prompt:            spec.Prompt,
		RPrompt:           spec.RPrompt,
		GlobalBindings:    spec.GlobalBindings,
		State:             spec.State,
	}
	if a.TTY == nil {
		a.TTY = NewTTY(os.Stdin, os.Stderr)
	}
	if a.MaxHeight == nil {
		a.MaxHeight = func() int { return -1 }
	}
	if a.RPromptPersistent == nil {
		a.RPromptPersistent = func() bool { return false }
	}
	if a.Highlighter == nil {
		a.Highlighter = dummyHighlighter{}
	}
	if a.Prompt == nil {
		a.Prompt = NewConstPrompt(nil)
	}
	if a.RPrompt == nil {
		a.RPrompt = NewConstPrompt(nil)
	}
	if a.GlobalBindings == nil {
		a.GlobalBindings = tk.DummyBindings{}
	}
	lp.HandleCb(a.handle)
	lp.RedrawCb(a.redraw)

	a.codeArea = tk.NewCodeArea(tk.CodeAreaSpec{
		Bindings:      spec.CodeAreaBindings,
		Highlighter:   a.Highlighter.Get,
		Prompt:        a.Prompt.Get,
		RPrompt:       a.RPrompt.Get,
		Abbreviations: spec.Abbreviations,
		QuotePaste:    spec.QuotePaste,
		OnSubmit:      a.CommitCode,
		State:         spec.CodeAreaState,

		SmallWordAbbreviations: spec.SmallWordAbbreviations,
	})

	return &a
}

func (a *app) MutateState(f func(*State)) {
	a.StateMutex.Lock()
	defer a.StateMutex.Unlock()
	f(&a.State)
}

func (a *app) CopyState() State {
	a.StateMutex.RLock()
	defer a.StateMutex.RUnlock()
	return State{
		append([]string(nil), a.State.Notes...),
		append([]tk.Widget(nil), a.State.Addons...),
	}
}

type closer interface {
	Close(bool)
}

func (a *app) PushAddon(w tk.Widget) {
	a.StateMutex.Lock()
	defer a.StateMutex.Unlock()
	a.State.Addons = append(a.State.Addons, w)
}

func (a *app) PopAddon(accept bool) {
	a.StateMutex.Lock()
	defer a.StateMutex.Unlock()
	if len(a.State.Addons) == 0 {
		return
	}
	if c, ok := a.State.Addons[len(a.State.Addons)-1].(closer); ok {
		c.Close(accept)
	}
	a.State.Addons = a.State.Addons[:len(a.State.Addons)-1]
}

func (a *app) ActiveWidget() tk.Widget {
	a.StateMutex.Lock()
	defer a.StateMutex.Unlock()
	if len(a.State.Addons) > 0 {
		return a.State.Addons[len(a.State.Addons)-1]
	}
	return a.codeArea
}

func (a *app) FocusedWidget() tk.Widget {
	a.StateMutex.Lock()
	defer a.StateMutex.Unlock()
	addons := a.State.Addons
	for i := len(addons) - 1; i >= 0; i-- {
		if hasFocus(addons[i]) {
			return addons[i]
		}
	}
	return a.codeArea
}

func (a *app) resetAllStates() {
	a.MutateState(func(s *State) { *s = State{} })
	a.codeArea.MutateState(
		func(s *tk.CodeAreaState) { *s = tk.CodeAreaState{} })
}

func (a *app) handle(e event) {
	switch e := e.(type) {
	case os.Signal:
		switch e {
		case syscall.SIGHUP:
			a.loop.Return("", io.EOF)
		case syscall.SIGINT:
			a.resetAllStates()
			a.triggerPrompts(true)
		case sys.SIGWINCH:
			a.RedrawFull()
		}
	case term.Event:
		target := a.ActiveWidget()
		handled := target.Handle(e)
		if !handled {
			a.GlobalBindings.Handle(target, e)
		}
		if !a.loop.HasReturned() {
			a.triggerPrompts(false)
			a.reqRead <- struct{}{}
		}
	}
}

func (a *app) triggerPrompts(force bool) {
	a.Prompt.Trigger(force)
	a.RPrompt.Trigger(force)
}

func (a *app) redraw(flag redrawFlag) {
	// Get the dimensions available.
	height, width := a.TTY.Size()
	if maxHeight := a.MaxHeight(); maxHeight > 0 && maxHeight < height {
		height = maxHeight
	}

	var notes []string
	var addons []tk.Widget
	a.MutateState(func(s *State) {
		notes = s.Notes
		s.Notes = nil
		addons = append([]tk.Widget(nil), s.Addons...)
	})

	bufNotes := renderNotes(notes, width)
	isFinalRedraw := flag&finalRedraw != 0
	if isFinalRedraw {
		hideRPrompt := !a.RPromptPersistent()
		if hideRPrompt {
			a.codeArea.MutateState(func(s *tk.CodeAreaState) { s.HideRPrompt = true })
		}
		bufMain := renderApp(a.codeArea, nil /* addon */, width, height)
		if hideRPrompt {
			a.codeArea.MutateState(func(s *tk.CodeAreaState) { s.HideRPrompt = false })
		}
		// Insert a newline after the buffer and position the cursor there.
		bufMain.Extend(term.NewBuffer(width), true)

		a.TTY.UpdateBuffer(bufNotes, bufMain, flag&fullRedraw != 0)
		a.TTY.ResetBuffer()
	} else {
		bufMain := renderApp(a.codeArea, addons, width, height)
		a.TTY.UpdateBuffer(bufNotes, bufMain, flag&fullRedraw != 0)
	}
}

// Renders notes. This does not respect height so that overflow notes end up in
// the scrollback buffer.
func renderNotes(notes []string, width int) *term.Buffer {
	if len(notes) == 0 {
		return nil
	}
	bb := term.NewBufferBuilder(width)
	for i, note := range notes {
		if i > 0 {
			bb.Newline()
		}
		bb.Write(note)
	}
	return bb.Buffer()
}

// Renders the codearea, and uses the rest of the height for the listing.
func renderApp(codeArea tk.Renderer, addons []tk.Widget, width, height int) *term.Buffer {
	buf := codeArea.Render(width, height)
	for _, w := range addons {
		if len(buf.Lines) >= height {
			break
		}
		bufListing := w.Render(width, height-len(buf.Lines))
		buf.Extend(bufListing, hasFocus(w))
	}
	return buf
}

func hasFocus(w interface{}) bool {
	if f, ok := w.(interface{ Focus() bool }); ok {
		return f.Focus()
	}
	return true
}

func (a *app) ReadCode() (string, error) {
	for _, f := range a.BeforeReadline {
		f()
	}
	defer func() {
		content := a.codeArea.CopyState().Buffer.Content
		for _, f := range a.AfterReadline {
			f(content)
		}
		a.resetAllStates()
	}()

	restore, err := a.TTY.Setup()
	if err != nil {
		return "", err
	}
	defer restore()

	var wg sync.WaitGroup
	defer wg.Wait()

	// Relay input events.
	a.reqRead = make(chan struct{}, 1)
	a.reqRead <- struct{}{}
	defer close(a.reqRead)
	defer a.TTY.CloseReader()
	wg.Add(1)
	go func() {
		defer wg.Done()
		for range a.reqRead {
			event, err := a.TTY.ReadEvent()
			if err == nil {
				a.loop.Input(event)
			} else if err == term.ErrStopped {
				return
			} else if term.IsReadErrorRecoverable(err) {
				a.loop.Input(term.NonfatalErrorEvent{Err: err})
			} else {
				a.loop.Input(term.FatalErrorEvent{Err: err})
				return
			}
		}
	}()

	// Relay signals.
	sigCh := a.TTY.NotifySignals()
	defer a.TTY.StopSignals()
	wg.Add(1)
	go func() {
		for sig := range sigCh {
			a.loop.Input(sig)
		}
		wg.Done()
	}()

	// Relay late updates from prompt, rprompt and highlighter.
	stopRelayLateUpdates := make(chan struct{})
	defer close(stopRelayLateUpdates)
	relayLateUpdates := func(ch <-chan struct{}) {
		if ch == nil {
			return
		}
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				select {
				case <-ch:
					a.Redraw()
				case <-stopRelayLateUpdates:
					return
				}
			}
		}()
	}

	relayLateUpdates(a.Prompt.LateUpdates())
	relayLateUpdates(a.RPrompt.LateUpdates())
	relayLateUpdates(a.Highlighter.LateUpdates())

	// Trigger an initial prompt update.
	a.triggerPrompts(true)

	return a.loop.Run()
}

func (a *app) Redraw() {
	a.loop.Redraw(false)
}

func (a *app) RedrawFull() {
	a.loop.Redraw(true)
}

func (a *app) CommitEOF() {
	a.loop.Return("", io.EOF)
}

func (a *app) CommitCode() {
	code := a.codeArea.CopyState().Buffer.Content
	a.loop.Return(code, nil)
}

func (a *app) Notify(note string) {
	a.MutateState(func(s *State) { s.Notes = append(s.Notes, note) })
	a.Redraw()
}
<|endoftext|>


<commit_before>package cmd

import (
	"net/http"
	"time"

	"github.com/codegangsta/cli"
	"github.com/golang/glog"
	"golang.org/x/net/context"

	"bitbucket.org/moovie/renderer/pkg/renderer"
	"bitbucket.org/moovie/renderer/pkg/web"
)

// Commands - List of renderer commands.
var Commands = []cli.Command{
	webCommand,
}

var webCommand = cli.Command{
	Name:  "web",
	Usage: "starts renderer web API",
	Flags: []cli.Flag{
		// Compiler options
		cli.StringFlag{
			Name:  "dir",
			Usage: "directory containing components",
		},
		cli.BoolFlag{
			Name:  "watch",
			Usage: "watch for changes in components",
		},

		// Web server options
		cli.StringFlag{
			Name:  "listen-addr",
			Usage: "web interface listening address",
			Value: "127.0.0.1:6660",
		},
		cli.DurationFlag{
			Name:  "render-timeout",
			Usage: "component render timeout",
			Value: 5 * time.Second,
		},
	},
	Action: func(c *cli.Context) {
		// Get components directory from --dir flag
		// Print fatal error if not set
		dirname := c.String("dir")
		if dirname == "" {
			glog.Fatal("Components directory needs to be set in --dir.")
		}

		// Create a new storage in directory from --dir flag
		storage, err := renderer.NewStorage(dirname, 15*time.Minute, 5*time.Minute)
		if err != nil {
			glog.Fatalf("[storage] %v", err)
		}
		defer storage.Close()

		// Create a compiler from storage
		compiler := renderer.NewCompiler(storage)

		// Base context
		ctx := context.Background()

		// Create a context with compiler
		ctx = renderer.CompilerCtx(ctx, compiler)

		// Create a web server http handler
		w := web.New(
			web.WithCtx(ctx),
		)

		glog.Infof("[api] starting on %s", c.String("listen-addr"))

		// Start http server
		err = http.ListenAndServe(c.String("listen-addr"), w)
		if err != nil {
			glog.Fatalf("[api] %v", err)
		}
	},
}
<commit_msg>fix(main): api construction<commit_after>package cmd

import (
	"net/http"
	"time"

	"github.com/codegangsta/cli"
	"github.com/golang/glog"
	"golang.org/x/net/context"

	"bitbucket.org/moovie/renderer/pkg/renderer"
	"bitbucket.org/moovie/renderer/pkg/web"
)

// Commands - List of renderer commands.
var Commands = []cli.Command{
	webCommand,
}

var webCommand = cli.Command{
	Name:  "web",
	Usage: "starts renderer web API",
	Flags: []cli.Flag{
		// Compiler options
		cli.StringFlag{
			Name:  "dir",
			Usage: "directory containing components",
		},
		cli.BoolFlag{
			Name:  "watch",
			Usage: "watch for changes in components",
		},

		// Web server options
		cli.StringFlag{
			Name:  "listen-addr",
			Usage: "web interface listening address",
			Value: "127.0.0.1:6660",
		},
		cli.DurationFlag{
			Name:  "render-timeout",
			Usage: "component render timeout",
			Value: 5 * time.Second,
		},
	},
	Action: func(c *cli.Context) {
		// Get components directory from --dir flag
		// Print fatal error if not set
		dirname := c.String("dir")
		if dirname == "" {
			glog.Fatal("Components directory needs to be set in --dir.")
		}

		// Create a new storage in directory from --dir flag
		storage, err := renderer.NewStorage(dirname, 15*time.Minute, 5*time.Minute)
		if err != nil {
			glog.Fatalf("[storage] %v", err)
		}
		defer storage.Close()

		// Create a compiler from storage
		compiler := renderer.NewCompiler(storage)

		// Base context
		ctx := context.Background()

		// Create a context with compiler
		ctx = renderer.CompilerCtx(ctx, compiler)

		// Create a web server http handler
		w := web.NewAPI(
			web.WithCtx(ctx),
		)

		glog.Infof("[api] starting on %s", c.String("listen-addr"))

		// Start http server
		err = http.ListenAndServe(c.String("listen-addr"), w)
		if err != nil {
			glog.Fatalf("[api] %v", err)
		}
	},
}
<|endoftext|>


<commit_before>package google

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/hashicorp/terraform/helper/schema"

	"crypto/md5"
	"encoding/base64"
	"google.golang.org/api/googleapi"
	"google.golang.org/api/storage/v1"
	"io/ioutil"
)

func resourceStorageBucketObject() *schema.Resource {
	return &schema.Resource{
		Create: resourceStorageBucketObjectCreate,
		Read:   resourceStorageBucketObjectRead,
		Delete: resourceStorageBucketObjectDelete,

		Schema: map[string]*schema.Schema{
			"bucket": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"cache_control": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
			},

			"content_disposition": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
			},

			"content_encoding": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
			},

			"content_language": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
			},

			"content_type": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"content": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ForceNew:      true,
				ConflictsWith: []string{"source"},
			},

			"crc32c": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"md5hash": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"predefined_acl": &schema.Schema{
				Type:     schema.TypeString,
				Removed:  "Please use resource \"storage_object_acl.predefined_acl\" instead.",
				Optional: true,
				ForceNew: true,
			},

			"source": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ForceNew:      true,
				ConflictsWith: []string{"content"},
			},

			// Detect changes to local file or changes made outside of Terraform to the file stored on the server.
			"detect_md5hash": &schema.Schema{
				Type: schema.TypeString,
				// This field is not Computed because it needs to trigger a diff.
				Optional: true,
				ForceNew: true,
				// Makes the diff message nicer:
				// detect_md5hash:       "1XcnP/iFw/hNrbhXi7QTmQ==" => "different hash" (forces new resource)
				// Instead of the more confusing:
				// detect_md5hash:       "1XcnP/iFw/hNrbhXi7QTmQ==" => "" (forces new resource)
				Default: "different hash",
				// 1. Compute the md5 hash of the local file
				// 2. Compare the computed md5 hash with the hash stored in Cloud Storage
				// 3. Don't suppress the diff iff they don't match
				DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
					localMd5Hash := ""
					if source, ok := d.GetOkExists("source"); ok {
						localMd5Hash = getFileMd5Hash(source.(string))
					}

					if content, ok := d.GetOkExists("content"); ok {
						localMd5Hash = getContentMd5Hash([]byte(content.(string)))
					}

					// If `source` or `content` is dynamically set, both field will be empty.
					// We should not suppress the diff to avoid the following error:
					// 'Mismatch reason: extra attributes: detect_md5hash'
					if localMd5Hash == "" {
						return false
					}

					// `old` is the md5 hash we retrieved from the server in the ReadFunc
					if old != localMd5Hash {
						return false
					}

					return true
				},
			},

			"storage_class": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},
		},
	}
}

func objectGetId(object *storage.Object) string {
	return object.Bucket + "-" + object.Name
}

func resourceStorageBucketObjectCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	bucket := d.Get("bucket").(string)
	name := d.Get("name").(string)
	var media io.Reader

	if v, ok := d.GetOk("source"); ok {
		err := error(nil)
		media, err = os.Open(v.(string))
		if err != nil {
			return err
		}
	} else if v, ok := d.GetOk("content"); ok {
		media = bytes.NewReader([]byte(v.(string)))
	} else {
		return fmt.Errorf("Error, either \"content\" or \"string\" must be specified")
	}

	objectsService := storage.NewObjectsService(config.clientStorage)
	object := &storage.Object{Bucket: bucket}

	if v, ok := d.GetOk("cache_control"); ok {
		object.CacheControl = v.(string)
	}

	if v, ok := d.GetOk("content_disposition"); ok {
		object.ContentDisposition = v.(string)
	}

	if v, ok := d.GetOk("content_encoding"); ok {
		object.ContentEncoding = v.(string)
	}

	if v, ok := d.GetOk("content_language"); ok {
		object.ContentLanguage = v.(string)
	}

	if v, ok := d.GetOk("content_type"); ok {
		object.ContentType = v.(string)
	}

	if v, ok := d.GetOk("storage_class"); ok {
		object.StorageClass = v.(string)
	}

	insertCall := objectsService.Insert(bucket, object)
	insertCall.Name(name)
	insertCall.Media(media)

	_, err := insertCall.Do()

	if err != nil {
		return fmt.Errorf("Error uploading object %s: %s", name, err)
	}

	return resourceStorageBucketObjectRead(d, meta)
}

func resourceStorageBucketObjectRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	bucket := d.Get("bucket").(string)
	name := d.Get("name").(string)

	objectsService := storage.NewObjectsService(config.clientStorage)
	getCall := objectsService.Get(bucket, name)

	res, err := getCall.Do()

	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("Storage Bucket Object %q", d.Get("name").(string)))
	}

	d.Set("md5hash", res.Md5Hash)
	d.Set("detect_md5hash", res.Md5Hash)
	d.Set("crc32c", res.Crc32c)
	d.Set("cache_control", res.CacheControl)
	d.Set("content_disposition", res.ContentDisposition)
	d.Set("content_encoding", res.ContentEncoding)
	d.Set("content_language", res.ContentLanguage)
	d.Set("content_type", res.ContentType)
	d.Set("storage_class", res.StorageClass)

	d.SetId(objectGetId(res))

	return nil
}

func resourceStorageBucketObjectDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	bucket := d.Get("bucket").(string)
	name := d.Get("name").(string)

	objectsService := storage.NewObjectsService(config.clientStorage)

	DeleteCall := objectsService.Delete(bucket, name)
	err := DeleteCall.Do()

	if err != nil {
		if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
			log.Printf("[WARN] Removing Bucket Object %q because it's gone", name)
			// The resource doesn't exist anymore
			d.SetId("")

			return nil
		}

		return fmt.Errorf("Error deleting contents of object %s: %s", name, err)
	}

	return nil
}

func getFileMd5Hash(filename string) string {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Printf("[WARN] Failed to read source file %q. Cannot compute md5 hash for it.", filename)
		return ""
	}

	return getContentMd5Hash(data)
}

func getContentMd5Hash(content []byte) string {
	h := md5.New()
	h.Write(content)
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
<commit_msg>Fix a typo in error message (#928)<commit_after>package google

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/hashicorp/terraform/helper/schema"

	"crypto/md5"
	"encoding/base64"
	"google.golang.org/api/googleapi"
	"google.golang.org/api/storage/v1"
	"io/ioutil"
)

func resourceStorageBucketObject() *schema.Resource {
	return &schema.Resource{
		Create: resourceStorageBucketObjectCreate,
		Read:   resourceStorageBucketObjectRead,
		Delete: resourceStorageBucketObjectDelete,

		Schema: map[string]*schema.Schema{
			"bucket": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"cache_control": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
			},

			"content_disposition": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
			},

			"content_encoding": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
			},

			"content_language": &schema.Schema{
				Type:     schema.TypeString,
				ForceNew: true,
				Optional: true,
			},

			"content_type": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"content": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ForceNew:      true,
				ConflictsWith: []string{"source"},
			},

			"crc32c": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"md5hash": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"predefined_acl": &schema.Schema{
				Type:     schema.TypeString,
				Removed:  "Please use resource \"storage_object_acl.predefined_acl\" instead.",
				Optional: true,
				ForceNew: true,
			},

			"source": &schema.Schema{
				Type:          schema.TypeString,
				Optional:      true,
				ForceNew:      true,
				ConflictsWith: []string{"content"},
			},

			// Detect changes to local file or changes made outside of Terraform to the file stored on the server.
			"detect_md5hash": &schema.Schema{
				Type: schema.TypeString,
				// This field is not Computed because it needs to trigger a diff.
				Optional: true,
				ForceNew: true,
				// Makes the diff message nicer:
				// detect_md5hash:       "1XcnP/iFw/hNrbhXi7QTmQ==" => "different hash" (forces new resource)
				// Instead of the more confusing:
				// detect_md5hash:       "1XcnP/iFw/hNrbhXi7QTmQ==" => "" (forces new resource)
				Default: "different hash",
				// 1. Compute the md5 hash of the local file
				// 2. Compare the computed md5 hash with the hash stored in Cloud Storage
				// 3. Don't suppress the diff iff they don't match
				DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool {
					localMd5Hash := ""
					if source, ok := d.GetOkExists("source"); ok {
						localMd5Hash = getFileMd5Hash(source.(string))
					}

					if content, ok := d.GetOkExists("content"); ok {
						localMd5Hash = getContentMd5Hash([]byte(content.(string)))
					}

					// If `source` or `content` is dynamically set, both field will be empty.
					// We should not suppress the diff to avoid the following error:
					// 'Mismatch reason: extra attributes: detect_md5hash'
					if localMd5Hash == "" {
						return false
					}

					// `old` is the md5 hash we retrieved from the server in the ReadFunc
					if old != localMd5Hash {
						return false
					}

					return true
				},
			},

			"storage_class": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},
		},
	}
}

func objectGetId(object *storage.Object) string {
	return object.Bucket + "-" + object.Name
}

func resourceStorageBucketObjectCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	bucket := d.Get("bucket").(string)
	name := d.Get("name").(string)
	var media io.Reader

	if v, ok := d.GetOk("source"); ok {
		err := error(nil)
		media, err = os.Open(v.(string))
		if err != nil {
			return err
		}
	} else if v, ok := d.GetOk("content"); ok {
		media = bytes.NewReader([]byte(v.(string)))
	} else {
		return fmt.Errorf("Error, either \"content\" or \"source\" must be specified")
	}

	objectsService := storage.NewObjectsService(config.clientStorage)
	object := &storage.Object{Bucket: bucket}

	if v, ok := d.GetOk("cache_control"); ok {
		object.CacheControl = v.(string)
	}

	if v, ok := d.GetOk("content_disposition"); ok {
		object.ContentDisposition = v.(string)
	}

	if v, ok := d.GetOk("content_encoding"); ok {
		object.ContentEncoding = v.(string)
	}

	if v, ok := d.GetOk("content_language"); ok {
		object.ContentLanguage = v.(string)
	}

	if v, ok := d.GetOk("content_type"); ok {
		object.ContentType = v.(string)
	}

	if v, ok := d.GetOk("storage_class"); ok {
		object.StorageClass = v.(string)
	}

	insertCall := objectsService.Insert(bucket, object)
	insertCall.Name(name)
	insertCall.Media(media)

	_, err := insertCall.Do()

	if err != nil {
		return fmt.Errorf("Error uploading object %s: %s", name, err)
	}

	return resourceStorageBucketObjectRead(d, meta)
}

func resourceStorageBucketObjectRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	bucket := d.Get("bucket").(string)
	name := d.Get("name").(string)

	objectsService := storage.NewObjectsService(config.clientStorage)
	getCall := objectsService.Get(bucket, name)

	res, err := getCall.Do()

	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("Storage Bucket Object %q", d.Get("name").(string)))
	}

	d.Set("md5hash", res.Md5Hash)
	d.Set("detect_md5hash", res.Md5Hash)
	d.Set("crc32c", res.Crc32c)
	d.Set("cache_control", res.CacheControl)
	d.Set("content_disposition", res.ContentDisposition)
	d.Set("content_encoding", res.ContentEncoding)
	d.Set("content_language", res.ContentLanguage)
	d.Set("content_type", res.ContentType)
	d.Set("storage_class", res.StorageClass)

	d.SetId(objectGetId(res))

	return nil
}

func resourceStorageBucketObjectDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	bucket := d.Get("bucket").(string)
	name := d.Get("name").(string)

	objectsService := storage.NewObjectsService(config.clientStorage)

	DeleteCall := objectsService.Delete(bucket, name)
	err := DeleteCall.Do()

	if err != nil {
		if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == 404 {
			log.Printf("[WARN] Removing Bucket Object %q because it's gone", name)
			// The resource doesn't exist anymore
			d.SetId("")

			return nil
		}

		return fmt.Errorf("Error deleting contents of object %s: %s", name, err)
	}

	return nil
}

func getFileMd5Hash(filename string) string {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Printf("[WARN] Failed to read source file %q. Cannot compute md5 hash for it.", filename)
		return ""
	}

	return getContentMd5Hash(data)
}

func getContentMd5Hash(content []byte) string {
	h := md5.New()
	h.Write(content)
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
<|endoftext|>


<commit_before>package scm

import (
	"fmt"
	"strings"

	"github.com/golang/glog"

	"github.com/openshift/source-to-image/pkg/build"
	"github.com/openshift/source-to-image/pkg/scm/empty"
	"github.com/openshift/source-to-image/pkg/scm/file"
	"github.com/openshift/source-to-image/pkg/scm/git"
	"github.com/openshift/source-to-image/pkg/util"
)

// DownloaderForSource determines what SCM plugin should be used for downloading
// the sources from the repository.
func DownloaderForSource(s string, forceCopy bool) (build.Downloader, string, error) {
	glog.V(4).Infof("DownloadForSource %s", s)

	if len(s) == 0 {
		return &empty.Noop{}, s, nil
	}

	details, mods := git.ParseFile(s)
	glog.V(4).Infof("return from ParseFile file exists %v proto specified %v use copy %v", details.FileExists, details.ProtoSpecified, details.UseCopy)

	if details.FileExists && details.BadRef {
		return nil, s, fmt.Errorf("local location referenced by %s exists but the input after the # is malformed", s)
	}

	if details.FileExists && mods != nil {
		glog.V(4).Infof("new source from parse file %s", mods.Path)
		if details.ProtoSpecified {
			s = mods.Path
		} else {
			// prepending with file:// is a precautionary step which previous incarnations of this code did; we
			// preserve that behavior (it is more explicit, if not absolutely necessary; but we do it here as was done before
			// vs. down in our generic git layer (which is leveraged separately in origin)
			s = "file://" + mods.Path
		}
	}

	if details.FileExists && (details.UseCopy || forceCopy) {
		return &file.File{util.NewFileSystem()}, s, nil
	}

	if CheckIsTFSSourceCode(s){
		return nil, s, fmt.Errorf("no downloader defined for the location: %q", s)
	} else {
		// If the source is valid  Git protocol (file://, ssh://, git://, git@, etc..) use Git
		// binary to download the sources
		g := git.New()
		if g.ValidCloneSpec(s) {
			return &git.Clone{g, util.NewFileSystem()}, s, nil
		}
	}

	return nil, s, fmt.Errorf("no downloader defined for location: %q", s)
}

func CheckIsTFSSourceCode(s string) bool{
	glog.Infof("CheckIsTFSSourceCode %q ...", s)
	return true
}<commit_msg>removed strings<commit_after>package scm

import (
	"fmt"

	"github.com/golang/glog"

	"github.com/openshift/source-to-image/pkg/build"
	"github.com/openshift/source-to-image/pkg/scm/empty"
	"github.com/openshift/source-to-image/pkg/scm/file"
	"github.com/openshift/source-to-image/pkg/scm/git"
	"github.com/openshift/source-to-image/pkg/util"
)

// DownloaderForSource determines what SCM plugin should be used for downloading
// the sources from the repository.
func DownloaderForSource(s string, forceCopy bool) (build.Downloader, string, error) {
	glog.V(4).Infof("DownloadForSource %s", s)

	if len(s) == 0 {
		return &empty.Noop{}, s, nil
	}

	details, mods := git.ParseFile(s)
	glog.V(4).Infof("return from ParseFile file exists %v proto specified %v use copy %v", details.FileExists, details.ProtoSpecified, details.UseCopy)

	if details.FileExists && details.BadRef {
		return nil, s, fmt.Errorf("local location referenced by %s exists but the input after the # is malformed", s)
	}

	if details.FileExists && mods != nil {
		glog.V(4).Infof("new source from parse file %s", mods.Path)
		if details.ProtoSpecified {
			s = mods.Path
		} else {
			// prepending with file:// is a precautionary step which previous incarnations of this code did; we
			// preserve that behavior (it is more explicit, if not absolutely necessary; but we do it here as was done before
			// vs. down in our generic git layer (which is leveraged separately in origin)
			s = "file://" + mods.Path
		}
	}

	if details.FileExists && (details.UseCopy || forceCopy) {
		return &file.File{util.NewFileSystem()}, s, nil
	}

	if CheckIsTFSSourceCode(s){
		return nil, s, fmt.Errorf("no downloader defined for the location: %q", s)
	} else {
		// If the source is valid  Git protocol (file://, ssh://, git://, git@, etc..) use Git
		// binary to download the sources
		g := git.New()
		if g.ValidCloneSpec(s) {
			return &git.Clone{g, util.NewFileSystem()}, s, nil
		}
	}

	return nil, s, fmt.Errorf("no downloader defined for location: %q", s)
}

func CheckIsTFSSourceCode(s string) bool{
	glog.Infof("CheckIsTFSSourceCode %q ...", s)
	return true
}<|endoftext|>