<commit_before>package nakadi

import (
	"bufio"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"gopkg.in/jarcoal/httpmock.v1"
)

func TestSimpleStreamOpener_openStream(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	createdAt := time.Time{}
	createdAt.UnmarshalText([]byte("2017-08-17T00:00:23+02:00"))

	sub := &Subscription{
		ID:                "4e6f4b42-5459-11e7-8b76-97cbdf1f5274",
		OwningApplication: "nakadi-client",
		EventTypes:        []string{"test"},
		ConsumerGroup:     "default",
		ReadFrom:          "end",
		CreatedAt:         createdAt}

	url := fmt.Sprintf("%s/subscriptions/%s/events", defaultNakadiURL, sub.ID)

	setupOpener := func() *simpleStreamOpener {
		client := &Client{
			nakadiURL:        defaultNakadiURL,
			httpClient:       http.DefaultClient,
			httpStreamClient: http.DefaultClient,
			tokenProvider:    func() (string, error) { return "token", nil }}
		return &simpleStreamOpener{
			client:         client,
			subscriptionID: sub.ID}
	}

	t.Run("fail retrieving token", func(t *testing.T) {
		opener := setupOpener()
		opener.client.tokenProvider = func() (string, error) { return "", assert.AnError }

		_, err := opener.openStream()
		require.Error(t, err)
		assert.Regexp(t, assert.AnError.Error(), err.Error())
	})

	t.Run("fail connect error", func(t *testing.T) {
		opener := setupOpener()
		httpmock.RegisterResponder("GET", url, httpmock.NewErrorResponder(assert.AnError))

		_, err := opener.openStream()
		require.Error(t, err)
		assert.Regexp(t, assert.AnError.Error(), err.Error())
	})

	t.Run("fail http error", func(t *testing.T) {
		problem := &problemJSON{Detail: "foo problem detail"}
		opener := setupOpener()
		responder, _ := httpmock.NewJsonResponder(400, &problem)
		httpmock.RegisterResponder("GET", url, responder)

		_, err := opener.openStream()
		require.Error(t, err)
		assert.Regexp(t, problem.Detail, err.Error())
	})
	t.Run("fail to read body", func(t *testing.T) {
		opener := setupOpener()
		responder := httpmock.ResponderFromResponse(&http.Response{
			Status:     strconv.Itoa(http.StatusBadRequest),
			StatusCode: http.StatusBadRequest,
			Body:       brokenBodyReader{},
		})
		httpmock.RegisterResponder("GET", url, responder)

		_, err := opener.openStream()
		require.Error(t, err)
		assert.Regexp(t, "unable to read response body", err.Error())
	})

	t.Run("success without token", func(t *testing.T) {
		opener := setupOpener()
		responder, _ := httpmock.NewJsonResponder(200, sub)
		httpmock.RegisterResponder("GET", url, responder)

		stream, err := opener.openStream()
		require.NoError(t, err)
		require.NotNil(t, stream)
	})

	t.Run("success with token", func(t *testing.T) {
		opener := setupOpener()
		responder, _ := httpmock.NewJsonResponder(200, sub)
		httpmock.RegisterResponder("GET", url, responder)

		stream, err := opener.openStream()
		require.NoError(t, err)
		require.NotNil(t, stream)
	})
}

func TestSimpleStream_nextEvents(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	id := "21e9e526-551e-11e7-bbe7-1f386750d2b6"
	url := fmt.Sprintf("%s/subscriptions/%s/events", defaultNakadiURL, id)

	setupStream := func(responder httpmock.Responder) *simpleStream {
		httpmock.RegisterResponder("GET", url, responder)
		response, err := http.DefaultClient.Get(url)
		require.NoError(t, err)
		return &simpleStream{
			nakadiStreamID: "stream-id",
			buffer:         bufio.NewReader(response.Body),
			closer:         response.Body}
	}

	t.Run("fail stream closed", func(t *testing.T) {
		stream := setupStream(httpmock.NewStringResponder(200, ""))
		stream.buffer = nil

		_, _, err := stream.nextEvents()
		require.Error(t, err)
		assert.Regexp(t, "stream is closed", err.Error())
	})

	t.Run("fail EOF", func(t *testing.T) {
		stream := setupStream(httpmock.NewStringResponder(200, ""))

		_, _, err := stream.nextEvents()
		require.Error(t, err)
		assert.Regexp(t, "EOF", err.Error())
	})

	t.Run("fail unmarshal event", func(t *testing.T) {
		stream := setupStream(httpmock.NewStringResponder(200, "not\n a\n event\n"))

		_, _, err := stream.nextEvents()
		require.Error(t, err)
		assert.Regexp(t, "failed to unmarshal next batch", err.Error())
	})

	t.Run("successfully read events", func(t *testing.T) {
		events := helperLoadTestData(t, "data-event-stream.json", nil)
		stream := setupStream(httpmock.NewStringResponder(200, string(events)))

		for i := 0; i < 5; i++ {
			cursor, _, err := stream.nextEvents()
			require.NoError(t, err)
			assert.Equal(t, "stream-id", cursor.NakadiStreamID)
		}
	})
}

type fakeCloser struct {
	Closed bool
}

func (fc *fakeCloser) Close() error {
	fc.Closed = true
	return nil
}

func TestSimpleStream_closeStream(t *testing.T) {
	closer := &fakeCloser{}
	stream := &simpleStream{
		nakadiStreamID: "stream-id",
		buffer:         bufio.NewReader(strings.NewReader("")),
		closer:         closer}

	err := stream.closeStream()
	assert.NoError(t, err)
	assert.Nil(t, stream.buffer)
	assert.True(t, closer.Closed)
}

func TestSimpleCommitter_commitEvents(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	id := "21e9e526-551e-11e7-bbe7-1f386750d2b6"
	url := fmt.Sprintf("%s/subscriptions/%s/cursors", defaultNakadiURL, id)

	setupCommitter := func(responder httpmock.Responder) *simpleCommitter {
		httpmock.RegisterResponder("POST", url, responder)
		client := &Client{
			nakadiURL:     defaultNakadiURL,
			httpClient:    http.DefaultClient,
			tokenProvider: func() (string, error) { return "token", nil }}
		return &simpleCommitter{
			client:         client,
			subscriptionID: id}
	}

	t.Run("fail retrieving token", func(t *testing.T) {
		stream := setupCommitter(httpmock.NewStringResponder(200, ""))
		stream.client.tokenProvider = func() (string, error) { return "", assert.AnError }

		err := stream.commitCursor(Cursor{})
		require.Error(t, err)
		assert.Regexp(t, assert.AnError, err)
	})

	t.Run("fail connect error", func(t *testing.T) {
		stream := setupCommitter(httpmock.NewErrorResponder(assert.AnError))

		err := stream.commitCursor(Cursor{})
		require.Error(t, err)
		assert.Regexp(t, assert.AnError, err)
	})

	t.Run("fail http error", func(t *testing.T) {
		problem := &problemJSON{Detail: "foo problem detail"}
		responder, _ := httpmock.NewJsonResponder(400, &problem)
		stream := setupCommitter(responder)

		err := stream.commitCursor(Cursor{})
		require.Error(t, err)
		assert.Regexp(t, problem.Detail, err)
	})
	t.Run("fail to read body", func(t *testing.T) {
		responder := httpmock.ResponderFromResponse(&http.Response{
			Status:     strconv.Itoa(http.StatusBadRequest),
			StatusCode: http.StatusBadRequest,
			Body:       brokenBodyReader{},
		})
		stream := setupCommitter(responder)

		err := stream.commitCursor(Cursor{})
		require.Error(t, err)
		assert.Regexp(t, "unable to read response body", err)
	})

	t.Run("successful commit", func(t *testing.T) {
		stream := setupCommitter(httpmock.NewStringResponder(200, ""))

		err := stream.commitCursor(Cursor{})
		require.NoError(t, err)
	})
}
<commit_msg>Changed tests to use buffer reader with minimal buffer size(16 bytes)<commit_after>package nakadi

import (
	"bufio"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"gopkg.in/jarcoal/httpmock.v1"
)

func TestSimpleStreamOpener_openStream(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	createdAt := time.Time{}
	createdAt.UnmarshalText([]byte("2017-08-17T00:00:23+02:00"))

	sub := &Subscription{
		ID:                "4e6f4b42-5459-11e7-8b76-97cbdf1f5274",
		OwningApplication: "nakadi-client",
		EventTypes:        []string{"test"},
		ConsumerGroup:     "default",
		ReadFrom:          "end",
		CreatedAt:         createdAt}

	url := fmt.Sprintf("%s/subscriptions/%s/events", defaultNakadiURL, sub.ID)

	setupOpener := func() *simpleStreamOpener {
		client := &Client{
			nakadiURL:        defaultNakadiURL,
			httpClient:       http.DefaultClient,
			httpStreamClient: http.DefaultClient,
			tokenProvider:    func() (string, error) { return "token", nil }}
		return &simpleStreamOpener{
			client:         client,
			subscriptionID: sub.ID}
	}

	t.Run("fail retrieving token", func(t *testing.T) {
		opener := setupOpener()
		opener.client.tokenProvider = func() (string, error) { return "", assert.AnError }

		_, err := opener.openStream()
		require.Error(t, err)
		assert.Regexp(t, assert.AnError.Error(), err.Error())
	})

	t.Run("fail connect error", func(t *testing.T) {
		opener := setupOpener()
		httpmock.RegisterResponder("GET", url, httpmock.NewErrorResponder(assert.AnError))

		_, err := opener.openStream()
		require.Error(t, err)
		assert.Regexp(t, assert.AnError.Error(), err.Error())
	})

	t.Run("fail http error", func(t *testing.T) {
		problem := &problemJSON{Detail: "foo problem detail"}
		opener := setupOpener()
		responder, _ := httpmock.NewJsonResponder(400, &problem)
		httpmock.RegisterResponder("GET", url, responder)

		_, err := opener.openStream()
		require.Error(t, err)
		assert.Regexp(t, problem.Detail, err.Error())
	})
	t.Run("fail to read body", func(t *testing.T) {
		opener := setupOpener()
		responder := httpmock.ResponderFromResponse(&http.Response{
			Status:     strconv.Itoa(http.StatusBadRequest),
			StatusCode: http.StatusBadRequest,
			Body:       brokenBodyReader{},
		})
		httpmock.RegisterResponder("GET", url, responder)

		_, err := opener.openStream()
		require.Error(t, err)
		assert.Regexp(t, "unable to read response body", err.Error())
	})

	t.Run("success without token", func(t *testing.T) {
		opener := setupOpener()
		responder, _ := httpmock.NewJsonResponder(200, sub)
		httpmock.RegisterResponder("GET", url, responder)

		stream, err := opener.openStream()
		require.NoError(t, err)
		require.NotNil(t, stream)
	})

	t.Run("success with token", func(t *testing.T) {
		opener := setupOpener()
		responder, _ := httpmock.NewJsonResponder(200, sub)
		httpmock.RegisterResponder("GET", url, responder)

		stream, err := opener.openStream()
		require.NoError(t, err)
		require.NotNil(t, stream)
	})
}

func TestSimpleStream_nextEvents(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	id := "21e9e526-551e-11e7-bbe7-1f386750d2b6"
	url := fmt.Sprintf("%s/subscriptions/%s/events", defaultNakadiURL, id)

	setupStream := func(responder httpmock.Responder) *simpleStream {
		httpmock.RegisterResponder("GET", url, responder)
		response, err := http.DefaultClient.Get(url)
		require.NoError(t, err)
		return &simpleStream{
			nakadiStreamID: "stream-id",
			buffer:         bufio.NewReaderSize(response.Body, 0),
			closer:         response.Body}
	}

	t.Run("fail stream closed", func(t *testing.T) {
		stream := setupStream(httpmock.NewStringResponder(200, ""))
		stream.buffer = nil

		_, _, err := stream.nextEvents()
		require.Error(t, err)
		assert.Regexp(t, "stream is closed", err.Error())
	})

	t.Run("fail EOF", func(t *testing.T) {
		stream := setupStream(httpmock.NewStringResponder(200, ""))

		_, _, err := stream.nextEvents()
		require.Error(t, err)
		assert.Regexp(t, "EOF", err.Error())
	})

	t.Run("fail unmarshal event", func(t *testing.T) {
		stream := setupStream(httpmock.NewStringResponder(200, "not\n a\n event\n"))

		_, _, err := stream.nextEvents()
		require.Error(t, err)
		assert.Regexp(t, "failed to unmarshal next batch", err.Error())
	})

	t.Run("successfully read events", func(t *testing.T) {
		events := helperLoadTestData(t, "data-event-stream.json", nil)
		stream := setupStream(httpmock.NewStringResponder(200, string(events)))

		for i := 0; i < 5; i++ {
			cursor, _, err := stream.nextEvents()
			require.NoError(t, err)
			assert.Equal(t, "stream-id", cursor.NakadiStreamID)
		}
	})
}

type fakeCloser struct {
	Closed bool
}

func (fc *fakeCloser) Close() error {
	fc.Closed = true
	return nil
}

func TestSimpleStream_closeStream(t *testing.T) {
	closer := &fakeCloser{}
	stream := &simpleStream{
		nakadiStreamID: "stream-id",
		buffer:         bufio.NewReader(strings.NewReader("")),
		closer:         closer}

	err := stream.closeStream()
	assert.NoError(t, err)
	assert.Nil(t, stream.buffer)
	assert.True(t, closer.Closed)
}

func TestSimpleCommitter_commitEvents(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	id := "21e9e526-551e-11e7-bbe7-1f386750d2b6"
	url := fmt.Sprintf("%s/subscriptions/%s/cursors", defaultNakadiURL, id)

	setupCommitter := func(responder httpmock.Responder) *simpleCommitter {
		httpmock.RegisterResponder("POST", url, responder)
		client := &Client{
			nakadiURL:     defaultNakadiURL,
			httpClient:    http.DefaultClient,
			tokenProvider: func() (string, error) { return "token", nil }}
		return &simpleCommitter{
			client:         client,
			subscriptionID: id}
	}

	t.Run("fail retrieving token", func(t *testing.T) {
		stream := setupCommitter(httpmock.NewStringResponder(200, ""))
		stream.client.tokenProvider = func() (string, error) { return "", assert.AnError }

		err := stream.commitCursor(Cursor{})
		require.Error(t, err)
		assert.Regexp(t, assert.AnError, err)
	})

	t.Run("fail connect error", func(t *testing.T) {
		stream := setupCommitter(httpmock.NewErrorResponder(assert.AnError))

		err := stream.commitCursor(Cursor{})
		require.Error(t, err)
		assert.Regexp(t, assert.AnError, err)
	})

	t.Run("fail http error", func(t *testing.T) {
		problem := &problemJSON{Detail: "foo problem detail"}
		responder, _ := httpmock.NewJsonResponder(400, &problem)
		stream := setupCommitter(responder)

		err := stream.commitCursor(Cursor{})
		require.Error(t, err)
		assert.Regexp(t, problem.Detail, err)
	})
	t.Run("fail to read body", func(t *testing.T) {
		responder := httpmock.ResponderFromResponse(&http.Response{
			Status:     strconv.Itoa(http.StatusBadRequest),
			StatusCode: http.StatusBadRequest,
			Body:       brokenBodyReader{},
		})
		stream := setupCommitter(responder)

		err := stream.commitCursor(Cursor{})
		require.Error(t, err)
		assert.Regexp(t, "unable to read response body", err)
	})

	t.Run("successful commit", func(t *testing.T) {
		stream := setupCommitter(httpmock.NewStringResponder(200, ""))

		err := stream.commitCursor(Cursor{})
		require.NoError(t, err)
	})
}
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildbucket

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"golang.org/x/net/context"

	"go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/auth/identity"
	"go.chromium.org/luci/common/data/strpair"
	"go.chromium.org/luci/common/errors"
)

const (
	// TagBuilder is the key of builder name tag.
	TagBuilder = "builder"
	// TagBuildAddress is the key of the build address tag.
	// See also Build.Address().
	TagBuildAddress = "build_address"
)

// Build is a buildbucket build.
// It is a more type-safe version of buildbucket.ApiCommonBuildMessage.
type Build struct {
	// fields set at the build creation time, immutable.

	ID           int64
	CreationTime time.Time
	CreatedBy    identity.Identity
	Project      string
	Bucket       string
	Builder      string
	// Number identifies the build within the builder.
	// Build numbers are monotonically increasing, mostly contiguous.
	//
	// The type is *int to prevent accidental confusion
	// of valid build number 0 with absence of the number (zero value).
	Number *int
	// BuildSets is parsed "buildset" tag values.
	//
	// If a buildset is present in tags, but not recognized
	// it won't be included here.
	BuildSets        []BuildSet
	Tags             strpair.Map
	Input            Input
	CanaryPreference CanaryPreference

	// fields that can change during build lifetime

	Status           Status
	StatusChangeTime time.Time
	URL              string
	StartTime        time.Time
	UpdateTime       time.Time
	Canary           bool

	// fields set on build completion

	CompletionTime time.Time
	Output         Output
}

// Address returns an alternative identifier of the build.
// If b has a number, the address is "<bucket>/<builder>/<number>".
// Otherwise it is "<id>".
//
// See also ParseBuildAddress.
func (b *Build) Address() string {
	if b.Number != nil {
		return fmt.Sprintf("%s/%s/%d", b.Bucket, b.Builder, *b.Number)
	}
	return strconv.FormatInt(b.ID, 10)
}

// RunDuration returns duration between build start and completion.
func (b *Build) RunDuration() (duration time.Duration, ok bool) {
	if b.StartTime.IsZero() || b.CompletionTime.IsZero() {
		return 0, false
	}
	return b.CompletionTime.Sub(b.StartTime), true
}

// SchedulingDuration returns duration between build creation and start.
func (b *Build) SchedulingDuration() (duration time.Duration, ok bool) {
	if b.CreationTime.IsZero() || b.StartTime.IsZero() {
		return 0, false
	}
	return b.StartTime.Sub(b.CreationTime), true
}

// Properties is data provided by users, opaque to LUCI services.
// The value must be JSON marshalable/unmarshalable into/out from a
// JSON object.
//
// When using an unmarshaling function, such as (*Build).ParseMessage,
// if the user knows the properties they need, they may set the
// value to a json-compatible struct. The unmarshaling function will try
// to unmarshal the properties into the struct. Otherwise, the unmarshaling
// function will use a generic type, e.g. map[string]interface{}.
//
// Example:
//
//   var props struct {
//     A string
//   }
//   var build buildbucket.Build
//   build.Input.Properties = &props
//   if err := build.ParseMessage(msg); err != nil {
//     return err
//   }
//   println(props.A)
type Properties interface{}

// Input is the input to the builder.
type Input struct {
	// Properties is opaque data passed to the build.
	// For recipe-based builds, this is build properties.
	Properties Properties
}

// Output is build output.
type Output struct {
	Properties Properties

	// TODO(nodir, iannucci): replace type "error" with a new type that
	// represents a stack of errors emitted by different layers of the system,
	// where each error has
	// - domain string, e.g. "kitchen"
	// - reason string, e.g. kitchen-specific error code
	// - message string: human readable error
	// - meta: a proto.Struct with random data provided by the layer
	// The new type must implement error so that the change is
	// backward compatible.
	Err error // populated in builds with status StatusError
}

// ParseMessage parses a build message to Build.
//
// Numeric values in JSON-formatted fields, e.g. property values, are parsed as
// json.Number.
//
// If an error is returned, the state of b is undefined.
func (b *Build) ParseMessage(msg *buildbucket.ApiCommonBuildMessage) error {
	status, err := ParseStatus(msg)
	if err != nil {
		return err
	}

	var createdBy identity.Identity
	if msg.CreatedBy != "" {
		createdBy, err = identity.MakeIdentity(msg.CreatedBy)
		if err != nil {
			return err
		}
	}

	tags := strpair.ParseMap(msg.Tags)
	builder := tags.Get(TagBuilder)

	canaryPref, err := parseEndpointsCanaryPreference(msg.CanaryPreference)
	if err != nil {
		return err
	}

	address := tags.Get(TagBuildAddress)
	var number *int
	if address == "" {
		address = strconv.FormatInt(msg.Id, 10)
	} else {
		parts := strings.Split(address, "/")
		if len(parts) != 3 {
			return fmt.Errorf("invalid build_address %q: expected exactly 2 slashes", address)
		}
		if msg.Bucket == "" {
			// this is a partial response message
			msg.Bucket = parts[0]
		} else if msg.Bucket != parts[0] {
			return fmt.Errorf("invalid build_address %q: expected first component to be %q", address, msg.Bucket)
		}
		if builder == "" {
			return fmt.Errorf("build_address tag is present, but builder tag is not")
		} else if parts[1] != builder {
			return fmt.Errorf("invalid build_address %q: expected second component to be %q", address, builder)
		}
		num, err := strconv.Atoi(parts[2])
		if err != nil {
			return fmt.Errorf("invalid build_address %q: expected third component to be a valid int32", address)
		}
		number = &num
	}

	input := struct{ Properties interface{} }{b.Input.Properties}
	if err := parseJSON(msg.ParametersJson, &input); err != nil {
		return errors.Annotate(err, "invalid msg.ParametersJson").Err()
	}

	output := struct {
		Properties interface{}
		Error      struct {
			Message string
		}
	}{Properties: b.Output.Properties}
	if err := parseJSON(msg.ResultDetailsJson, &output); err != nil {
		return errors.Annotate(err, "invalid msg.ResultDetailsJson").Err()
	}
	var outErr error
	if output.Error.Message != "" {
		outErr = errors.New(output.Error.Message)
	}

	var bs []BuildSet
	for _, t := range tags[TagBuildSet] {
		if parsed := ParseBuildSet(t); parsed != nil {
			bs = append(bs, parsed)
		}
	}

	project := msg.Project
	if project == "" {
		// old builds do not have project attribute.
		project = projectFromBucket(msg.Bucket)
	}

	*b = Build{
		ID:               msg.Id,
		CreationTime:     ParseTimestamp(msg.CreatedTs),
		CreatedBy:        createdBy,
		Project:          msg.Project,
		Bucket:           msg.Bucket,
		Builder:          builder,
		Number:           number,
		BuildSets:        bs,
		Tags:             tags,
		CanaryPreference: canaryPref,
		Input: Input{
			Properties: input.Properties,
		},

		Status:           status,
		StatusChangeTime: ParseTimestamp(msg.StatusChangedTs),
		URL:              msg.Url,
		StartTime:        ParseTimestamp(msg.StartedTs),
		UpdateTime:       ParseTimestamp(msg.UpdatedTs),
		Canary:           msg.Canary,

		CompletionTime: ParseTimestamp(msg.CompletedTs),
		Output: Output{
			Properties: output.Properties,
			Err:        outErr,
		},
	}
	return nil
}

// PutRequest converts b to a build creation request.
//
// If a buildset is present in both b.BuildSets and b.Map, it is deduped.
// Returned value has zero ClientOperationId.
// Returns an error if properties could not be marshaled to JSON.
func (b *Build) PutRequest() (*buildbucket.ApiPutRequestMessage, error) {
	tags := b.Tags.Copy()
	tags.Del(TagBuilder) // buildbucket adds it automatically
	for _, bs := range b.BuildSets {
		s := bs.String()
		if !tags.Contains(TagBuildSet, s) {
			tags.Add(TagBuildSet, s)
		}
	}

	canaryPref, err := b.CanaryPreference.endpointsString()
	if err != nil {
		return nil, err
	}
	msg := &buildbucket.ApiPutRequestMessage{
		Bucket:           b.Bucket,
		Tags:             tags.Format(),
		CanaryPreference: canaryPref,
	}

	parameters := map[string]interface{}{
		"builder_name": b.Builder,
		"properties":   b.Input.Properties,
		// keep this synced with marshaling error annotation
	}
	if data, err := json.Marshal(parameters); err != nil {
		// realistically, only properties may cause this.
		return nil, errors.Annotate(err, "marshaling properties").Err()
	} else {
		msg.ParametersJson = string(data)
	}

	return msg, nil
}

func parseJSON(data string, v interface{}) error {
	if data == "" {
		return nil
	}
	dec := json.NewDecoder(strings.NewReader(data))
	dec.UseNumber()
	return dec.Decode(v)
}

// ParseTimestamp parses a buildbucket timestamp.
func ParseTimestamp(usec int64) time.Time {
	if usec == 0 {
		return time.Time{}
	}
	// TODO(nodir): will start to overflow in year ~2250+, fix it then.
	return time.Unix(0, usec*1000).UTC()
}

// FormatTimestamp converts t to a buildbucket timestamp.
func FormatTimestamp(t time.Time) int64 {
	return t.UnixNano() / 1000
}

// projectFromBucket tries to retrieve project id from bucket name.
// Returns "" on failure.
func projectFromBucket(bucket string) string {
	// Buildbucket guarantees that buckets that start with "luci."
	// have "luci.<project id>." prefix.
	parts := strings.Split(bucket, ".")
	if len(parts) >= 3 && parts[0] == "luci" {
		return parts[1]
	}
	return ""
}

// ParseBuildAddress parses a build address returned by Build.Address().
//
// If id is non-zero, project, bucket and builder are zero.
// If bucket is non-zero, id is zero.
func ParseBuildAddress(address string) (id int64, project, bucket, builder string, number int, err error) {
	parts := strings.Split(address, "/")
	switch len(parts) {
	case 4:
		// future-proof
		project = parts[0]
		parts = parts[1:]
		fallthrough
	case 3:
		var numberStr string
		bucket, builder, numberStr = parts[0], parts[1], parts[2]
		if project == "" {
			project = projectFromBucket(bucket)
		}
		number, err = strconv.Atoi(numberStr)
	case 1:
		id, err = strconv.ParseInt(parts[0], 10, 64)
	default:
		err = fmt.Errorf("unrecognized build address format %q", address)
	}
	return
}

// ValidateBuildAddress returns an error if the build address is invalid.
func ValidateBuildAddress(address string) error {
	_, _, _, _, _, err := ParseBuildAddress(address)
	return err
}

// GetByAddress fetches a build by its address.
// Returns (nil, nil) if build is not found.
func GetByAddress(c context.Context, client *buildbucket.Service, address string) (*Build, error) {
	id, _, _, _, _, err := ParseBuildAddress(address)
	if err != nil {
		return nil, err
	}
	var msg *buildbucket.ApiCommonBuildMessage
	if id != 0 {
		res, err := client.Get(id).Context(c).Do()
		switch {
		case err != nil:
			return nil, err
		case res.Error.Reason == buildbucket.ReasonNotFound:
			return nil, nil
		default:
			msg = res.Build
		}
	} else {
		msgs, err := client.Search().
			Context(c).
			Tag(strpair.Format(TagBuildAddress, address)).
			Fetch(1, nil)
		switch {
		case err != nil:
			return nil, err
		case len(msgs) == 0:
			return nil, nil
		default:
			msg = msgs[0]
		}
	}

	var build Build
	err = build.ParseMessage(msg)
	return &build, err
}
<commit_msg>luci/buildbucket/build.go: Fix panic<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package buildbucket

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"golang.org/x/net/context"

	"go.chromium.org/luci/common/api/buildbucket/buildbucket/v1"
	"go.chromium.org/luci/common/auth/identity"
	"go.chromium.org/luci/common/data/strpair"
	"go.chromium.org/luci/common/errors"
)

const (
	// TagBuilder is the key of builder name tag.
	TagBuilder = "builder"
	// TagBuildAddress is the key of the build address tag.
	// See also Build.Address().
	TagBuildAddress = "build_address"
)

// Build is a buildbucket build.
// It is a more type-safe version of buildbucket.ApiCommonBuildMessage.
type Build struct {
	// fields set at the build creation time, immutable.

	ID           int64
	CreationTime time.Time
	CreatedBy    identity.Identity
	Project      string
	Bucket       string
	Builder      string
	// Number identifies the build within the builder.
	// Build numbers are monotonically increasing, mostly contiguous.
	//
	// The type is *int to prevent accidental confusion
	// of valid build number 0 with absence of the number (zero value).
	Number *int
	// BuildSets is parsed "buildset" tag values.
	//
	// If a buildset is present in tags, but not recognized
	// it won't be included here.
	BuildSets        []BuildSet
	Tags             strpair.Map
	Input            Input
	CanaryPreference CanaryPreference

	// fields that can change during build lifetime

	Status           Status
	StatusChangeTime time.Time
	URL              string
	StartTime        time.Time
	UpdateTime       time.Time
	Canary           bool

	// fields set on build completion

	CompletionTime time.Time
	Output         Output
}

// Address returns an alternative identifier of the build.
// If b has a number, the address is "<bucket>/<builder>/<number>".
// Otherwise it is "<id>".
//
// See also ParseBuildAddress.
func (b *Build) Address() string {
	if b.Number != nil {
		return fmt.Sprintf("%s/%s/%d", b.Bucket, b.Builder, *b.Number)
	}
	return strconv.FormatInt(b.ID, 10)
}

// RunDuration returns duration between build start and completion.
func (b *Build) RunDuration() (duration time.Duration, ok bool) {
	if b.StartTime.IsZero() || b.CompletionTime.IsZero() {
		return 0, false
	}
	return b.CompletionTime.Sub(b.StartTime), true
}

// SchedulingDuration returns duration between build creation and start.
func (b *Build) SchedulingDuration() (duration time.Duration, ok bool) {
	if b.CreationTime.IsZero() || b.StartTime.IsZero() {
		return 0, false
	}
	return b.StartTime.Sub(b.CreationTime), true
}

// Properties is data provided by users, opaque to LUCI services.
// The value must be JSON marshalable/unmarshalable into/out from a
// JSON object.
//
// When using an unmarshaling function, such as (*Build).ParseMessage,
// if the user knows the properties they need, they may set the
// value to a json-compatible struct. The unmarshaling function will try
// to unmarshal the properties into the struct. Otherwise, the unmarshaling
// function will use a generic type, e.g. map[string]interface{}.
//
// Example:
//
//   var props struct {
//     A string
//   }
//   var build buildbucket.Build
//   build.Input.Properties = &props
//   if err := build.ParseMessage(msg); err != nil {
//     return err
//   }
//   println(props.A)
type Properties interface{}

// Input is the input to the builder.
type Input struct {
	// Properties is opaque data passed to the build.
	// For recipe-based builds, this is build properties.
	Properties Properties
}

// Output is build output.
type Output struct {
	Properties Properties

	// TODO(nodir, iannucci): replace type "error" with a new type that
	// represents a stack of errors emitted by different layers of the system,
	// where each error has
	// - domain string, e.g. "kitchen"
	// - reason string, e.g. kitchen-specific error code
	// - message string: human readable error
	// - meta: a proto.Struct with random data provided by the layer
	// The new type must implement error so that the change is
	// backward compatible.
	Err error // populated in builds with status StatusError
}

// ParseMessage parses a build message to Build.
//
// Numeric values in JSON-formatted fields, e.g. property values, are parsed as
// json.Number.
//
// If an error is returned, the state of b is undefined.
func (b *Build) ParseMessage(msg *buildbucket.ApiCommonBuildMessage) error {
	status, err := ParseStatus(msg)
	if err != nil {
		return err
	}

	var createdBy identity.Identity
	if msg.CreatedBy != "" {
		createdBy, err = identity.MakeIdentity(msg.CreatedBy)
		if err != nil {
			return err
		}
	}

	tags := strpair.ParseMap(msg.Tags)
	builder := tags.Get(TagBuilder)

	canaryPref, err := parseEndpointsCanaryPreference(msg.CanaryPreference)
	if err != nil {
		return err
	}

	address := tags.Get(TagBuildAddress)
	var number *int
	if address == "" {
		address = strconv.FormatInt(msg.Id, 10)
	} else {
		parts := strings.Split(address, "/")
		if len(parts) != 3 {
			return fmt.Errorf("invalid build_address %q: expected exactly 2 slashes", address)
		}
		if msg.Bucket == "" {
			// this is a partial response message
			msg.Bucket = parts[0]
		} else if msg.Bucket != parts[0] {
			return fmt.Errorf("invalid build_address %q: expected first component to be %q", address, msg.Bucket)
		}
		if builder == "" {
			return fmt.Errorf("build_address tag is present, but builder tag is not")
		} else if parts[1] != builder {
			return fmt.Errorf("invalid build_address %q: expected second component to be %q", address, builder)
		}
		num, err := strconv.Atoi(parts[2])
		if err != nil {
			return fmt.Errorf("invalid build_address %q: expected third component to be a valid int32", address)
		}
		number = &num
	}

	input := struct{ Properties interface{} }{b.Input.Properties}
	if err := parseJSON(msg.ParametersJson, &input); err != nil {
		return errors.Annotate(err, "invalid msg.ParametersJson").Err()
	}

	output := struct {
		Properties interface{}
		Error      struct {
			Message string
		}
	}{Properties: b.Output.Properties}
	if err := parseJSON(msg.ResultDetailsJson, &output); err != nil {
		return errors.Annotate(err, "invalid msg.ResultDetailsJson").Err()
	}
	var outErr error
	if output.Error.Message != "" {
		outErr = errors.New(output.Error.Message)
	}

	var bs []BuildSet
	for _, t := range tags[TagBuildSet] {
		if parsed := ParseBuildSet(t); parsed != nil {
			bs = append(bs, parsed)
		}
	}

	project := msg.Project
	if project == "" {
		// old builds do not have project attribute.
		project = projectFromBucket(msg.Bucket)
	}

	*b = Build{
		ID:               msg.Id,
		CreationTime:     ParseTimestamp(msg.CreatedTs),
		CreatedBy:        createdBy,
		Project:          msg.Project,
		Bucket:           msg.Bucket,
		Builder:          builder,
		Number:           number,
		BuildSets:        bs,
		Tags:             tags,
		CanaryPreference: canaryPref,
		Input: Input{
			Properties: input.Properties,
		},

		Status:           status,
		StatusChangeTime: ParseTimestamp(msg.StatusChangedTs),
		URL:              msg.Url,
		StartTime:        ParseTimestamp(msg.StartedTs),
		UpdateTime:       ParseTimestamp(msg.UpdatedTs),
		Canary:           msg.Canary,

		CompletionTime: ParseTimestamp(msg.CompletedTs),
		Output: Output{
			Properties: output.Properties,
			Err:        outErr,
		},
	}
	return nil
}

// PutRequest converts b to a build creation request.
//
// If a buildset is present in both b.BuildSets and b.Map, it is deduped.
// Returned value has zero ClientOperationId.
// Returns an error if properties could not be marshaled to JSON.
func (b *Build) PutRequest() (*buildbucket.ApiPutRequestMessage, error) {
	tags := b.Tags.Copy()
	tags.Del(TagBuilder) // buildbucket adds it automatically
	for _, bs := range b.BuildSets {
		s := bs.String()
		if !tags.Contains(TagBuildSet, s) {
			tags.Add(TagBuildSet, s)
		}
	}

	canaryPref, err := b.CanaryPreference.endpointsString()
	if err != nil {
		return nil, err
	}
	msg := &buildbucket.ApiPutRequestMessage{
		Bucket:           b.Bucket,
		Tags:             tags.Format(),
		CanaryPreference: canaryPref,
	}

	parameters := map[string]interface{}{
		"builder_name": b.Builder,
		"properties":   b.Input.Properties,
		// keep this synced with marshaling error annotation
	}
	if data, err := json.Marshal(parameters); err != nil {
		// realistically, only properties may cause this.
		return nil, errors.Annotate(err, "marshaling properties").Err()
	} else {
		msg.ParametersJson = string(data)
	}

	return msg, nil
}

func parseJSON(data string, v interface{}) error {
	if data == "" {
		return nil
	}
	dec := json.NewDecoder(strings.NewReader(data))
	dec.UseNumber()
	return dec.Decode(v)
}

// ParseTimestamp parses a buildbucket timestamp.
func ParseTimestamp(usec int64) time.Time {
	if usec == 0 {
		return time.Time{}
	}
	// TODO(nodir): will start to overflow in year ~2250+, fix it then.
	return time.Unix(0, usec*1000).UTC()
}

// FormatTimestamp converts t to a buildbucket timestamp.
func FormatTimestamp(t time.Time) int64 {
	return t.UnixNano() / 1000
}

// projectFromBucket tries to retrieve project id from bucket name.
// Returns "" on failure.
func projectFromBucket(bucket string) string {
	// Buildbucket guarantees that buckets that start with "luci."
	// have "luci.<project id>." prefix.
	parts := strings.Split(bucket, ".")
	if len(parts) >= 3 && parts[0] == "luci" {
		return parts[1]
	}
	return ""
}

// ParseBuildAddress parses a build address returned by Build.Address().
//
// If id is non-zero, project, bucket and builder are zero.
// If bucket is non-zero, id is zero.
func ParseBuildAddress(address string) (id int64, project, bucket, builder string, number int, err error) {
	parts := strings.Split(address, "/")
	switch len(parts) {
	case 4:
		// future-proof
		project = parts[0]
		parts = parts[1:]
		fallthrough
	case 3:
		var numberStr string
		bucket, builder, numberStr = parts[0], parts[1], parts[2]
		if project == "" {
			project = projectFromBucket(bucket)
		}
		number, err = strconv.Atoi(numberStr)
	case 1:
		id, err = strconv.ParseInt(parts[0], 10, 64)
	default:
		err = fmt.Errorf("unrecognized build address format %q", address)
	}
	return
}

// ValidateBuildAddress returns an error if the build address is invalid.
func ValidateBuildAddress(address string) error {
	_, _, _, _, _, err := ParseBuildAddress(address)
	return err
}

// GetByAddress fetches a build by its address.
// Returns (nil, nil) if build is not found.
func GetByAddress(c context.Context, client *buildbucket.Service, address string) (*Build, error) {
	id, _, _, _, _, err := ParseBuildAddress(address)
	if err != nil {
		return nil, err
	}
	var msg *buildbucket.ApiCommonBuildMessage
	if id != 0 {
		res, err := client.Get(id).Context(c).Do()
		switch {
		case err != nil:
			return nil, err
		case res.Error != nil && res.Error.Reason == buildbucket.ReasonNotFound:
			return nil, nil
		default:
			msg = res.Build
		}
	} else {
		msgs, err := client.Search().
			Context(c).
			Tag(strpair.Format(TagBuildAddress, address)).
			Fetch(1, nil)
		switch {
		case err != nil:
			return nil, err
		case len(msgs) == 0:
			return nil, nil
		default:
			msg = msgs[0]
		}
	}

	var build Build
	err = build.ParseMessage(msg)
	return &build, err
}
<|endoftext|>


<commit_before>// Copyright 2015 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten

import (
	"image/color"
)

// A Lines represents the set of lines.
type Lines interface {
	Len() int
	Points(i int) (x0, y0, x1, y1 int)
	Color(i int) color.Color
}

type line struct {
	x0, y0 int
	x1, y1 int
	color  color.Color
}

func (l *line) Len() int {
	return 1
}

func (l *line) Points(i int) (x0, y0, x1, y1 int) {
	return l.x0, l.y0, l.x1, l.y1
}

func (l *line) Color(i int) color.Color {
	return l.color
}

type rectsAsLines struct {
	Rects
}

func (r *rectsAsLines) Len() int {
	return r.Rects.Len() * 4
}

func (r *rectsAsLines) Points(i int) (x0, y0, x1, y1 int) {
	x, y, w, h := r.Rects.Rect(i / 4)
	switch i % 4 {
	case 0:
		return x, y, x + w, y
	case 1:
		return x, y + 1, x, y + h - 1
	case 2:
		return x, y + h - 1, x + w, y + h - 1
	case 3:
		return x + w - 1, y + 1, x + w - 1, y + h - 1
	}
	panic("not reach")
}

func (r *rectsAsLines) Color(i int) color.Color {
	return r.Rects.Color(i / 4)
}

// A Rects represents the set of rectangles.
type Rects interface {
	Len() int
	Rect(i int) (x, y, width, height int)
	Color(i int) color.Color
}

type rect struct {
	x, y          int
	width, height int
	color         color.Color
}

func (r *rect) Len() int {
	return 1
}

func (r *rect) Rect(i int) (x, y, width, height int) {
	return r.x, r.y, r.width, r.height
}

func (r *rect) Color(i int) color.Color {
	return r.color
}
<commit_msg>image: Remove Rects and Lines<commit_after><|endoftext|>


<commit_before>/*Package shodan is an interface for the Shodan API*/
package shodan

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
)

// APIHost is the URL of the Shodan API.
var (
	APIHost = "https://api.shodan.io"
)

// Client stores shared data that is used to interact with the API.
// Key is our Shodan API Key.
type Client struct {
	Key string
}

// HostOptions are used to pass parameters to the Host function.
type HostOptions struct {
	IP      string
	History bool
	Minify  bool
}

// Host is used to unmarshal the JSON response from '/shodan/host/{ip}'.
type Host struct {
	RegionCode  interface{}   `json:"region_code"`
	IP          string        `json:"ip"`
	AreaCode    interface{}   `json:"area_code"`
	CountryName string        `json:"country_name"`
	Hostnames   []interface{} `json:"hostnames"`
	PostalCode  interface{}   `json:"postal_code"`
	DmaCode     interface{}   `json:"dma_code"`
	CountryCode string        `json:"country_code"`
	Data        []struct {
		Product    string        `json:"product"`
		Os         interface{}   `json:"os"`
		Timestamp  string        `json:"timestamp"`
		Isp        string        `json:"isp"`
		Asn        string        `json:"asn"`
		Banner     string        `json:"banner"`
		Hostnames  []interface{} `json:"hostnames"`
		Devicetype string        `json:"devicetype"`
		Location   struct {
			City         interface{} `json:"city"`
			RegionCode   interface{} `json:"region_code"`
			AreaCode     interface{} `json:"area_code"`
			Longitude    int         `json:"longitude"`
			CountryCode3 string      `json:"country_code3"`
			CountryName  string      `json:"country_name"`
			PostalCode   interface{} `json:"postal_code"`
			DmaCode      interface{} `json:"dma_code"`
			CountryCode  string      `json:"country_code"`
			Latitude     int         `json:"latitude"`
		} `json:"location"`
		IP      string        `json:"ip"`
		Domains []interface{} `json:"domains"`
		Org     string        `json:"org"`
		Port    int           `json:"port"`
		Opts    struct {
		} `json:"opts"`
	} `json:"data"`
	City         interface{} `json:"city"`
	Longitude    int         `json:"longitude"`
	CountryCode3 string      `json:"country_code3"`
	Latitude     int         `json:"latitude"`
	Os           interface{} `json:"os"`
	Ports        []int       `json:"ports"`
}

// HostCount is used to unmarshal the JSON response from '/shodan/host/count'.
type HostCount struct {
	Matches []interface{} `json:"matches"`
	Facets  struct {
		Org []struct {
			Count int    `json:"count"`
			Value string `json:"value"`
		} `json:"org"`
	} `json:"facets"`
	Total int `json:"total"`
}

// HostSearch is used to unmarshal the JSON response from '/shodan/host/search'.
type HostSearch struct {
	Matches []struct {
		Os        interface{}   `json:"os"`
		Timestamp string        `json:"timestamp"`
		Isp       string        `json:"isp"`
		Asn       string        `json:"asn"`
		Hostnames []interface{} `json:"hostnames"`
		Location  struct {
			City         interface{} `json:"city"`
			RegionCode   interface{} `json:"region_code"`
			AreaCode     interface{} `json:"area_code"`
			Longitude    int         `json:"longitude"`
			CountryCode3 string      `json:"country_code3"`
			CountryName  string      `json:"country_name"`
			PostalCode   interface{} `json:"postal_code"`
			DmaCode      interface{} `json:"dma_code"`
			CountryCode  string      `json:"country_code"`
			Latitude     int         `json:"latitude"`
		} `json:"location"`
		IP      int64         `json:"ip"`
		Domains []interface{} `json:"domains"`
		Data    string        `json:"data"`
		Org     string        `json:"org"`
		Port    int           `json:"port"`
		IPStr   string        `json:"ip_str"`
	} `json:"matches"`
	Facets struct {
		Org []struct {
			Count int    `json:"count"`
			Value string `json:"value"`
		} `json:"org"`
	} `json:"facets"`
	Total int `json:"total"`
}

// HostSearchTokens is used to unmarshal the JSON response from '/shodan/host/search/tokens'.
type HostSearchTokens struct {
	Attributes struct {
		Ports []int `json:"ports"`
	} `json:"attributes"`
	Errors  []interface{} `json:"errors"`
	String  string        `json:"string"`
	Filters []string      `json:"filters"`
}

// Scan is used to unmarshal the JSON response from '/shodan/scan'.
type Scan struct {
	ID          string `json:"id"`
	Count       int    `json:"count"`
	CreditsLeft int    `json:"credits_left"`
}

// ScanInternet is used to unmarshal the JSON response from '/shodan/scan/internet'.
type ScanInternet struct {
	ID string `json:"id"`
}

// Query is used to unmarshal the JSON response from '/shodan/query/{search}'.
type Query struct {
	Total   int `json:"total"`
	Matches []struct {
		Votes       int      `json:"votes"`
		Description string   `json:"description"`
		Title       string   `json:"title"`
		Timestamp   string   `json:"timestamp"`
		Tags        []string `json:"tags"`
		Query       string   `json:"query"`
	} `json:"matches"`
}

// QueryTags is used to unmarshal the JSON response from '/shodan/query/tags'.
type QueryTags struct {
	Total   int `json:"total"`
	Matches []struct {
		Value string `json:"value"`
		Count int    `json:"count"`
	} `json:"matches"`
}

// APIInfo is used to unmarshal the JSON response from '/shodan/api-info'.
type APIInfo struct {
	QueryCredits int    `json:"query_credits"`
	ScanCredits  int    `json:"scan_credits"`
	Telnet       bool   `json:"telnet"`
	Plan         string `json:"plan"`
	HTTPS        bool   `json:"https"`
	Unlocked     bool   `json:"unlocked"`
}

// Error used to unmarshal the JSOn response of an error.
type Error struct {
	Error string `json:"error"`
}

// New returns a new Client.
func New(key string) *Client {
	return &Client{
		Key: key,
	}
}

// Host calls '/shodan/host/{ip}' and returns the unmarshaled response.
func (c *Client) Host(opts *HostOptions) (*Host, error) {
	h := &Host{}
	client := &http.Client{}
	req, err := http.NewRequest("GET", APIHost+"/shodan/host/"+opts.IP+"&key="+c.Key, nil)
	if err != nil {
		return h, err
	}
	if opts.Minify {
		req.URL.Query().Set("minify", "true")
	}
	if opts.History {
		req.URL.Query().Set("history", "true")
	}
	resp, err := client.Do(req)
	if err != nil {
		return h, err
	}
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return h, err
	}
	if resp.StatusCode >= 300 {
		e := &Error{}
		if err := json.Unmarshal(data, &e); err != nil {
			return h, err
		}
		return h, errors.New(e.Error)
	}
	return h, nil
}
<commit_msg>Adds /dns/resolve and /dns/reverse<commit_after>/*Package shodan is an interface for the Shodan API*/
package shodan

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"strings"
)

// APIHost is the URL of the Shodan API.
var (
	APIHost = "https://api.shodan.io"
)

// Client stores shared data that is used to interact with the API.
// Key is our Shodan API Key.
type Client struct {
	Key string
}

// HostOptions are used to pass parameters to the Host function.
type HostOptions struct {
	IP      string
	History bool
	Minify  bool
}

// Host is used to unmarshal the JSON response from '/shodan/host/{ip}'.
type Host struct {
	RegionCode  interface{}   `json:"region_code"`
	IP          string        `json:"ip"`
	AreaCode    interface{}   `json:"area_code"`
	CountryName string        `json:"country_name"`
	Hostnames   []interface{} `json:"hostnames"`
	PostalCode  interface{}   `json:"postal_code"`
	DmaCode     interface{}   `json:"dma_code"`
	CountryCode string        `json:"country_code"`
	Data        []struct {
		Product    string        `json:"product"`
		Os         interface{}   `json:"os"`
		Timestamp  string        `json:"timestamp"`
		Isp        string        `json:"isp"`
		Asn        string        `json:"asn"`
		Banner     string        `json:"banner"`
		Hostnames  []interface{} `json:"hostnames"`
		Devicetype string        `json:"devicetype"`
		Location   struct {
			City         interface{} `json:"city"`
			RegionCode   interface{} `json:"region_code"`
			AreaCode     interface{} `json:"area_code"`
			Longitude    int         `json:"longitude"`
			CountryCode3 string      `json:"country_code3"`
			CountryName  string      `json:"country_name"`
			PostalCode   interface{} `json:"postal_code"`
			DmaCode      interface{} `json:"dma_code"`
			CountryCode  string      `json:"country_code"`
			Latitude     int         `json:"latitude"`
		} `json:"location"`
		IP      string        `json:"ip"`
		Domains []interface{} `json:"domains"`
		Org     string        `json:"org"`
		Port    int           `json:"port"`
		Opts    struct {
		} `json:"opts"`
	} `json:"data"`
	City         interface{} `json:"city"`
	Longitude    int         `json:"longitude"`
	CountryCode3 string      `json:"country_code3"`
	Latitude     int         `json:"latitude"`
	Os           interface{} `json:"os"`
	Ports        []int       `json:"ports"`
}

// HostCount is used to unmarshal the JSON response from '/shodan/host/count'.
type HostCount struct {
	Matches []interface{} `json:"matches"`
	Facets  struct {
		Org []struct {
			Count int    `json:"count"`
			Value string `json:"value"`
		} `json:"org"`
	} `json:"facets"`
	Total int `json:"total"`
}

// HostSearch is used to unmarshal the JSON response from '/shodan/host/search'.
type HostSearch struct {
	Matches []struct {
		Os        interface{}   `json:"os"`
		Timestamp string        `json:"timestamp"`
		Isp       string        `json:"isp"`
		Asn       string        `json:"asn"`
		Hostnames []interface{} `json:"hostnames"`
		Location  struct {
			City         interface{} `json:"city"`
			RegionCode   interface{} `json:"region_code"`
			AreaCode     interface{} `json:"area_code"`
			Longitude    int         `json:"longitude"`
			CountryCode3 string      `json:"country_code3"`
			CountryName  string      `json:"country_name"`
			PostalCode   interface{} `json:"postal_code"`
			DmaCode      interface{} `json:"dma_code"`
			CountryCode  string      `json:"country_code"`
			Latitude     int         `json:"latitude"`
		} `json:"location"`
		IP      int64         `json:"ip"`
		Domains []interface{} `json:"domains"`
		Data    string        `json:"data"`
		Org     string        `json:"org"`
		Port    int           `json:"port"`
		IPStr   string        `json:"ip_str"`
	} `json:"matches"`
	Facets struct {
		Org []struct {
			Count int    `json:"count"`
			Value string `json:"value"`
		} `json:"org"`
	} `json:"facets"`
	Total int `json:"total"`
}

// HostSearchTokens is used to unmarshal the JSON response from '/shodan/host/search/tokens'.
type HostSearchTokens struct {
	Attributes struct {
		Ports []int `json:"ports"`
	} `json:"attributes"`
	Errors  []interface{} `json:"errors"`
	String  string        `json:"string"`
	Filters []string      `json:"filters"`
}

// Scan is used to unmarshal the JSON response from '/shodan/scan'.
type Scan struct {
	ID          string `json:"id"`
	Count       int    `json:"count"`
	CreditsLeft int    `json:"credits_left"`
}

// ScanInternet is used to unmarshal the JSON response from '/shodan/scan/internet'.
type ScanInternet struct {
	ID string `json:"id"`
}

// Query is used to unmarshal the JSON response from '/shodan/query/{search}'.
type Query struct {
	Total   int `json:"total"`
	Matches []struct {
		Votes       int      `json:"votes"`
		Description string   `json:"description"`
		Title       string   `json:"title"`
		Timestamp   string   `json:"timestamp"`
		Tags        []string `json:"tags"`
		Query       string   `json:"query"`
	} `json:"matches"`
}

// QueryTags is used to unmarshal the JSON response from '/shodan/query/tags'.
type QueryTags struct {
	Total   int `json:"total"`
	Matches []struct {
		Value string `json:"value"`
		Count int    `json:"count"`
	} `json:"matches"`
}

// APIInfo is used to unmarshal the JSON response from '/shodan/api-info'.
type APIInfo struct {
	QueryCredits int    `json:"query_credits"`
	ScanCredits  int    `json:"scan_credits"`
	Telnet       bool   `json:"telnet"`
	Plan         string `json:"plan"`
	HTTPS        bool   `json:"https"`
	Unlocked     bool   `json:"unlocked"`
}

// DNSResolve is used to transform the map[string]string response from '/dns/resolve'
// into a struct that is a bit easier to work with.
type DNSResolve struct {
	Hostname string
	IP       string
}

// DNSReverse is used to transform the map[string][]string response from '/dns/reverse'
// into a struct that is a bit easier to work with.
type DNSReverse struct {
	IP        string
	Hostnames []string
}

// Error used to unmarshal the JSON response of an error.
type Error struct {
	Error string `json:"error"`
}

// New returns a new Client.
func New(key string) *Client {
	return &Client{
		Key: key,
	}
}

// Host calls '/shodan/host/{ip}' and returns the unmarshaled response.
func (c *Client) Host(opts *HostOptions) (*Host, error) {
	h := &Host{}
	client := &http.Client{}
	req, err := http.NewRequest("GET", APIHost+"/shodan/host/"+opts.IP+"&key="+c.Key, nil)
	if err != nil {
		return h, err
	}
	if opts.Minify {
		req.URL.Query().Set("minify", "true")
	}
	if opts.History {
		req.URL.Query().Set("history", "true")
	}
	resp, err := client.Do(req)
	if err != nil {
		return h, err
	}
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return h, err
	}
	if err := checkError(resp, data); err != nil {
		return h, err
	}
	return h, nil
}

// DNSResolve calls '/dns/resolve' and returns the unmarshaled response.
func (c *Client) DNSResolve(hostnames []string) ([]DNSResolve, error) {
	d := []DNSResolve{}
	resp, err := http.Get(APIHost + "/dns/resolve/?key=" + c.Key + "&hostnames=" + strings.Join(hostnames, ","))
	if err != nil {
		return d, err
	}
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return d, err
	}
	if err := checkError(resp, data); err != nil {
		return d, err
	}
	m := make(map[string]string)
	if err := json.Unmarshal(data, m); err != nil {
		return d, err
	}
	for k, v := range m {
		d = append(d, DNSResolve{
			Hostname: k,
			IP:       v,
		})
	}
	return d, nil
}

// DNSReverse calls '/dns/reverse' and returns the unmarshaled response.
func (c *Client) DNSReverse(ips []string) ([]DNSReverse, error) {
	d := []DNSReverse{}
	resp, err := http.Get(APIHost + "/dns/resolve/?key=" + c.Key + "ips=" + strings.Join(ips, ","))
	if err != nil {
		return d, err
	}
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return d, err
	}
	if err := checkError(resp, data); err != nil {
		return d, err
	}
	m := make(map[string][]string)
	if err := json.Unmarshal(data, m); err != nil {
		return d, err
	}
	for k, v := range m {
		r := DNSReverse{IP: k}
		for _, n := range v {
			r.Hostnames = append(r.Hostnames, n)
		}
		d = append(d, r)
	}
	return d, nil
}

func checkError(resp *http.Response, data []byte) error {
	if resp.StatusCode >= 300 {
		e := &Error{}
		if err := json.Unmarshal(data, &e); err != nil {
			return err
		}
		return errors.New(e.Error)
	}
	return nil
}
<|endoftext|>


<commit_before>package PEncoding

import (
	"bytes"
	"fmt"
	. "github.com/xitongsys/parquet-go/ParquetType"
	"testing"
)

func TestReadPlainINT32(t *testing.T) {
	testData := []struct {
		expected   []interface{}
		byteReader *bytes.Reader
	}{
		{[]interface{}{}, bytes.NewReader([]byte{})},
		{[]interface{}{INT32(0)}, bytes.NewReader([]byte{0, 0, 0, 0})},
		{[]interface{}{INT32(0), INT32(1), INT32(2)}, bytes.NewReader([]byte{0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0})},
	}

	for _, data := range testData {
		res := ReadPlainINT32(data.byteReader, uint64(len(data.expected)))
		if fmt.Sprintf("%v", res) != fmt.Sprintf("%v", data.expected) {
			t.Errorf("ReadPlainINT32 error, expect %v, get %v", data.expected, res)
		}
	}
}
<commit_msg>adding encoding test<commit_after>package PEncoding

import (
	"bytes"
	"fmt"
	. "github.com/xitongsys/parquet-go/ParquetType"
	"testing"
)

func TestReadPlainINT32(t *testing.T) {
	testData := []struct {
		expected   []interface{}
		byteReader *bytes.Reader
	}{
		{[]interface{}{}, bytes.NewReader([]byte{})},
		{[]interface{}{INT32(0)}, bytes.NewReader([]byte{0, 0, 0, 0})},
		{[]interface{}{INT32(0), INT32(1), INT32(2)}, bytes.NewReader([]byte{0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0})},
	}

	for _, data := range testData {
		res := ReadPlainINT32(data.byteReader, uint64(len(data.expected)))
		if fmt.Sprintf("%v", res) != fmt.Sprintf("%v", data.expected) {
			t.Errorf("ReadPlainINT32 error, expect %v, get %v", data.expected, res)
		}
	}
}

func TestReadPlainINT64(t *testing.T) {
	testData := []struct {
		expected   []interface{}
		byteReader *bytes.Reader
	}{
		{[]interface{}{}, bytes.NewReader([]byte{})},
		{[]interface{}{INT64(0)}, bytes.NewReader([]byte{0, 0, 0, 0})},
		{[]interface{}{INT64(0), INT64(1), INT64(2)}, bytes.NewReader([]byte{0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0})},
	}

	for _, data := range testData {
		res := ReadPlainINT64(data.byteReader, uint64(len(data.expected)))
		if fmt.Sprintf("%v", res) != fmt.Sprintf("%v", data.expected) {
			t.Errorf("ReadPlainINT64 error, expect %v, get %v", data.expected, res)
		}
	}
}
<|endoftext|>


<commit_before>// Package bytebuffer implements a java like bytebuffer for go
//
// initially tried to use bytes.Buffer but the main restriction with that is that
// it does not allow the freedom to move around in the buffer. Further, it always
// writes at the end of the buffer
//
// another attempt was to maintain a position identifier that was always passed
// to any function that wrote anything and the function *had* to return the
// next writable location, which resulted in calls like
//
//	pos = writeString(buffer, pos, "mmv")
//
// which became unmaintainable after a while, and along with all the side
// maintainance looked extremely ugly
//
// this (tries) to implement a minimal buffer wrapper that gives the freedom
// to move around and write anywhere you want
package bytebuffer

type Buffer interface {
	Bytes() []byte
	Pos() int
	SetPos(int)
	Len() int
	Write([]byte)
	WriteString(string)
	WriteInt(int)
	WriteInt32(int32)
	WriteInt64(int64)
	WriteUint32(uint32)
	WriteUint64(uint64)
}
<commit_msg>bytebuffer: redefine Buffer to be io.Writer and return errors<commit_after>// Package bytebuffer implements a java like bytebuffer for go
//
// initially tried to use bytes.Buffer but the main restriction with that is that
// it does not allow the freedom to move around in the buffer. Further, it always
// writes at the end of the buffer
//
// another attempt was to maintain a position identifier that was always passed
// to any function that wrote anything and the function *had* to return the
// next writable location, which resulted in calls like
//
//	pos = writeString(buffer, pos, "mmv")
//
// which became unmaintainable after a while, and along with all the side
// maintainance looked extremely ugly
//
// this (tries) to implement a minimal buffer wrapper that gives the freedom
// to move around and write anywhere you want
package bytebuffer

import "io"

// Buffer defines an abstraction for an object that allows writing of binary
// values anywhere within a fixed range
type Buffer interface {
	io.Writer
	Bytes() []byte
	Pos() int
	SetPos(int) error
	Len() int
	WriteVal(val interface{}) error
	WriteString(string) error
	WriteInt32(int32) error
	WriteInt64(int64) error
	WriteUint32(uint32) error
	WriteUint64(uint64) error
	WriteFloat32(float32) error
	WriteFloat64(float64) error
}
<|endoftext|>


<commit_before>/*
 * This file is a part of linuxdeploy - tool for
 * creating standalone applications for Linux
 *
 * Copyright (C) 2017 Taras Kushnir <kushnirTV@gmail.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the MIT License.

 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 */

package main

import (
  "log"
  "os"
  "strings"
  "sync"
  "path/filepath"
  "fmt"
  "bufio"
)

const (
  LDD_DEPENDENCY_FLAG Bitmask = 1 << iota // check ldd deps for given item
  FIX_RPATH_FLAG // fix rpath for qt-related libs/plugins
  DEPLOY_ONLY_LIBRARIES_FLAG
)

const (
  LDD_AND_RPATH_FLAG = LDD_DEPENDENCY_FLAG | FIX_RPATH_FLAG
  LIBRARIES_AND_RPATH_FLAG = FIX_RPATH_FLAG | DEPLOY_ONLY_LIBRARIES_FLAG
)

type DeployRequest struct {
  sourcePath string // relative or absolute path of file to process
  sourceRoot string // if empty then sourcePath is absolute path
  targetPath string // target *relative* path
  flags Bitmask // deployment flags
}

func (dp *DeployRequest) FullPath() string {
  if len(dp.sourceRoot) == 0 {
    return dp.sourcePath
  } else {
    return filepath.Join(dp.sourceRoot, dp.sourcePath)
  }
}

func (dp *DeployRequest) Basename() string {
  return filepath.Base(dp.sourcePath)
}

func (dp *DeployRequest) SourceDir() string {
  return filepath.Dir(dp.sourcePath)
}

func (dp *DeployRequest) RequiresRPathFix() bool {
  return dp.flags.HasFlag(FIX_RPATH_FLAG)
}

func (dp *DeployRequest) IsLddDependency() bool {
  return dp.flags.HasFlag(LDD_DEPENDENCY_FLAG)
}

type AppDeployer struct {
  waitGroup sync.WaitGroup
  processedLibs map[string]bool

  libsChannel chan *DeployRequest
  copyChannel chan *DeployRequest
  stripChannel chan string
  rpathChannel chan string
  qtChannel chan string

  qtDeployer *QtDeployer
  additionalLibPaths []string
  destinationRoot string
  targetExePath string
  destinationExePath string
  iconFilename string
}

func (ad *AppDeployer) DeployApp() {
  if err := ad.qtDeployer.queryQtEnv(); err != nil {
    log.Println(err)
  }

  ad.waitGroup.Add(1)
  go ad.processMainExe()

  go ad.processCopyTasks()
  go ad.processFixRPathTasks()
  go ad.processStripTasks()
  go ad.processQtLibTasks()

  blacklist := generateLibsBlacklist()

  log.Printf("Waiting for tasks processing to finish")
  ad.waitGroup.Wait()
  log.Printf("Tasks have been processed")

  close(ad.libsChannel)
  close(ad.copyChannel)
  close(ad.qtChannel)
  close(ad.rpathChannel)
  close(ad.stripChannel)

  var wg sync.WaitGroup
  wg.Add(1)
  go ad.deployQtTranslations(filepath.Join(ad.destinationRoot, "translations"), &wg)

  err := cleanupBlacklistedLibs(ad.LibsPath(), blacklist)
  if err != nil { log.Printf("Error while removing blacklisted libs: %v", err) }

  wg.Wait()
}

func (ad *AppDeployer) LibsPath() string {
  return filepath.Join(ad.destinationRoot, "lib")
}

func (ad *AppDeployer) addLibTask(sourceRoot, sourcePath, targetPath string, flags Bitmask) {
  ad.waitGroup.Add(1)
  go func() {
    ad.libsChannel <- &DeployRequest{
      sourceRoot: sourceRoot,
      sourcePath: sourcePath,
      targetPath: targetPath,
      flags: flags,
    }
  }()
}

func (ad *AppDeployer) addCopyTask(sourceRoot, sourcePath, targetPath string, flags Bitmask) {
  ad.waitGroup.Add(1)
  go func() {
    ad.copyChannel <- &DeployRequest{
      sourceRoot: sourceRoot,
      sourcePath: sourcePath,
      targetPath: targetPath,
      flags: flags,
    }
  }()
}

func (ad *AppDeployer) accountLibrary(libpath string) {
  log.Printf("Processed library %v", libpath)
  ad.processedLibs[libpath] = true
}

func (ad *AppDeployer) isLibraryDeployed(libpath string) bool {
  _, ok := ad.processedLibs[libpath]
  return ok
}

func (ad *AppDeployer) processMainExe() {
  defer ad.waitGroup.Done()

  go ad.copyMainExe()

  dependencies, err := ad.findLddDependencies(filepath.Base(ad.targetExePath), ad.targetExePath)
  if err != nil { log.Fatal(err) }

  for _, dependPath := range dependencies {
    if !ad.isLibraryDeployed(dependPath) {
      ad.addLibTask("", dependPath, "lib", LDD_AND_RPATH_FLAG)
    } else {
      log.Printf("Dependency seems to be processed: %v", dependPath)
    }
  }

  go ad.processLibTasks()

  log.Println("Main exe processing finished")
}

func (ad *AppDeployer) copyMainExe() {
  destinationPath := filepath.Join(ad.destinationRoot, filepath.Base(ad.targetExePath))
  ensureDirExists(destinationPath)

  err := copyFile(ad.targetExePath, destinationPath)
  if err != nil {
    log.Fatal("Error while copying main exe [%v] to [%v]: %v", ad.targetExePath, destinationPath, err)
  }

  ad.destinationExePath = destinationPath
  log.Printf("Destination path of main exe is %v", destinationPath)

  ad.addFixRPathTask(destinationPath)

  if generateAppImg() {
    ad.createAppLink()
  }

  ad.copyIcon()

  if *generateDesktopFlag {
    ad.generateDesktopFile()
  }
}

func (ad *AppDeployer) createAppLink() {
  appname := filepath.Base(ad.destinationExePath)
  symlinkPath := filepath.Join(ad.destinationRoot, "AppRun")
  err := os.Symlink(appname, symlinkPath)
  if err != nil {
    log.Printf("Error creating symlink: %v", err)
  }
}

func (ad *AppDeployer) copyIcon() {
  if len(*iconPathFlag) == 0 { return }

  if _, err := os.Stat(*iconPathFlag); err != nil {
    log.Printf("Cannot process icon %v: %v", *iconPathFlag, err)
    return
  }

  iconFilename := filepath.Base(*iconPathFlag)
  iconDestinationPath := filepath.Join(ad.destinationRoot, iconFilename)
  err := copyFile(*iconPathFlag, iconDestinationPath)
  if err != nil {
    log.Printf("Error while copying icon %v", err)
  }

  ad.iconFilename = iconFilename
}

func (ad *AppDeployer) generateDesktopFile() {
  exeFilename := filepath.Base(ad.destinationExePath)
  desktopFilepath := filepath.Join(ad.destinationRoot, fmt.Sprintf("%s.desktop", exeFilename))

  desktopFile, err := os.OpenFile(desktopFilepath, os.O_CREATE | os.O_RDWR | os.O_TRUNC, 0777)
  if err != nil { return }
	writer := bufio.NewWriter(desktopFile)
	defer desktopFile.Close()

	fmt.Fprintln(writer, "[Desktop Entry]")
  fmt.Fprintln(writer, "Type=Application")
  fmt.Fprintf(writer, "Name=%s\n", exeFilename)

  if generateAppImg() {
    fmt.Fprintln(writer, "Exec=./AppRun %F")
  } else {
    fmt.Fprintf(writer, "Exec=%s\n", exeFilename)
  }

  if len(ad.iconFilename) > 0 {
    fmt.Fprintf(writer, "Icon=%s\n", ad.iconFilename)
  }

  fmt.Fprintln(writer, "Terminal=false")
  fmt.Fprintln(writer, "StartupNotify=true")
  fmt.Fprintln(writer, "Encoding=UTF-8")

  writer.Flush()
}

func (ad *AppDeployer) addFixRPathTask(fullpath string) {
  ad.waitGroup.Add(1)
  go func() {
    ad.rpathChannel <- fullpath
  }()
}

func (ad *AppDeployer) addQtLibTask(fullpath string) {
  if !ad.qtDeployer.qtEnvironmentSet {
    log.Println("Qt environment is not set!")
    return
  }

  ad.waitGroup.Add(1)
  go func() {
    ad.qtChannel <- fullpath
  }()
}

// copies everything without inspection
func (ad *AppDeployer) copyRecursively(sourceRoot, sourcePath, targetPath string) error {
  // rescue agains premature finish of the main loop
  ad.waitGroup.Add(1)
  defer ad.waitGroup.Done()

  var emptyFlags Bitmask = 0

  rootpath := filepath.Join(sourceRoot, sourcePath)
  log.Printf("Copying recursively %v into %v", rootpath, targetPath)

  err := filepath.Walk(rootpath, func(path string, info os.FileInfo, err error) error {
    if err != nil {
      return err
    }

    if !info.Mode().IsRegular() {
      return nil
    }

    relativePath, err := filepath.Rel(sourceRoot, path)
    if err != nil {
      log.Println(err)
    }

    ad.addCopyTask(sourceRoot, relativePath, targetPath, emptyFlags)

    return nil
  })

  return err
}

// inspects libraries for dependencies and copies other files
func (ad *AppDeployer) deployRecursively(sourceRoot, sourcePath, targetPath string, flags Bitmask) error {
  // rescue agains premature finish of the main loop
  ad.waitGroup.Add(1)
  defer ad.waitGroup.Done()

  rootpath := filepath.Join(sourceRoot, sourcePath)
  log.Printf("Deploying recursively %v in %v to %v", sourcePath, sourceRoot, targetPath)

  onlyLibraries := flags.HasFlag(DEPLOY_ONLY_LIBRARIES_FLAG)
  var emptyFlags Bitmask = 0

  err := filepath.Walk(rootpath, func(path string, info os.FileInfo, err error) error {
    if err != nil {
      return err
    }

    if !info.Mode().IsRegular() {
      return nil
    }

    basename := filepath.Base(path)
    isLibrary := strings.HasPrefix(basename, "lib") && strings.Contains(basename, ".so")

    if !isLibrary && onlyLibraries {
      return nil
    }

    relativePath, err := filepath.Rel(sourceRoot, path)
    if err != nil {
      log.Println(err)
    }

    if isLibrary {
      ad.addLibTask(sourceRoot, relativePath, targetPath, flags | LDD_DEPENDENCY_FLAG)
    } else {
      ad.addCopyTask(sourceRoot, relativePath, targetPath, emptyFlags)
    }

    return nil
  })

  return err
}
<commit_msg>More logging for desktop file<commit_after>/*
 * This file is a part of linuxdeploy - tool for
 * creating standalone applications for Linux
 *
 * Copyright (C) 2017 Taras Kushnir <kushnirTV@gmail.com>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the MIT License.

 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 */

package main

import (
  "log"
  "os"
  "strings"
  "sync"
  "path/filepath"
  "fmt"
  "bufio"
)

const (
  LDD_DEPENDENCY_FLAG Bitmask = 1 << iota // check ldd deps for given item
  FIX_RPATH_FLAG // fix rpath for qt-related libs/plugins
  DEPLOY_ONLY_LIBRARIES_FLAG
)

const (
  LDD_AND_RPATH_FLAG = LDD_DEPENDENCY_FLAG | FIX_RPATH_FLAG
  LIBRARIES_AND_RPATH_FLAG = FIX_RPATH_FLAG | DEPLOY_ONLY_LIBRARIES_FLAG
)

type DeployRequest struct {
  sourcePath string // relative or absolute path of file to process
  sourceRoot string // if empty then sourcePath is absolute path
  targetPath string // target *relative* path
  flags Bitmask // deployment flags
}

func (dp *DeployRequest) FullPath() string {
  if len(dp.sourceRoot) == 0 {
    return dp.sourcePath
  } else {
    return filepath.Join(dp.sourceRoot, dp.sourcePath)
  }
}

func (dp *DeployRequest) Basename() string {
  return filepath.Base(dp.sourcePath)
}

func (dp *DeployRequest) SourceDir() string {
  return filepath.Dir(dp.sourcePath)
}

func (dp *DeployRequest) RequiresRPathFix() bool {
  return dp.flags.HasFlag(FIX_RPATH_FLAG)
}

func (dp *DeployRequest) IsLddDependency() bool {
  return dp.flags.HasFlag(LDD_DEPENDENCY_FLAG)
}

type AppDeployer struct {
  waitGroup sync.WaitGroup
  processedLibs map[string]bool

  libsChannel chan *DeployRequest
  copyChannel chan *DeployRequest
  stripChannel chan string
  rpathChannel chan string
  qtChannel chan string

  qtDeployer *QtDeployer
  additionalLibPaths []string
  destinationRoot string
  targetExePath string
  destinationExePath string
  iconFilename string
}

func (ad *AppDeployer) DeployApp() {
  if err := ad.qtDeployer.queryQtEnv(); err != nil {
    log.Println(err)
  }

  ad.waitGroup.Add(1)
  go ad.processMainExe()

  go ad.processCopyTasks()
  go ad.processFixRPathTasks()
  go ad.processStripTasks()
  go ad.processQtLibTasks()

  blacklist := generateLibsBlacklist()

  log.Printf("Waiting for tasks processing to finish")
  ad.waitGroup.Wait()
  log.Printf("Tasks have been processed")

  close(ad.libsChannel)
  close(ad.copyChannel)
  close(ad.qtChannel)
  close(ad.rpathChannel)
  close(ad.stripChannel)

  var wg sync.WaitGroup
  wg.Add(1)
  go ad.deployQtTranslations(filepath.Join(ad.destinationRoot, "translations"), &wg)

  err := cleanupBlacklistedLibs(ad.LibsPath(), blacklist)
  if err != nil { log.Printf("Error while removing blacklisted libs: %v", err) }

  wg.Wait()
}

func (ad *AppDeployer) LibsPath() string {
  return filepath.Join(ad.destinationRoot, "lib")
}

func (ad *AppDeployer) addLibTask(sourceRoot, sourcePath, targetPath string, flags Bitmask) {
  ad.waitGroup.Add(1)
  go func() {
    ad.libsChannel <- &DeployRequest{
      sourceRoot: sourceRoot,
      sourcePath: sourcePath,
      targetPath: targetPath,
      flags: flags,
    }
  }()
}

func (ad *AppDeployer) addCopyTask(sourceRoot, sourcePath, targetPath string, flags Bitmask) {
  ad.waitGroup.Add(1)
  go func() {
    ad.copyChannel <- &DeployRequest{
      sourceRoot: sourceRoot,
      sourcePath: sourcePath,
      targetPath: targetPath,
      flags: flags,
    }
  }()
}

func (ad *AppDeployer) accountLibrary(libpath string) {
  log.Printf("Processed library %v", libpath)
  ad.processedLibs[libpath] = true
}

func (ad *AppDeployer) isLibraryDeployed(libpath string) bool {
  _, ok := ad.processedLibs[libpath]
  return ok
}

func (ad *AppDeployer) processMainExe() {
  defer ad.waitGroup.Done()

  go ad.copyMainExe()

  dependencies, err := ad.findLddDependencies(filepath.Base(ad.targetExePath), ad.targetExePath)
  if err != nil { log.Fatal(err) }

  for _, dependPath := range dependencies {
    if !ad.isLibraryDeployed(dependPath) {
      ad.addLibTask("", dependPath, "lib", LDD_AND_RPATH_FLAG)
    } else {
      log.Printf("Dependency seems to be processed: %v", dependPath)
    }
  }

  go ad.processLibTasks()

  log.Println("Main exe processing finished")
}

func (ad *AppDeployer) copyMainExe() {
  destinationPath := filepath.Join(ad.destinationRoot, filepath.Base(ad.targetExePath))
  ensureDirExists(destinationPath)

  err := copyFile(ad.targetExePath, destinationPath)
  if err != nil {
    log.Fatal("Error while copying main exe [%v] to [%v]: %v", ad.targetExePath, destinationPath, err)
  }

  ad.destinationExePath = destinationPath
  log.Printf("Destination path of main exe is %v", destinationPath)

  ad.addFixRPathTask(destinationPath)

  if generateAppImg() {
    ad.createAppLink()
  }

  ad.copyIcon()

  if *generateDesktopFlag {
    ad.generateDesktopFile()
  }
}

func (ad *AppDeployer) createAppLink() {
  appname := filepath.Base(ad.destinationExePath)
  symlinkPath := filepath.Join(ad.destinationRoot, "AppRun")
  err := os.Symlink(appname, symlinkPath)
  if err != nil {
    log.Printf("Error creating symlink: %v", err)
  }
}

func (ad *AppDeployer) copyIcon() {
  if len(*iconPathFlag) == 0 { return }

  if _, err := os.Stat(*iconPathFlag); err != nil {
    log.Printf("Cannot process icon %v: %v", *iconPathFlag, err)
    return
  }

  iconFilename := filepath.Base(*iconPathFlag)
  iconDestinationPath := filepath.Join(ad.destinationRoot, iconFilename)
  err := copyFile(*iconPathFlag, iconDestinationPath)
  if err != nil {
    log.Printf("Error while copying icon %v", err)
  }

  ad.iconFilename = iconFilename
}

func (ad *AppDeployer) generateDesktopFile() {
  exeFilename := filepath.Base(ad.destinationExePath)
  desktopFilepath := filepath.Join(ad.destinationRoot, fmt.Sprintf("%s.desktop", exeFilename))

  desktopFile, err := os.OpenFile(desktopFilepath, os.O_CREATE | os.O_RDWR | os.O_TRUNC, 0777)
  if err != nil {
    log.Printf("Failed to create desktop file: %v", err)
    return
  }

	writer := bufio.NewWriter(desktopFile)
	defer desktopFile.Close()

	fmt.Fprintln(writer, "[Desktop Entry]")
  fmt.Fprintln(writer, "Type=Application")
  fmt.Fprintf(writer, "Name=%s\n", exeFilename)

  if generateAppImg() {
    fmt.Fprintln(writer, "Exec=./AppRun %F")
  } else {
    fmt.Fprintf(writer, "Exec=%s\n", exeFilename)
  }

  if len(ad.iconFilename) > 0 {
    fmt.Fprintf(writer, "Icon=%s\n", ad.iconFilename)
  }

  fmt.Fprintln(writer, "Terminal=false")
  fmt.Fprintln(writer, "StartupNotify=true")
  fmt.Fprintln(writer, "Encoding=UTF-8")

  writer.Flush()

  log.Println("Desktop file generated")
}

func (ad *AppDeployer) addFixRPathTask(fullpath string) {
  ad.waitGroup.Add(1)
  go func() {
    ad.rpathChannel <- fullpath
  }()
}

func (ad *AppDeployer) addQtLibTask(fullpath string) {
  if !ad.qtDeployer.qtEnvironmentSet {
    log.Println("Qt environment is not set!")
    return
  }

  ad.waitGroup.Add(1)
  go func() {
    ad.qtChannel <- fullpath
  }()
}

// copies everything without inspection
func (ad *AppDeployer) copyRecursively(sourceRoot, sourcePath, targetPath string) error {
  // rescue agains premature finish of the main loop
  ad.waitGroup.Add(1)
  defer ad.waitGroup.Done()

  var emptyFlags Bitmask = 0

  rootpath := filepath.Join(sourceRoot, sourcePath)
  log.Printf("Copying recursively %v into %v", rootpath, targetPath)

  err := filepath.Walk(rootpath, func(path string, info os.FileInfo, err error) error {
    if err != nil {
      return err
    }

    if !info.Mode().IsRegular() {
      return nil
    }

    relativePath, err := filepath.Rel(sourceRoot, path)
    if err != nil {
      log.Println(err)
    }

    ad.addCopyTask(sourceRoot, relativePath, targetPath, emptyFlags)

    return nil
  })

  return err
}

// inspects libraries for dependencies and copies other files
func (ad *AppDeployer) deployRecursively(sourceRoot, sourcePath, targetPath string, flags Bitmask) error {
  // rescue agains premature finish of the main loop
  ad.waitGroup.Add(1)
  defer ad.waitGroup.Done()

  rootpath := filepath.Join(sourceRoot, sourcePath)
  log.Printf("Deploying recursively %v in %v to %v", sourcePath, sourceRoot, targetPath)

  onlyLibraries := flags.HasFlag(DEPLOY_ONLY_LIBRARIES_FLAG)
  var emptyFlags Bitmask = 0

  err := filepath.Walk(rootpath, func(path string, info os.FileInfo, err error) error {
    if err != nil {
      return err
    }

    if !info.Mode().IsRegular() {
      return nil
    }

    basename := filepath.Base(path)
    isLibrary := strings.HasPrefix(basename, "lib") && strings.Contains(basename, ".so")

    if !isLibrary && onlyLibraries {
      return nil
    }

    relativePath, err := filepath.Rel(sourceRoot, path)
    if err != nil {
      log.Println(err)
    }

    if isLibrary {
      ad.addLibTask(sourceRoot, relativePath, targetPath, flags | LDD_DEPENDENCY_FLAG)
    } else {
      ad.addCopyTask(sourceRoot, relativePath, targetPath, emptyFlags)
    }

    return nil
  })

  return err
}
<|endoftext|>


<commit_before>package lidar

import (
	"bufio"
	"fmt"
	"log"
	"net"

	"github.com/tarm/serial"
)

var port *serial.Port
var reader *bufio.Reader

var client net.Conn

var clientConnected bool

type LidarData struct {
	Distance        uint16
	Invalid         bool
	StrengthWarning bool
	SignalStrength  uint16
}

type Packet struct {
	Start    uint8
	Index    uint8
	Speed    uint16
	Data     [4]LidarData
	Checksum uint16
}

func parsePacket(a [22]byte) Packet {
	var packet Packet

	// Start packet should always be 0xFA, TODO: check that
	packet.Start = a[0]

	// Index byte goes from 0xA0 (packet 0, reading 0 - 3) to 0xF9 (packet 89, readings 356 - 359)
	packet.Index = a[1] - 0xA0

	// Rotational speed in RPM, TODO: I think there's more precision available here than just dividing by 64
	// TODO: Add a PID control loop to maintain constant speed
	packet.Speed = ((uint16(a[3]) << 8) + uint16(a[2])) / 64

	// Parse the four data readings
	for i := 0; i < 4; i++ {
		b0 := uint32(a[4*i+4])
		b1 := uint32(a[4*i+5])
		b2 := uint32(a[4*i+6])
		b3 := uint32(a[4*i+7])

		// Distance
		packet.Data[i].Distance = uint16(((b1 & 0x00111111) << 8) + b0)

		// Invalid flag
		packet.Data[i].Invalid = ((b1 & 0x10000000) >> 7) == 1

		// Strength warning flag
		packet.Data[i].StrengthWarning = ((b1 & 0x01000000) >> 6) == 1

		// Signal strength
		packet.Data[i].SignalStrength = uint16((b3 << 8) + b2)
	}

	// TODO: add the checksum calculation

	return packet
}

func printPacket(p Packet) {
	fmt.Printf("%X %d %d\n", p.Start, p.Index, p.Speed)
	for i := 0; i < 4; i++ {
		fmt.Printf("%t %t %d %d\n", p.Data[i].Invalid, p.Data[i].StrengthWarning, p.Data[i].Distance, p.Data[i].SignalStrength)
	}
}

func printPacketCSV(p Packet) string {
	var r string
	for i := 0; i < 4; i++ {
		r += fmt.Sprintf("%d,%d,%d\n", int(p.Index)*4+i, p.Data[i].Distance, p.Data[i].SignalStrength)
	}
	return r
}

func SocketServer() {
	fmt.Println("Starting socket server...")
	ln, _ := net.Listen("tcp", ":9000")

	clientConnected = false

	for {
		client, _ = ln.Accept()
		fmt.Println("Client connected")
		clientConnected = true
	}

}

func Connect() {
	fmt.Println("Connecting to Lidar...")
	c := &serial.Config{Name: "/dev/ttyUSB1", Baud: 115200}
	var err error
	port, err = serial.OpenPort(c)

	if err != nil {
		log.Fatal(err)
	} else {
		reader = bufio.NewReader(port)
	}
}

// Disconnect from the serial port
func Disconnect() {
	fmt.Println("Disconnecting from Lidar...")
	port.Flush()
	port.Close()
	fmt.Println("Disconnected from Lidar")
}

func Read() {
	var r [22]byte
	var i int

	fmt.Println("Reading LIDAR data...")

	for {
		b, err := reader.ReadByte()

		if err != nil {
			log.Print(err)
		} else {

			if b == 0xFA {
				// clear byte array
				for j := 0; j < 22; j++ {
					r[j] = 0
				}
				i = 0

			}
			r[i] = b
			i++

			if i == 22 {
				// got there
				packet := parsePacket(r)

				if clientConnected {
					for i := 0; i < 4; i++ {
						client.Write([]byte(fmt.Sprintf("*%d,%d,%d!\n", int(packet.Index)*4+i, packet.Data[i].Distance, packet.Data[i].SignalStrength)))
					}
				}

				// clear byte array
				for j := 0; j < 22; j++ {
					r[j] = 0
				}
				i = 0
			}

		}
	}
}
<commit_msg>Added serial connection state and check to LIDAR<commit_after>package lidar

import (
	"bufio"
	"fmt"
	"log"
	"net"

	"github.com/tarm/serial"
)

var port *serial.Port
var reader *bufio.Reader

var serialConnected bool

var client net.Conn

var clientConnected bool

type LidarData struct {
	Distance        uint16
	Invalid         bool
	StrengthWarning bool
	SignalStrength  uint16
}

type Packet struct {
	Start    uint8
	Index    uint8
	Speed    uint16
	Data     [4]LidarData
	Checksum uint16
}

func parsePacket(a [22]byte) Packet {
	var packet Packet

	// Start packet should always be 0xFA, TODO: check that
	packet.Start = a[0]

	// Index byte goes from 0xA0 (packet 0, reading 0 - 3) to 0xF9 (packet 89, readings 356 - 359)
	packet.Index = a[1] - 0xA0

	// Rotational speed in RPM, TODO: I think there's more precision available here than just dividing by 64
	// TODO: Add a PID control loop to maintain constant speed
	packet.Speed = ((uint16(a[3]) << 8) + uint16(a[2])) / 64

	// Parse the four data readings
	for i := 0; i < 4; i++ {
		b0 := uint32(a[4*i+4])
		b1 := uint32(a[4*i+5])
		b2 := uint32(a[4*i+6])
		b3 := uint32(a[4*i+7])

		// Distance
		packet.Data[i].Distance = uint16(((b1 & 0x00111111) << 8) + b0)

		// Invalid flag
		packet.Data[i].Invalid = ((b1 & 0x10000000) >> 7) == 1

		// Strength warning flag
		packet.Data[i].StrengthWarning = ((b1 & 0x01000000) >> 6) == 1

		// Signal strength
		packet.Data[i].SignalStrength = uint16((b3 << 8) + b2)
	}

	// TODO: add the checksum calculation

	return packet
}

func printPacket(p Packet) {
	fmt.Printf("%X %d %d\n", p.Start, p.Index, p.Speed)
	for i := 0; i < 4; i++ {
		fmt.Printf("%t %t %d %d\n", p.Data[i].Invalid, p.Data[i].StrengthWarning, p.Data[i].Distance, p.Data[i].SignalStrength)
	}
}

func printPacketCSV(p Packet) string {
	var r string
	for i := 0; i < 4; i++ {
		r += fmt.Sprintf("%d,%d,%d\n", int(p.Index)*4+i, p.Data[i].Distance, p.Data[i].SignalStrength)
	}
	return r
}

func SocketServer() {
	fmt.Println("Starting socket server...")
	ln, _ := net.Listen("tcp", ":9000")

	clientConnected = false

	for {
		client, _ = ln.Accept()
		fmt.Println("Client connected")
		clientConnected = true
	}

}

func Connect() {
	fmt.Println("Connecting to Lidar...")
	c := &serial.Config{Name: "/dev/ttyUSB1", Baud: 115200}
	var err error
	port, err = serial.OpenPort(c)

	if err != nil {
		log.Fatal(err)
	} else {
		reader = bufio.NewReader(port)
		serialConnected = true
	}
}

// Disconnect from the serial port
func Disconnect() {
	fmt.Println("Disconnecting from Lidar...")
	port.Flush()
	port.Close()
	serialConnected = false
	fmt.Println("Disconnected from Lidar")
}

func Read() {
	var r [22]byte
	var i int

	fmt.Println("Reading LIDAR data...")

	for {
		if serialConnected {
			b, err := reader.ReadByte()

			if err != nil {
				log.Print(err)
			} else {

				if b == 0xFA {
					// clear byte array
					for j := 0; j < 22; j++ {
						r[j] = 0
					}
					i = 0

				}
				r[i] = b
				i++

				if i == 22 {
					// got there
					packet := parsePacket(r)

					if clientConnected {
						for i := 0; i < 4; i++ {
							client.Write([]byte(fmt.Sprintf("*%d,%d,%d!\n", int(packet.Index)*4+i, packet.Data[i].Distance, packet.Data[i].SignalStrength)))
						}
					}

					// clear byte array
					for j := 0; j < 22; j++ {
						r[j] = 0
					}
					i = 0
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015-2016, RadiantBlue Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package functions

import (
	"bytes"
	"encoding/json"
	"errors"
	"os/exec"
)

// InfoOptions defines options for the Info function.
type InfoOptions struct {
	// compute hexagonal boundary that contains all points
	Boundary bool `json:"boundary"`
	// dump metadata associated with the input file
	Metadata bool `json:"metadata"`
	// dump the schema of the internal point storage
	Schema bool `json:"schema"`
}

// NewInfoOptions constructs InfoOptions with default values.
func NewInfoOptions() *InfoOptions {
	return &InfoOptions{
		Boundary: false,
		Metadata: false,
		Schema:   false,
	}
}

// Info implements pdal info.
func Info(i, o string, options *json.RawMessage) ([]byte, error) {
	opts := NewInfoOptions()
	if options != nil {
		if err := json.Unmarshal(*options, &opts); err != nil {
			return nil, errors.New("Error with json.Unmarshal() " + err.Error())
		}
	}

	var args []string
	args = append(args, "info")
	args = append(args, i)
	if opts.Boundary {
		args = append(args, "--boundary")
	}
	if opts.Metadata {
		args = append(args, "--metadata")
	}
	if opts.Schema {
		args = append(args, "--schema")
	}

	out, err := exec.Command("pdal", args...).CombinedOutput()
	if err != nil {
		return nil, errors.New("Error with exec.Command() " + err.Error())
	}

	// Trim whitespace
	buffer := new(bytes.Buffer)
	if err := json.Compact(buffer, out); err != nil {
		return nil, errors.New("Error with json.Compact() " + err.Error())
	}

	return buffer.Bytes(), nil
}
<commit_msg>more log messages<commit_after>/*
Copyright 2015-2016, RadiantBlue Technologies, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package functions

import (
	"bytes"
	"encoding/json"
	"errors"
	"log"
	"os/exec"
)

// InfoOptions defines options for the Info function.
type InfoOptions struct {
	// compute hexagonal boundary that contains all points
	Boundary bool `json:"boundary"`
	// dump metadata associated with the input file
	Metadata bool `json:"metadata"`
	// dump the schema of the internal point storage
	Schema bool `json:"schema"`
}

// NewInfoOptions constructs InfoOptions with default values.
func NewInfoOptions() *InfoOptions {
	return &InfoOptions{
		Boundary: false,
		Metadata: false,
		Schema:   false,
	}
}

// Info implements pdal info.
func Info(i, o string, options *json.RawMessage) ([]byte, error) {
	opts := NewInfoOptions()
	if options != nil {
		if err := json.Unmarshal(*options, &opts); err != nil {
			return nil, errors.New("Error with json.Unmarshal() " + err.Error())
		}
	}
	log.Println("detected info options")

	var args []string
	args = append(args, "info")
	args = append(args, i)
	if opts.Boundary {
		args = append(args, "--boundary")
	}
	if opts.Metadata {
		args = append(args, "--metadata")
	}
	if opts.Schema {
		args = append(args, "--schema")
	}
	log.Println("set args " + args...)

	out, err := exec.Command("pdal", args...).CombinedOutput()
	if err != nil {
		return nil, errors.New("Error with exec.Command() " + err.Error())
	}
	log.Println("ran the command")

	// Trim whitespace
	buffer := new(bytes.Buffer)
	if err := json.Compact(buffer, out); err != nil {
		return nil, errors.New("Error with json.Compact() " + err.Error())
	}
	log.Println("compact buffer")

	return buffer.Bytes(), nil
}
<|endoftext|>


<commit_before>package cache

import (
	"sync"

	"github.com/lfq7413/tomato/types"
	"github.com/lfq7413/tomato/utils"
)

const mainSchema = "__MAIN_SCHEMA"
const schemaCachePrefix = "__SCHEMA"
const allKeys = "__ALL_KEYS"

// SchemaCache ...
type SchemaCache struct {
	ttl    int
	prefix string
	mu     sync.Mutex
}

// NewSchemaCache ...
func NewSchemaCache(ttl int) *SchemaCache {
	if adapter == nil {
		adapter = newInMemoryCacheAdapter(5)
	}
	return &SchemaCache{
		ttl:    ttl,
		prefix: schemaCachePrefix + utils.CreateToken(),
	}
}

// Put ...
func (s *SchemaCache) Put(key string, value interface{}) {
	var keys map[string]bool
	v := get(s.prefix + allKeys)
	if v == nil {
		keys = map[string]bool{}
	} else {
		if m, ok := v.(map[string]bool); ok {
			keys = m
		} else {
			keys = map[string]bool{}
		}
	}
	if _, ok := keys[key]; ok == false {
		s.mu.Lock()
		keys[key] = true
		s.mu.Unlock()
	}
	put(s.prefix+allKeys, keys, int64(s.ttl))
	put(key, value, int64(s.ttl))
}

// GetAllClasses ...
func (s *SchemaCache) GetAllClasses() []types.M {
	if s.ttl < 0 {
		return nil
	}
	v := get(s.prefix + mainSchema)
	if r, ok := v.([]types.M); ok {
		return r
	}
	return []types.M{}
}

// SetAllClasses ...
func (s *SchemaCache) SetAllClasses(schema []types.M) {
	if s.ttl < 0 {
		return
	}
	s.Put(s.prefix+mainSchema, schema)
}

// SetOneSchema ...
func (s *SchemaCache) SetOneSchema(className string, schema types.M) {
	if s.ttl < 0 {
		return
	}
	s.Put(s.prefix+className, schema)
}

// GetOneSchema ...
func (s *SchemaCache) GetOneSchema(className string) types.M {
	if s.ttl < 0 {
		return nil
	}
	v := get(s.prefix + className)
	return utils.M(v)
}

// Clear ...
func (s *SchemaCache) Clear() {
	var keys map[string]bool
	v := get(s.prefix + allKeys)
	if v == nil {
		return
	}
	if m, ok := v.(map[string]bool); ok {
		keys = m
	} else {
		return
	}

	for key := range keys {
		del(key)
	}
	del(s.prefix + allKeys)
}
<commit_msg>修改数据类型，支持 redis 返回的数据<commit_after>package cache

import (
	"sync"

	"github.com/lfq7413/tomato/types"
	"github.com/lfq7413/tomato/utils"
)

const mainSchema = "__MAIN_SCHEMA"
const schemaCachePrefix = "__SCHEMA"
const allKeys = "__ALL_KEYS"

// SchemaCache ...
type SchemaCache struct {
	ttl    int
	prefix string
	mu     sync.Mutex
}

// NewSchemaCache ...
func NewSchemaCache(ttl int) *SchemaCache {
	if adapter == nil {
		adapter = newInMemoryCacheAdapter(5)
	}
	return &SchemaCache{
		ttl:    ttl,
		prefix: schemaCachePrefix + utils.CreateToken(),
	}
}

// Put ...
func (s *SchemaCache) Put(key string, value interface{}) {
	s.mu.Lock()
	defer s.mu.Unlock()
	var keys map[string]interface{}
	v := get(s.prefix + allKeys)
	if v == nil {
		keys = map[string]interface{}{}
	} else {
		if m, ok := v.(map[string]interface{}); ok {
			keys = m
		} else {
			keys = map[string]interface{}{}
		}
	}
	if _, ok := keys[key]; ok == false {
		keys[key] = true
	}
	put(s.prefix+allKeys, keys, int64(s.ttl))
	put(key, value, int64(s.ttl))
}

// GetAllClasses ...
func (s *SchemaCache) GetAllClasses() []types.M {
	if s.ttl < 0 {
		return nil
	}
	v := get(s.prefix + mainSchema)
	if r, ok := v.([]types.M); ok {
		return r
	} else if r, ok := v.([]interface{}); ok {
		res := []types.M{}
		for _, m := range r {
			if subv := utils.M(m); subv != nil {
				res = append(res, subv)
			}
		}
		return res
	}
	return []types.M{}
}

// SetAllClasses ...
func (s *SchemaCache) SetAllClasses(schema []types.M) {
	if s.ttl < 0 {
		return
	}
	s.Put(s.prefix+mainSchema, schema)
}

// SetOneSchema ...
func (s *SchemaCache) SetOneSchema(className string, schema types.M) {
	if s.ttl < 0 {
		return
	}
	s.Put(s.prefix+className, schema)
}

// GetOneSchema ...
func (s *SchemaCache) GetOneSchema(className string) types.M {
	if s.ttl < 0 {
		return nil
	}
	v := get(s.prefix + className)
	return utils.M(v)
}

// Clear ...
func (s *SchemaCache) Clear() {
	var keys map[string]interface{}
	v := get(s.prefix + allKeys)
	if v == nil {
		return
	}
	if m, ok := v.(map[string]interface{}); ok {
		keys = m
	} else {
		return
	}

	for key := range keys {
		del(key)
	}
	del(s.prefix + allKeys)
}
<|endoftext|>