<commit_before>package kafka

import (
        "github.com/Shopify/sarama"
        "encoding/json"
        "reflect"
        "log"
)

type Producer struct {
  syncProducer sarama.SyncProducer
}

// Constructor
func NewProducer(brokers []string) *Producer {
  config := sarama.NewConfig()
  config.Producer.RequiredAcks = sarama.WaitForAll
  config.Producer.Retry.Max = 10

  syncProducer, err := sarama.NewSyncProducer(brokers, config)
  if err != nil {
    log.Fatalln("Failed to start Sarama producer:", err)
    panic(err)
  }

  return &Producer{ syncProducer : syncProducer }
}

func (this *Producer) SendEventToTopic( event interface{}, topic string ) error {

  // marshal event
  json, err := json.Marshal(event)

  if err != nil {
    return err
  }

  log.Println(string(json))

  // send event
  _, _, err = this.syncProducer.SendMessage(&sarama.ProducerMessage {
      Topic: topic,
      Value: sarama.StringEncoder(reflect.TypeOf(event).Name() + "," + string(json)),
    })

  if err != nil {
    return err
  }

  log.Println("event sent")
  return nil
}

func (this *Producer) Close() {
  if err := this.syncProducer.Close(); err != nil {
    log.Println("Failed to shut down kafka producer cleanly", err)
  }
}
<commit_msg>Remove useless log<commit_after>package kafka

import (
        "github.com/Shopify/sarama"
        "encoding/json"
        "reflect"
        "log"
)

type Producer struct {
  syncProducer sarama.SyncProducer
}

// Constructor
func NewProducer(brokers []string) *Producer {
  config := sarama.NewConfig()
  config.Producer.RequiredAcks = sarama.WaitForAll
  config.Producer.Retry.Max = 10

  syncProducer, err := sarama.NewSyncProducer(brokers, config)
  if err != nil {
    log.Fatalln("Failed to start Sarama producer:", err)
    panic(err)
  }

  return &Producer{ syncProducer : syncProducer }
}

func (this *Producer) SendEventToTopic( event interface{}, topic string ) error {

  // marshal event
  json, err := json.Marshal(event)

  if err != nil {
    return err
  }

  // send event
  _, _, err = this.syncProducer.SendMessage(&sarama.ProducerMessage {
      Topic: topic,
      Value: sarama.StringEncoder(reflect.TypeOf(event).Name() + "," + string(json)),
    })

  if err != nil {
    return err
  }

  log.Println("event sent")
  return nil
}

func (this *Producer) Close() {
  if err := this.syncProducer.Close(); err != nil {
    log.Println("Failed to shut down kafka producer cleanly", err)
  }
}
<|endoftext|>


<commit_before>package clicommand

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/buildkite/agent/v3/api"
	"github.com/buildkite/agent/v3/cliconfig"
	"github.com/buildkite/roko"
	"github.com/urfave/cli"
)

type OidcTokenConfig struct {
	Audience string `cli:"audience"`
	Job      string `cli:"job"      validate:"required"`

	// Global flags
	Debug       bool     `cli:"debug"`
	LogLevel    string   `cli:"log-level"`
	NoColor     bool     `cli:"no-color"`
	Experiments []string `cli:"experiment" normalize:"list"`
	Profile     string   `cli:"profile"`

	// API config
	DebugHTTP        bool   `cli:"debug-http"`
	AgentAccessToken string `cli:"agent-access-token" validate:"required"`
	Endpoint         string `cli:"endpoint"           validate:"required"`
	NoHTTP2          bool   `cli:"no-http2"`
}

const (
	oidcTokenDescription = `Usage:

   buildkite-agent oidc token [options...]

Description:
   Requests and prints an OIDC token from Buildkite that claims the Job ID and
	 the specified audience. If no audience is specified, the endpoint's default
	 audience will be claimed.

Example:
   $ buildkite-agent oidc token --audience sts.amazonaws.com

   Prints the environment passed into the process
`
	backoffSeconds = 2
	maxAttempts    = 5
)

var OidcTokenCommand = cli.Command{
	Name:        "token",
	Usage:       "Requests and prints an OIDC token from Buildkite with the specified audience,",
	Description: oidcTokenDescription,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "audience",
			Value: "",
			Usage: "The audience that will consume the OIDC token",
		},
		cli.StringFlag{
			Name:   "job",
			Value:  "",
			Usage:  "Buildkite Job Id to claim in the OIDC token",
			EnvVar: "BUILDKITE_JOB_ID",
		},

		// API Flags
		AgentAccessTokenFlag,
		EndpointFlag,
		NoHTTP2Flag,
		DebugHTTPFlag,

		// Global flags
		NoColorFlag,
		DebugFlag,
		LogLevelFlag,
		ExperimentsFlag,
		ProfileFlag,
	},
	Action: func(c *cli.Context) error {
		// The configuration will be loaded into this struct
		cfg := OidcTokenConfig{}

		loader := cliconfig.Loader{CLI: c, Config: &cfg}
		warnings, err := loader.Load()
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(1)
		}

		l := CreateLogger(&cfg)

		// Now that we have a logger, log out the warnings that loading config generated
		for _, warning := range warnings {
			l.Warn("%s", warning)
		}

		// Setup any global configuration options
		done := HandleGlobalFlags(l, cfg)
		defer done()

		// Create the API client
		client := api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken"))

		// Find the meta data value
		var token *api.OidcToken
		var resp *api.Response

		if err := roko.NewRetrier(
			roko.WithMaxAttempts(maxAttempts),
			roko.WithStrategy(roko.Exponential(backoffSeconds*time.Second, 0)),
		).Do(func(r *roko.Retrier) error {
			var audience []string
			if len(cfg.Audience) > 0 {
				audience = []string{cfg.Audience}
			}

			token, resp, err = client.OidcToken(cfg.Job, audience...)
			if resp != nil {
				fmt.Println(resp.StatusCode, r.ShouldGiveUp())

				switch resp.StatusCode {
				// Don't bother retrying if the response was one of these statuses
				case http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden, http.StatusUnprocessableEntity:
					r.Break()
					return err
				}
			}

			if err != nil {
				l.Warn("%s (%s)", err, r)
			}

			return err
		}); err != nil {
			if len(cfg.Audience) > 0 {
				l.Error("Could not obtain OIDC token for audience %s", cfg.Audience)
			} else {
				l.Error("Could not obtain OIDC token for default audience")
			}
			return err
		}

		fmt.Println(token.Token)
		return nil
	},
}
<commit_msg>Remove a debug print<commit_after>package clicommand

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/buildkite/agent/v3/api"
	"github.com/buildkite/agent/v3/cliconfig"
	"github.com/buildkite/roko"
	"github.com/urfave/cli"
)

type OidcTokenConfig struct {
	Audience string `cli:"audience"`
	Job      string `cli:"job"      validate:"required"`

	// Global flags
	Debug       bool     `cli:"debug"`
	LogLevel    string   `cli:"log-level"`
	NoColor     bool     `cli:"no-color"`
	Experiments []string `cli:"experiment" normalize:"list"`
	Profile     string   `cli:"profile"`

	// API config
	DebugHTTP        bool   `cli:"debug-http"`
	AgentAccessToken string `cli:"agent-access-token" validate:"required"`
	Endpoint         string `cli:"endpoint"           validate:"required"`
	NoHTTP2          bool   `cli:"no-http2"`
}

const (
	oidcTokenDescription = `Usage:

   buildkite-agent oidc token [options...]

Description:
   Requests and prints an OIDC token from Buildkite that claims the Job ID and
	 the specified audience. If no audience is specified, the endpoint's default
	 audience will be claimed.

Example:
   $ buildkite-agent oidc token --audience sts.amazonaws.com

   Prints the environment passed into the process
`
	backoffSeconds = 2
	maxAttempts    = 5
)

var OidcTokenCommand = cli.Command{
	Name:        "token",
	Usage:       "Requests and prints an OIDC token from Buildkite with the specified audience,",
	Description: oidcTokenDescription,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "audience",
			Value: "",
			Usage: "The audience that will consume the OIDC token",
		},
		cli.StringFlag{
			Name:   "job",
			Value:  "",
			Usage:  "Buildkite Job Id to claim in the OIDC token",
			EnvVar: "BUILDKITE_JOB_ID",
		},

		// API Flags
		AgentAccessTokenFlag,
		EndpointFlag,
		NoHTTP2Flag,
		DebugHTTPFlag,

		// Global flags
		NoColorFlag,
		DebugFlag,
		LogLevelFlag,
		ExperimentsFlag,
		ProfileFlag,
	},
	Action: func(c *cli.Context) error {
		// The configuration will be loaded into this struct
		cfg := OidcTokenConfig{}

		loader := cliconfig.Loader{CLI: c, Config: &cfg}
		warnings, err := loader.Load()
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s\n", err)
			os.Exit(1)
		}

		l := CreateLogger(&cfg)

		// Now that we have a logger, log out the warnings that loading config generated
		for _, warning := range warnings {
			l.Warn("%s", warning)
		}

		// Setup any global configuration options
		done := HandleGlobalFlags(l, cfg)
		defer done()

		// Create the API client
		client := api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken"))

		// Find the meta data value
		var token *api.OidcToken
		var resp *api.Response

		if err := roko.NewRetrier(
			roko.WithMaxAttempts(maxAttempts),
			roko.WithStrategy(roko.Exponential(backoffSeconds*time.Second, 0)),
		).Do(func(r *roko.Retrier) error {
			var audience []string
			if len(cfg.Audience) > 0 {
				audience = []string{cfg.Audience}
			}

			token, resp, err = client.OidcToken(cfg.Job, audience...)
			if resp != nil {
				switch resp.StatusCode {
				// Don't bother retrying if the response was one of these statuses
				case http.StatusBadRequest, http.StatusUnauthorized, http.StatusForbidden, http.StatusUnprocessableEntity:
					r.Break()
					return err
				}
			}

			if err != nil {
				l.Warn("%s (%s)", err, r)
			}

			return err
		}); err != nil {
			if len(cfg.Audience) > 0 {
				l.Error("Could not obtain OIDC token for audience %s", cfg.Audience)
			} else {
				l.Error("Could not obtain OIDC token for default audience")
			}
			return err
		}

		fmt.Println(token.Token)
		return nil
	},
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package bigqueryio provides transformations and utilities to interact with
// Google BigQuery. See also: https://cloud.google.com/bigquery/docs.
package bigqueryio

import (
	"context"
	"fmt"
	"net/http"
	"reflect"
	"strings"
	"time"

	"cloud.google.com/go/bigquery"
	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
	bq "google.golang.org/api/bigquery/v2"
	"google.golang.org/api/googleapi"
	"google.golang.org/api/iterator"
)

// writeSizeLimit is the maximum number of rows allowed by BQ in a write.
const writeRowLimit = 10000
// writeSizeLimit is the maximum number of bytes allowed in BQ write.
const writeSizeLimit = 10485760

func init() {
	beam.RegisterType(reflect.TypeOf((*queryFn)(nil)).Elem())
	beam.RegisterType(reflect.TypeOf((*writeFn)(nil)).Elem())
}

// QualifiedTableName is a fully qualified name of a bigquery table.
type QualifiedTableName struct {
	// Project is the Google Cloud project ID.
	Project string `json:"project"`
	// Dataset is the dataset ID within the project.
	Dataset string `json:"dataset"`
	// Table is the table ID within the dataset.
	Table string `json:"table"`
}

// String formats the qualified name as "<project>:<dataset>.<table>".
func (qn QualifiedTableName) String() string {
	return fmt.Sprintf("%v:%v.%v", qn.Project, qn.Dataset, qn.Table)
}

// NewQualifiedTableName parses "<project>:<dataset>.<table>" into a QualifiedTableName.
func NewQualifiedTableName(s string) (QualifiedTableName, error) {
	c := strings.LastIndex(s, ":")
	d := strings.LastIndex(s, ".")
	if c == -1 || d == -1 || d < c {
		return QualifiedTableName{}, fmt.Errorf("table name missing components: %v", s)
	}

	project := s[:c]
	dataset := s[c+1 : d]
	table := s[d+1:]
	if strings.TrimSpace(project) == "" || strings.TrimSpace(dataset) == "" || strings.TrimSpace(table) == "" {
		return QualifiedTableName{}, fmt.Errorf("table name has empty components: %v", s)
	}
	return QualifiedTableName{Project: project, Dataset: dataset, Table: table}, nil
}

// Read reads all rows from the given table. The table must have a schema
// compatible with the given type, t, and Read returns a PCollection<t>. If the
// table has more rows than t, then Read is implicitly a projection.
func Read(s beam.Scope, project, table string, t reflect.Type) beam.PCollection {
	mustParseTable(table)

	s = s.Scope("bigquery.Read")

	// TODO(herohde) 7/13/2017: using * is probably too inefficient. We could infer
	// a focused query from the type.
	return query(s, project, fmt.Sprintf("SELECT * from [%v]", table), t)
}

// Query executes a query. The output must have a schema compatible with the given
// type, t. It returns a PCollection<t>.
func Query(s beam.Scope, project, q string, t reflect.Type) beam.PCollection {
	s = s.Scope("bigquery.Query")
	return query(s, project, q, t)
}

func query(s beam.Scope, project, query string, t reflect.Type) beam.PCollection {
	mustInferSchema(t)

	imp := beam.Impulse(s)
	return beam.ParDo(s, &queryFn{Project: project, Query: query, Type: beam.EncodedType{T: t}}, imp, beam.TypeDefinition{Var: beam.XType, T: t})
}

type queryFn struct {
	// Project is the project
	Project string `json:"project"`
	// Table is the table identifier.
	Query string `json:"query"`
	// Type is the encoded schema type.
	Type beam.EncodedType `json:"type"`
}

func (f *queryFn) ProcessElement(ctx context.Context, _ []byte, emit func(beam.X)) error {
	client, err := bigquery.NewClient(ctx, f.Project)
	if err != nil {
		return err
	}
	defer client.Close()

	q := client.Query(f.Query)
	q.UseLegacySQL = true

	it, err := q.Read(ctx)
	if err != nil {
		return err
	}

	for {
		val := reflect.New(f.Type.T).Interface() // val : *T
		if err := it.Next(val); err != nil {
			if err == iterator.Done {
				break
			}
			return err
		}

		emit(reflect.ValueOf(val).Elem().Interface()) // emit(*val)
	}
	return nil
}

func mustInferSchema(t reflect.Type) bigquery.Schema {
	if t.Kind() != reflect.Struct {
		panic(fmt.Sprintf("schema type must be struct: %v", t))
	}
	schema, err := bigquery.InferSchema(reflect.Zero(t).Interface())
	if err != nil {
		panic(fmt.Sprintf("invalid schema type: %v", err))
	}
	return schema
}

func mustParseTable(table string) QualifiedTableName {
	qn, err := NewQualifiedTableName(table)
	if err != nil {
		panic(err)
	}
	return qn
}

// TODO(herohde) 7/14/2017: allow CreateDispositions and WriteDispositions. The default
// is not quite what the Dataflow examples do.

// Write writes the elements of the given PCollection<T> to bigquery. T is required
// to be the schema type.
func Write(s beam.Scope, project, table string, col beam.PCollection) {
	t := col.Type().Type()
	mustInferSchema(t)
	qn := mustParseTable(table)

	s = s.Scope("bigquery.Write")

	// TODO(BEAM-3860) 3/15/2018: use side input instead of GBK.

	pre := beam.AddFixedKey(s, col)
	post := beam.GroupByKey(s, pre)
	beam.ParDo0(s, &writeFn{Project: project, Table: qn, Type: beam.EncodedType{T: t}}, post)
}

type writeFn struct {
	// Project is the project
	Project string `json:"project"`
	// Table is the qualified table identifier.
	Table QualifiedTableName `json:"table"`
	// Type is the encoded schema type.
	Type beam.EncodedType `json:"type"`
}

// Approximate the size of an element as it would appear in a BQ insert request.
func getInsertSize(v interface{}, schema bigquery.Schema) (int, error) {
	saver := bigquery.StructSaver{
		InsertID: strings.Repeat("0", 27),
		Struct:   v,
		Schema:   schema,
	}
	row, id, err := saver.Save()
	if err != nil {
		return 0, err
	}
	m := make(map[string]bq.JsonValue)
	for k, v := range row {
		m[k] = bq.JsonValue(v)
	}
	req := bq.TableDataInsertAllRequestRows{
		InsertId: id,
		Json: m,
	}
	data, err := req.MarshalJSON()
	if (err != nil) {
		return 0, err
	}
	// Add 1 for comma separator between elements.
	return len(data) + 1, err
}

func (f *writeFn) ProcessElement(ctx context.Context, _ int, iter func(*beam.X) bool) error {
	client, err := bigquery.NewClient(ctx, f.Project)
	if err != nil {
		return err
	}
	defer client.Close()

	// TODO(herohde) 7/14/2017: should we create datasets? For now, "no".

	dataset := client.DatasetInProject(f.Table.Project, f.Table.Dataset)
	if _, err := dataset.Metadata(ctx); err != nil {
		return err
	}

	schema := mustInferSchema(f.Type.T)
	table := dataset.Table(f.Table.Table)
	if _, err := table.Metadata(ctx); err != nil {
		if !isNotFound(err) {
			return err
		}
		if err := table.Create(ctx, &bigquery.TableMetadata{Schema: schema}); err != nil {
			return err
		}
	}

	var data []reflect.Value
	// This stores the running byte size estimate of a BQ request. Initializing to 1KB
	// as an estimate for overall message overhead.
	var size = 1024
	var val beam.X
	for iter(&val) {
		current, err := getInsertSize(val.(interface{}), schema)
		if err != nil {
			return fmt.Errorf("biquery write error: %v", err)
		}
		if len(data) + 1 > writeRowLimit || size + current > writeSizeLimit {
			// Write rows in batches to comply with BQ limits.
			if err := put(ctx, table, f.Type.T, data); err != nil {
				return fmt.Errorf("bigquery write error [len=%d, size=%d]: %v", len(data), size, err)
			}
			data = nil
			size = 1024
		} else {
			data = append(data, reflect.ValueOf(val.(interface{})))
			size += current
		}
	}
	if len(data) == 0 {
		return nil
	}
	if err := put(ctx, table, f.Type.T, data); err != nil {
		return fmt.Errorf("bigquery write error [len=%d, size=%d]: %v", len(data), size, err)
	}
	return nil
}

func put(ctx context.Context, table *bigquery.Table, t reflect.Type, data []reflect.Value) error {
	// list : []T to allow Put to infer the schema
	list := reflectx.MakeSlice(t, data...).Interface()

	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
	defer cancel()

	return table.Uploader().Put(ctx, list)
}

func isNotFound(err error) bool {
	e, ok := err.(*googleapi.Error)
	return ok && e.Code == http.StatusNotFound
}
<commit_msg>Setting the write overhead to a constant.<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package bigqueryio provides transformations and utilities to interact with
// Google BigQuery. See also: https://cloud.google.com/bigquery/docs.
package bigqueryio

import (
	"context"
	"fmt"
	"net/http"
	"reflect"
	"strings"
	"time"

	"cloud.google.com/go/bigquery"
	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
	bq "google.golang.org/api/bigquery/v2"
	"google.golang.org/api/googleapi"
	"google.golang.org/api/iterator"
)

// writeSizeLimit is the maximum number of rows allowed by BQ in a write.
const writeRowLimit = 10000
// writeSizeLimit is the maximum number of bytes allowed in BQ write.
const writeSizeLimit = 10485760
// Estimate for overall message overhead.for a write message in bytes.
const writeOverheadBytes = 1024

func init() {
	beam.RegisterType(reflect.TypeOf((*queryFn)(nil)).Elem())
	beam.RegisterType(reflect.TypeOf((*writeFn)(nil)).Elem())
}

// QualifiedTableName is a fully qualified name of a bigquery table.
type QualifiedTableName struct {
	// Project is the Google Cloud project ID.
	Project string `json:"project"`
	// Dataset is the dataset ID within the project.
	Dataset string `json:"dataset"`
	// Table is the table ID within the dataset.
	Table string `json:"table"`
}

// String formats the qualified name as "<project>:<dataset>.<table>".
func (qn QualifiedTableName) String() string {
	return fmt.Sprintf("%v:%v.%v", qn.Project, qn.Dataset, qn.Table)
}

// NewQualifiedTableName parses "<project>:<dataset>.<table>" into a QualifiedTableName.
func NewQualifiedTableName(s string) (QualifiedTableName, error) {
	c := strings.LastIndex(s, ":")
	d := strings.LastIndex(s, ".")
	if c == -1 || d == -1 || d < c {
		return QualifiedTableName{}, fmt.Errorf("table name missing components: %v", s)
	}

	project := s[:c]
	dataset := s[c+1 : d]
	table := s[d+1:]
	if strings.TrimSpace(project) == "" || strings.TrimSpace(dataset) == "" || strings.TrimSpace(table) == "" {
		return QualifiedTableName{}, fmt.Errorf("table name has empty components: %v", s)
	}
	return QualifiedTableName{Project: project, Dataset: dataset, Table: table}, nil
}

// Read reads all rows from the given table. The table must have a schema
// compatible with the given type, t, and Read returns a PCollection<t>. If the
// table has more rows than t, then Read is implicitly a projection.
func Read(s beam.Scope, project, table string, t reflect.Type) beam.PCollection {
	mustParseTable(table)

	s = s.Scope("bigquery.Read")

	// TODO(herohde) 7/13/2017: using * is probably too inefficient. We could infer
	// a focused query from the type.
	return query(s, project, fmt.Sprintf("SELECT * from [%v]", table), t)
}

// Query executes a query. The output must have a schema compatible with the given
// type, t. It returns a PCollection<t>.
func Query(s beam.Scope, project, q string, t reflect.Type) beam.PCollection {
	s = s.Scope("bigquery.Query")
	return query(s, project, q, t)
}

func query(s beam.Scope, project, query string, t reflect.Type) beam.PCollection {
	mustInferSchema(t)

	imp := beam.Impulse(s)
	return beam.ParDo(s, &queryFn{Project: project, Query: query, Type: beam.EncodedType{T: t}}, imp, beam.TypeDefinition{Var: beam.XType, T: t})
}

type queryFn struct {
	// Project is the project
	Project string `json:"project"`
	// Table is the table identifier.
	Query string `json:"query"`
	// Type is the encoded schema type.
	Type beam.EncodedType `json:"type"`
}

func (f *queryFn) ProcessElement(ctx context.Context, _ []byte, emit func(beam.X)) error {
	client, err := bigquery.NewClient(ctx, f.Project)
	if err != nil {
		return err
	}
	defer client.Close()

	q := client.Query(f.Query)
	q.UseLegacySQL = true

	it, err := q.Read(ctx)
	if err != nil {
		return err
	}

	for {
		val := reflect.New(f.Type.T).Interface() // val : *T
		if err := it.Next(val); err != nil {
			if err == iterator.Done {
				break
			}
			return err
		}

		emit(reflect.ValueOf(val).Elem().Interface()) // emit(*val)
	}
	return nil
}

func mustInferSchema(t reflect.Type) bigquery.Schema {
	if t.Kind() != reflect.Struct {
		panic(fmt.Sprintf("schema type must be struct: %v", t))
	}
	schema, err := bigquery.InferSchema(reflect.Zero(t).Interface())
	if err != nil {
		panic(fmt.Sprintf("invalid schema type: %v", err))
	}
	return schema
}

func mustParseTable(table string) QualifiedTableName {
	qn, err := NewQualifiedTableName(table)
	if err != nil {
		panic(err)
	}
	return qn
}

// TODO(herohde) 7/14/2017: allow CreateDispositions and WriteDispositions. The default
// is not quite what the Dataflow examples do.

// Write writes the elements of the given PCollection<T> to bigquery. T is required
// to be the schema type.
func Write(s beam.Scope, project, table string, col beam.PCollection) {
	t := col.Type().Type()
	mustInferSchema(t)
	qn := mustParseTable(table)

	s = s.Scope("bigquery.Write")

	// TODO(BEAM-3860) 3/15/2018: use side input instead of GBK.

	pre := beam.AddFixedKey(s, col)
	post := beam.GroupByKey(s, pre)
	beam.ParDo0(s, &writeFn{Project: project, Table: qn, Type: beam.EncodedType{T: t}}, post)
}

type writeFn struct {
	// Project is the project
	Project string `json:"project"`
	// Table is the qualified table identifier.
	Table QualifiedTableName `json:"table"`
	// Type is the encoded schema type.
	Type beam.EncodedType `json:"type"`
}

// Approximate the size of an element as it would appear in a BQ insert request.
func getInsertSize(v interface{}, schema bigquery.Schema) (int, error) {
	saver := bigquery.StructSaver{
		InsertID: strings.Repeat("0", 27),
		Struct:   v,
		Schema:   schema,
	}
	row, id, err := saver.Save()
	if err != nil {
		return 0, err
	}
	m := make(map[string]bq.JsonValue)
	for k, v := range row {
		m[k] = bq.JsonValue(v)
	}
	req := bq.TableDataInsertAllRequestRows{
		InsertId: id,
		Json: m,
	}
	data, err := req.MarshalJSON()
	if (err != nil) {
		return 0, err
	}
	// Add 1 for comma separator between elements.
	return len(data) + 1, err
}

func (f *writeFn) ProcessElement(ctx context.Context, _ int, iter func(*beam.X) bool) error {
	client, err := bigquery.NewClient(ctx, f.Project)
	if err != nil {
		return err
	}
	defer client.Close()

	// TODO(herohde) 7/14/2017: should we create datasets? For now, "no".

	dataset := client.DatasetInProject(f.Table.Project, f.Table.Dataset)
	if _, err := dataset.Metadata(ctx); err != nil {
		return err
	}

	schema := mustInferSchema(f.Type.T)
	table := dataset.Table(f.Table.Table)
	if _, err := table.Metadata(ctx); err != nil {
		if !isNotFound(err) {
			return err
		}
		if err := table.Create(ctx, &bigquery.TableMetadata{Schema: schema}); err != nil {
			return err
		}
	}

	var data []reflect.Value
	// This stores the running byte size estimate of a BQ request.
	size := writeOverheadBytes

	var val beam.X
	for iter(&val) {
		current, err := getInsertSize(val.(interface{}), schema)
		if err != nil {
			return fmt.Errorf("biquery write error: %v", err)
		}
		if len(data) + 1 > writeRowLimit || size + current > writeSizeLimit {
			// Write rows in batches to comply with BQ limits.
			if err := put(ctx, table, f.Type.T, data); err != nil {
				return fmt.Errorf("bigquery write error [len=%d, size=%d]: %v", len(data), size, err)
			}
			data = nil
			size = writeOverheadBytes
		} else {
			data = append(data, reflect.ValueOf(val.(interface{})))
			size += current
		}
	}
	if len(data) == 0 {
		return nil
	}
	if err := put(ctx, table, f.Type.T, data); err != nil {
		return fmt.Errorf("bigquery write error [len=%d, size=%d]: %v", len(data), size, err)
	}
	return nil
}

func put(ctx context.Context, table *bigquery.Table, t reflect.Type, data []reflect.Value) error {
	// list : []T to allow Put to infer the schema
	list := reflectx.MakeSlice(t, data...).Interface()

	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
	defer cancel()

	return table.Uploader().Put(ctx, list)
}

func isNotFound(err error) bool {
	e, ok := err.(*googleapi.Error)
	return ok && e.Code == http.StatusNotFound
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package tests_test

import (
	"flag"
	"fmt"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"

	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("User Access", func() {

	flag.Parse()

	BeforeEach(func() {
		tests.BeforeTestCleanup()
	})

	Describe("With default kubevirt service accounts", func() {
		table.DescribeTable("should verify permissions are correct for view, edit, and admin", func(resource string) {
			tests.SkipIfNoCmd("kubectl")

			view := tests.ViewServiceAccountName
			edit := tests.EditServiceAccountName
			admin := tests.AdminServiceAccountName

			viewVerbs := make(map[string]string)
			editVerbs := make(map[string]string)
			adminVerbs := make(map[string]string)

			// GET
			viewVerbs["get"] = "yes"
			editVerbs["get"] = "yes"
			adminVerbs["get"] = "yes"

			// List
			viewVerbs["list"] = "yes"
			editVerbs["list"] = "yes"
			adminVerbs["list"] = "yes"

			// WATCH
			viewVerbs["watch"] = "yes"
			editVerbs["watch"] = "yes"
			adminVerbs["watch"] = "yes"

			// DELETE
			viewVerbs["delete"] = "no"
			editVerbs["delete"] = "yes"
			adminVerbs["delete"] = "yes"

			// CREATE
			viewVerbs["create"] = "no"
			editVerbs["create"] = "yes"
			adminVerbs["create"] = "yes"

			// UPDATE
			viewVerbs["update"] = "no"
			editVerbs["update"] = "yes"
			adminVerbs["update"] = "yes"

			// PATCH
			viewVerbs["patch"] = "no"
			editVerbs["patch"] = "yes"
			adminVerbs["patch"] = "yes"

			// DELETE COllECTION
			viewVerbs["deleteCollection"] = "no"
			editVerbs["deleteCollection"] = "no"
			adminVerbs["deleteCollection"] = "yes"

			namespace := tests.NamespaceTestDefault
			verbs := []string{"get", "list", "watch", "delete", "create", "update", "patch", "deletecollection"}

			for _, verb := range verbs {
				// VIEW
				By(fmt.Sprintf("verifying VIEW sa for verb %s", verb))
				expectedRes, _ := viewVerbs[verb]
				as := fmt.Sprintf("system:serviceaccount:%s:%s", namespace, view)
				result, _, _ := tests.RunCommand("kubectl", "auth", "can-i", "--as", as, verb, resource)
				Expect(result).To(ContainSubstring(expectedRes))

				// EDIT
				By(fmt.Sprintf("verifying EDIT sa for verb %s", verb))
				expectedRes, _ = editVerbs[verb]
				as = fmt.Sprintf("system:serviceaccount:%s:%s", namespace, edit)
				result, _, _ = tests.RunCommand("kubectl", "auth", "can-i", "--as", as, verb, resource)
				Expect(result).To(ContainSubstring(expectedRes))

				// ADMIN
				By(fmt.Sprintf("verifying ADMIN sa for verb %s", verb))
				expectedRes, _ = adminVerbs[verb]
				as = fmt.Sprintf("system:serviceaccount:%s:%s", namespace, admin)
				result, _, _ = tests.RunCommand("kubectl", "auth", "can-i", "--as", as, verb, resource)
				Expect(result).To(ContainSubstring(expectedRes))

				// DEFAULT - the default should always return 'no' for ever verb.
				// This is primarily a sanity check.
				By(fmt.Sprintf("verifying DEFAULT sa for verb %s", verb))
				expectedRes = "no"
				as = fmt.Sprintf("system:serviceaccount:%s:default", namespace)
				result, _, _ = tests.RunCommand("kubectl", "auth", "can-i", "--as", as, verb, resource)
				Expect(result).To(ContainSubstring(expectedRes))
			}
		},
			table.Entry("given a vmi", "virtualmachineinstances"),
			table.Entry("given an vm", "virtualmachines"),
			table.Entry("given a vmi preset", "virtualmachineinstancepresets"),
			table.Entry("given a vmi replica set", "virtualmachineinstancereplicasets"),
		)
	})
})
<commit_msg>What this PR does / why we need it: Changed all test levels in api_validation_test to focus (FDescribe, FContext, FIt) added rfe id, test id, criticality etc<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package tests_test

import (
	"flag"
	"fmt"

	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"

	"kubevirt.io/kubevirt/tests"
)

var _ = Describe("[rfe_id:500, crit:high, vendor:cnv-qe@redhat.com, level:component]User Access", func() {

	flag.Parse()

	BeforeEach(func() {
		tests.BeforeTestCleanup()
	})

	Describe("With default kubevirt service accounts", func() {
		table.DescribeTable("should verify permissions are correct for view, edit, and admin", func(resource string) {
			tests.SkipIfNoCmd("kubectl")

			view := tests.ViewServiceAccountName
			edit := tests.EditServiceAccountName
			admin := tests.AdminServiceAccountName

			viewVerbs := make(map[string]string)
			editVerbs := make(map[string]string)
			adminVerbs := make(map[string]string)

			// GET
			viewVerbs["get"] = "yes"
			editVerbs["get"] = "yes"
			adminVerbs["get"] = "yes"

			// List
			viewVerbs["list"] = "yes"
			editVerbs["list"] = "yes"
			adminVerbs["list"] = "yes"

			// WATCH
			viewVerbs["watch"] = "yes"
			editVerbs["watch"] = "yes"
			adminVerbs["watch"] = "yes"

			// DELETE
			viewVerbs["delete"] = "no"
			editVerbs["delete"] = "yes"
			adminVerbs["delete"] = "yes"

			// CREATE
			viewVerbs["create"] = "no"
			editVerbs["create"] = "yes"
			adminVerbs["create"] = "yes"

			// UPDATE
			viewVerbs["update"] = "no"
			editVerbs["update"] = "yes"
			adminVerbs["update"] = "yes"

			// PATCH
			viewVerbs["patch"] = "no"
			editVerbs["patch"] = "yes"
			adminVerbs["patch"] = "yes"

			// DELETE COllECTION
			viewVerbs["deleteCollection"] = "no"
			editVerbs["deleteCollection"] = "no"
			adminVerbs["deleteCollection"] = "yes"

			namespace := tests.NamespaceTestDefault
			verbs := []string{"get", "list", "watch", "delete", "create", "update", "patch", "deletecollection"}

			for _, verb := range verbs {
				// VIEW
				By(fmt.Sprintf("verifying VIEW sa for verb %s", verb))
				expectedRes, _ := viewVerbs[verb]
				as := fmt.Sprintf("system:serviceaccount:%s:%s", namespace, view)
				result, _, _ := tests.RunCommand("kubectl", "auth", "can-i", "--as", as, verb, resource)
				Expect(result).To(ContainSubstring(expectedRes))

				// EDIT
				By(fmt.Sprintf("verifying EDIT sa for verb %s", verb))
				expectedRes, _ = editVerbs[verb]
				as = fmt.Sprintf("system:serviceaccount:%s:%s", namespace, edit)
				result, _, _ = tests.RunCommand("kubectl", "auth", "can-i", "--as", as, verb, resource)
				Expect(result).To(ContainSubstring(expectedRes))

				// ADMIN
				By(fmt.Sprintf("verifying ADMIN sa for verb %s", verb))
				expectedRes, _ = adminVerbs[verb]
				as = fmt.Sprintf("system:serviceaccount:%s:%s", namespace, admin)
				result, _, _ = tests.RunCommand("kubectl", "auth", "can-i", "--as", as, verb, resource)
				Expect(result).To(ContainSubstring(expectedRes))

				// DEFAULT - the default should always return 'no' for ever verb.
				// This is primarily a sanity check.
				By(fmt.Sprintf("verifying DEFAULT sa for verb %s", verb))
				expectedRes = "no"
				as = fmt.Sprintf("system:serviceaccount:%s:default", namespace)
				result, _, _ = tests.RunCommand("kubectl", "auth", "can-i", "--as", as, verb, resource)
				Expect(result).To(ContainSubstring(expectedRes))
			}
		},
			table.Entry("[test_id:526]given a vmi", "virtualmachineinstances"),
			table.Entry("[test_id:527]given an vm", "virtualmachines"),
			table.Entry("[test_id:528]given a vmi preset", "virtualmachineinstancepresets"),
			table.Entry("[test_id:529, crit:low]given a vmi replica set", "virtualmachineinstancereplicasets"),
		)
	})
})
<|endoftext|>


<commit_before>// The quantile package implements Effective Computation of Biased Quantiles over Data Streams
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
//
// This package is useful for calculating targeted quantiles for large datasets within low memory and cpu bounds.
package quantile

import (
	"container/list"
	"math"
)

type Interface interface {
	// Query returns the calculated qth percentiles value. Calling Query
	// with q not in the set quantiles given to New will have non-deterministic
	// results.
	Query(q float64) float64

	// Insert inserts v into the list.
	Insert(v float64)

	// Merge merges samples into the list. This handy when
	// merging multiple streams from seperate threads.
	Merge(samples Samples)

	// Samples returns a copy of the list of samples kept from the data
	// stream.
	Samples() Samples

	// Count returns the total number of samples observed in the stream
	// since initialization.
	Count() int

	// Init initializes or clears the list.
	Init()

	// Min returns the minimum value observed in the list.
	Min() float64

	// Max returns the maximum value observed in the list.
	Max() float64
}

type stream struct {
	e   float64
	q   []float64
	n   float64
	l   *list.List
	max float64
}

// New returns an initialized stream targeted at quantiles for error e. e is usually 0.01.
func New(e float64, quantiles ...float64) Interface {
	x := &stream{e: e, q: quantiles, l: list.New()}
	return &buffer{x, make(Samples, 0, 500)}
}

func (qt *stream) Init() {
	qt.l.Init()
	qt.n = 0
}

func (qt *stream) ƒ(r float64) float64 {
	var m float64 = math.MaxFloat64
	var f float64
	for _, q := range qt.q {
		if q*qt.n <= r {
			f = (2 * qt.e * r) / q
		} else {
			f = (2 * qt.e * (qt.n - r)) / (1 - q)
		}
		m = math.Min(m, f)
	}
	return m
}

func (qt *stream) Insert(v float64) {
	fn := qt.mergeFunc()
	fn(v, 1)
}

func (qt *stream) Merge(samples Samples) {
	fn := qt.mergeFunc()
	for _, s := range samples {
		fn(s.Value, s.Width)
	}
}

func (qt *stream) mergeFunc() func(v, w float64) {
	// NOTE: I used a goto over defer because it bought me a few extra
	// nanoseconds. I know. I know.
	var r float64
	e := qt.l.Front()
	return func(v, w float64) {
		if v > qt.max {
			qt.max = v
		}

		for ; e != nil; e = e.Next() {
			c := e.Value.(*Sample)
			if c.Value > v {
				s := &Sample{v, w, math.Floor(qt.ƒ(r)) - 1}
				qt.l.InsertBefore(s, e)
				goto inserted
			}
			r += c.Width
		}
		qt.l.PushBack(&Sample{v, w, 0})
	inserted:
		qt.n += w
	}
}

func (qt *stream) Count() int {
	return int(qt.n)
}

func (qt *stream) Query(q float64) float64 {
	e := qt.l.Front()
	t := math.Ceil(q * qt.n)
	t += math.Ceil(qt.ƒ(t) / 2)
	p := e.Value.(*Sample)
	e = e.Next()
	r := float64(0)
	for e != nil {
		c := e.Value.(*Sample)
		if r+c.Width+c.Delta > t {
			return p.Value
		}
		r += p.Width
		p = c
		e = e.Next()
	}
	return p.Value
}

func (qt *stream) compress() {
	if qt.l.Len() < 2 {
		return
	}
	e := qt.l.Back()
	x := e.Value.(*Sample)
	r := qt.n - 1 - x.Width
	e = e.Prev()
	for e != nil {
		c := e.Value.(*Sample)
		if c.Width+x.Width+x.Delta <= qt.ƒ(r) {
			x.Width += c.Width
			o := e
			e = e.Prev()
			qt.l.Remove(o)
		} else {
			x = c
			e = e.Prev()
		}
		r -= c.Width
	}
}

func (qt *stream) Samples() Samples {
	samples := make(Samples, 0, qt.l.Len())
	for e := qt.l.Front(); e != nil; e = e.Next() {
		samples = append(samples, *e.Value.(*Sample))
	}
	return samples
}

// Min returns the mininmul value observed in the stream.
func (qt *stream) Min() float64 {
	if e := qt.l.Front(); e != nil {
		return e.Value.(*Sample).Value
	}
	return math.NaN()
}

// Max returns the maximum value observed in the stream within the error epsilon.
func (qt *stream) Max() float64 {
	if qt.l.Len() > 0 {
		return qt.max
	}
	return math.NaN()
}
<commit_msg>more docs<commit_after>// The quantile package implements Effective Computation of Biased Quantiles
// over Data Streams http://www.cs.rutgers.edu/~muthu/bquant.pdf
//
// This package is useful for calculating targeted quantiles for large datasets
// within low memory and cpu bounds. This means your trading a small amount of
// accuracy in rank selection, for efficiency.
//
// NOTE: Multiple streams can be merged before a Query, allowing clients to be distributed across threads.
package quantile

import (
	"container/list"
	"math"
)

type Interface interface {
	// Query returns the calculated qth percentiles value. Calling Query
	// with q not in the set quantiles given to New will have non-deterministic
	// results.
	Query(q float64) float64

	// Insert inserts v into the list.
	Insert(v float64)

	// Merge merges samples into the list. This handy when
	// merging multiple streams from seperate threads.
	Merge(samples Samples)

	// Samples returns a copy of the list of samples kept from the data
	// stream.
	Samples() Samples

	// Count returns the total number of samples observed in the stream
	// since initialization.
	Count() int

	// Init initializes or clears the list.
	Init()

	// Min returns the minimum value observed in the list.
	Min() float64

	// Max returns the maximum value observed in the list.
	Max() float64
}

type stream struct {
	e   float64
	q   []float64
	n   float64
	l   *list.List
	max float64
}

// New returns an initialized stream targeted at quantiles for error e. e is usually 0.01.
func New(e float64, quantiles ...float64) Interface {
	x := &stream{e: e, q: quantiles, l: list.New()}
	return &buffer{x, make(Samples, 0, 500)}
}

func (qt *stream) Init() {
	qt.l.Init()
	qt.n = 0
}

func (qt *stream) ƒ(r float64) float64 {
	var m float64 = math.MaxFloat64
	var f float64
	for _, q := range qt.q {
		if q*qt.n <= r {
			f = (2 * qt.e * r) / q
		} else {
			f = (2 * qt.e * (qt.n - r)) / (1 - q)
		}
		m = math.Min(m, f)
	}
	return m
}

func (qt *stream) Insert(v float64) {
	fn := qt.mergeFunc()
	fn(v, 1)
}

func (qt *stream) Merge(samples Samples) {
	fn := qt.mergeFunc()
	for _, s := range samples {
		fn(s.Value, s.Width)
	}
}

func (qt *stream) mergeFunc() func(v, w float64) {
	// NOTE: I used a goto over defer because it bought me a few extra
	// nanoseconds. I know. I know.
	var r float64
	e := qt.l.Front()
	return func(v, w float64) {
		if v > qt.max {
			qt.max = v
		}

		for ; e != nil; e = e.Next() {
			c := e.Value.(*Sample)
			if c.Value > v {
				s := &Sample{v, w, math.Floor(qt.ƒ(r)) - 1}
				qt.l.InsertBefore(s, e)
				goto inserted
			}
			r += c.Width
		}
		qt.l.PushBack(&Sample{v, w, 0})
	inserted:
		qt.n += w
	}
}

func (qt *stream) Count() int {
	return int(qt.n)
}

func (qt *stream) Query(q float64) float64 {
	e := qt.l.Front()
	t := math.Ceil(q * qt.n)
	t += math.Ceil(qt.ƒ(t) / 2)
	p := e.Value.(*Sample)
	e = e.Next()
	r := float64(0)
	for e != nil {
		c := e.Value.(*Sample)
		if r+c.Width+c.Delta > t {
			return p.Value
		}
		r += p.Width
		p = c
		e = e.Next()
	}
	return p.Value
}

func (qt *stream) compress() {
	if qt.l.Len() < 2 {
		return
	}
	e := qt.l.Back()
	x := e.Value.(*Sample)
	r := qt.n - 1 - x.Width
	e = e.Prev()
	for e != nil {
		c := e.Value.(*Sample)
		if c.Width+x.Width+x.Delta <= qt.ƒ(r) {
			x.Width += c.Width
			o := e
			e = e.Prev()
			qt.l.Remove(o)
		} else {
			x = c
			e = e.Prev()
		}
		r -= c.Width
	}
}

func (qt *stream) Samples() Samples {
	samples := make(Samples, 0, qt.l.Len())
	for e := qt.l.Front(); e != nil; e = e.Next() {
		samples = append(samples, *e.Value.(*Sample))
	}
	return samples
}

// Min returns the mininmul value observed in the stream.
func (qt *stream) Min() float64 {
	if e := qt.l.Front(); e != nil {
		return e.Value.(*Sample).Value
	}
	return math.NaN()
}

// Max returns the maximum value observed in the stream within the error epsilon.
func (qt *stream) Max() float64 {
	if qt.l.Len() > 0 {
		return qt.max
	}
	return math.NaN()
}
<|endoftext|>


<commit_before>package workers

import (
	"fmt"
	"github.com/CiscoCloud/distributive/tabular"
	"github.com/CiscoCloud/distributive/wrkutils"
	"log"
	"os/exec"
	"regexp"
	"strings"
)

// RegisterPackage registers these checks so they can be used.
func RegisterPackage() {
	wrkutils.RegisterCheck("installed", installed, 1)
	wrkutils.RegisterCheck("repoexists", repoExists, 2)
	wrkutils.RegisterCheck("repoexistsuri", repoExistsURI, 2)
	wrkutils.RegisterCheck("pacmanignore", pacmanIgnore, 1)
}

// getKeys returns the string keys from a string -> string map
func getKeys(m map[string]string) []string {
	keys := make([]string, len(m))
	i := 0
	for key := range managers {
		keys[i] = key
		i++
	}
	return keys
}

// package managers and their options for queries
var managers = map[string]string{
	"dpkg":   "-s",
	"rpm":    "-q",
	"pacman": "-Qs",
}
var keys = getKeys(managers)

// getManager returns package manager as a string
func getManager() string {
	for _, program := range keys {
		cmd := exec.Command(program, "--version")
		err := cmd.Start()
		// as long as the command was found, return that manager
		message := ""
		if err != nil {
			message = err.Error()
		}
		if strings.Contains(message, "not found") == false {
			return program
		}
	}
	log.Fatal("No package manager found. Attempted: " + fmt.Sprint(managers))
	return "" // never reaches this return
}

// repo is a unified interface for pacman, apt, and yum repos
type repo struct {
	ID     string
	Name   string // yum
	URL    string // apt, pacman
	Status string
}

// repoToString converts a repo struct into a representable, printable string
func repoToString(r repo) (str string) {
	str += "Name: " + r.Name
	str += " ID: " + r.ID
	str += " URL: " + r.URL
	str += " Status: " + r.Status
	return str
}

// getYumRepos constructs Repos from the yum.conf file at path. Gives non-zero
// Names, Fullnames, and URLs.
func getYumRepos() (repos []repo) {
	// safeAccess allows access w/o fear of a panic into a slice of strings
	safeAccess := func(slc []string, index int) string {
		if len(slc) > index {
			return slc[index]
		}
		return ""
	}

	// get output of `yum repolist`
	cmd := exec.Command("yum", "repolist")
	out, err := cmd.Output()
	outstr := string(out)
	if err != nil {
		wrkutils.ExecError(cmd, outstr, err)
	}
	// parse output
	slc := tabular.ProbabalisticSplit(outstr)
	ids := tabular.GetColumnByHeader("repo id", slc)
	ids = ids[:len(ids)-2] // has extra line at end
	names := tabular.GetColumnByHeader("repo name", slc)
	statuses := tabular.GetColumnByHeader("status", slc)
	if len(ids) != len(names) || len(names) != len(statuses) {
		fmt.Println("Warning: could not fetch complete metadata for every repo.")
		fmt.Println("Names: " + fmt.Sprint(len(names)))
		fmt.Println("IDs: " + fmt.Sprint(len(ids)))
		fmt.Println("Statuses: " + fmt.Sprint(len(statuses)))
	}
	// Construct repos
	for i := range ids {
		name := safeAccess(names, i)
		id := safeAccess(ids, i)
		status := safeAccess(statuses, i)
		repo := repo{Name: name, ID: id, Status: status}
		repos = append(repos, repo)
	}
	return repos
}

// getAptrepos constructs repos from the sources.list file at path. Gives
// non-zero URLs
func getAptRepos() (repos []repo) {
	// getAptSources returns all the urls of all apt sources (including source
	// code repositories
	getAptSources := func() (urls []string) {
		otherLists := wrkutils.GetFilesWithExtension("/etc/apt/sources.list.d", ".list")
		sourceLists := append([]string{"/etc/apt/sources.list"}, otherLists...)
		for _, f := range sourceLists {
			split := tabular.ProbabalisticSplit(wrkutils.FileToString(f))
			// filter out comments
			commentRegex := regexp.MustCompile("^\\s*#.*")
			for _, line := range split {
				if len(line) > 1 && !(commentRegex.MatchString(line[0])) {
					urls = append(urls, line[1])
				}
			}
		}
		return urls
	}
	for _, src := range getAptSources() {
		repos = append(repos, repo{URL: src})
	}
	return repos
}

// getPacmanRepos constructs repos from the pacman.conf file at path. Gives
// non-zero Names and URLs
func getPacmanRepos(path string) (repos []repo) {
	data := wrkutils.FileToLines(path)
	// match words and dashes in brackets without comments
	nameRegex := regexp.MustCompile("[^#]\\[(\\w|\\-)+\\]")
	// match lines that start with Include= or Server= and anything after that
	urlRegex := regexp.MustCompile("[^#](Include|Server)\\s*=\\s*.*")
	var names []string
	var urls []string
	for _, line := range data {
		if nameRegex.Match(line) {
			names = append(names, string(nameRegex.Find(line)))
		}
		if urlRegex.Match(line) {
			urls = append(urls, string(urlRegex.Find(line)))
		}
	}
	for i, name := range names {
		if len(urls) > i {
			repos = append(repos, repo{Name: name, URL: urls[i]})
		}
	}
	return repos
}

// getRepos simply returns a list of repos based on the manager chosen
func getRepos(manager string) (repos []repo) {
	switch manager {
	case "yum":
		return getYumRepos()
	case "apt":
		return getAptRepos()
	case "pacman":
		return getPacmanRepos("/etc/pacman.conf")
	default:
		msg := "Cannot find repos of unsupported package manager: "
		_, message := wrkutils.GenericError(msg, manager, []string{getManager()})
		log.Fatal(message)
	}
	return []repo{} // will never reach here b/c of default case
}

// existsRepoWithProperty is an abstraction of YumRepoExists and YumRepoURL.
// It takes a struct field name to check, and an expected value. If the expected
// value is found in the field of a repo, it returns 0, "" else an error message.
// Valid choices for prop: "URL" | "Name" | "Name"
func existsRepoWithProperty(prop string, val *regexp.Regexp, manager string) (int, string) {
	var properties []string
	for _, repo := range getRepos(manager) {
		switch prop {
		case "URL":
			properties = append(properties, repo.URL)
		case "Name":
			properties = append(properties, repo.Name)
		case "Status":
			properties = append(properties, repo.Status)
		case "ID":
			properties = append(properties, repo.ID)
		default:
			log.Fatal("Repos don't have the requested property: " + prop)
		}
	}
	if tabular.ReIn(val, properties) {
		return 0, ""
	}
	msg := "Repo with given " + prop + " not found"
	return wrkutils.GenericError(msg, val.String(), properties)
}

// repoExists checks to see that a given repo is listed in the appropriate
// configuration file
func repoExists(parameters []string) (exitCode int, exitMessage string) {
	re := wrkutils.ParseUserRegex(parameters[1])
	return existsRepoWithProperty("Name", re, parameters[0])
}

// repoExistsURI checks to see if the repo with the given URI is listed in the
// appropriate configuration file
func repoExistsURI(parameters []string) (exitCode int, exitMessage string) {
	re := wrkutils.ParseUserRegex(parameters[1])
	return existsRepoWithProperty("URL", re, parameters[0])
}

// pacmanIgnore checks to see whether a given package is in /etc/pacman.conf's
// IgnorePkg setting
func pacmanIgnore(parameters []string) (exitCode int, exitMessage string) {
	pkg := parameters[0]
	data := wrkutils.FileToString("/etc/pacman.conf")
	re := regexp.MustCompile("[^#]IgnorePkg\\s+=\\s+.+")
	find := re.FindString(data)
	var packages []string
	if find != "" {
		spl := strings.Split(find, " ")
		if len(spl) > 2 {
			packages = spl[2:] // first two are "IgnorePkg" and "="
			if tabular.StrIn(pkg, packages) {
				return 0, ""
			}
		}
	}
	msg := "Couldn't find package in IgnorePkg"
	return wrkutils.GenericError(msg, pkg, packages)
}

// installed detects whether the OS is using dpkg, rpm, or pacman, queries
// a package accoringly, and returns an error if it is not installed.
func installed(parameters []string) (exitCode int, exitMessage string) {
	pkg := parameters[0]
	name := getManager()
	options := managers[name]
	out, _ := exec.Command(name, options, pkg).Output()
	if strings.Contains(string(out), pkg) {
		return 0, ""
	}
	msg := "Package was not found:"
	msg += "\n\tPackage name: " + pkg
	msg += "\n\tPackage manager: " + name
	return 1, msg
}
<commit_msg>Fixed slice error in yum repolist<commit_after>package workers

import (
	"fmt"
	"github.com/CiscoCloud/distributive/tabular"
	"github.com/CiscoCloud/distributive/wrkutils"
	log "github.com/Sirupsen/logrus"
	"os/exec"
	"regexp"
	"strings"
)

// RegisterPackage registers these checks so they can be used.
func RegisterPackage() {
	wrkutils.RegisterCheck("installed", installed, 1)
	wrkutils.RegisterCheck("repoexists", repoExists, 2)
	wrkutils.RegisterCheck("repoexistsuri", repoExistsURI, 2)
	wrkutils.RegisterCheck("pacmanignore", pacmanIgnore, 1)
}

// getKeys returns the string keys from a string -> string map
func getKeys(m map[string]string) []string {
	keys := make([]string, len(m))
	i := 0
	for key := range managers {
		keys[i] = key
		i++
	}
	return keys
}

// package managers and their options for queries
var managers = map[string]string{
	"dpkg":   "-s",
	"rpm":    "-q",
	"pacman": "-Qs",
}
var keys = getKeys(managers)

// getManager returns package manager as a string
func getManager() string {
	for _, program := range keys {
		cmd := exec.Command(program, "--version")
		err := cmd.Start()
		// as long as the command was found, return that manager
		message := ""
		if err != nil {
			message = err.Error()
		}
		if strings.Contains(message, "not found") == false {
			return program
		}
	}
	log.Fatal("No package manager found. Attempted: " + fmt.Sprint(managers))
	return "" // never reaches this return
}

// repo is a unified interface for pacman, apt, and yum repos
type repo struct {
	ID     string
	Name   string // yum
	URL    string // apt, pacman
	Status string
}

// repoToString converts a repo struct into a representable, printable string
func repoToString(r repo) (str string) {
	str += "Name: " + r.Name
	str += " ID: " + r.ID
	str += " URL: " + r.URL
	str += " Status: " + r.Status
	return str
}

// getYumRepos constructs Repos from the yum.conf file at path. Gives non-zero
// Names, Fullnames, and URLs.
func getYumRepos() (repos []repo) {
	// safeAccess allows access w/o fear of a panic into a slice of strings
	safeAccess := func(slc []string, index int) string {
		// catch runtime panic
		defer func() {
			if err := recover(); err != nil {
				log.WithFields(log.Fields{
					"slice":  slc,
					"length": len(slc),
					"index":  index,
				}).Warn("Accessing out-of-bounds index. Please report.")
			}
		}() // invoke inside defer
		if len(slc) > index {
			return slc[index]
		}
		return ""
	}

	// get output of `yum repolist`
	cmd := exec.Command("yum", "repolist")
	out, err := cmd.Output()
	outstr := string(out)
	if err != nil {
		wrkutils.ExecError(cmd, outstr, err)
	}
	// parse output
	slc := tabular.ProbabalisticSplit(outstr)
	ids := tabular.GetColumnByHeader("repo id", slc)
	if len(ids) > 2 {
		ids = ids[:len(ids)-2] // has extra line at end
	}
	names := tabular.GetColumnByHeader("repo name", slc)
	statuses := tabular.GetColumnByHeader("status", slc)
	if len(ids) != len(names) || len(names) != len(statuses) {
		log.WithFields(log.Fields{
			"names":    len(names),
			"ids":      len(ids),
			"statuses": len(statuses),
		}).Warn("Could not fetch complete metadata for every repo.")
	}
	// Construct repos
	for i := range ids {
		name := safeAccess(names, i)
		id := safeAccess(ids, i)
		status := safeAccess(statuses, i)
		repo := repo{Name: name, ID: id, Status: status}
		repos = append(repos, repo)
	}
	return repos
}

// getAptrepos constructs repos from the sources.list file at path. Gives
// non-zero URLs
func getAptRepos() (repos []repo) {
	// getAptSources returns all the urls of all apt sources (including source
	// code repositories
	getAptSources := func() (urls []string) {
		otherLists := wrkutils.GetFilesWithExtension("/etc/apt/sources.list.d", ".list")
		sourceLists := append([]string{"/etc/apt/sources.list"}, otherLists...)
		for _, f := range sourceLists {
			split := tabular.ProbabalisticSplit(wrkutils.FileToString(f))
			// filter out comments
			commentRegex := regexp.MustCompile("^\\s*#.*")
			for _, line := range split {
				if len(line) > 1 && !(commentRegex.MatchString(line[0])) {
					urls = append(urls, line[1])
				}
			}
		}
		return urls
	}
	for _, src := range getAptSources() {
		repos = append(repos, repo{URL: src})
	}
	return repos
}

// getPacmanRepos constructs repos from the pacman.conf file at path. Gives
// non-zero Names and URLs
func getPacmanRepos(path string) (repos []repo) {
	data := wrkutils.FileToLines(path)
	// match words and dashes in brackets without comments
	nameRegex := regexp.MustCompile("[^#]\\[(\\w|\\-)+\\]")
	// match lines that start with Include= or Server= and anything after that
	urlRegex := regexp.MustCompile("[^#](Include|Server)\\s*=\\s*.*")
	var names []string
	var urls []string
	for _, line := range data {
		if nameRegex.Match(line) {
			names = append(names, string(nameRegex.Find(line)))
		}
		if urlRegex.Match(line) {
			urls = append(urls, string(urlRegex.Find(line)))
		}
	}
	for i, name := range names {
		if len(urls) > i {
			repos = append(repos, repo{Name: name, URL: urls[i]})
		}
	}
	return repos
}

// getRepos simply returns a list of repos based on the manager chosen
func getRepos(manager string) (repos []repo) {
	switch manager {
	case "yum":
		return getYumRepos()
	case "apt":
		return getAptRepos()
	case "pacman":
		return getPacmanRepos("/etc/pacman.conf")
	default:
		msg := "Cannot find repos of unsupported package manager: "
		_, message := wrkutils.GenericError(msg, manager, []string{getManager()})
		log.Fatal(message)
	}
	return []repo{} // will never reach here b/c of default case
}

// existsRepoWithProperty is an abstraction of YumRepoExists and YumRepoURL.
// It takes a struct field name to check, and an expected value. If the expected
// value is found in the field of a repo, it returns 0, "" else an error message.
// Valid choices for prop: "URL" | "Name" | "Name"
func existsRepoWithProperty(prop string, val *regexp.Regexp, manager string) (int, string) {
	var properties []string
	for _, repo := range getRepos(manager) {
		switch prop {
		case "URL":
			properties = append(properties, repo.URL)
		case "Name":
			properties = append(properties, repo.Name)
		case "Status":
			properties = append(properties, repo.Status)
		case "ID":
			properties = append(properties, repo.ID)
		default:
			log.Fatal("Repos don't have the requested property: " + prop)
		}
	}
	if tabular.ReIn(val, properties) {
		return 0, ""
	}
	msg := "Repo with given " + prop + " not found"
	return wrkutils.GenericError(msg, val.String(), properties)
}

// repoExists checks to see that a given repo is listed in the appropriate
// configuration file
func repoExists(parameters []string) (exitCode int, exitMessage string) {
	re := wrkutils.ParseUserRegex(parameters[1])
	return existsRepoWithProperty("Name", re, parameters[0])
}

// repoExistsURI checks to see if the repo with the given URI is listed in the
// appropriate configuration file
func repoExistsURI(parameters []string) (exitCode int, exitMessage string) {
	re := wrkutils.ParseUserRegex(parameters[1])
	return existsRepoWithProperty("URL", re, parameters[0])
}

// pacmanIgnore checks to see whether a given package is in /etc/pacman.conf's
// IgnorePkg setting
func pacmanIgnore(parameters []string) (exitCode int, exitMessage string) {
	pkg := parameters[0]
	data := wrkutils.FileToString("/etc/pacman.conf")
	re := regexp.MustCompile("[^#]IgnorePkg\\s+=\\s+.+")
	find := re.FindString(data)
	var packages []string
	if find != "" {
		spl := strings.Split(find, " ")
		if len(spl) > 2 {
			packages = spl[2:] // first two are "IgnorePkg" and "="
			if tabular.StrIn(pkg, packages) {
				return 0, ""
			}
		}
	}
	msg := "Couldn't find package in IgnorePkg"
	return wrkutils.GenericError(msg, pkg, packages)
}

// installed detects whether the OS is using dpkg, rpm, or pacman, queries
// a package accoringly, and returns an error if it is not installed.
func installed(parameters []string) (exitCode int, exitMessage string) {
	pkg := parameters[0]
	name := getManager()
	options := managers[name]
	out, _ := exec.Command(name, options, pkg).Output()
	if strings.Contains(string(out), pkg) {
		return 0, ""
	}
	msg := "Package was not found:"
	msg += "\n\tPackage name: " + pkg
	msg += "\n\tPackage manager: " + name
	return 1, msg
}
<|endoftext|>


<commit_before>package cfschema

import (
	"encoding/json"
)

const (
	PropertyFormatDate                = "date"
	PropertyFormatDateTime            = "date-time"
	PropertyFormatEmail               = "email"
	PropertyFormatHostname            = "hostname"
	PropertyFormatIdnEmail            = "idn-email"
	PropertyFormatIdnHostname         = "idn-hostname"
	PropertyFormatIpv4                = "ipv4"
	PropertyFormatIpv6                = "ipv6"
	PropertyFormatIri                 = "iri"
	PropertyFormatIriReference        = "iri-reference"
	PropertyFormatJsonPointer         = "json-pointer"
	PropertyFormatRegex               = "regex"
	PropertyFormatRelativeJsonPointer = "relative-json-pointer"
	PropertyFormatTime                = "time"
	PropertyFormatUri                 = "uri"
	PropertyFormatUriReference        = "uri-reference"
	PropertyFormatUriTemplate         = "uri-template"
)

const (
	PropertyTypeArray   = "array"
	PropertyTypeBoolean = "boolean"
	PropertyTypeInteger = "integer"
	PropertyTypeNull    = "null"
	PropertyTypeNumber  = "number"
	PropertyTypeObject  = "object"
	PropertyTypeString  = "string"
)

// Property represents the CloudFormation Resource Schema customization for Definitions and Properties.
type Property struct {
	AdditionalProperties *bool                `json:"additionalProperties,omitempty"`
	AllOf                []*PropertySubschema `json:"allOf,omitempty"`
	AnyOf                []*PropertySubschema `json:"anyOf,omitempty"`
	Comment              *string              `json:"$comment,omitempty"`
	Default              interface{}          `json:"default,omitempty"`
	Description          *string              `json:"description,omitempty"`
	Enum                 []interface{}        `json:"enum,omitempty"`
	Examples             []interface{}        `json:"examples,omitempty"`
	Format               *string              `json:"format,omitempty"`
	InsertionOrder       *bool                `json:"insertionOrder,omitempty"`
	Items                *Property            `json:"items,omitempty"`
	Maximum              *int                 `json:"maximum,omitempty"`
	MaxItems             *int                 `json:"maxItems,omitempty"`
	MaxLength            *int                 `json:"maxLength,omitempty"`
	Minimum              *int                 `json:"minimum,omitempty"`
	MinItems             *int                 `json:"minItems,omitempty"`
	MinLength            *int                 `json:"minLength,omitempty"`
	OneOf                []*PropertySubschema `json:"oneOf,omitempty"`
	Pattern              *string              `json:"pattern,omitempty"`
	PatternProperties    map[string]*Property `json:"patternProperties,omitempty"`
	Properties           map[string]*Property `json:"properties,omitempty"`
	Ref                  *Reference           `json:"$ref,omitempty"`
	Required             []string             `json:"required,omitempty"`
	Type                 *Type                `json:"type,omitempty"`
	UniqueItems          *bool                `json:"uniqueItems,omitempty"`
}

// String returns a string representation of Property.
func (p *Property) String() string {
	if p == nil {
		return ""
	}

	b, _ := json.MarshalIndent(p, "", "  ")

	return string(b)
}

func (p *Property) IsRequired(name string) bool {
	if p == nil {
		return false
	}

	for _, req := range p.Required {
		if req == name {
			return true
		}
	}

	return false
}
<commit_msg>'Property.Maximum' and 'Property.Minimum'; 'int' -> 'int64'.<commit_after>package cfschema

import (
	"encoding/json"
)

const (
	PropertyFormatDate                = "date"
	PropertyFormatDateTime            = "date-time"
	PropertyFormatEmail               = "email"
	PropertyFormatHostname            = "hostname"
	PropertyFormatIdnEmail            = "idn-email"
	PropertyFormatIdnHostname         = "idn-hostname"
	PropertyFormatIpv4                = "ipv4"
	PropertyFormatIpv6                = "ipv6"
	PropertyFormatIri                 = "iri"
	PropertyFormatIriReference        = "iri-reference"
	PropertyFormatJsonPointer         = "json-pointer"
	PropertyFormatRegex               = "regex"
	PropertyFormatRelativeJsonPointer = "relative-json-pointer"
	PropertyFormatTime                = "time"
	PropertyFormatUri                 = "uri"
	PropertyFormatUriReference        = "uri-reference"
	PropertyFormatUriTemplate         = "uri-template"
)

const (
	PropertyTypeArray   = "array"
	PropertyTypeBoolean = "boolean"
	PropertyTypeInteger = "integer"
	PropertyTypeNull    = "null"
	PropertyTypeNumber  = "number"
	PropertyTypeObject  = "object"
	PropertyTypeString  = "string"
)

// Property represents the CloudFormation Resource Schema customization for Definitions and Properties.
type Property struct {
	AdditionalProperties *bool                `json:"additionalProperties,omitempty"`
	AllOf                []*PropertySubschema `json:"allOf,omitempty"`
	AnyOf                []*PropertySubschema `json:"anyOf,omitempty"`
	Comment              *string              `json:"$comment,omitempty"`
	Default              interface{}          `json:"default,omitempty"`
	Description          *string              `json:"description,omitempty"`
	Enum                 []interface{}        `json:"enum,omitempty"`
	Examples             []interface{}        `json:"examples,omitempty"`
	Format               *string              `json:"format,omitempty"`
	InsertionOrder       *bool                `json:"insertionOrder,omitempty"`
	Items                *Property            `json:"items,omitempty"`
	Maximum              *int64               `json:"maximum,omitempty"`
	MaxItems             *int                 `json:"maxItems,omitempty"`
	MaxLength            *int                 `json:"maxLength,omitempty"`
	Minimum              *int64               `json:"minimum,omitempty"`
	MinItems             *int                 `json:"minItems,omitempty"`
	MinLength            *int                 `json:"minLength,omitempty"`
	OneOf                []*PropertySubschema `json:"oneOf,omitempty"`
	Pattern              *string              `json:"pattern,omitempty"`
	PatternProperties    map[string]*Property `json:"patternProperties,omitempty"`
	Properties           map[string]*Property `json:"properties,omitempty"`
	Ref                  *Reference           `json:"$ref,omitempty"`
	Required             []string             `json:"required,omitempty"`
	Type                 *Type                `json:"type,omitempty"`
	UniqueItems          *bool                `json:"uniqueItems,omitempty"`
}

// String returns a string representation of Property.
func (p *Property) String() string {
	if p == nil {
		return ""
	}

	b, _ := json.MarshalIndent(p, "", "  ")

	return string(b)
}

func (p *Property) IsRequired(name string) bool {
	if p == nil {
		return false
	}

	for _, req := range p.Required {
		if req == name {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package main

/*
#include <stdlib.h>
#include <string.h>
typedef enum {
  PADDLE_ELEMENT_TYPE_INT32   = 0,
  PADDLE_ELEMENT_TYPE_UINT32  = 1,
  PADDLE_ELEMENT_TYPE_INT64   = 2,
  PADDLE_ELEMENT_TYPE_UINT64  = 3,
  PADDLE_ELEMENT_TYPE_FLOAT32 = 4,
  PADDLE_ELEMENT_TYPE_FLOAT64 = 5,
} paddle_element_type;

typedef struct {
  char*               name;
  paddle_element_type element_type;
  unsigned char*      content;
  int                 content_len;
} paddle_parameter, paddle_gradient;

typedef int paddle_pserver_client;
*/
import "C"

import (
	"log"
	"strings"
	"sync"
	"unsafe"

	"github.com/PaddlePaddle/Paddle/go/pserver"
)

var nullPtr = unsafe.Pointer(uintptr(0))
var mu sync.Mutex
var handleMap = make(map[C.paddle_pserver_client]*pserver.Client)
var curHandle C.paddle_pserver_client

func add(c *pserver.Client) C.paddle_pserver_client {
	mu.Lock()
	defer mu.Unlock()
	client := curHandle
	curHandle++
	handleMap[client] = c
	return client
}

func get(client C.paddle_pserver_client) *pserver.Client {
	mu.Lock()
	defer mu.Unlock()
	return handleMap[client]
}

func remove(client C.paddle_pserver_client) *pserver.Client {
	mu.Lock()
	defer mu.Unlock()
	h := handleMap[client]
	delete(handleMap, client)
	return h
}

func cArrayToSlice(p unsafe.Pointer, len int) []byte {
	if p == nullPtr {
		return nil
	}

	// create a Go clice backed by a C array, reference:
	// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
	//
	// Go garbage collector will not interact with this data, need
	// to be freed properly.
	return (*[1 << 30]byte)(p)[:len:len]
}

type selector bool

func (s selector) Select() bool {
	return bool(s)
}

type lister []pserver.Server

func (l lister) List() []pserver.Server {
	return l
}

//export paddle_new_pserver_client
func paddle_new_pserver_client(addrs *C.char, selected int) C.paddle_pserver_client {
	a := C.GoString(addrs)
	as := strings.Split(a, ",")
	servers := make([]pserver.Server, len(as))
	for i := range as {
		servers[i].Index = i
		servers[i].Addr = as[i]
	}
	c := pserver.NewClient(lister(servers), len(as), selector(selected != 0))
	return add(c)
}

//export paddle_new_etcd_pserver_client
func paddle_new_etcd_pserver_client(etcd_addr *C.char) C.paddle_pserver_client {
	// TODO(helin): fault tolerant pserver client using etcd.
	panic("not implemented.")
}

//export paddle_pserver_client_release
func paddle_pserver_client_release(client C.paddle_pserver_client) {
	remove(client)
}

//export paddle_begin_init_params
func paddle_begin_init_params(client C.paddle_pserver_client) C.int {
	c := get(client)
	if selected := c.BeginInitParams(); selected {
		return 1
	}
	return 0
}

//export paddle_init_param
func paddle_init_param(client C.paddle_pserver_client, param C.paddle_parameter, param_config unsafe.Pointer, config_len C.int) C.int {
	et := pserver.ElementType(param.element_type)
	name := C.GoString(param.name)
	content := cArrayToSlice(unsafe.Pointer(param.content), int(param.content_len))
	pc := pserver.ParameterWithConfig{
		Param:  pserver.Parameter{Name: name, ElementType: et, Content: content},
		Config: cArrayToSlice(param_config, int(config_len)),
	}
	c := get(client)
	err := c.InitParam(pc)

	if err != nil {
		if err.Error() == pserver.AlreadyInitialized {
			log.Println("parameter", name, "already initialized, treat paddle_init_param as sucessful.")
			return 0
		}
		log.Println(err)
		return -1
	}

	return 0
}

//export paddle_finish_init_params
func paddle_finish_init_params(client C.paddle_pserver_client) C.int {
	c := get(client)
	err := c.FinishInitParams()
	if err != nil {
		if err.Error() == pserver.AlreadyInitialized {
			log.Println("parameters already initialized, treat paddle_finish_init_params as sucessful.")
			return 0
		}

		log.Println(err)
		return -1
	}

	return 0
}

//export paddle_send_grads
func paddle_send_grads(client C.paddle_pserver_client, grads *C.paddle_gradient, total C.int) C.int {
	var gs []pserver.Gradient
	for i := 0; i < int(total); i++ {
		grad := (*C.paddle_gradient)(unsafe.Pointer((uintptr(unsafe.Pointer(grads)) + uintptr(i)*unsafe.Sizeof(*grads))))
		et := pserver.ElementType(grad.element_type)
		name := C.GoString(grad.name)
		content := cArrayToSlice(unsafe.Pointer(grad.content), int(grad.content_len))
		gs = append(gs, pserver.Gradient{Name: name, ElementType: et, Content: content})
	}

	c := get(client)
	err := c.SendGrads(gs)
	if err != nil {
		log.Println(err)
		return -1
	}

	return 0
}

//export paddle_get_params
func paddle_get_params(client C.paddle_pserver_client, dst **C.paddle_parameter, total C.int) C.int {
	var ns []string
	for i := 0; i < int(total); i++ {
		param := *(**C.paddle_parameter)(unsafe.Pointer((uintptr(unsafe.Pointer(dst)) + uintptr(i)*unsafe.Sizeof(*dst))))
		ns = append(ns, C.GoString(param.name))
	}
	c := get(client)
	ps, err := c.GetParams(ns)
	if err != nil {
		log.Println(err)
		return -1
	}

	if len(ps) != len(ns) {
		return -1
	}

	for i := range ps {
		if ns[i] != ps[i].Name {
			return -1
		}
	}

	for i := 0; i < int(total); i++ {
		p := ps[i]
		param := *(**C.paddle_parameter)(unsafe.Pointer((uintptr(unsafe.Pointer(dst)) + uintptr(i)*unsafe.Sizeof(*dst))))

		if unsafe.Pointer(param) == nullPtr {
			log.Println("Error: must pre-allocate parameter.")
			return -1
		} else {
			if unsafe.Pointer(param.content) != nullPtr {
				if int(param.content_len) != len(p.Content) {
					log.Println("Error: the pre-allocated content len does not match parameter content len.", param.content_len, len(p.Content))
					return -1
				}
			}
		}

		C.memcpy(unsafe.Pointer(param.content), unsafe.Pointer(&p.Content[0]), C.size_t(len(p.Content)))
		param.content_len = C.int(len(p.Content))
		param.element_type = C.paddle_element_type(p.ElementType)
	}

	return 0
}

//export paddle_save_model
func paddle_save_model(client C.paddle_pserver_client, path *C.char) C.int {
	p := C.GoString(path)
	c := get(client)
	err := c.Save(p)
	if err != nil {
		log.Println(err)
		return -1
	}

	return 0
}

func main() {} // Required but ignored
<commit_msg>better logging<commit_after>package main

/*
#include <stdlib.h>
#include <string.h>
typedef enum {
  PADDLE_ELEMENT_TYPE_INT32   = 0,
  PADDLE_ELEMENT_TYPE_UINT32  = 1,
  PADDLE_ELEMENT_TYPE_INT64   = 2,
  PADDLE_ELEMENT_TYPE_UINT64  = 3,
  PADDLE_ELEMENT_TYPE_FLOAT32 = 4,
  PADDLE_ELEMENT_TYPE_FLOAT64 = 5,
} paddle_element_type;

typedef struct {
  char*               name;
  paddle_element_type element_type;
  unsigned char*      content;
  int                 content_len;
} paddle_parameter, paddle_gradient;

typedef int paddle_pserver_client;
*/
import "C"

import (
	"log"
	"strings"
	"sync"
	"unsafe"

	"github.com/PaddlePaddle/Paddle/go/pserver"
)

var nullPtr = unsafe.Pointer(uintptr(0))
var mu sync.Mutex
var handleMap = make(map[C.paddle_pserver_client]*pserver.Client)
var curHandle C.paddle_pserver_client

func add(c *pserver.Client) C.paddle_pserver_client {
	mu.Lock()
	defer mu.Unlock()
	client := curHandle
	curHandle++
	handleMap[client] = c
	return client
}

func get(client C.paddle_pserver_client) *pserver.Client {
	mu.Lock()
	defer mu.Unlock()
	return handleMap[client]
}

func remove(client C.paddle_pserver_client) *pserver.Client {
	mu.Lock()
	defer mu.Unlock()
	h := handleMap[client]
	delete(handleMap, client)
	return h
}

func cArrayToSlice(p unsafe.Pointer, len int) []byte {
	if p == nullPtr {
		return nil
	}

	// create a Go clice backed by a C array, reference:
	// https://github.com/golang/go/wiki/cgo#turning-c-arrays-into-go-slices
	//
	// Go garbage collector will not interact with this data, need
	// to be freed properly.
	return (*[1 << 30]byte)(p)[:len:len]
}

type selector bool

func (s selector) Select() bool {
	return bool(s)
}

type lister []pserver.Server

func (l lister) List() []pserver.Server {
	return l
}

//export paddle_new_pserver_client
func paddle_new_pserver_client(addrs *C.char, selected int) C.paddle_pserver_client {
	a := C.GoString(addrs)
	as := strings.Split(a, ",")
	servers := make([]pserver.Server, len(as))
	for i := range as {
		servers[i].Index = i
		servers[i].Addr = as[i]
	}
	c := pserver.NewClient(lister(servers), len(as), selector(selected != 0))
	return add(c)
}

//export paddle_new_etcd_pserver_client
func paddle_new_etcd_pserver_client(etcd_addr *C.char) C.paddle_pserver_client {
	// TODO(helin): fault tolerant pserver client using etcd.
	panic("not implemented.")
}

//export paddle_pserver_client_release
func paddle_pserver_client_release(client C.paddle_pserver_client) {
	remove(client)
}

//export paddle_begin_init_params
func paddle_begin_init_params(client C.paddle_pserver_client) C.int {
	c := get(client)
	if selected := c.BeginInitParams(); selected {
		return 1
	}
	return 0
}

//export paddle_init_param
func paddle_init_param(client C.paddle_pserver_client, param C.paddle_parameter, param_config unsafe.Pointer, config_len C.int) C.int {
	et := pserver.ElementType(param.element_type)
	name := C.GoString(param.name)
	content := cArrayToSlice(unsafe.Pointer(param.content), int(param.content_len))
	pc := pserver.ParameterWithConfig{
		Param:  pserver.Parameter{Name: name, ElementType: et, Content: content},
		Config: cArrayToSlice(param_config, int(config_len)),
	}
	c := get(client)
	err := c.InitParam(pc)

	if err != nil {
		if err.Error() == pserver.AlreadyInitialized {
			log.Printf("parameter %s already initialized, treat paddle_init_param as sucessful.\n", name)
			return 0
		}
		log.Println(err)
		return -1
	}

	return 0
}

//export paddle_finish_init_params
func paddle_finish_init_params(client C.paddle_pserver_client) C.int {
	c := get(client)
	err := c.FinishInitParams()
	if err != nil {
		if err.Error() == pserver.AlreadyInitialized {
			log.Println("parameters already initialized, treat paddle_finish_init_params as sucessful.")
			return 0
		}

		log.Println(err)
		return -1
	}

	return 0
}

//export paddle_send_grads
func paddle_send_grads(client C.paddle_pserver_client, grads *C.paddle_gradient, total C.int) C.int {
	var gs []pserver.Gradient
	for i := 0; i < int(total); i++ {
		grad := (*C.paddle_gradient)(unsafe.Pointer((uintptr(unsafe.Pointer(grads)) + uintptr(i)*unsafe.Sizeof(*grads))))
		et := pserver.ElementType(grad.element_type)
		name := C.GoString(grad.name)
		content := cArrayToSlice(unsafe.Pointer(grad.content), int(grad.content_len))
		gs = append(gs, pserver.Gradient{Name: name, ElementType: et, Content: content})
	}

	c := get(client)
	err := c.SendGrads(gs)
	if err != nil {
		log.Println(err)
		return -1
	}

	return 0
}

//export paddle_get_params
func paddle_get_params(client C.paddle_pserver_client, dst **C.paddle_parameter, total C.int) C.int {
	var ns []string
	for i := 0; i < int(total); i++ {
		param := *(**C.paddle_parameter)(unsafe.Pointer((uintptr(unsafe.Pointer(dst)) + uintptr(i)*unsafe.Sizeof(*dst))))
		ns = append(ns, C.GoString(param.name))
	}
	c := get(client)
	ps, err := c.GetParams(ns)
	if err != nil {
		log.Println(err)
		return -1
	}

	names := func() (string, string) {
		retNames := ""
		for _, p := range ps {
			if retNames == "" {
				retNames = p.Name
			} else {
				retNames = ", " + p.Name
			}
		}

		requestedNames := ""
		for _, n := range ns {
			if requestedNames == "" {
				requestedNames = n
			} else {
				requestedNames = ", " + n
			}
		}

		return requestedNames, retNames
	}

	if len(ps) != len(ns) {
		requestedNames, retNames := names()
		log.Printf("pserver returned wrong number of parameters. Requested: %s, returned: %s.\n", retNames, requestedNames)
		return -1
	}

	for i := range ps {
		if ns[i] != ps[i].Name {
			requestedNames, retNames := names()
			log.Printf("pserver returned wrong parameters, or not in requested order. Requested: %s, returned: %s.\n", retNames, requestedNames)
			return -1
		}
	}

	for i := 0; i < int(total); i++ {
		p := ps[i]
		param := *(**C.paddle_parameter)(unsafe.Pointer((uintptr(unsafe.Pointer(dst)) + uintptr(i)*unsafe.Sizeof(*dst))))

		if unsafe.Pointer(param) == nullPtr {
			log.Println("must pre-allocate parameter.")
			return -1
		} else {
			if unsafe.Pointer(param.content) != nullPtr {
				if int(param.content_len) != len(p.Content) {
					log.Printf("the pre-allocated content len does not match parameter content len. Pre-allocated len: %d, returned len: %d", param.content_len, len(p.Content))
					return -1
				}
			}
		}

		C.memcpy(unsafe.Pointer(param.content), unsafe.Pointer(&p.Content[0]), C.size_t(len(p.Content)))
		param.content_len = C.int(len(p.Content))
		param.element_type = C.paddle_element_type(p.ElementType)
	}

	return 0
}

//export paddle_save_model
func paddle_save_model(client C.paddle_pserver_client, path *C.char) C.int {
	p := C.GoString(path)
	c := get(client)
	err := c.Save(p)
	if err != nil {
		log.Println(err)
		return -1
	}

	return 0
}

func main() {} // Required but ignored
<|endoftext|>


<commit_before>package impl

import (
	s "../"
	m "../../model"
	"strconv"
	"time"
)

type tradingService struct {
	sessionDAO m.SessionDAO
	tradingDAO m.TradingDAO
	envDAO     m.EnvDAO
	seqDAO     m.SeqDAO
}

func NewTradingSerivce(s m.SessionDAO, t m.TradingDAO, models *m.Models) *tradingService {
	return &tradingService{
		sessionDAO: s,
		tradingDAO: t,
		envDAO:     models.Env,
		seqDAO:     models.Seq,
	}
}

func (s *tradingService) GetListByUser(token string) s.Result {
	// input check
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// get
	tradings, err := s.tradingDAO.GetListByUser(session.UserId)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	list := make([]interface{}, 0)
	for _, t := range tradings {
		list = append(list, s.toJson(t))
	}
	body := map[string]interface{}{
		"tradings": list,
	}
	return jsonResult(200, body)
}

func (s *tradingService) Create(token, companyId, subject, product, memo string, titleType int, workFrom, workTo, total, quotationDate, billDate, deliveryDate int64, taxRate float32) s.Result {
	// input check
	if len(companyId) == 0 {
		return errorResult(400, MSG_ERR_COMPANY_ID_EMPTY)
	}
	if len(subject) == 0 {
		return errorResult(400, MSG_ERR_SUBJECT_EMPTY)
	}

	// get session
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// create
	item, err := s.tradingDAO.Create(companyId, subject, titleType, workFrom, workTo, total, quotationDate, billDate, deliveryDate, taxRate, session.UserId, product, memo)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}

	body := map[string]interface{}{
		"id": item.Id,
	}
	return jsonResult(201, body)
}

func (s *tradingService) Update(token string, trading s.Trading) s.Result {
	// input check
	if len(trading.Id) == 0 {
		return errorResult(400, MSG_ERR_ID_EMPTY)
	}
	if len(trading.CompanyId) == 0 {
		return errorResult(400, MSG_ERR_COMPANY_ID_EMPTY)
	}
	if len(trading.Subject) == 0 {
		return errorResult(400, MSG_ERR_SUBJECT_EMPTY)
	}
	if len(trading.Product) == 0 {
		return errorResult(400, MSG_ERR_PRODUCT_EMPTY)
	}

	// get session
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}

	// get item
	item, err := s.tradingDAO.GetById(trading.Id, session.UserId)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if item == nil {
		return errorResult(404, MSG_TRADING_NOT_FOUND)
	}
	// update
	item2, err := s.tradingDAO.Update(trading.Trading)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}

	body := map[string]interface{}{
		"id": item2.Id,
	}
	return jsonResult(200, body)
}

func (o *tradingService) Delete(token, tradingId string) s.Result {
	// input check
	if len(tradingId) == 0 {
		return errorResult(400, MSG_ERR_ID_EMPTY)
	}

	// get session
	session, err := o.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}

	// delete
	err = o.tradingDAO.Delete(tradingId)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}

	return &result{
		status: 204,
		body:   "",
	}
}

func (s *tradingService) GetItemListByTradingId(token, tradingId string) s.Result {
	// input check
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// get trading
	// get
	items, err := s.tradingDAO.GetItemsById(tradingId)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	list := make([]interface{}, 0)
	for _, t := range items {
		list = append(list, map[string]interface{}{
			"id":         t.Id,
			"subject":    t.Subject,
			"unit_price": t.UnitPrice,
			"amount":     t.Amount,
			"degree":     t.Degree,
			"tax_type":   t.TaxType,
			"memo":       t.Memo,
		})
	}
	body := map[string]interface{}{
		"items": list,
	}
	return jsonResult(200, body)
}

func (s *tradingService) CreateItem(token, tradingId, subject, degree, memo string, sortOrder, unitPrice, amount, taxType int) s.Result {
	// input check
	// get session
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// get trading
	// create
	item, err := s.tradingDAO.CreateItem(tradingId, subject, degree, memo, sortOrder, unitPrice, amount, taxType)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	body := map[string]interface{}{
		"id": item.Id,
	}
	return jsonResult(201, body)
}

func (s *tradingService) UpdateItem(token, id, tradingId, subject, degree, memo string, sortOrder, unitPrice, amount, taxType int) s.Result {
	// input check
	// get session
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// get trading
	// create
	item, err := s.tradingDAO.UpdateItem(id, tradingId, subject, degree, memo, sortOrder, unitPrice, amount, taxType)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	body := map[string]interface{}{
		"id": item.Id,
	}
	return jsonResult(200, body)
}

func (s *tradingService) DeleteItem(token, id, tradingId string) s.Result {
	// input check
	// get session
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// soft delete
	err = s.tradingDAO.SoftDeleteItem(id, tradingId)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	return &result{
		status: 204,
		body:   "",
	}
}

func (o *tradingService) GetNextNumber(token, seqType string, date int64) s.Result {
	// input check
	if isEmpty(token) {
		return errorResult(401, s.ERR_TOKEN_EMPTY)
	}
	var seqTypeInt m.SeqType
	switch seqType {
	case "quotation":
		seqTypeInt = m.SeqType_Quotation
		break
	case "delivery":
		seqTypeInt = m.SeqType_Delivery
		break
	case "bill":
		seqTypeInt = m.SeqType_Bill
		break
	default:
		return errorResult(400, s.ERR_INVALID_SEQUENCE_TYPE)
	}
	// get session
	session, err := o.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// determine year
	t := time.Unix(date/1000, 0)
	year := t.Year()
	month := t.Month()
	env, err := o.envDAO.Get("closing_month")
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	intVal, _ := strconv.Atoi(env.Value)
	if int(month) <= intVal {
		year--
	}
	// get next sequence
	seq, err := o.seqDAO.Next(seqTypeInt, year)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}

	body := map[string]interface{}{
		"number": year*10000 + seq.Value,
	}
	return jsonResult(200, body)
}

func (s *tradingService) toJson(t *m.Trading) map[string]interface{} {
	return map[string]interface{}{
		"id":               t.Id,
		"company_id":       t.CompanyId,
		"subject":          t.Subject,
		"title_type":       t.TitleType,
		"work_from":        t.WorkFrom,
		"work_to":          t.WorkTo,
		"total":            t.Total,
		"quotation_date":   t.QuotationDate,
		"quotation_number": t.QuotationNumber,
		"bill_date":        t.BillDate,
		"bill_number":      t.BillNumber,
		"delivery_date":    t.DeliveryDate,
		"delivery_number":  t.DeliveryNumber,
		"tax_rate":         t.TaxRate,
		"assignee":         t.AssigneeId,
		"product":          t.Product,
		"memo":             t.Memo,
		"modified_time":    t.ModifiedTime * 1000,
	}
}
<commit_msg>remove product check<commit_after>package impl

import (
	s "../"
	m "../../model"
	"strconv"
	"time"
)

type tradingService struct {
	sessionDAO m.SessionDAO
	tradingDAO m.TradingDAO
	envDAO     m.EnvDAO
	seqDAO     m.SeqDAO
}

func NewTradingSerivce(s m.SessionDAO, t m.TradingDAO, models *m.Models) *tradingService {
	return &tradingService{
		sessionDAO: s,
		tradingDAO: t,
		envDAO:     models.Env,
		seqDAO:     models.Seq,
	}
}

func (s *tradingService) GetListByUser(token string) s.Result {
	// input check
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// get
	tradings, err := s.tradingDAO.GetListByUser(session.UserId)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	list := make([]interface{}, 0)
	for _, t := range tradings {
		list = append(list, s.toJson(t))
	}
	body := map[string]interface{}{
		"tradings": list,
	}
	return jsonResult(200, body)
}

func (s *tradingService) Create(token, companyId, subject, product, memo string, titleType int, workFrom, workTo, total, quotationDate, billDate, deliveryDate int64, taxRate float32) s.Result {
	// input check
	if len(companyId) == 0 {
		return errorResult(400, MSG_ERR_COMPANY_ID_EMPTY)
	}
	if len(subject) == 0 {
		return errorResult(400, MSG_ERR_SUBJECT_EMPTY)
	}

	// get session
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// create
	item, err := s.tradingDAO.Create(companyId, subject, titleType, workFrom, workTo, total, quotationDate, billDate, deliveryDate, taxRate, session.UserId, product, memo)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}

	body := map[string]interface{}{
		"id": item.Id,
	}
	return jsonResult(201, body)
}

func (s *tradingService) Update(token string, trading s.Trading) s.Result {
	// input check
	if len(trading.Id) == 0 {
		return errorResult(400, MSG_ERR_ID_EMPTY)
	}
	if len(trading.CompanyId) == 0 {
		return errorResult(400, MSG_ERR_COMPANY_ID_EMPTY)
	}
	if len(trading.Subject) == 0 {
		return errorResult(400, MSG_ERR_SUBJECT_EMPTY)
	}

	// get session
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}

	// get item
	item, err := s.tradingDAO.GetById(trading.Id, session.UserId)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if item == nil {
		return errorResult(404, MSG_TRADING_NOT_FOUND)
	}
	// update
	item2, err := s.tradingDAO.Update(trading.Trading)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}

	body := map[string]interface{}{
		"id": item2.Id,
	}
	return jsonResult(200, body)
}

func (o *tradingService) Delete(token, tradingId string) s.Result {
	// input check
	if len(tradingId) == 0 {
		return errorResult(400, MSG_ERR_ID_EMPTY)
	}

	// get session
	session, err := o.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}

	// delete
	err = o.tradingDAO.Delete(tradingId)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}

	return &result{
		status: 204,
		body:   "",
	}
}

func (s *tradingService) GetItemListByTradingId(token, tradingId string) s.Result {
	// input check
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// get trading
	// get
	items, err := s.tradingDAO.GetItemsById(tradingId)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	list := make([]interface{}, 0)
	for _, t := range items {
		list = append(list, map[string]interface{}{
			"id":         t.Id,
			"subject":    t.Subject,
			"unit_price": t.UnitPrice,
			"amount":     t.Amount,
			"degree":     t.Degree,
			"tax_type":   t.TaxType,
			"memo":       t.Memo,
		})
	}
	body := map[string]interface{}{
		"items": list,
	}
	return jsonResult(200, body)
}

func (s *tradingService) CreateItem(token, tradingId, subject, degree, memo string, sortOrder, unitPrice, amount, taxType int) s.Result {
	// input check
	// get session
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// get trading
	// create
	item, err := s.tradingDAO.CreateItem(tradingId, subject, degree, memo, sortOrder, unitPrice, amount, taxType)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	body := map[string]interface{}{
		"id": item.Id,
	}
	return jsonResult(201, body)
}

func (s *tradingService) UpdateItem(token, id, tradingId, subject, degree, memo string, sortOrder, unitPrice, amount, taxType int) s.Result {
	// input check
	// get session
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// get trading
	// create
	item, err := s.tradingDAO.UpdateItem(id, tradingId, subject, degree, memo, sortOrder, unitPrice, amount, taxType)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	body := map[string]interface{}{
		"id": item.Id,
	}
	return jsonResult(200, body)
}

func (s *tradingService) DeleteItem(token, id, tradingId string) s.Result {
	// input check
	// get session
	session, err := s.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// soft delete
	err = s.tradingDAO.SoftDeleteItem(id, tradingId)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	return &result{
		status: 204,
		body:   "",
	}
}

func (o *tradingService) GetNextNumber(token, seqType string, date int64) s.Result {
	// input check
	if isEmpty(token) {
		return errorResult(401, s.ERR_TOKEN_EMPTY)
	}
	var seqTypeInt m.SeqType
	switch seqType {
	case "quotation":
		seqTypeInt = m.SeqType_Quotation
		break
	case "delivery":
		seqTypeInt = m.SeqType_Delivery
		break
	case "bill":
		seqTypeInt = m.SeqType_Bill
		break
	default:
		return errorResult(400, s.ERR_INVALID_SEQUENCE_TYPE)
	}
	// get session
	session, err := o.sessionDAO.GetByToken(token)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	if session == nil {
		return errorResult(401, MSG_WRONG_TOKEN)
	}
	// determine year
	t := time.Unix(date/1000, 0)
	year := t.Year()
	month := t.Month()
	env, err := o.envDAO.Get("closing_month")
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}
	intVal, _ := strconv.Atoi(env.Value)
	if int(month) <= intVal {
		year--
	}
	// get next sequence
	seq, err := o.seqDAO.Next(seqTypeInt, year)
	if err != nil {
		return errorResult(500, MSG_SERVER_ERROR)
	}

	body := map[string]interface{}{
		"number": year*10000 + seq.Value,
	}
	return jsonResult(200, body)
}

func (s *tradingService) toJson(t *m.Trading) map[string]interface{} {
	return map[string]interface{}{
		"id":               t.Id,
		"company_id":       t.CompanyId,
		"subject":          t.Subject,
		"title_type":       t.TitleType,
		"work_from":        t.WorkFrom,
		"work_to":          t.WorkTo,
		"total":            t.Total,
		"quotation_date":   t.QuotationDate,
		"quotation_number": t.QuotationNumber,
		"bill_date":        t.BillDate,
		"bill_number":      t.BillNumber,
		"delivery_date":    t.DeliveryDate,
		"delivery_number":  t.DeliveryNumber,
		"tax_rate":         t.TaxRate,
		"assignee":         t.AssigneeId,
		"product":          t.Product,
		"memo":             t.Memo,
		"modified_time":    t.ModifiedTime * 1000,
	}
}
<|endoftext|>