<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package components

import (
	"fmt"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/loader"
)

// AWSCloudControllerManagerOptionsBuilder adds options for the kubernetes controller manager to the model.
type AWSCloudControllerManagerOptionsBuilder struct {
	*OptionsContext
}

var _ loader.OptionsBuilder = &AWSCloudControllerManagerOptionsBuilder{}

// BuildOptions generates the configurations used for the AWS cloud controller manager manifest
func (b *AWSCloudControllerManagerOptionsBuilder) BuildOptions(o interface{}) error {
	clusterSpec := o.(*kops.ClusterSpec)

	if clusterSpec.GetCloudProvider() != kops.CloudProviderAWS {
		return nil
	}

	if clusterSpec.ExternalCloudControllerManager == nil && b.IsKubernetesGTE("1.24") {
		clusterSpec.ExternalCloudControllerManager = &kops.CloudControllerManagerConfig{}
	}

	eccm := clusterSpec.ExternalCloudControllerManager

	if eccm == nil {
		return nil
	}

	// No significant downside to always doing a leader election.
	// Also, having multiple control plane nodes requires leader election.
	eccm.LeaderElection = &kops.LeaderElectionConfiguration{LeaderElect: fi.Bool(true)}

	eccm.ClusterName = b.ClusterName

	eccm.ClusterCIDR = clusterSpec.NonMasqueradeCIDR

	eccm.AllocateNodeCIDRs = fi.Bool(true)
	eccm.ConfigureCloudRoutes = fi.Bool(false)

	// TODO: we want to consolidate this with the logic from KCM
	networking := clusterSpec.Networking
	if networking == nil {
		eccm.ConfigureCloudRoutes = fi.Bool(true)
	} else if networking.Kubenet != nil {
		eccm.ConfigureCloudRoutes = fi.Bool(true)
	} else if networking.GCE != nil {
		eccm.ConfigureCloudRoutes = fi.Bool(false)
		eccm.CIDRAllocatorType = fi.String("CloudAllocator")

		if eccm.ClusterCIDR == "" {
			eccm.ClusterCIDR = clusterSpec.PodCIDR
		}
	} else if networking.External != nil {
		eccm.ConfigureCloudRoutes = fi.Bool(false)
	} else if UsesCNI(networking) {
		eccm.ConfigureCloudRoutes = fi.Bool(false)
	} else if networking.Kopeio != nil {
		// Kopeio is based on kubenet / external
		eccm.ConfigureCloudRoutes = fi.Bool(false)
	} else {
		return fmt.Errorf("no networking mode set")
	}

	if eccm.Image == "" {
		// See https://us.gcr.io/k8s-artifacts-prod/provider-aws/cloud-controller-manager
		switch b.KubernetesVersion.Minor {
		case 19:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.19.0-alpha.1"
		case 20:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.20.0-alpha.0"
		case 21:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.21.0-alpha.0"
		case 22:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.22.0"
		case 23:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.23.0-alpha.0"
		case 24:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.23.0-alpha.0"
		case 25:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.23.0-alpha.0"
		default:
			eccm.Image = "gcr.io/k8s-staging-provider-aws/cloud-controller-manager:latest"
		}
	}

	if b.IsKubernetesGTE("1.24") && b.IsKubernetesLT("1.25") {
		eccm.EnableLeaderMigration = fi.Bool(true)
	}

	return nil
}
<commit_msg>Use latest staging AWS CCM for k8s 1.24+<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package components

import (
	"fmt"

	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/loader"
)

// AWSCloudControllerManagerOptionsBuilder adds options for the kubernetes controller manager to the model.
type AWSCloudControllerManagerOptionsBuilder struct {
	*OptionsContext
}

var _ loader.OptionsBuilder = &AWSCloudControllerManagerOptionsBuilder{}

// BuildOptions generates the configurations used for the AWS cloud controller manager manifest
func (b *AWSCloudControllerManagerOptionsBuilder) BuildOptions(o interface{}) error {
	clusterSpec := o.(*kops.ClusterSpec)

	if clusterSpec.GetCloudProvider() != kops.CloudProviderAWS {
		return nil
	}

	if clusterSpec.ExternalCloudControllerManager == nil && b.IsKubernetesGTE("1.24") {
		clusterSpec.ExternalCloudControllerManager = &kops.CloudControllerManagerConfig{}
	}

	eccm := clusterSpec.ExternalCloudControllerManager

	if eccm == nil {
		return nil
	}

	// No significant downside to always doing a leader election.
	// Also, having multiple control plane nodes requires leader election.
	eccm.LeaderElection = &kops.LeaderElectionConfiguration{LeaderElect: fi.Bool(true)}

	eccm.ClusterName = b.ClusterName

	eccm.ClusterCIDR = clusterSpec.NonMasqueradeCIDR

	eccm.AllocateNodeCIDRs = fi.Bool(true)
	eccm.ConfigureCloudRoutes = fi.Bool(false)

	// TODO: we want to consolidate this with the logic from KCM
	networking := clusterSpec.Networking
	if networking == nil {
		eccm.ConfigureCloudRoutes = fi.Bool(true)
	} else if networking.Kubenet != nil {
		eccm.ConfigureCloudRoutes = fi.Bool(true)
	} else if networking.GCE != nil {
		eccm.ConfigureCloudRoutes = fi.Bool(false)
		eccm.CIDRAllocatorType = fi.String("CloudAllocator")

		if eccm.ClusterCIDR == "" {
			eccm.ClusterCIDR = clusterSpec.PodCIDR
		}
	} else if networking.External != nil {
		eccm.ConfigureCloudRoutes = fi.Bool(false)
	} else if UsesCNI(networking) {
		eccm.ConfigureCloudRoutes = fi.Bool(false)
	} else if networking.Kopeio != nil {
		// Kopeio is based on kubenet / external
		eccm.ConfigureCloudRoutes = fi.Bool(false)
	} else {
		return fmt.Errorf("no networking mode set")
	}

	if eccm.Image == "" {
		// See https://us.gcr.io/k8s-artifacts-prod/provider-aws/cloud-controller-manager
		switch b.KubernetesVersion.Minor {
		case 19:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.19.0-alpha.1"
		case 20:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.20.0-alpha.0"
		case 21:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.21.0-alpha.0"
		case 22:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.22.0"
		case 23:
			eccm.Image = "registry.k8s.io/provider-aws/cloud-controller-manager:v1.23.0-alpha.0"
		default:
			eccm.Image = "gcr.io/k8s-staging-provider-aws/cloud-controller-manager:latest"
		}
	}

	if b.IsKubernetesGTE("1.24") && b.IsKubernetesLT("1.25") {
		eccm.EnableLeaderMigration = fi.Bool(true)
	}

	return nil
}
<|endoftext|>


<commit_before>package migrations

import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator"

func addDashboardMigration(mg *Migrator) {
	var dashboardV1 = Table{
		Name: "dashboard",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "version", Type: DB_Int, Nullable: false},
			{Name: "slug", Type: DB_NVarchar, Length: 190, Nullable: false},
			{Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false},
			{Name: "data", Type: DB_Text, Nullable: false},
			{Name: "account_id", Type: DB_BigInt, Nullable: false},
			{Name: "created", Type: DB_DateTime, Nullable: false},
			{Name: "updated", Type: DB_DateTime, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"account_id"}},
			{Cols: []string{"account_id", "slug"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("create dashboard table", NewAddTableMigration(dashboardV1))

	//-------  indexes ------------------
	mg.AddMigration("add index dashboard.account_id", NewAddIndexMigration(dashboardV1, dashboardV1.Indices[0]))
	mg.AddMigration("add unique index dashboard_account_id_slug", NewAddIndexMigration(dashboardV1, dashboardV1.Indices[1]))

	dashboardTagV1 := Table{
		Name: "dashboard_tag",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "dashboard_id", Type: DB_BigInt, Nullable: false},
			{Name: "term", Type: DB_NVarchar, Length: 50, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"dashboard_id", "term"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("create dashboard_tag table", NewAddTableMigration(dashboardTagV1))
	mg.AddMigration("add unique index dashboard_tag.dasboard_id_term", NewAddIndexMigration(dashboardTagV1, dashboardTagV1.Indices[0]))

	// ---------------------
	// account -> org changes

	//-------  drop dashboard indexes ------------------
	addDropAllIndicesMigrations(mg, "v1", dashboardTagV1)
	//------- rename table ------------------
	addTableRenameMigration(mg, "dashboard", "dashboard_v1", "v1")

	// dashboard v2
	var dashboardV2 = Table{
		Name: "dashboard",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "version", Type: DB_Int, Nullable: false},
			{Name: "slug", Type: DB_NVarchar, Length: 189, Nullable: false},
			{Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false},
			{Name: "data", Type: DB_Text, Nullable: false},
			{Name: "org_id", Type: DB_BigInt, Nullable: false},
			{Name: "created", Type: DB_DateTime, Nullable: false},
			{Name: "updated", Type: DB_DateTime, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"org_id"}},
			{Cols: []string{"org_id", "slug"}, Type: UniqueIndex},
		},
	}

	// recreate table
	mg.AddMigration("create dashboard v2", NewAddTableMigration(dashboardV2))
	// recreate indices
	addTableIndicesMigrations(mg, "v2", dashboardV2)
	// copy data
	mg.AddMigration("copy dashboard v1 to v2", NewCopyTableDataMigration("dashboard", "dashboard_v1", map[string]string{
		"id":      "id",
		"version": "version",
		"slug":    "slug",
		"title":   "title",
		"data":    "data",
		"org_id":  "account_id",
		"created": "created",
		"updated": "updated",
	}))

	mg.AddMigration("drop table dashboard_v1", NewDropTableMigration("dashboard_v1"))

	// change column type of dashboard.data
	mg.AddMigration("alter dashboard.data to mediumtext v1", new(RawSqlMigration).
		Sqlite("SELECT 0 WHERE 0;").
		Postgres("SELECT 0;").
		Mysql("ALTER TABLE dashboard MODIFY data MEDIUMTEXT;"))

	// add column to store updater of a dashboard
	mg.AddMigration("Add column updated_by in dashboard - v2", NewAddColumnMigration(dashboardV2, &Column{
		Name: "updated_by", Type: DB_Int, Nullable: true,
	}))

	// add column to store creator of a dashboard
	mg.AddMigration("Add column created_by in dashboard - v2", NewAddColumnMigration(dashboardV2, &Column{
		Name: "created_by", Type: DB_Int, Nullable: true,
	}))

	// add column to store gnetId
	mg.AddMigration("Add column gnetId in dashboard", NewAddColumnMigration(dashboardV2, &Column{
		Name: "gnet_id", Type: DB_BigInt, Nullable: true,
	}))

	mg.AddMigration("Add index for gnetId in dashboard", NewAddIndexMigration(dashboardV2, &Index{
		Cols: []string{"gnet_id"}, Type: IndexType,
	}))

	// add column to store plugin_id
	mg.AddMigration("Add column plugin_id in dashboard", NewAddColumnMigration(dashboardV2, &Column{
		Name: "plugin_id", Type: DB_NVarchar, Nullable: true, Length: 255,
	}))

	mg.AddMigration("Add index for plugin_id in dashboard", NewAddIndexMigration(dashboardV2, &Index{
		Cols: []string{"org_id", "plugin_id"}, Type: IndexType,
	}))

	// dashboard_id index for dashboard_tag table
	mg.AddMigration("Add index for dashboard_id in dashboard_tag", NewAddIndexMigration(dashboardTagV1, &Index{
		Cols: []string{"dashboard_id"}, Type: IndexType,
	}))

	mg.AddMigration("Update dashboard table charset", NewTableCharsetMigration("dashboard", []*Column{
		{Name: "slug", Type: DB_NVarchar, Length: 189, Nullable: false},
		{Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false},
		{Name: "plugin_id", Type: DB_NVarchar, Nullable: true, Length: 255},
		{Name: "data", Type: DB_MediumText, Nullable: false},
	}))

	mg.AddMigration("Update dashboard_tag table charset", NewTableCharsetMigration("dashboard_tag", []*Column{
		{Name: "term", Type: DB_NVarchar, Length: 50, Nullable: false},
	}))
}
<commit_msg>reduce length of dashboard columns used in compound indexes (#8507)<commit_after>package migrations

import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator"

func addDashboardMigration(mg *Migrator) {
	var dashboardV1 = Table{
		Name: "dashboard",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "version", Type: DB_Int, Nullable: false},
			{Name: "slug", Type: DB_NVarchar, Length: 189, Nullable: false},
			{Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false},
			{Name: "data", Type: DB_Text, Nullable: false},
			{Name: "account_id", Type: DB_BigInt, Nullable: false},
			{Name: "created", Type: DB_DateTime, Nullable: false},
			{Name: "updated", Type: DB_DateTime, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"account_id"}},
			{Cols: []string{"account_id", "slug"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("create dashboard table", NewAddTableMigration(dashboardV1))

	//-------  indexes ------------------
	mg.AddMigration("add index dashboard.account_id", NewAddIndexMigration(dashboardV1, dashboardV1.Indices[0]))
	mg.AddMigration("add unique index dashboard_account_id_slug", NewAddIndexMigration(dashboardV1, dashboardV1.Indices[1]))

	dashboardTagV1 := Table{
		Name: "dashboard_tag",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "dashboard_id", Type: DB_BigInt, Nullable: false},
			{Name: "term", Type: DB_NVarchar, Length: 50, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"dashboard_id", "term"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("create dashboard_tag table", NewAddTableMigration(dashboardTagV1))
	mg.AddMigration("add unique index dashboard_tag.dasboard_id_term", NewAddIndexMigration(dashboardTagV1, dashboardTagV1.Indices[0]))

	// ---------------------
	// account -> org changes

	//-------  drop dashboard indexes ------------------
	addDropAllIndicesMigrations(mg, "v1", dashboardTagV1)
	//------- rename table ------------------
	addTableRenameMigration(mg, "dashboard", "dashboard_v1", "v1")

	// dashboard v2
	var dashboardV2 = Table{
		Name: "dashboard",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "version", Type: DB_Int, Nullable: false},
			{Name: "slug", Type: DB_NVarchar, Length: 189, Nullable: false},
			{Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false},
			{Name: "data", Type: DB_Text, Nullable: false},
			{Name: "org_id", Type: DB_BigInt, Nullable: false},
			{Name: "created", Type: DB_DateTime, Nullable: false},
			{Name: "updated", Type: DB_DateTime, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"org_id"}},
			{Cols: []string{"org_id", "slug"}, Type: UniqueIndex},
		},
	}

	// recreate table
	mg.AddMigration("create dashboard v2", NewAddTableMigration(dashboardV2))
	// recreate indices
	addTableIndicesMigrations(mg, "v2", dashboardV2)
	// copy data
	mg.AddMigration("copy dashboard v1 to v2", NewCopyTableDataMigration("dashboard", "dashboard_v1", map[string]string{
		"id":      "id",
		"version": "version",
		"slug":    "slug",
		"title":   "title",
		"data":    "data",
		"org_id":  "account_id",
		"created": "created",
		"updated": "updated",
	}))

	mg.AddMigration("drop table dashboard_v1", NewDropTableMigration("dashboard_v1"))

	// change column type of dashboard.data
	mg.AddMigration("alter dashboard.data to mediumtext v1", new(RawSqlMigration).
		Sqlite("SELECT 0 WHERE 0;").
		Postgres("SELECT 0;").
		Mysql("ALTER TABLE dashboard MODIFY data MEDIUMTEXT;"))

	// add column to store updater of a dashboard
	mg.AddMigration("Add column updated_by in dashboard - v2", NewAddColumnMigration(dashboardV2, &Column{
		Name: "updated_by", Type: DB_Int, Nullable: true,
	}))

	// add column to store creator of a dashboard
	mg.AddMigration("Add column created_by in dashboard - v2", NewAddColumnMigration(dashboardV2, &Column{
		Name: "created_by", Type: DB_Int, Nullable: true,
	}))

	// add column to store gnetId
	mg.AddMigration("Add column gnetId in dashboard", NewAddColumnMigration(dashboardV2, &Column{
		Name: "gnet_id", Type: DB_BigInt, Nullable: true,
	}))

	mg.AddMigration("Add index for gnetId in dashboard", NewAddIndexMigration(dashboardV2, &Index{
		Cols: []string{"gnet_id"}, Type: IndexType,
	}))

	// add column to store plugin_id
	mg.AddMigration("Add column plugin_id in dashboard", NewAddColumnMigration(dashboardV2, &Column{
		Name: "plugin_id", Type: DB_NVarchar, Nullable: true, Length: 189,
	}))

	mg.AddMigration("Add index for plugin_id in dashboard", NewAddIndexMigration(dashboardV2, &Index{
		Cols: []string{"org_id", "plugin_id"}, Type: IndexType,
	}))

	// dashboard_id index for dashboard_tag table
	mg.AddMigration("Add index for dashboard_id in dashboard_tag", NewAddIndexMigration(dashboardTagV1, &Index{
		Cols: []string{"dashboard_id"}, Type: IndexType,
	}))

	mg.AddMigration("Update dashboard table charset", NewTableCharsetMigration("dashboard", []*Column{
		{Name: "slug", Type: DB_NVarchar, Length: 189, Nullable: false},
		{Name: "title", Type: DB_NVarchar, Length: 255, Nullable: false},
		{Name: "plugin_id", Type: DB_NVarchar, Nullable: true, Length: 189},
		{Name: "data", Type: DB_MediumText, Nullable: false},
	}))

	mg.AddMigration("Update dashboard_tag table charset", NewTableCharsetMigration("dashboard_tag", []*Column{
		{Name: "term", Type: DB_NVarchar, Length: 50, Nullable: false},
	}))
}
<|endoftext|>


<commit_before>package worker

import (
	"encoding/json"
	"fmt"
	"time"

	gocontext "context"

	"github.com/bitly/go-simplejson"
	"github.com/streadway/amqp"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/context"
	"github.com/travis-ci/worker/metrics"
)

type amqpJob struct {
	conn            *amqp.Connection
	delivery        amqp.Delivery
	payload         *JobPayload
	rawPayload      *simplejson.Json
	startAttributes *backend.StartAttributes
	received        time.Time
	started         time.Time
}

func (j *amqpJob) GoString() string {
	return fmt.Sprintf("&amqpJob{conn: %#v, delivery: %#v, payload: %#v, startAttributes: %#v}",
		j.conn, j.delivery, j.payload, j.startAttributes)
}

func (j *amqpJob) Payload() *JobPayload {
	return j.payload
}

func (j *amqpJob) RawPayload() *simplejson.Json {
	return j.rawPayload
}

func (j *amqpJob) StartAttributes() *backend.StartAttributes {
	return j.startAttributes
}

func (j *amqpJob) Error(ctx gocontext.Context, errMessage string) error {
	log, err := j.LogWriter(ctx, time.Minute)
	if err != nil {
		return err
	}

	_, err = log.WriteAndClose([]byte(errMessage))
	if err != nil {
		return err
	}

	return j.Finish(ctx, FinishStateErrored)
}

func (j *amqpJob) Requeue(ctx gocontext.Context) error {
	context.LoggerFromContext(ctx).Info("requeueing job")

	metrics.Mark("worker.job.requeue")

	err := j.sendStateUpdate("job:test:reset", map[string]interface{}{
		"id":    j.Payload().Job.ID,
		"state": "reset",
	})
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) Received() error {
	j.received = time.Now()

	payload := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       "received",
		"received_at": j.received.UTC().Format(time.RFC3339),
	}

	if j.Payload().Job.QueuedAt != nil {
		metrics.TimeSince("travis.worker.job.queue_time", *j.payload.Job.QueuedAt)
		payload["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}

	return j.sendStateUpdate("job:test:receive", payload)
}

func (j *amqpJob) Started() error {
	j.started = time.Now()

	metrics.TimeSince("travis.worker.job.start_time", j.received)

	payload := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       "started",
		"received_at": j.received.UTC().Format(time.RFC3339),
		"started_at":  j.started.UTC().Format(time.RFC3339),
	}

	if j.Payload().Job.QueuedAt != nil {
		payload["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}

	return j.sendStateUpdate("job:test:start", payload)
}

func (j *amqpJob) Finish(ctx gocontext.Context, state FinishState) error {
	context.LoggerFromContext(ctx).WithField("state", state).Info("finishing job")

	finishedAt := time.Now()
	receivedAt := j.received
	if receivedAt.IsZero() {
		receivedAt = finishedAt
	}
	startedAt := j.started
	if startedAt.IsZero() {
		startedAt = finishedAt
	}

	metrics.Mark(fmt.Sprintf("travis.worker.job.finish.%s", state))
	metrics.Mark("travis.worker.job.finish")

	payload := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       state,
		"received_at": receivedAt.UTC().Format(time.RFC3339),
		"started_at":  startedAt.UTC().Format(time.RFC3339),
		"finished_at": finishedAt.UTC().Format(time.RFC3339),
	}

	if j.Payload().Job.QueuedAt != nil {
		payload["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}

	err := j.sendStateUpdate("job:test:finish", payload)
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) LogWriter(ctx gocontext.Context, defaultLogTimeout time.Duration) (LogWriter, error) {
	logTimeout := time.Duration(j.payload.Timeouts.LogSilence) * time.Second
	if logTimeout == 0 {
		logTimeout = defaultLogTimeout
	}

	return newAMQPLogWriter(ctx, j.conn, j.payload.Job.ID, logTimeout)
}

func (j *amqpJob) sendStateUpdate(event string, body map[string]interface{}) error {
	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	return amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now().UTC(),
		Type:         event,
		Body:         bodyBytes,
	})
}
<commit_msg>amqp-job: send timestamps on requeue<commit_after>package worker

import (
	"encoding/json"
	"fmt"
	"time"

	gocontext "context"

	"github.com/bitly/go-simplejson"
	"github.com/streadway/amqp"
	"github.com/travis-ci/worker/backend"
	"github.com/travis-ci/worker/context"
	"github.com/travis-ci/worker/metrics"
)

type amqpJob struct {
	conn            *amqp.Connection
	delivery        amqp.Delivery
	payload         *JobPayload
	rawPayload      *simplejson.Json
	startAttributes *backend.StartAttributes
	received        time.Time
	started         time.Time
}

func (j *amqpJob) GoString() string {
	return fmt.Sprintf("&amqpJob{conn: %#v, delivery: %#v, payload: %#v, startAttributes: %#v}",
		j.conn, j.delivery, j.payload, j.startAttributes)
}

func (j *amqpJob) Payload() *JobPayload {
	return j.payload
}

func (j *amqpJob) RawPayload() *simplejson.Json {
	return j.rawPayload
}

func (j *amqpJob) StartAttributes() *backend.StartAttributes {
	return j.startAttributes
}

func (j *amqpJob) Error(ctx gocontext.Context, errMessage string) error {
	log, err := j.LogWriter(ctx, time.Minute)
	if err != nil {
		return err
	}

	_, err = log.WriteAndClose([]byte(errMessage))
	if err != nil {
		return err
	}

	return j.Finish(ctx, FinishStateErrored)
}

func (j *amqpJob) Requeue(ctx gocontext.Context) error {
	context.LoggerFromContext(ctx).Info("requeueing job")

	metrics.Mark("worker.job.requeue")

	payload := map[string]interface{}{
		"id":    j.Payload().Job.ID,
		"state": "reset",
	}

	if j.Payload().Job.QueuedAt != nil {
		payload["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}

	if !j.received.IsZero() {
		payload["received_at"] = j.received.UTC().Format(time.RFC3339)
	}
	if !j.started.IsZero() {
		payload["started_at"] = j.started.UTC().Format(time.RFC3339)
	}

	err := j.sendStateUpdate("job:test:reset", payload)
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) Received() error {
	j.received = time.Now()

	payload := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       "received",
		"received_at": j.received.UTC().Format(time.RFC3339),
	}

	if j.Payload().Job.QueuedAt != nil {
		metrics.TimeSince("travis.worker.job.queue_time", *j.payload.Job.QueuedAt)
		payload["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}

	return j.sendStateUpdate("job:test:receive", payload)
}

func (j *amqpJob) Started() error {
	j.started = time.Now()

	metrics.TimeSince("travis.worker.job.start_time", j.received)

	payload := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       "started",
		"received_at": j.received.UTC().Format(time.RFC3339),
		"started_at":  j.started.UTC().Format(time.RFC3339),
	}

	if j.Payload().Job.QueuedAt != nil {
		payload["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}

	return j.sendStateUpdate("job:test:start", payload)
}

func (j *amqpJob) Finish(ctx gocontext.Context, state FinishState) error {
	context.LoggerFromContext(ctx).WithField("state", state).Info("finishing job")

	finishedAt := time.Now()
	receivedAt := j.received
	if receivedAt.IsZero() {
		receivedAt = finishedAt
	}
	startedAt := j.started
	if startedAt.IsZero() {
		startedAt = finishedAt
	}

	metrics.Mark(fmt.Sprintf("travis.worker.job.finish.%s", state))
	metrics.Mark("travis.worker.job.finish")

	payload := map[string]interface{}{
		"id":          j.Payload().Job.ID,
		"state":       state,
		"received_at": receivedAt.UTC().Format(time.RFC3339),
		"started_at":  startedAt.UTC().Format(time.RFC3339),
		"finished_at": finishedAt.UTC().Format(time.RFC3339),
	}

	if j.Payload().Job.QueuedAt != nil {
		payload["queued_at"] = j.Payload().Job.QueuedAt.UTC().Format(time.RFC3339)
	}

	err := j.sendStateUpdate("job:test:finish", payload)
	if err != nil {
		return err
	}

	return j.delivery.Ack(false)
}

func (j *amqpJob) LogWriter(ctx gocontext.Context, defaultLogTimeout time.Duration) (LogWriter, error) {
	logTimeout := time.Duration(j.payload.Timeouts.LogSilence) * time.Second
	if logTimeout == 0 {
		logTimeout = defaultLogTimeout
	}

	return newAMQPLogWriter(ctx, j.conn, j.payload.Job.ID, logTimeout)
}

func (j *amqpJob) sendStateUpdate(event string, body map[string]interface{}) error {
	amqpChan, err := j.conn.Channel()
	if err != nil {
		return err
	}
	defer amqpChan.Close()

	bodyBytes, err := json.Marshal(body)
	if err != nil {
		return err
	}

	_, err = amqpChan.QueueDeclare("reporting.jobs.builds", true, false, false, false, nil)
	if err != nil {
		return err
	}

	return amqpChan.Publish("", "reporting.jobs.builds", false, false, amqp.Publishing{
		ContentType:  "application/json",
		DeliveryMode: amqp.Persistent,
		Timestamp:    time.Now().UTC(),
		Type:         event,
		Body:         bodyBytes,
	})
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"
	"time"

	"github.com/gopherjs/gopherjs/js"
	"github.com/gopherjs/jquery"
)

// AjaxError is returned on the error channel after a call to an Ajax method.
type AjaxError struct {
	StatusCode int
	Message    string
}

//AjaxPut behaves indentically to AjaxPost other than using the method PUT.
func AjaxPut(ptrToStruct interface{}, path string) (chan interface{}, chan AjaxError) {
	return putOrPost(ptrToStruct, path, "PUT")
}

//AjaxPost sends an instance of a wire type to the server.  The first argument
//should be a wire type and must be a pointer to a struct or this function
//will panic. The value sent to the server is supplied in the first argument.
//The two returned values are a content channel and an error channel.  If the
//call succeeds, the content channel will be sent a different instance of the
//same type as the first argument. If the result from the server cannot be understood
//as the type of the first argument, the special error code 418 will be sent
//on the error channel.  If we fail to encode the object to be sent, the error
//code 420 will be sent on the error channel and no call to the server is made.
func AjaxPost(ptrToStruct interface{}, path string) (chan interface{}, chan AjaxError) {
	return putOrPost(ptrToStruct, path, "POST")
}

//AjaxIndex retreives a collection of wire types from the server.
//If the first argument is not a pointer to a slice of pointer to struct,
//it will panic.  The first element should be a slice of wire types.
//The returned values are a content channel and an error channel.  The content
//channel will receive the same type as your first argument if anything.  The error
//channel is used for non-200 http responses and the special error code 418
//is used to indicate that the received json from the server could not be successfully
//parsed as the type of the first argument.
func AjaxIndex(ptrToSliceOfPtrToStruct interface{}, path string) (chan interface{}, chan AjaxError) {
	isPointerToSliceOfPointerToStructOrPanic(ptrToSliceOfPtrToStruct)
	contentCh := make(chan interface{})
	errCh := make(chan AjaxError)
	ajaxRawChannels(ptrToSliceOfPtrToStruct, "", contentCh, errCh, "GET", path)
	return contentCh, errCh
}

//AjaxGet retreives an instance of a wire type from the server and decodes the result as
//Json. If the first argument is not a pointer to a struct, it will panic.
//The first argument should be a wire type that you expect to receive in the success
//case.  The returned values are a content channel and an error channel.  The content
//channel will receive the same type as your first argument if anything.  The error
//channel is used for non-200 http responses and the special error code 418
//is used to indicate that the received json from the server could not be successfully
//parsed as the type of the first argument.
func AjaxGet(ptrToStruct interface{}, path string) (chan interface{}, chan AjaxError) {
	isPointerToStructOrPanic(ptrToStruct)
	contentCh := make(chan interface{})
	errCh := make(chan AjaxError)
	ajaxRawChannels(ptrToStruct, "", contentCh, errCh, "GET", path)
	return contentCh, errCh
}

func ajaxRawChannels(output interface{}, body string, contentChan chan interface{}, errChan chan AjaxError,
	method string, path string) error {

	m := map[string]interface{}{
		"contentType": "application/json",
		"dataType":    "text",
		"type":        method,
		"url":         path,
		"cache":       false,
	}
	if body != "" {
		m["data"] = body
	}

	jquery.Ajax(m).
		Then(func(valueCreated *js.Object) {
		rd := strings.NewReader(valueCreated.String())
		dec := json.NewDecoder(rd)
		if err := dec.Decode(output); err != nil {
			go func() {
				errChan <- AjaxError{418, err.Error()}
			}()
			return
		}
		go func() {
			contentChan <- output
		}()
	}).
		Fail(func(p1 *js.Object) {
		go func() {
			errChan <- AjaxError{p1.Get("status").Int(), p1.Get("responseText").String()}
		}()
	})

	return nil
}

//
// HELPERS
//

func typeToUrlName(i interface{}) string {
	name, ok := i.(string)
	if !ok {
		name = fmt.Sprintf("%T", i)
	}
	pair := strings.Split(name, ".")
	if len(pair) != 2 {
		panic(fmt.Sprintf("unable to understand type name: %s", name))
	}
	return strings.ToLower(pair[1])
}

func encodeBody(i interface{}) (string, error) {
	//encode body
	var w bytes.Buffer
	enc := json.NewEncoder(&w)
	err := enc.Encode(i)
	if err != nil {
		return "", fmt.Errorf("error encoding body: %v ", err)
	}
	return w.String(), nil
}

func isPointerToStructOrPanic(i interface{}) reflect.Type {
	t := reflect.TypeOf(i)
	if t.Kind() != reflect.Ptr {
		panic(fmt.Sprintf("expected ptr to struct but got %T", i))
	}
	if t.Elem().Kind() != reflect.Struct {
		panic(fmt.Sprintf("expected ptr to struct but got ptr to %v", t.Elem().Kind()))
	}
	return t
}

func isPointerToSliceOfPointerToStructOrPanic(i interface{}) reflect.Type {
	t := reflect.TypeOf(i)
	if t.Kind() != reflect.Ptr {
		panic(fmt.Sprintf("expected ptr to slice of ptr to struct but got %T", i))
	}
	if t.Elem().Kind() != reflect.Slice {
		panic(fmt.Sprintf("expected ptr to SLICE of ptr to struct but got ptr to %v", t.Elem().Kind()))
	}
	if t.Elem().Elem().Kind() != reflect.Ptr {
		panic(fmt.Sprintf("expected ptr to slice of PTR to struct but got ptr to slice of %v", t.Elem().Elem().Kind()))
	}
	if t.Elem().Elem().Elem().Kind() != reflect.Struct {
		panic(fmt.Sprintf("expected ptr to slice of ptr to STRUCT but got ptr to slice of ptr to %v", t.Elem().Elem().Elem().Kind()))
	}
	return t
}

func putOrPost(ptrToStruct interface{}, path string, method string) (chan interface{}, chan AjaxError) {
	t := isPointerToStructOrPanic(ptrToStruct)
	output := reflect.New(t.Elem())
	contentCh := make(chan interface{})
	errCh := make(chan AjaxError)
	body, err := encodeBody(ptrToStruct)
	if err != nil {
		go func() {
			errCh <- AjaxError{420, err.Error()}
		}()
	} else {
		ajaxRawChannels(output.Interface(), body, contentCh, errCh, method, path)
	}
	return contentCh, errCh
}

func getFieldName(f reflect.StructField) string {
	name := f.Tag.Get("json")
	jsonPreferred := ""
	if name != "" {
		parts := strings.Split(name, ",")
		for _, part := range parts {
			if part == "-" {
				return "-"
			}
			if part == "omitempty" {
				continue
			}
			jsonPreferred = part
		}
	}
	if jsonPreferred != "" {
		return jsonPreferred
	}
	return f.Name
}

//
// DEPRECATED
//

//UnpackJson has been deprecated in favor of the Ajax methods.  This method
//is a naive json unpacker that uses reflection on the go struct to convert
//javascript values.  It cannot handle arbitrary types of fields, cannot handle
//nested structures, nor can it handle the UnmarshalJson interface.
func UnpackJson(ptrToStruct interface{}, jsonBlob *js.Object) error {
	t := reflect.TypeOf(ptrToStruct)
	if t.Kind() != reflect.Ptr {
		return fmt.Errorf("expected pointer to struct, but got %v", t.Kind())
	}
	elem := t.Elem()
	if elem.Kind() != reflect.Struct {
		return fmt.Errorf("expected pointer to struct, but got pointer to  %v", elem.Kind())
	}
	v := reflect.ValueOf(ptrToStruct)
	v = v.Elem()
	for i := 0; i < elem.NumField(); i++ {
		f := v.Field(i)
		fn := getFieldName(elem.Field(i))
		if fn == "-" || jsonBlob.Get(fn) == js.Undefined || jsonBlob.Get(fn) == nil {
			continue
		}

		//
		// Time is really useful
		//
		if f.Type().Name() == "Time" && f.Type().PkgPath() == "time" {
			str := jsonBlob.Get(fn).String()
			//2015-01-17T17:48:30.346218Z
			//2006-01-02T15:04:05.999999999Z
			t, err := time.Parse(time.RFC3339Nano, str)
			if err != nil {
				print("warning: could not convert string", str, ":", err)
			} else {
				f.Set(reflect.ValueOf(t))
			}
			continue
		}
		switch f.Type().Kind() {
		case reflect.Int64:
			f.SetInt(jsonBlob.Get(fn).Int64())
		case reflect.Int:
			f.SetInt(int64(jsonBlob.Get(fn).Int()))
		case reflect.String:
			f.SetString(jsonBlob.Get(fn).String())
		case reflect.Float64:
			f.SetFloat(jsonBlob.Get(fn).Float())
		case reflect.Bool:
			f.SetBool(jsonBlob.Get(fn).Bool())
		default:
			//print("warning: %s", fn, " has a type other than int64, string, float64 or bool")
		}
	}
	return nil
}
<commit_msg>safety<commit_after>package client

import (
	"bytes"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"
	"time"

	"github.com/gopherjs/gopherjs/js"
	"github.com/gopherjs/jquery"
)

// AjaxError is returned on the error channel after a call to an Ajax method.
type AjaxError struct {
	StatusCode int
	Message    string
}

//AjaxPut behaves indentically to AjaxPost other than using the method PUT.
func AjaxPut(ptrToStruct interface{}, path string) (chan interface{}, chan AjaxError) {
	return putPostDel(ptrToStruct, path, "PUT", true)
}

//AjaxDelete behaves indentically to AjaxPost other than using the method DELETE
//and not sending the object to be deleted's contents, just its id. First parameter
//here is just for the type.
func AjaxDelete(ptrToStruct interface{}, path string) (chan interface{}, chan AjaxError) {
	return putPostDel(ptrToStruct, path, "DELETE", false)
}

//AjaxPost sends an instance of a wire type to the server.  The first argument
//should be a wire type and must be a pointer to a struct or this function
//will panic. The value sent to the server is supplied in the first argument.
//The two returned values are a content channel and an error channel.  If the
//call succeeds, the content channel will be sent a different instance of the
//same type as the first argument. If the result from the server cannot be understood
//as the type of the first argument, the special error code 418 will be sent
//on the error channel.  If we fail to encode the object to be sent, the error
//code 420 will be sent on the error channel and no call to the server is made.
func AjaxPost(ptrToStruct interface{}, path string) (chan interface{}, chan AjaxError) {
	return putPostDel(ptrToStruct, path, "POST", true)
}

//AjaxIndex retreives a collection of wire types from the server.
//If the first argument is not a pointer to a slice of pointer to struct,
//it will panic.  The first element should be a slice of wire types.
//The returned values are a content channel and an error channel.  The content
//channel will receive the same type as your first argument if anything.  The error
//channel is used for non-200 http responses and the special error code 418
//is used to indicate that the received json from the server could not be successfully
//parsed as the type of the first argument.
func AjaxIndex(ptrToSliceOfPtrToStruct interface{}, path string) (chan interface{}, chan AjaxError) {
	isPointerToSliceOfPointerToStructOrPanic(ptrToSliceOfPtrToStruct)
	contentCh := make(chan interface{})
	errCh := make(chan AjaxError)
	ajaxRawChannels(ptrToSliceOfPtrToStruct, "", contentCh, errCh, "GET", path)
	return contentCh, errCh
}

//AjaxGet retreives an instance of a wire type from the server and decodes the result as
//Json. If the first argument is not a pointer to a struct, it will panic.
//The first argument should be a wire type that you expect to receive in the success
//case.  The returned values are a content channel and an error channel.  The content
//channel will receive the same type as your first argument if anything.  The error
//channel is used for non-200 http responses and the special error code 418
//is used to indicate that the received json from the server could not be successfully
//parsed as the type of the first argument.
func AjaxGet(ptrToStruct interface{}, path string) (chan interface{}, chan AjaxError) {
	isPointerToStructOrPanic(ptrToStruct)
	contentCh := make(chan interface{})
	errCh := make(chan AjaxError)
	ajaxRawChannels(ptrToStruct, "", contentCh, errCh, "GET", path)
	return contentCh, errCh
}

func ajaxRawChannels(output interface{}, body string, contentChan chan interface{}, errChan chan AjaxError,
	method string, path string) error {

	m := map[string]interface{}{
		"contentType": "application/json",
		"dataType":    "text",
		"type":        method,
		"url":         path,
		"cache":       false,
	}
	if body != "" {
		m["data"] = body
	}

	jquery.Ajax(m).
		Then(func(valueCreated *js.Object) {
		rd := strings.NewReader(valueCreated.String())
		dec := json.NewDecoder(rd)
		if err := dec.Decode(output); err != nil {
			go func() {
				errChan <- AjaxError{418, err.Error()}
			}()
			return
		}
		go func() {
			contentChan <- output
		}()
	}).
		Fail(func(p1 *js.Object) {
		go func() {
			errChan <- AjaxError{p1.Get("status").Int(), p1.Get("responseText").String()}
		}()
	})

	return nil
}

//
// HELPERS
//

func typeToUrlName(i interface{}) string {
	name, ok := i.(string)
	if !ok {
		name = fmt.Sprintf("%T", i)
	}
	pair := strings.Split(name, ".")
	if len(pair) != 2 {
		panic(fmt.Sprintf("unable to understand type name: %s", name))
	}
	return strings.ToLower(pair[1])
}

func encodeBody(i interface{}) (string, error) {
	//encode body
	var w bytes.Buffer
	enc := json.NewEncoder(&w)
	err := enc.Encode(i)
	if err != nil {
		return "", fmt.Errorf("error encoding body: %v ", err)
	}
	return w.String(), nil
}

func isPointerToStructOrPanic(i interface{}) reflect.Type {
	t := reflect.TypeOf(i)
	if t.Kind() != reflect.Ptr {
		panic(fmt.Sprintf("expected ptr to struct but got %T", i))
	}
	if t.Elem().Kind() != reflect.Struct {
		panic(fmt.Sprintf("expected ptr to struct but got ptr to %v", t.Elem().Kind()))
	}
	return t
}

func isPointerToSliceOfPointerToStructOrPanic(i interface{}) reflect.Type {
	t := reflect.TypeOf(i)
	if t.Kind() != reflect.Ptr {
		panic(fmt.Sprintf("expected ptr to slice of ptr to struct but got %T", i))
	}
	if t.Elem().Kind() != reflect.Slice {
		panic(fmt.Sprintf("expected ptr to SLICE of ptr to struct but got ptr to %v", t.Elem().Kind()))
	}
	if t.Elem().Elem().Kind() != reflect.Ptr {
		panic(fmt.Sprintf("expected ptr to slice of PTR to struct but got ptr to slice of %v", t.Elem().Elem().Kind()))
	}
	if t.Elem().Elem().Elem().Kind() != reflect.Struct {
		panic(fmt.Sprintf("expected ptr to slice of ptr to STRUCT but got ptr to slice of ptr to %v", t.Elem().Elem().Elem().Kind()))
	}
	return t
}

func putPostDel(ptrToStruct interface{}, path string, method string, sendBody bool) (chan interface{}, chan AjaxError) {
	t := isPointerToStructOrPanic(ptrToStruct)
	output := reflect.New(t.Elem())
	var body string
	if sendBody {
		body, err := encodeBody(ptrToStruct)
		if err != nil {
			go func() {
				errCh <- AjaxError{420, err.Error()}
			}()
			return
		}
	}
	contentCh := make(chan interface{})
	errCh := make(chan AjaxError)
	ajaxRawChannels(output.Interface(), body, contentCh, errCh, method, path)
	return contentCh, errCh
}

func getFieldName(f reflect.StructField) string {
	name := f.Tag.Get("json")
	jsonPreferred := ""
	if name != "" {
		parts := strings.Split(name, ",")
		for _, part := range parts {
			if part == "-" {
				return "-"
			}
			if part == "omitempty" {
				continue
			}
			jsonPreferred = part
		}
	}
	if jsonPreferred != "" {
		return jsonPreferred
	}
	return f.Name
}

//
// DEPRECATED
//

//UnpackJson has been deprecated in favor of the Ajax methods.  This method
//is a naive json unpacker that uses reflection on the go struct to convert
//javascript values.  It cannot handle arbitrary types of fields, cannot handle
//nested structures, nor can it handle the UnmarshalJson interface.
func UnpackJson(ptrToStruct interface{}, jsonBlob *js.Object) error {
	t := reflect.TypeOf(ptrToStruct)
	if t.Kind() != reflect.Ptr {
		return fmt.Errorf("expected pointer to struct, but got %v", t.Kind())
	}
	elem := t.Elem()
	if elem.Kind() != reflect.Struct {
		return fmt.Errorf("expected pointer to struct, but got pointer to  %v", elem.Kind())
	}
	v := reflect.ValueOf(ptrToStruct)
	v = v.Elem()
	for i := 0; i < elem.NumField(); i++ {
		f := v.Field(i)
		fn := getFieldName(elem.Field(i))
		if fn == "-" || jsonBlob.Get(fn) == js.Undefined || jsonBlob.Get(fn) == nil {
			continue
		}

		//
		// Time is really useful
		//
		if f.Type().Name() == "Time" && f.Type().PkgPath() == "time" {
			str := jsonBlob.Get(fn).String()
			//2015-01-17T17:48:30.346218Z
			//2006-01-02T15:04:05.999999999Z
			t, err := time.Parse(time.RFC3339Nano, str)
			if err != nil {
				print("warning: could not convert string", str, ":", err)
			} else {
				f.Set(reflect.ValueOf(t))
			}
			continue
		}
		switch f.Type().Kind() {
		case reflect.Int64:
			f.SetInt(jsonBlob.Get(fn).Int64())
		case reflect.Int:
			f.SetInt(int64(jsonBlob.Get(fn).Int()))
		case reflect.String:
			f.SetString(jsonBlob.Get(fn).String())
		case reflect.Float64:
			f.SetFloat(jsonBlob.Get(fn).Float())
		case reflect.Bool:
			f.SetBool(jsonBlob.Get(fn).Bool())
		default:
			//print("warning: %s", fn, " has a type other than int64, string, float64 or bool")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"
	"time"

	"github.com/gopherjs/gopherjs/js"
	"github.com/gopherjs/jquery"
)

type FailureFunc func(int, string)
type SuccessNewFunc func(int64)
type SuccessPutFunc func(js.Object)

func getFieldName(f reflect.StructField) string {
	name := f.Tag.Get("json")
	jsonPreferred := ""
	if name != "" {
		parts := strings.Split(name, ",")
		for _, part := range parts {
			if part == "-" {
				return "-"
			}
			if part == "omitempty" {
				continue
			}
			jsonPreferred = part
		}
	}
	if jsonPreferred != "" {
		return jsonPreferred
	}
	return f.Name
}
func UnpackJson(ptrToStruct interface{}, jsonBlob js.Object) error {
	t := reflect.TypeOf(ptrToStruct)
	if t.Kind() != reflect.Ptr {
		return fmt.Errorf("expected pointer to struct, but got %v", t.Kind())
	}
	elem := t.Elem()
	if elem.Kind() != reflect.Struct {
		return fmt.Errorf("expected pointer to struct, but got pointer to  %v", elem.Kind())
	}
	v := reflect.ValueOf(ptrToStruct)
	v = v.Elem()
	for i := 0; i < elem.NumField(); i++ {
		f := v.Field(i)
		fn := getFieldName(elem.Field(i))
		if fn == "-" || jsonBlob.Get(fn) == js.Undefined || jsonBlob.Get(fn) == nil {
			continue
		}

		//
		// Time is really useful
		//
		if f.Type().Name() == "Time" && f.Type().PkgPath() == "time" {
			str := jsonBlob.Get(fn).Str()
			//2015-01-17T17:48:30.346218Z
			//2006-01-02T15:04:05.999999999Z
			t, err := time.Parse(time.RFC3339Nano, str)
			if err != nil {
				print("warning: could not convert string", str, ":", err)
			} else {
				f.Set(reflect.ValueOf(t))
			}
			continue
		}
		switch f.Type().Kind() {
		case reflect.Int64:
			f.SetInt(jsonBlob.Get(fn).Int64())
		case reflect.Int:
			f.SetInt(int64(jsonBlob.Get(fn).Int()))
		case reflect.String:
			f.SetString(jsonBlob.Get(fn).Str())
		case reflect.Float64:
			f.SetFloat(jsonBlob.Get(fn).Float())
		case reflect.Bool:
			f.SetBool(jsonBlob.Get(fn).Bool())
		default:
			print("warning: %s", fn, " has a type other than int64, string, float64 or bool")
		}
	}
	return nil
}

//PutExisting sends a new version of the object given by id to the server.
//It returns an error only if the put could not be started, otherwise success
//or failure are communicated throught the callback functions.  The root should
//be the root of the rest heirarchy, probably "/rest".  The id is not examined
//by this routine, it can be the string representation of an integer or a UDID.
func PutExisting(i interface{}, root string, id string,
	success SuccessPutFunc, failure FailureFunc) error {

	var w bytes.Buffer
	enc := json.NewEncoder(&w)
	err := enc.Encode(i)
	if err != nil {
		return fmt.Errorf("error encoding put msg: %v ", err)
	}

	urlname := typeToUrlName(i)
	jquery.Ajax(
		map[string]interface{}{
			"contentType": "application/json",
			"dataType":    "json",
			"type":        "PUT",
			"url":         fmt.Sprintf("%s/%s/%s", root, urlname, id),
			"data":        w.String(),
			"cache":       false,
		}).
		Then(func(v js.Object) {
		success(v)
	}).
		Fail(func(p1 js.Object) {
		if failure != nil {
			failure(p1.Get("status").Int(), p1.Get("responseText").Str())
		}
	})

	return nil
}
func typeToUrlName(i interface{}) string {
	name := fmt.Sprintf("%T", i)
	pair := strings.Split(name, ".")
	if len(pair) != 2 {
		panic(fmt.Sprintf("unable to understand type name: %s", name))
	}
	return strings.ToLower(pair[1])
}

//PostNew sends an instance of a wire type to the server.  It returns an error
//only if the Post could not be sent.  The success or failure are indications are
//communicated through the callback functions.
func PostNew(i interface{}, root string, success SuccessNewFunc, failure FailureFunc) error {
	var w bytes.Buffer
	enc := json.NewEncoder(&w)
	err := enc.Encode(i)
	if err != nil {
		return fmt.Errorf("error encoding post msg: %v ", err)
	}
	urlname := typeToUrlName(i)
	jquery.Ajax(
		map[string]interface{}{
			"contentType": "application/json",
			"dataType":    "json",
			"type":        "POST",
			"url":         fmt.Sprintf("%s/%s", root, urlname),
			"data":        w.String(),
			"cache":       false,
		}).
		Then(func(valueCreated js.Object) {
		id := valueCreated.Get("Id").Int64()
		if success != nil {
			success(id)
		}
	}).
		Fail(func(p1 js.Object) {
		if failure != nil {
			failure(p1.Get("status").Int(), p1.Get("responseText").Str())
		}
	})

	return nil
}
<commit_msg>This is the Str()->String() change, for the last time.<commit_after>package client

import (
	"bytes"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"
	"time"

	"github.com/gopherjs/gopherjs/js"
	"github.com/gopherjs/jquery"
)

type FailureFunc func(int, string)
type SuccessNewFunc func(int64)
type SuccessPutFunc func(js.Object)

func getFieldName(f reflect.StructField) string {
	name := f.Tag.Get("json")
	jsonPreferred := ""
	if name != "" {
		parts := strings.Split(name, ",")
		for _, part := range parts {
			if part == "-" {
				return "-"
			}
			if part == "omitempty" {
				continue
			}
			jsonPreferred = part
		}
	}
	if jsonPreferred != "" {
		return jsonPreferred
	}
	return f.Name
}
func UnpackJson(ptrToStruct interface{}, jsonBlob js.Object) error {
	t := reflect.TypeOf(ptrToStruct)
	if t.Kind() != reflect.Ptr {
		return fmt.Errorf("expected pointer to struct, but got %v", t.Kind())
	}
	elem := t.Elem()
	if elem.Kind() != reflect.Struct {
		return fmt.Errorf("expected pointer to struct, but got pointer to  %v", elem.Kind())
	}
	v := reflect.ValueOf(ptrToStruct)
	v = v.Elem()
	for i := 0; i < elem.NumField(); i++ {
		f := v.Field(i)
		fn := getFieldName(elem.Field(i))
		if fn == "-" || jsonBlob.Get(fn) == js.Undefined || jsonBlob.Get(fn) == nil {
			continue
		}

		//
		// Time is really useful
		//
		if f.Type().Name() == "Time" && f.Type().PkgPath() == "time" {
			str := jsonBlob.Get(fn).String()
			//2015-01-17T17:48:30.346218Z
			//2006-01-02T15:04:05.999999999Z
			t, err := time.Parse(time.RFC3339Nano, str)
			if err != nil {
				print("warning: could not convert string", str, ":", err)
			} else {
				f.Set(reflect.ValueOf(t))
			}
			continue
		}
		switch f.Type().Kind() {
		case reflect.Int64:
			f.SetInt(jsonBlob.Get(fn).Int64())
		case reflect.Int:
			f.SetInt(int64(jsonBlob.Get(fn).Int()))
		case reflect.String:
			f.SetString(jsonBlob.Get(fn).String())
		case reflect.Float64:
			f.SetFloat(jsonBlob.Get(fn).Float())
		case reflect.Bool:
			f.SetBool(jsonBlob.Get(fn).Bool())
		default:
			//print("warning: %s", fn, " has a type other than int64, string, float64 or bool")
		}
	}
	return nil
}

//PutExisting sends a new version of the object given by id to the server.
//It returns an error only if the put could not be started, otherwise success
//or failure are communicated throught the callback functions.  The root should
//be the root of the rest heirarchy, probably "/rest".  The id is not examined
//by this routine, it can be the string representation of an integer or a UDID.
func PutExisting(i interface{}, root string, id string,
	success SuccessPutFunc, failure FailureFunc) error {

	var w bytes.Buffer
	enc := json.NewEncoder(&w)
	err := enc.Encode(i)
	if err != nil {
		return fmt.Errorf("error encoding put msg: %v ", err)
	}

	urlname := typeToUrlName(i)
	jquery.Ajax(
		map[string]interface{}{
			"contentType": "application/json",
			"dataType":    "json",
			"type":        "PUT",
			"url":         fmt.Sprintf("%s/%s/%s", root, urlname, id),
			"data":        w.String(),
			"cache":       false,
		}).
		Then(func(v js.Object) {
		success(v)
	}).
		Fail(func(p1 js.Object) {
		if failure != nil {
			failure(p1.Get("status").Int(), p1.Get("responseText").String())
		}
	})

	return nil
}
func typeToUrlName(i interface{}) string {
	name := fmt.Sprintf("%T", i)
	pair := strings.Split(name, ".")
	if len(pair) != 2 {
		panic(fmt.Sprintf("unable to understand type name: %s", name))
	}
	return strings.ToLower(pair[1])
}

//PostNew sends an instance of a wire type to the server.  It returns an error
//only if the Post could not be sent.  The success or failure are indications are
//communicated through the callback functions.
func PostNew(i interface{}, root string, success SuccessNewFunc, failure FailureFunc) error {
	var w bytes.Buffer
	enc := json.NewEncoder(&w)
	err := enc.Encode(i)
	if err != nil {
		return fmt.Errorf("error encoding post msg: %v ", err)
	}
	urlname := typeToUrlName(i)
	jquery.Ajax(
		map[string]interface{}{
			"contentType": "application/json",
			"dataType":    "json",
			"type":        "POST",
			"url":         fmt.Sprintf("%s/%s", root, urlname),
			"data":        w.String(),
			"cache":       false,
		}).
		Then(func(valueCreated js.Object) {
		id := valueCreated.Get("Id").Int64()
		if success != nil {
			success(id)
		}
	}).
		Fail(func(p1 js.Object) {
		if failure != nil {
			failure(p1.Get("status").Int(), p1.Get("responseText").String())
		}
	})

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/build"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"text/template"
)

// A Package holds the typedef, function, and enum definitions for a Go package.
type Package struct {
	Name    string
	API     string
	Version Version
	Profile string
	TmplDir string

	Typedefs  []*Typedef
	Enums     map[string]*Enum
	Functions map[string]*PackageFunction

	SpecRev string
}

// A PackageFunction is a package-specific Function wrapper.
type PackageFunction struct {
	Function
	Required bool
	Doc      string
}

// UniqueName returns a globally unique Go-compatible name for this package.
func (pkg *Package) UniqueName() string {
	version := strings.Replace(pkg.Version.String(), ".", "", -1)
	return fmt.Sprintf("%s%s%s", pkg.API, pkg.Profile, version)
}

// GeneratePackage writes a Go package to specified directory.
func (pkg *Package) GeneratePackage(dir string) error {
	if err := os.MkdirAll(dir, 0755); err != nil {
		return err
	}

	if err := pkg.generateFile("package", dir); err != nil {
		return err
	}
	if err := pkg.generateFile("conversions", dir); err != nil {
		return err
	}
	if err := pkg.generateFile("procaddr", dir); err != nil {
		return err
	}
	if pkg.HasDebugCallbackFeature() {
		if err := pkg.generateFile("debug", dir); err != nil {
			return err
		}
	}

	// gofmt the generated .go files.
	if err := exec.Command("gofmt", "-w", dir).Run(); err != nil {
		return fmt.Errorf("gofmt error: %v", err)
	}

	return nil
}

func (pkg *Package) generateFile(file, dir string) error {
	out, err := os.Create(filepath.Join(dir, file+".go"))
	if err != nil {
		return err
	}
	defer out.Close()

	fns := template.FuncMap{
		"replace": strings.Replace,
		"toUpper": strings.ToUpper,
	}

	tmpl := template.Must(template.New(file + ".tmpl").Funcs(fns).ParseFiles(filepath.Join(pkg.TmplDir, file+".tmpl")))

	return tmpl.Execute(NewBlankLineStrippingWriter(out), pkg)
}

// HasDebugCallbackFeature returns whether this package exposes the ability to
// set a debug callback. Used to determine whether to include the necessary
// GL-specific callback code.
func (pkg *Package) HasDebugCallbackFeature() bool {
	for _, fn := range pkg.Functions {
		for _, param := range fn.Parameters {
			if param.Type.IsDebugProc() {
				return true
			}
		}
	}
	return false
}

// HasRequiredFunctions returns true if at least one function in this package
// is required.
func (pkg *Package) HasRequiredFunctions() bool {
	for _, fn := range pkg.Functions {
		if fn.Required {
			return true
		}
	}
	return false
}

// Filter removes any enums, or functions found in this package that are not
// listed in the given lookup maps. If either of the maps has a length of zero,
// filtering does not occur for that type (e.g. all functions are left intact).
func (pkg *Package) Filter(enums, functions map[string]bool) {
	if len(enums) > 0 {
		// Remove any enum not listed in the enums lookup map.
		for name := range pkg.Enums {
			_, valid := enums[name]
			if !valid {
				delete(pkg.Enums, name)
			}
		}
	}

	if len(functions) > 0 {
		// Remove any function not listed in the functions lookup map.
		for name := range pkg.Functions {
			_, valid := functions[name]
			if !valid {
				delete(pkg.Functions, name)
			}
		}
	}
}

// importPathToDir resolves the absolute path to an import path.
// There doesn't need to be a valid Go package inside that import path,
// but the directory must exist.
func importPathToDir(importPath string) string {
	p, err := build.Import(importPath, "", build.FindOnly)
	if err != nil {
		log.Fatalln(err)
	}
	return p.Dir
}
<commit_msg>Documentation fix and elaboration.<commit_after>package main

import (
	"fmt"
	"go/build"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"text/template"
)

// A Package holds the typedef, function, and enum definitions for a Go package.
type Package struct {
	Name    string
	API     string
	Version Version
	Profile string
	TmplDir string

	Typedefs  []*Typedef
	Enums     map[string]*Enum
	Functions map[string]*PackageFunction

	SpecRev string
}

// A PackageFunction is a package-specific Function wrapper.
type PackageFunction struct {
	Function
	Required bool
	Doc      string
}

// UniqueName returns a globally unique Go-compatible name for this package.
func (pkg *Package) UniqueName() string {
	version := strings.Replace(pkg.Version.String(), ".", "", -1)
	return fmt.Sprintf("%s%s%s", pkg.API, pkg.Profile, version)
}

// GeneratePackage writes a Go package to specified directory.
func (pkg *Package) GeneratePackage(dir string) error {
	if err := os.MkdirAll(dir, 0755); err != nil {
		return err
	}

	if err := pkg.generateFile("package", dir); err != nil {
		return err
	}
	if err := pkg.generateFile("conversions", dir); err != nil {
		return err
	}
	if err := pkg.generateFile("procaddr", dir); err != nil {
		return err
	}
	if pkg.HasDebugCallbackFeature() {
		if err := pkg.generateFile("debug", dir); err != nil {
			return err
		}
	}

	// gofmt the generated .go files.
	if err := exec.Command("gofmt", "-w", dir).Run(); err != nil {
		return fmt.Errorf("gofmt error: %v", err)
	}

	return nil
}

func (pkg *Package) generateFile(file, dir string) error {
	out, err := os.Create(filepath.Join(dir, file+".go"))
	if err != nil {
		return err
	}
	defer out.Close()

	fns := template.FuncMap{
		"replace": strings.Replace,
		"toUpper": strings.ToUpper,
	}

	tmpl := template.Must(template.New(file + ".tmpl").Funcs(fns).ParseFiles(filepath.Join(pkg.TmplDir, file+".tmpl")))

	return tmpl.Execute(NewBlankLineStrippingWriter(out), pkg)
}

// HasDebugCallbackFeature returns whether this package exposes the ability to
// set a debug callback. Used to determine whether to include the necessary
// GL-specific callback code.
func (pkg *Package) HasDebugCallbackFeature() bool {
	for _, fn := range pkg.Functions {
		for _, param := range fn.Parameters {
			if param.Type.IsDebugProc() {
				return true
			}
		}
	}
	return false
}

// HasRequiredFunctions returns true if at least one function in this package
// is required.
func (pkg *Package) HasRequiredFunctions() bool {
	for _, fn := range pkg.Functions {
		if fn.Required {
			return true
		}
	}
	return false
}

// Filter removes any enums, or functions found in this package that are not
// listed in the given lookup maps. If either of the maps has a length of zero,
// filtering does not occur for that type (e.g. all functions are left intact).
func (pkg *Package) Filter(enums, functions map[string]bool) {
	if len(enums) > 0 {
		// Remove any enum not listed in the enums lookup map.
		for name := range pkg.Enums {
			_, valid := enums[name]
			if !valid {
				delete(pkg.Enums, name)
			}
		}
	}

	if len(functions) > 0 {
		// Remove any function not listed in the functions lookup map.
		for name := range pkg.Functions {
			_, valid := functions[name]
			if !valid {
				delete(pkg.Functions, name)
			}
		}
	}
}

// importPathToDir resolves the absolute path from importPath.
// There doesn't need to be a valid Go package inside that import path,
// but the directory must exist. It calls log.Fatalln if it fails.
func importPathToDir(importPath string) string {
	p, err := build.Import(importPath, "", build.FindOnly)
	if err != nil {
		log.Fatalln(err)
	}
	return p.Dir
}
<|endoftext|>


<commit_before>package gogo

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

// Package describes a Go package.
type Package struct {
	// The Context that resolved this package.
	*Context

	// The name of the package
	name string

	// The import path of the package.
	importPath string

	// The path to the source of this package relative to Project.Root()
	srcdir string

	// Source files
	GoFiles        []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
	CgoFiles       []string // .go source files that import "C"
	SFiles         []string // .s source files
	IgnoredGoFiles []string // .go source files ignored for this build

	// Cgo directives
	CgoPkgConfig []string // Cgo pkg-config directives
	CgoCFLAGS    []string // Cgo CFLAGS directives
	CgoLDFLAGS   []string // Cgo LDFLAGS directives

	// Test information
	TestGoFiles  []string // _test.go files in package
	XTestGoFiles []string // _test.go files outside package

	Imports []*Package
}

// newPackage constructs a new Package for the Context context.
func newPackage(context *Context, path string) (*Package, error) {
	pkg := &Package{
		Context:    context,
		importPath: path,
		srcdir:     filepath.Join("src", path),
	}
	files, err := ioutil.ReadDir(pkg.Srcdir())
	if err != nil {
		return nil, err
	}
	return pkg, pkg.scanFiles(files)
}

// Name returns the name of the package.
func (p *Package) Name() string { return p.name }

// ImportPath returns the import path that would is used to import this package into another.
func (p *Package) ImportPath() string { return p.importPath }

// Srcdir returns the path to this package.
func (p *Package) Srcdir() string { return filepath.Join(p.Project.Root(), p.srcdir) }

func (p *Package) openFile(name string) (io.ReadCloser, error) {
	return os.Open(filepath.Join(p.Srcdir(), name))
}

// scanFiles scans the Package recording all source files relevant to the
// current Context.
func (p *Package) scanFiles(files []os.FileInfo) error {
	imports := make(map[string]struct{})
	fset := token.NewFileSet()
	var firstFile string
	for _, file := range files {
		if file.IsDir() {
			// skip
			continue
		}
		filename := file.Name()
		if strings.HasPrefix(filename, "_") || strings.HasPrefix(filename, ".") {
			continue
		}

		ext := filepath.Ext(filename)

		if !p.goodOSArchFile(filename) {
			if ext == ".go" {
				p.IgnoredGoFiles = append(p.IgnoredGoFiles, filename)
			}
			continue
		}

		switch ext {
		case ".go", ".c", ".s", ".h", ".S", ".swig", ".swigcxx":
			// tentatively okay - read to make sure
		default:
			// skip
			continue
		}

		r, err := p.openFile(filename)
		if err != nil {
			return err
		}
		var data []byte
		if strings.HasSuffix(filename, ".go") {
			data, err = readImports(r, false)
		} else {
			data, err = readComments(r)
		}
		r.Close()
		if err != nil {
			return err
		}

		// Look for +build comments to accept or reject the file.
		if !p.shouldBuild(data) {
			if ext == ".go" {
				p.IgnoredGoFiles = append(p.IgnoredGoFiles, filename)
			}
			continue
		}

		switch ext {
		case ".s":
			p.SFiles = append(p.SFiles, filename)
			continue
		}

		pf, err := parser.ParseFile(fset, filename, data, parser.ImportsOnly|parser.ParseComments)
		if err != nil {
			return err
		}

		pkg := pf.Name.Name
		if pkg == "documentation" {
			p.IgnoredGoFiles = append(p.IgnoredGoFiles, filename)
			continue
		}

		isTest := strings.HasSuffix(filename, "_test.go")
		var isXTest bool
		if isTest && strings.HasSuffix(pkg, "_test") {
			isXTest = true
			pkg = pkg[:len(pkg)-len("_test")]
		}
		if p.name == "" {
			p.name = pkg
			firstFile = filename
		} else if pkg != p.name {
			return fmt.Errorf("found packages %s (%s) and %s (%s) in %s", p.name, firstFile, pkg, filename, p.importPath)
		}
		var isCgo bool
		for _, decl := range pf.Decls {
			switch decl := decl.(type) {
			case *ast.GenDecl:
				for _, spec := range decl.Specs {
					switch spec := spec.(type) {
					case *ast.ImportSpec:
						quoted := spec.Path.Value
						path, err := strconv.Unquote(quoted)
						if err != nil {
							return err
						}
						switch path {
						case "":
							return fmt.Errorf("package %q imported blank path: %s", spec.Pos())
						case ".":
							return fmt.Errorf("package %q imported dot path: %s", spec.Pos())
						case "C":
							if isTest {
								return fmt.Errorf("use of cgo in test %s not supported", filename)
							}
							cg := spec.Doc
							if cg == nil && len(decl.Specs) == 1 {
								cg = decl.Doc
							}
							if cg != nil {
								if err := p.saveCgo(filename, cg); err != nil {
									return err
								}
							}
							isCgo = true
						default:
							if !isXTest {
								imports[path] = struct{}{}
							}
						}
					default:
						// skip
					}
				}
			default:
				// skip

			}
		}
		if isCgo {
			if p.cgoEnabled {
				p.CgoFiles = append(p.CgoFiles, filename)
			}
		} else if isXTest {
			p.XTestGoFiles = append(p.XTestGoFiles, filename)
		} else if isTest {
			p.TestGoFiles = append(p.TestGoFiles, filename)
		} else {
			p.GoFiles = append(p.GoFiles, filename)
		}
	}
	if p.name == "" {
		return &build.NoGoError{p.importPath}
	}
	for i := range imports {
		if stdlib[i] {
			// skip
			continue
		}
		pkg, err := p.ResolvePackage(i)
		if err != nil {
			return err
		}
		p.Imports = append(p.Imports, pkg)
	}
	return nil
}

// from $GOROOT/src/pkg/go/build/build.go

var slashslash = []byte("//")

// shouldBuild reports whether it is okay to use this file,
// The rule is that in the file's leading run of // comments
// and blank lines, which must be followed by a blank line
// (to avoid including a Go package clause doc comment),
// lines beginning with '// +build' are taken as build directives.
//
// The file is accepted only if each such line lists something
// matching the file.  For example:
//
//      // +build windows linux
//
// marks the file as applicable only on Windows and Linux.
//
func (ctxt *Context) shouldBuild(content []byte) bool {
	// Pass 1. Identify leading run of // comments and blank lines,
	// which must be followed by a blank line.
	end := 0
	p := content
	for len(p) > 0 {
		line := p
		if i := bytes.IndexByte(line, '\n'); i >= 0 {
			line, p = line[:i], p[i+1:]
		} else {
			p = p[len(p):]
		}
		line = bytes.TrimSpace(line)
		if len(line) == 0 { // Blank line
			end = len(content) - len(p)
			continue
		}
		if !bytes.HasPrefix(line, slashslash) { // Not comment line
			break
		}
	}
	content = content[:end]

	// Pass 2.  Process each line in the run.
	p = content
	for len(p) > 0 {
		line := p
		if i := bytes.IndexByte(line, '\n'); i >= 0 {
			line, p = line[:i], p[i+1:]
		} else {
			p = p[len(p):]
		}
		line = bytes.TrimSpace(line)
		if bytes.HasPrefix(line, slashslash) {
			line = bytes.TrimSpace(line[len(slashslash):])
			if len(line) > 0 && line[0] == '+' {
				// Looks like a comment +line.
				f := strings.Fields(string(line))
				if f[0] == "+build" {
					ok := false
					for _, tok := range f[1:] {
						if ctxt.match(tok) {
							ok = true
							break
						}
					}
					if !ok {
						return false // this one doesn't match
					}
				}
			}
		}
	}
	return true // everything matches
}

// saveCgo saves the information from the #cgo lines in the import "C" comment.
// These lines set CFLAGS and LDFLAGS and pkg-config directives that affect
// the way cgo's C code is built.
//
// TODO(rsc): This duplicates code in cgo.
// Once the dust settles, remove this code from cgo.
func (p *Package) saveCgo(filename string, cg *ast.CommentGroup) error {
	text := cg.Text()
	for _, line := range strings.Split(text, "\n") {
		orig := line

		// Line is
		//      #cgo [GOOS/GOARCH...] LDFLAGS: stuff
		//
		line = strings.TrimSpace(line)
		if len(line) < 5 || line[:4] != "#cgo" || (line[4] != ' ' && line[4] != '\t') {
			continue
		}

		// Split at colon.
		line = strings.TrimSpace(line[4:])
		i := strings.Index(line, ":")
		if i < 0 {
			return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig)
		}
		line, argstr := line[:i], line[i+1:]

		// Parse GOOS/GOARCH stuff.
		f := strings.Fields(line)
		if len(f) < 1 {
			return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig)
		}

		cond, verb := f[:len(f)-1], f[len(f)-1]
		if len(cond) > 0 {
			ok := false
			for _, c := range cond {
				if p.Context.match(c) {
					ok = true
					break
				}
			}
			if !ok {
				continue
			}
		}

		args, err := splitQuoted(argstr)
		if err != nil {
			return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig)
		}
		for _, arg := range args {
			if !safeName(arg) {
				return fmt.Errorf("%s: malformed #cgo argument: %s", filename, arg)
			}
		}

		switch verb {
		case "CFLAGS":
			p.CgoCFLAGS = append(p.CgoCFLAGS, args...)
		case "LDFLAGS":
			p.CgoLDFLAGS = append(p.CgoLDFLAGS, args...)
		case "pkg-config":
			p.CgoPkgConfig = append(p.CgoPkgConfig, args...)
		default:
			return fmt.Errorf("%s: invalid #cgo verb: %s", filename, orig)
		}
	}
	return nil
}

// Objdir returns the destination for object files compiled for this Package.
func (p *Package) Objdir() string {
	return filepath.Join(p.Context.workdir, filepath.FromSlash(p.ImportPath()), "_obj")
}

// TestObjDir returns the destination for test object files compiled for this Package.
func (p *Package) TestObjdir() string {
	return filepath.Join(p.Context.workdir, filepath.FromSlash(p.ImportPath()), "_test")
}
<commit_msg>go vet<commit_after>package gogo

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

// Package describes a Go package.
type Package struct {
	// The Context that resolved this package.
	*Context

	// The name of the package
	name string

	// The import path of the package.
	importPath string

	// The path to the source of this package relative to Project.Root()
	srcdir string

	// Source files
	GoFiles        []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
	CgoFiles       []string // .go source files that import "C"
	SFiles         []string // .s source files
	IgnoredGoFiles []string // .go source files ignored for this build

	// Cgo directives
	CgoPkgConfig []string // Cgo pkg-config directives
	CgoCFLAGS    []string // Cgo CFLAGS directives
	CgoLDFLAGS   []string // Cgo LDFLAGS directives

	// Test information
	TestGoFiles  []string // _test.go files in package
	XTestGoFiles []string // _test.go files outside package

	Imports []*Package
}

// newPackage constructs a new Package for the Context context.
func newPackage(context *Context, path string) (*Package, error) {
	pkg := &Package{
		Context:    context,
		importPath: path,
		srcdir:     filepath.Join("src", path),
	}
	files, err := ioutil.ReadDir(pkg.Srcdir())
	if err != nil {
		return nil, err
	}
	return pkg, pkg.scanFiles(files)
}

// Name returns the name of the package.
func (p *Package) Name() string { return p.name }

// ImportPath returns the import path that would is used to import this package into another.
func (p *Package) ImportPath() string { return p.importPath }

// Srcdir returns the path to this package.
func (p *Package) Srcdir() string { return filepath.Join(p.Project.Root(), p.srcdir) }

func (p *Package) openFile(name string) (io.ReadCloser, error) {
	return os.Open(filepath.Join(p.Srcdir(), name))
}

// scanFiles scans the Package recording all source files relevant to the
// current Context.
func (p *Package) scanFiles(files []os.FileInfo) error {
	imports := make(map[string]struct{})
	fset := token.NewFileSet()
	var firstFile string
	for _, file := range files {
		if file.IsDir() {
			// skip
			continue
		}
		filename := file.Name()
		if strings.HasPrefix(filename, "_") || strings.HasPrefix(filename, ".") {
			continue
		}

		ext := filepath.Ext(filename)

		if !p.goodOSArchFile(filename) {
			if ext == ".go" {
				p.IgnoredGoFiles = append(p.IgnoredGoFiles, filename)
			}
			continue
		}

		switch ext {
		case ".go", ".c", ".s", ".h", ".S", ".swig", ".swigcxx":
			// tentatively okay - read to make sure
		default:
			// skip
			continue
		}

		r, err := p.openFile(filename)
		if err != nil {
			return err
		}
		var data []byte
		if strings.HasSuffix(filename, ".go") {
			data, err = readImports(r, false)
		} else {
			data, err = readComments(r)
		}
		r.Close()
		if err != nil {
			return err
		}

		// Look for +build comments to accept or reject the file.
		if !p.shouldBuild(data) {
			if ext == ".go" {
				p.IgnoredGoFiles = append(p.IgnoredGoFiles, filename)
			}
			continue
		}

		switch ext {
		case ".s":
			p.SFiles = append(p.SFiles, filename)
			continue
		}

		pf, err := parser.ParseFile(fset, filename, data, parser.ImportsOnly|parser.ParseComments)
		if err != nil {
			return err
		}

		pkg := pf.Name.Name
		if pkg == "documentation" {
			p.IgnoredGoFiles = append(p.IgnoredGoFiles, filename)
			continue
		}

		isTest := strings.HasSuffix(filename, "_test.go")
		var isXTest bool
		if isTest && strings.HasSuffix(pkg, "_test") {
			isXTest = true
			pkg = pkg[:len(pkg)-len("_test")]
		}
		if p.name == "" {
			p.name = pkg
			firstFile = filename
		} else if pkg != p.name {
			return fmt.Errorf("found packages %s (%s) and %s (%s) in %s", p.name, firstFile, pkg, filename, p.importPath)
		}
		var isCgo bool
		for _, decl := range pf.Decls {
			switch decl := decl.(type) {
			case *ast.GenDecl:
				for _, spec := range decl.Specs {
					switch spec := spec.(type) {
					case *ast.ImportSpec:
						quoted := spec.Path.Value
						path, err := strconv.Unquote(quoted)
						if err != nil {
							return err
						}
						switch path {
						case "":
							return fmt.Errorf("package %q imported blank path: %s", p.Name(), spec.Pos())
						case ".":
							return fmt.Errorf("package %q imported dot path: %s", p.Name(), spec.Pos())
						case "C":
							if isTest {
								return fmt.Errorf("use of cgo in test %s not supported", filename)
							}
							cg := spec.Doc
							if cg == nil && len(decl.Specs) == 1 {
								cg = decl.Doc
							}
							if cg != nil {
								if err := p.saveCgo(filename, cg); err != nil {
									return err
								}
							}
							isCgo = true
						default:
							if !isXTest {
								imports[path] = struct{}{}
							}
						}
					default:
						// skip
					}
				}
			default:
				// skip

			}
		}
		if isCgo {
			if p.cgoEnabled {
				p.CgoFiles = append(p.CgoFiles, filename)
			}
		} else if isXTest {
			p.XTestGoFiles = append(p.XTestGoFiles, filename)
		} else if isTest {
			p.TestGoFiles = append(p.TestGoFiles, filename)
		} else {
			p.GoFiles = append(p.GoFiles, filename)
		}
	}
	if p.name == "" {
		return &build.NoGoError{p.importPath}
	}
	for i := range imports {
		if stdlib[i] {
			// skip
			continue
		}
		pkg, err := p.ResolvePackage(i)
		if err != nil {
			return err
		}
		p.Imports = append(p.Imports, pkg)
	}
	return nil
}

// from $GOROOT/src/pkg/go/build/build.go

var slashslash = []byte("//")

// shouldBuild reports whether it is okay to use this file,
// The rule is that in the file's leading run of // comments
// and blank lines, which must be followed by a blank line
// (to avoid including a Go package clause doc comment),
// lines beginning with '// +build' are taken as build directives.
//
// The file is accepted only if each such line lists something
// matching the file.  For example:
//
//      // +build windows linux
//
// marks the file as applicable only on Windows and Linux.
//
func (ctxt *Context) shouldBuild(content []byte) bool {
	// Pass 1. Identify leading run of // comments and blank lines,
	// which must be followed by a blank line.
	end := 0
	p := content
	for len(p) > 0 {
		line := p
		if i := bytes.IndexByte(line, '\n'); i >= 0 {
			line, p = line[:i], p[i+1:]
		} else {
			p = p[len(p):]
		}
		line = bytes.TrimSpace(line)
		if len(line) == 0 { // Blank line
			end = len(content) - len(p)
			continue
		}
		if !bytes.HasPrefix(line, slashslash) { // Not comment line
			break
		}
	}
	content = content[:end]

	// Pass 2.  Process each line in the run.
	p = content
	for len(p) > 0 {
		line := p
		if i := bytes.IndexByte(line, '\n'); i >= 0 {
			line, p = line[:i], p[i+1:]
		} else {
			p = p[len(p):]
		}
		line = bytes.TrimSpace(line)
		if bytes.HasPrefix(line, slashslash) {
			line = bytes.TrimSpace(line[len(slashslash):])
			if len(line) > 0 && line[0] == '+' {
				// Looks like a comment +line.
				f := strings.Fields(string(line))
				if f[0] == "+build" {
					ok := false
					for _, tok := range f[1:] {
						if ctxt.match(tok) {
							ok = true
							break
						}
					}
					if !ok {
						return false // this one doesn't match
					}
				}
			}
		}
	}
	return true // everything matches
}

// saveCgo saves the information from the #cgo lines in the import "C" comment.
// These lines set CFLAGS and LDFLAGS and pkg-config directives that affect
// the way cgo's C code is built.
//
// TODO(rsc): This duplicates code in cgo.
// Once the dust settles, remove this code from cgo.
func (p *Package) saveCgo(filename string, cg *ast.CommentGroup) error {
	text := cg.Text()
	for _, line := range strings.Split(text, "\n") {
		orig := line

		// Line is
		//      #cgo [GOOS/GOARCH...] LDFLAGS: stuff
		//
		line = strings.TrimSpace(line)
		if len(line) < 5 || line[:4] != "#cgo" || (line[4] != ' ' && line[4] != '\t') {
			continue
		}

		// Split at colon.
		line = strings.TrimSpace(line[4:])
		i := strings.Index(line, ":")
		if i < 0 {
			return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig)
		}
		line, argstr := line[:i], line[i+1:]

		// Parse GOOS/GOARCH stuff.
		f := strings.Fields(line)
		if len(f) < 1 {
			return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig)
		}

		cond, verb := f[:len(f)-1], f[len(f)-1]
		if len(cond) > 0 {
			ok := false
			for _, c := range cond {
				if p.Context.match(c) {
					ok = true
					break
				}
			}
			if !ok {
				continue
			}
		}

		args, err := splitQuoted(argstr)
		if err != nil {
			return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig)
		}
		for _, arg := range args {
			if !safeName(arg) {
				return fmt.Errorf("%s: malformed #cgo argument: %s", filename, arg)
			}
		}

		switch verb {
		case "CFLAGS":
			p.CgoCFLAGS = append(p.CgoCFLAGS, args...)
		case "LDFLAGS":
			p.CgoLDFLAGS = append(p.CgoLDFLAGS, args...)
		case "pkg-config":
			p.CgoPkgConfig = append(p.CgoPkgConfig, args...)
		default:
			return fmt.Errorf("%s: invalid #cgo verb: %s", filename, orig)
		}
	}
	return nil
}

// Objdir returns the destination for object files compiled for this Package.
func (p *Package) Objdir() string {
	return filepath.Join(p.Context.workdir, filepath.FromSlash(p.ImportPath()), "_obj")
}

// TestObjDir returns the destination for test object files compiled for this Package.
func (p *Package) TestObjdir() string {
	return filepath.Join(p.Context.workdir, filepath.FromSlash(p.ImportPath()), "_test")
}
<|endoftext|>


<commit_before>package client

/*
 Copyleft 2016 Alexander I.Grafov <grafov@gmail.com>

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.

 ॐ तारे तुत्तारे तुरे स्व
*/

import (
	"encoding/json"
	"fmt"
	"net/url"
	"time"

	"github.com/grafov/autograf/grafana"
)

type BoardMeta struct {
	IsStarred  bool      `json:"isStarred,omitempty"`
	IsHome     bool      `json:"isHome,omitempty"`
	IsSnapshot bool      `json:"isSnapshot,omitempty"`
	Type       string    `json:"type,omitempty"`
	CanSave    bool      `json:"canSave"`
	CanEdit    bool      `json:"canEdit"`
	CanStar    bool      `json:"canStar"`
	Slug       string    `json:"slug"`
	Expires    time.Time `json:"expires"`
	Created    time.Time `json:"created"`
	Updated    time.Time `json:"updated"`
	UpdatedBy  string    `json:"updatedBy"`
	CreatedBy  string    `json:"createdBy"`
	Version    int       `json:"version"`
}

type BoardWithMeta struct {
	Meta  BoardMeta     `json:"meta"`
	Board grafana.Board `json:"dashboard"`
}

func (r *Instance) GetDashboard(slug string) (BoardWithMeta, error) {
	var (
		raw   []byte
		board BoardWithMeta
		err   error
	)
	if raw, err = r.get(fmt.Sprintf("api/dashboards/%s", slug), nil); err != nil {
		return BoardWithMeta{}, err
	}
	err = json.Unmarshal(raw, &board)
	return board, err
}

type FoundBoard struct {
	ID        uint     `json:"id"`
	Title     string   `json:"title"`
	URI       string   `json:"uri"`
	Type      string   `json:"type"`
	Tags      []string `json:"tags"`
	IsStarred bool     `json:"isStarred"`
}

// SearchDashboards search dashboards by query substring. Il allows restrict the result set with
// only starred dashboards and only for tags (logical OR applied to multiple tags).
func (r *Instance) SearchDashboards(query string, starred bool, tags ...string) ([]FoundBoard, error) {
	var (
		raw    []byte
		boards []FoundBoard
		err    error
	)
	u := url.URL{}
	q := u.Query()
	if query != "" {
		q.Set("query", query)
	}
	if starred {
		q.Set("starred", "true")
	}
	for _, tag := range tags {
		q.Add("tag", tag)
	}
	if raw, err = r.get("api/search", q); err != nil {
		return nil, err
	}
	err = json.Unmarshal(raw, &boards)
	return boards, err
}

// SetDashboard updates existing dashboard or creates a new one.
// Set dasboard ID to nil to create a new dashboard.
// Set overwrite to true if you want to overwrite existing dashboard with
// newer version or with same dashboard title.
func (r *Instance) SetDashboard(board grafana.Board, overwrite bool) error {
	var (
		newBoard struct {
			Dashboard grafana.Board `json:"dashboard"`
			Overwrite bool          `json:"overwrite"`
		}
		raw  []byte
		resp StatusMessage
		code int
		err  error
	)
	newBoard.Dashboard = board
	newBoard.Overwrite = overwrite
	if !overwrite {
		newBoard.Dashboard.ID = 0
	}
	if raw, err = json.Marshal(newBoard); err != nil {
		return err
	}
	if raw, code, err = r.post("api/dashboards/db", nil, raw); err != nil {
		return err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return err
	}
	switch code {
	case 401:
		return fmt.Errorf("%d %s", code, *resp.Message)
	case 412:
		return fmt.Errorf("%d %s", code, *resp.Message)
	}
	return nil
}

// DeleteDashboard deletes dashboard that selected by slug string.
func (r *Instance) DeleteDashboard(slug string) (StatusMessage, error) {
	var (
		raw   []byte
		reply StatusMessage
		err   error
	)
	if raw, err = r.delete(fmt.Sprintf("api/dashboards/db/%s", slug)); err != nil {
		return StatusMessage{}, err
	}
	err = json.Unmarshal(raw, &reply)
	return reply, err
}
<commit_msg>Simplify datatype in client.<commit_after>package client

/*
 Copyleft 2016 Alexander I.Grafov <grafov@gmail.com>

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
 the Free Software Foundation, either version 3 of the License, or
 (at your option) any later version.

 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.

 You should have received a copy of the GNU General Public License
 along with this program.  If not, see <http://www.gnu.org/licenses/>.

 ॐ तारे तुत्तारे तुरे स्व
*/

import (
	"encoding/json"
	"fmt"
	"net/url"
	"time"

	"github.com/grafov/autograf/grafana"
)

type BoardProperties struct {
	IsStarred  bool      `json:"isStarred,omitempty"`
	IsHome     bool      `json:"isHome,omitempty"`
	IsSnapshot bool      `json:"isSnapshot,omitempty"`
	Type       string    `json:"type,omitempty"`
	CanSave    bool      `json:"canSave"`
	CanEdit    bool      `json:"canEdit"`
	CanStar    bool      `json:"canStar"`
	Slug       string    `json:"slug"`
	Expires    time.Time `json:"expires"`
	Created    time.Time `json:"created"`
	Updated    time.Time `json:"updated"`
	UpdatedBy  string    `json:"updatedBy"`
	CreatedBy  string    `json:"createdBy"`
	Version    int       `json:"version"`
}

func (r *Instance) GetDashboard(slug string) (grafana.Board, BoardProperties, error) {
	var (
		raw    []byte
		result struct {
			Meta  BoardProperties `json:"meta"`
			Board grafana.Board   `json:"dashboard"`
		}
		err error
	)
	if raw, err = r.get(fmt.Sprintf("api/dashboards/%s", slug), nil); err != nil {
		return grafana.Board{}, BoardProperties{}, err
	}
	if err = json.Unmarshal(raw, &result); err != nil {
		return grafana.Board{}, BoardProperties{}, err
	}
	return result.Board, result.Meta, err
}

type FoundBoard struct {
	ID        uint     `json:"id"`
	Title     string   `json:"title"`
	URI       string   `json:"uri"`
	Type      string   `json:"type"`
	Tags      []string `json:"tags"`
	IsStarred bool     `json:"isStarred"`
}

// SearchDashboards search dashboards by query substring. Il allows restrict the result set with
// only starred dashboards and only for tags (logical OR applied to multiple tags).
func (r *Instance) SearchDashboards(query string, starred bool, tags ...string) ([]FoundBoard, error) {
	var (
		raw    []byte
		boards []FoundBoard
		err    error
	)
	u := url.URL{}
	q := u.Query()
	if query != "" {
		q.Set("query", query)
	}
	if starred {
		q.Set("starred", "true")
	}
	for _, tag := range tags {
		q.Add("tag", tag)
	}
	if raw, err = r.get("api/search", q); err != nil {
		return nil, err
	}
	err = json.Unmarshal(raw, &boards)
	return boards, err
}

// SetDashboard updates existing dashboard or creates a new one.
// Set dasboard ID to nil to create a new dashboard.
// Set overwrite to true if you want to overwrite existing dashboard with
// newer version or with same dashboard title.
func (r *Instance) SetDashboard(board grafana.Board, overwrite bool) error {
	var (
		newBoard struct {
			Dashboard grafana.Board `json:"dashboard"`
			Overwrite bool          `json:"overwrite"`
		}
		raw  []byte
		resp StatusMessage
		code int
		err  error
	)
	newBoard.Dashboard = board
	newBoard.Overwrite = overwrite
	if !overwrite {
		newBoard.Dashboard.ID = 0
	}
	if raw, err = json.Marshal(newBoard); err != nil {
		return err
	}
	if raw, code, err = r.post("api/dashboards/db", nil, raw); err != nil {
		return err
	}
	if err = json.Unmarshal(raw, &resp); err != nil {
		return err
	}
	switch code {
	case 401:
		return fmt.Errorf("%d %s", code, *resp.Message)
	case 412:
		return fmt.Errorf("%d %s", code, *resp.Message)
	}
	return nil
}

// DeleteDashboard deletes dashboard that selected by slug string.
func (r *Instance) DeleteDashboard(slug string) (StatusMessage, error) {
	var (
		raw   []byte
		reply StatusMessage
		err   error
	)
	if raw, err = r.delete(fmt.Sprintf("api/dashboards/db/%s", slug)); err != nil {
		return StatusMessage{}, err
	}
	err = json.Unmarshal(raw, &reply)
	return reply, err
}
<|endoftext|>


<commit_before>package util

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// Getenv retrieves the value of the environment variable named by the key.
// If that variable is not present, it iterates over the given aliases until
// it finds one that is. If none are present, the empty string is returned.
func Getenv(key string, aliases ...string) string {
	val := os.Getenv(key)
	if val != "" {
		return val
	}
	for _, alias := range aliases {
		val = os.Getenv(alias)
		if val != "" {
			return val
		}
	}
	return ""
}

// GetModDir gets directory of the module containing the package with path `pkgpath`. It passes the
// `go list` command `modflag`, which should be of the form `-mod=<mod mode>`, as described by `go
// help modules`.
func GetModDir(pkgpath string, modflag string) string {
	var cmd *exec.Cmd
	if modflag != "" {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Module}}", modflag, pkgpath)
	} else {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Module}}", pkgpath)
	}
	mod, err := cmd.Output()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			log.Printf("Warning: go list command failed, output below:\n%s%s", mod, err.Stderr)
		} else {
			log.Printf("Warning: Failed to run go list: %s", err.Error())
		}

		return ""
	}

	if strings.TrimSpace(string(mod)) == "<nil>" {
		// if modules aren't being used, return nothing
		return ""
	}

	if modflag != "" {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Module.Dir}}", modflag, pkgpath)
	} else {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Module.Dir}}", pkgpath)
	}
	modDir, err := cmd.Output()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			log.Printf("Warning: go list command failed, output below:\n%s%s", modDir, err.Stderr)
		} else {
			log.Printf("Warning: Failed to run go list: %s", err.Error())
		}

		return ""
	}

	trimmed := strings.TrimSpace(string(modDir))
	abs, err := filepath.Abs(trimmed)
	if err != nil {
		log.Printf("Warning: unable to make %s absolute: %s", trimmed, err.Error())
	}
	return abs
}

// GetPkgDir gets directory containing the package with path `pkgpath`. It passes the `go list`
// command `modflag`, which should be of the form `-mod=<mod mode>`, as described by `go help
// modules`.
func GetPkgDir(pkgpath string, modflag string) string {
	var cmd *exec.Cmd
	if modflag != "" {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Dir}}", modflag, pkgpath)
	} else {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Dir}}", pkgpath)
	}
	pkgDir, err := cmd.Output()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			log.Printf("Warning: go list command failed, output below:\n%s%s", pkgDir, err.Stderr)
		} else {
			log.Printf("Warning: Failed to run go list: %s", err.Error())
		}

		return ""
	}

	trimmed := strings.TrimSpace(string(pkgDir))
	abs, err := filepath.Abs(trimmed)
	if err != nil {
		log.Printf("Warning: unable to make %s absolute: %s", trimmed, err.Error())
	}
	return abs
}

// FileExists tests whether the file at `filename` exists.
func FileExists(filename string) bool {
	info, err := os.Stat(filename)
	if err != nil && !os.IsNotExist(err) {
		log.Printf("Unable to stat %s: %s\n", filename, err.Error())
	}
	return err == nil && !info.IsDir()
}
<commit_msg>Clarify some comments<commit_after>package util

import (
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// Getenv retrieves the value of the environment variable named by the key.
// If that variable is not present, it iterates over the given aliases until
// it finds one that is. If none are present, the empty string is returned.
func Getenv(key string, aliases ...string) string {
	val := os.Getenv(key)
	if val != "" {
		return val
	}
	for _, alias := range aliases {
		val = os.Getenv(alias)
		if val != "" {
			return val
		}
	}
	return ""
}

// GetModDir gets directory of the module containing the package with path `pkgpath`. It passes the
// `go list` command `modflag`, which should be of the form `-mod=<mod mode>`, as described by `go
// help modules`.
func GetModDir(pkgpath string, modflag string) string {
	var cmd *exec.Cmd
	if modflag != "" {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Module}}", modflag, pkgpath)
	} else {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Module}}", pkgpath)
	}
	mod, err := cmd.Output()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			log.Printf("Warning: go list command failed, output below:\n%s%s", mod, err.Stderr)
		} else {
			log.Printf("Warning: Failed to run go list: %s", err.Error())
		}

		return ""
	}

	if strings.TrimSpace(string(mod)) == "<nil>" {
		// if modules aren't being used, return the empty string
		return ""
	}

	if modflag != "" {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Module.Dir}}", modflag, pkgpath)
	} else {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Module.Dir}}", pkgpath)
	}
	modDir, err := cmd.Output()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			log.Printf("Warning: go list command failed, output below:\n%s%s", modDir, err.Stderr)
		} else {
			log.Printf("Warning: Failed to run go list: %s", err.Error())
		}

		return ""
	}

	trimmed := strings.TrimSpace(string(modDir))
	abs, err := filepath.Abs(trimmed)
	if err != nil {
		log.Printf("Warning: unable to make %s absolute: %s", trimmed, err.Error())
	}
	return abs
}

// GetPkgDir gets directory containing the package with path `pkgpath`. It passes the `go list`
// command `modflag`, which should be of the form `-mod=<mod mode>`, as described by `go help
// modules`.
func GetPkgDir(pkgpath string, modflag string) string {
	var cmd *exec.Cmd
	if modflag != "" {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Dir}}", modflag, pkgpath)
	} else {
		cmd = exec.Command("go", "list", "-e", "-f", "{{.Dir}}", pkgpath)
	}
	pkgDir, err := cmd.Output()
	if err != nil {
		if err, ok := err.(*exec.ExitError); ok {
			log.Printf("Warning: go list command failed, output below:\n%s%s", pkgDir, err.Stderr)
		} else {
			log.Printf("Warning: Failed to run go list: %s", err.Error())
		}

		return ""
	}

	trimmed := strings.TrimSpace(string(pkgDir))
	abs, err := filepath.Abs(trimmed)
	if err != nil {
		log.Printf("Warning: unable to make %s absolute: %s", trimmed, err.Error())
	}
	return abs
}

// FileExists tests whether the file at `filename` exists and is not a directory.
func FileExists(filename string) bool {
	info, err := os.Stat(filename)
	if err != nil && !os.IsNotExist(err) {
		log.Printf("Unable to stat %s: %s\n", filename, err.Error())
	}
	return err == nil && !info.IsDir()
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "io"
  "io/ioutil"
  "net/http"
  "os"
  "strings"
)

func downloadLatestCloudbleed() {
  url := "https://raw.githubusercontent.com/pirate/sites-using-cloudflare/master/sorted_unique_cf.txt"
  tokens := strings.Split(url, "/")
  fileName := tokens[len(tokens)-1]
  fmt.Println("Downloading latest Cloudbleed file...")

  output, err := os.Create(fileName)
  if err != nil {
    fmt.Println("Error while creating", fileName, "-", err)
    return
  }

  defer output.Close()

  response, err := http.Get(url)
  if err != nil {
    fmt.Println("Error while downloading", url, "-", err)
    return
  }

  defer response.Body.Close()

  n, err := io.Copy(output, response.Body)
  if err != nil {
    fmt.Println("Error while downloading", url, "-", err)
    return
  }

  fmt.Println(n, "bytes downloaded.")
}

func checkDomain(domain string) string {
  if _, err := os.Stat("sorted_unique_cf.txt"); os.IsNotExist(err) {
    downloadLatestCloudbleed()
  }

  b, err := ioutil.ReadFile("sorted_unique_cf.txt")
  if(err != nil) {
    panic(err)
  }

  s := string(b)

  if(strings.Contains(s, "\n" + strings.ToLower(domain) + "\n")) {
    return domain + " is in the Cloudflare directory"
  } else {
    return domain + " is not in the Cloudflare directory"
  }
}

func main() {
  if _, err := os.Stat("sorted_unique_cf.txt"); os.IsNotExist(err) {
    downloadLatestCloudbleed()
  }

  var domain string
  fmt.Print("What would you like to search in Cloudflare? -> ")
  fmt.Scanln(&domain)

  fmt.Println(checkDomain(domain))
}
<commit_msg>Add new check<commit_after>package main

import (
  "fmt"
  "io"
  "io/ioutil"
  "net/http"
  "os"
  "strings"
)

func downloadLatestCloudbleed() {
  url := "https://raw.githubusercontent.com/pirate/sites-using-cloudflare/master/sorted_unique_cf.txt"
  tokens := strings.Split(url, "/")
  fileName := tokens[len(tokens)-1]
  fmt.Println("Downloading latest Cloudbleed file...")

  output, err := os.Create(fileName)
  if err != nil {
    fmt.Println("Error while creating", fileName, "-", err)
    return
  }

  defer output.Close()

  response, err := http.Get(url)
  if err != nil {
    fmt.Println("Error while downloading", url, "-", err)
    return
  }

  defer response.Body.Close()

  n, err := io.Copy(output, response.Body)
  if err != nil {
    fmt.Println("Error while downloading", url, "-", err)
    return
  }

  fmt.Println(n, "bytes downloaded.")
}

func checkDomain(domain string) string {
  if _, err := os.Stat("sorted_unique_cf.txt"); os.IsNotExist(err) {
    downloadLatestCloudbleed()
  }

  b, err := ioutil.ReadFile("sorted_unique_cf.txt")
  if(err != nil) {
    panic(err)
  }

  s := string(b)

  if(strings.Contains(s, "\n" + strings.ToLower(domain) + "\n")) {
    return domain + " is in the Cloudflare directory"
  } else if(strings.Contains(s, strings.ToLower(domain))) {
    return domain + " is not specifically in the Cloudflare directory, but there are domains that contain " + domain + " as a substring"
  } else {
    return domain + " is not in the Cloudflare directory"
  }
}

func main() {
  if _, err := os.Stat("sorted_unique_cf.txt"); os.IsNotExist(err) {
    downloadLatestCloudbleed()
  }

  var domain string
  fmt.Print("What would you like to search in Cloudflare? -> ")
  fmt.Scanln(&domain)

  fmt.Println(checkDomain(domain))
}
<|endoftext|>