<commit_before>// Copyright 2017 Thibault Chataigner <thibault.chataigner@gmail.com>
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graphite

import (
	"bytes"
	"fmt"
	"net/url"
	"reflect"
	"testing"

	"github.com/go-kit/kit/log"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/prompb"

	"golang.org/x/net/context"
)

var (
	expectedLabels = []*prompb.Label{
		&prompb.Label{Name: model.MetricNameLabel, Value: "test"},
		&prompb.Label{Name: "owner", Value: "team-X"},
	}
	expectedSamples = []*prompb.Sample{
		&prompb.Sample{Value: float64(18), Timestamp: int64(0)},
		&prompb.Sample{Value: float64(42), Timestamp: int64(300000)},
	}
)

func fakeFetchExpandURL(ctx context.Context, l log.Logger, u *url.URL) ([]byte, error) {
	var body bytes.Buffer
	if u.String() == "http://fakeHost:6666/metrics/expand?format=json&leavesOnly=1&query=prometheus-prefix.test.%2A%2A" {
		body.WriteString("{\"results\": [\"prometheus-prefix.test.owner.team-X\", \"prometheus-prefix.test.owner.team-Y\"]}")
	}
	return body.Bytes(), nil
}

func fakeFetchRenderURL(ctx context.Context, l log.Logger, u *url.URL) ([]byte, error) {
	var body bytes.Buffer
	if u.String() == "http://fakeHost:6666/render/?format=json&from=0&target=prometheus-prefix.test.owner.team-X&until=300" {
		body.WriteString("[{\"target\": \"prometheus-prefix.test.owner.team-X\", \"datapoints\": [[18,0], [42,300]]}]")
	} else if u.String() == "http://fakeHost:6666/render/?format=json&from=0&target=seriesByTag%28%22name%3Dprometheus-prefix.test%22%2C%22owner%3Dteam-x%22%29&until=300" {
		body.WriteString("[")
		body.WriteString("{\"target\": \"prometheus-prefix.test\", \"tags\": {\"owner\": \"team-X\", \"name\": \"prometheus-prefix.test\"}, \"datapoints\": [[18,0], [42,300]]},")
		body.WriteString("{\"target\": \"prometheus-prefix.test\", \"tags\": {\"owner\": \"team-X\", \"name\": \"prometheus-prefix.test\", \"foo\": \"bar\"}, \"datapoints\": [[18,0], [42,300]]}")
		body.WriteString("]")
	}
	return body.Bytes(), nil
}

func TestQueryToTargets(t *testing.T) {
	fetchURL = fakeFetchExpandURL
	expectedTargets := []string{"prometheus-prefix.test.owner.team-X", "prometheus-prefix.test.owner.team-Y"}

	labelMatchers := []*prompb.LabelMatcher{
		// Query a specific metric.
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_EQ, Name: model.MetricNameLabel, Value: "test"},
		// Validate that we can match labels.
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_RE, Name: "owner", Value: "team.*"},
		// Also check that we are not equal to a fake label.
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_NEQ, Name: "invalid.", Value: "fake"},
	}
	query := &prompb.Query{
		StartTimestampMs: int64(0),
		EndTimestampMs:   int64(300),
		Matchers:         labelMatchers,
	}

	actualTargets, _ := testClient.queryToTargets(nil, query, testClient.cfg.DefaultPrefix)
	if !reflect.DeepEqual(expectedTargets, actualTargets) {
		t.Errorf("Expected %s, got %s", expectedTargets, actualTargets)
	}
}

func TestInvalidQueryToTargets(t *testing.T) {
	expectedErr := fmt.Errorf("Invalid remote query: no %s label provided", model.MetricNameLabel)

	labelMatchers := []*prompb.LabelMatcher{
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_EQ, Name: "labelname", Value: "labelvalue"},
	}
	invalidQuery := &prompb.Query{
		StartTimestampMs: int64(0),
		EndTimestampMs:   int64(300),
		Matchers:         labelMatchers,
	}

	_, err := testClient.queryToTargets(nil, invalidQuery, testClient.cfg.DefaultPrefix)
	if !reflect.DeepEqual(err, expectedErr) {
		t.Errorf("Error from queryToTargets not returned.  Expected %v, got %v", expectedErr, err)
	}
}

func TestTargetToTimeseries(t *testing.T) {
	fetchURL = fakeFetchRenderURL
	expectedTs := &prompb.TimeSeries{
		Labels:  expectedLabels,
		Samples: expectedSamples,
	}

	actualTs, err := testClient.targetToTimeseries(nil, "prometheus-prefix.test.owner.team-X", "0", "300", testClient.cfg.DefaultPrefix)
	if !reflect.DeepEqual(err, nil) {
		t.Errorf("Expected err: %s, got %s", nil, err)
	}
	if !reflect.DeepEqual(expectedTs, actualTs[0]) {
		t.Errorf("Expected %s, got %s", expectedTs, actualTs[0])
	}
}

func TestQueryTargetsWithTags(t *testing.T) {
	fetchURL = fakeFetchRenderURL

	labelMatchers := []*prompb.LabelMatcher{
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_EQ, Name: model.MetricNameLabel, Value: "test"},
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_EQ, Name: "owner", Value: "team-x"},
	}
	query := &prompb.Query{
		StartTimestampMs: int64(0),
		EndTimestampMs:   int64(300),
		Matchers:         labelMatchers,
	}

	expectedTargets := []string{
		"seriesByTag(\"name=prometheus-prefix.test\",\"owner=team-x\")",
	}

	expectedTs := []*prompb.TimeSeries{
		&prompb.TimeSeries{
			Labels:  expectedLabels,
			Samples: expectedSamples,
		},
		&prompb.TimeSeries{
			Labels: []*prompb.Label{
				&prompb.Label{Name: "foo", Value: "bar"},
				expectedLabels[0],
				expectedLabels[1],
			},
			Samples: expectedSamples,
		},
	}

	testClient.cfg.EnableTags = true
	targets, err := testClient.queryToTargetsWithTags(nil, query, testClient.cfg.DefaultPrefix)
	if err != nil {
		t.Errorf("Unexpected err: %s", err)
	}
	if !reflect.DeepEqual(expectedTargets, targets) {
		t.Errorf("Expected %s, got %s", expectedTargets, targets)
	}

	actualTs, err := testClient.targetToTimeseries(nil, targets[0], "0", "300", testClient.cfg.DefaultPrefix)
	testClient.cfg.EnableTags = false
	if err != nil {
		t.Errorf("Unexpected err: %s", err)
	}
	if !reflect.DeepEqual(expectedTs, actualTs) {
		t.Errorf("Expected %s, got %s", expectedTs, actualTs)
	}
}
<commit_msg>Fix test error<commit_after>// Copyright 2017 Thibault Chataigner <thibault.chataigner@gmail.com>
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graphite

import (
	"bytes"
	"fmt"
	"net/url"
	"reflect"
	"testing"

	"github.com/go-kit/kit/log"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/prompb"

	"golang.org/x/net/context"
)

var (
	expectedLabels = []*prompb.Label{
		&prompb.Label{Name: model.MetricNameLabel, Value: "test"},
		&prompb.Label{Name: "owner", Value: "team-X"},
	}
	expectedSamples = []*prompb.Sample{
		&prompb.Sample{Value: float64(18), Timestamp: int64(0)},
		&prompb.Sample{Value: float64(42), Timestamp: int64(300000)},
	}
)

func fakeFetchExpandURL(ctx context.Context, l log.Logger, u *url.URL) ([]byte, error) {
	var body bytes.Buffer
	if u.String() == "http://fakeHost:6666/metrics/expand?format=json&leavesOnly=1&query=prometheus-prefix.test.%2A%2A" {
		body.WriteString("{\"results\": [\"prometheus-prefix.test.owner.team-X\", \"prometheus-prefix.test.owner.team-Y\"]}")
	}
	return body.Bytes(), nil
}

func fakeFetchRenderURL(ctx context.Context, l log.Logger, u *url.URL) ([]byte, error) {
	var body bytes.Buffer
	if u.String() == "http://fakeHost:6666/render/?format=json&from=0&target=prometheus-prefix.test.owner.team-X&until=300" {
		body.WriteString("[{\"target\": \"prometheus-prefix.test.owner.team-X\", \"datapoints\": [[18,0], [42,300]]}]")
	} else if u.String() == "http://fakeHost:6666/render/?format=json&from=0&target=seriesByTag%28%22name%3Dprometheus-prefix.test%22%2C%22owner%3Dteam-x%22%29&until=300" {
		body.WriteString("[")
		body.WriteString("{\"target\": \"prometheus-prefix.test\", \"tags\": {\"owner\": \"team-X\", \"name\": \"prometheus-prefix.test\"}, \"datapoints\": [[18,0], [42,300]]},")
		body.WriteString("{\"target\": \"prometheus-prefix.test\", \"tags\": {\"owner\": \"team-X\", \"name\": \"prometheus-prefix.test\", \"foo\": \"bar\"}, \"datapoints\": [[18,0], [42,300]]}")
		body.WriteString("]")
	}
	return body.Bytes(), nil
}

func TestQueryToTargets(t *testing.T) {
	fetchURL = fakeFetchExpandURL
	expectedTargets := []string{"prometheus-prefix.test.owner.team-X", "prometheus-prefix.test.owner.team-Y"}

	labelMatchers := []*prompb.LabelMatcher{
		// Query a specific metric.
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_EQ, Name: model.MetricNameLabel, Value: "test"},
		// Validate that we can match labels.
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_RE, Name: "owner", Value: "team.*"},
		// Also check that we are not equal to a fake label.
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_NEQ, Name: "invalid.", Value: "fake"},
	}
	query := &prompb.Query{
		StartTimestampMs: int64(0),
		EndTimestampMs:   int64(300),
		Matchers:         labelMatchers,
	}

	actualTargets, _ := testClient.queryToTargets(nil, query, testClient.cfg.DefaultPrefix)
	if !reflect.DeepEqual(expectedTargets, actualTargets) {
		t.Errorf("Expected %s, got %s", expectedTargets, actualTargets)
	}
}

func TestInvalidQueryToTargets(t *testing.T) {
	expectedErr := fmt.Errorf("Invalid remote query: no %s label provided", model.MetricNameLabel)

	labelMatchers := []*prompb.LabelMatcher{
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_EQ, Name: "labelname", Value: "labelvalue"},
	}
	invalidQuery := &prompb.Query{
		StartTimestampMs: int64(0),
		EndTimestampMs:   int64(300),
		Matchers:         labelMatchers,
	}

	_, err := testClient.queryToTargets(nil, invalidQuery, testClient.cfg.DefaultPrefix)
	if !reflect.DeepEqual(err, expectedErr) {
		t.Errorf("Error from queryToTargets not returned.  Expected %v, got %v", expectedErr, err)
	}
}

func TestTargetToTimeseries(t *testing.T) {
	fetchURL = fakeFetchRenderURL
	expectedTs := &prompb.TimeSeries{
		Labels:  expectedLabels,
		Samples: expectedSamples,
	}

	actualTs, err := testClient.targetToTimeseries(nil, "prometheus-prefix.test.owner.team-X", "0", "300", testClient.cfg.DefaultPrefix)
	if !reflect.DeepEqual(err, nil) {
		t.Errorf("Expected no err, got %s", err)
	}
	if !reflect.DeepEqual(expectedTs, actualTs[0]) {
		t.Errorf("Expected %s, got %s", expectedTs, actualTs[0])
	}
}

func TestQueryTargetsWithTags(t *testing.T) {
	fetchURL = fakeFetchRenderURL

	labelMatchers := []*prompb.LabelMatcher{
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_EQ, Name: model.MetricNameLabel, Value: "test"},
		&prompb.LabelMatcher{Type: prompb.LabelMatcher_EQ, Name: "owner", Value: "team-x"},
	}
	query := &prompb.Query{
		StartTimestampMs: int64(0),
		EndTimestampMs:   int64(300),
		Matchers:         labelMatchers,
	}

	expectedTargets := []string{
		"seriesByTag(\"name=prometheus-prefix.test\",\"owner=team-x\")",
	}

	expectedTs := []*prompb.TimeSeries{
		&prompb.TimeSeries{
			Labels:  expectedLabels,
			Samples: expectedSamples,
		},
		&prompb.TimeSeries{
			Labels: []*prompb.Label{
				&prompb.Label{Name: "foo", Value: "bar"},
				expectedLabels[0],
				expectedLabels[1],
			},
			Samples: expectedSamples,
		},
	}

	testClient.cfg.EnableTags = true
	targets, err := testClient.queryToTargetsWithTags(nil, query, testClient.cfg.DefaultPrefix)
	if err != nil {
		t.Errorf("Unexpected err: %s", err)
	}
	if !reflect.DeepEqual(expectedTargets, targets) {
		t.Errorf("Expected %s, got %s", expectedTargets, targets)
	}

	actualTs, err := testClient.targetToTimeseries(nil, targets[0], "0", "300", testClient.cfg.DefaultPrefix)
	testClient.cfg.EnableTags = false
	if err != nil {
		t.Errorf("Unexpected err: %s", err)
	}
	if !reflect.DeepEqual(expectedTs, actualTs) {
		t.Errorf("Expected %s, got %s", expectedTs, actualTs)
	}
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"net"
	"strings"
	"testing"
	"time"

	"github.com/docker/machine/libmachine/state"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/test/integration/util"
)

func TestStartStop(t *testing.T) {
	tests := []struct {
		name string
		args []string
	}{
		{"nocache_oldest", []string{
			"--cache-images=false",
			fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion),
		}},
		{"feature_gates_newest_cni", []string{
			"--feature-gates",
			"ServerSideApply=true",
			"--network-plugin=cni",
			"--extra-config=kubelet.network-plugin=cni",
			fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion),
		}},
		{"containerd_and_non_default_apiserver_port", []string{
			"--container-runtime=containerd",
			"--docker-opt containerd=/var/run/containerd/containerd.sock",
			"--apiserver-port=8444",
		}},
		{"crio_ignore_preflights", []string{
			"--container-runtime=crio",
			"--extra-config",
			"kubeadm.ignore-preflight-errors=SystemVerification",
		}},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			r := NewMinikubeRunner(t)
			if !strings.Contains(test.name, "docker") && usingNoneDriver(r) {
				t.Skipf("skipping %s - incompatible with none driver", test.name)
			}

			r.RunCommand("config set WantReportErrorPrompt false", true)
			r.RunCommand("delete", false)
			r.CheckStatus(state.None.String())
			r.Start(test.args...)
			r.CheckStatus(state.Running.String())

			ip := r.RunCommand("ip", true)
			ip = strings.TrimRight(ip, "\n")
			if net.ParseIP(ip) == nil {
				t.Fatalf("IP command returned an invalid address: %s", ip)
			}

			// check for the current-context before and after the stop
			kubectlRunner := util.NewKubectlRunner(t)
			currentContext, err := kubectlRunner.RunCommand([]string{"config", "current-context"})
			if err != nil {
				t.Fatalf("Failed to fetch current-context")
			}
			if strings.TrimRight(string(currentContext), "\n") != "minikube" {
				t.Fatalf("got current-context - %q, want  current-context %q", string(currentContext), "minikube")
			}

			checkStop := func() error {
				r.RunCommand("stop", true)
				return r.CheckStatusNoFail(state.Stopped.String())
			}

			currentContext, err = kubectlRunner.RunCommand([]string{"config", "current-context"})
			if err != nil {
				t.Fatalf("Failed to fetch current-context")
			}
			if strings.TrimRight(string(currentContext), "\n") != "" {
				t.Fatalf("Failed to unset the current-context")
			}

			if err := util.Retry(t, checkStop, 5*time.Second, 6); err != nil {
				t.Fatalf("timed out while checking stopped status: %v", err)
			}

			r.Start(test.args...)
			r.CheckStatus(state.Running.String())

			r.RunCommand("delete", true)
			r.CheckStatus(state.None.String())
		})
	}
}
<commit_msg>Added a param in the test case to find the current-context after stop<commit_after>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"net"
	"strings"
	"testing"
	"time"

	"github.com/docker/machine/libmachine/state"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/test/integration/util"
)

func TestStartStop(t *testing.T) {
	tests := []struct {
		name string
		args []string
	}{
		{"nocache_oldest", []string{
			"--cache-images=false",
			fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion),
		}},
		{"feature_gates_newest_cni", []string{
			"--feature-gates",
			"ServerSideApply=true",
			"--network-plugin=cni",
			"--extra-config=kubelet.network-plugin=cni",
			fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion),
		}},
		{"containerd_and_non_default_apiserver_port", []string{
			"--container-runtime=containerd",
			"--docker-opt containerd=/var/run/containerd/containerd.sock",
			"--apiserver-port=8444",
		}},
		{"crio_ignore_preflights", []string{
			"--container-runtime=crio",
			"--extra-config",
			"kubeadm.ignore-preflight-errors=SystemVerification",
		}},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			r := NewMinikubeRunner(t)
			if !strings.Contains(test.name, "docker") && usingNoneDriver(r) {
				t.Skipf("skipping %s - incompatible with none driver", test.name)
			}

			r.RunCommand("config set WantReportErrorPrompt false", true)
			r.RunCommand("delete", false)
			r.CheckStatus(state.None.String())
			r.Start(test.args...)
			r.CheckStatus(state.Running.String())

			ip := r.RunCommand("ip", true)
			ip = strings.TrimRight(ip, "\n")
			if net.ParseIP(ip) == nil {
				t.Fatalf("IP command returned an invalid address: %s", ip)
			}

			// check for the current-context before and after the stop
			kubectlRunner := util.NewKubectlRunner(t)
			currentContext, err := kubectlRunner.RunCommand([]string{"config", "current-context"})
			if err != nil {
				t.Fatalf("Failed to fetch current-context")
			}
			if strings.TrimRight(string(currentContext), "\n") != "minikube" {
				t.Fatalf("got current-context - %q, want  current-context %q", string(currentContext), "minikube")
			}

			checkStop := func() error {
				r.RunCommand("stop", true)
				return r.CheckStatusNoFail(state.Stopped.String())
			}

			currentContext, err = kubectlRunner.RunCommand([]string{"config", "current-context"})
			if err != nil {
				t.Fatalf("Failed to fetch current-context")
			}
			if strings.TrimRight(string(currentContext), "\n") != "" {
				t.Fatalf("Failed to unset the current-context %q", string(currentContext))
			}

			if err := util.Retry(t, checkStop, 5*time.Second, 6); err != nil {
				t.Fatalf("timed out while checking stopped status: %v", err)
			}

			r.Start(test.args...)
			r.CheckStatus(state.Running.String())

			r.RunCommand("delete", true)
			r.CheckStatus(state.None.String())
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/shifr/imgwizard/cache"

	"github.com/gorilla/mux"
	"github.com/shifr/vips"
)

type Context struct {
	Path      string
	Format    string
	CachePath string
	Storage   string
	Width     int
	Height    int
}

type Settings struct {
	ListenAddr    string
	CacheDir      string
	Scheme        string
	Local404Thumb string
	AllowedSizes  []string
	AllowedMedia  []string
	Directories   []string
	UrlTemplate   string

	Context Context
	Options vips.Options
}

const (
	WEBP_HEADER = "image/webp"
)

var (
	settings         Settings
	supportedFormats = []string{"jpg", "jpeg", "png"}
	listenAddr       = flag.String("l", "127.0.0.1:8070", "Address to listen on")
	allowedMedia     = flag.String("m", "", "comma separated list of allowed media server hosts")
	allowedSizes     = flag.String("s", "", "comma separated list of allowed sizes")
	cacheDir         = flag.String("c", "/tmp/imgwizard", "directory for cached files")
	dirsToSearch     = flag.String("d", "", "comma separated list of directories to search requested file")
	local404Thumb    = flag.String("thumb", "/tmp/404.jpg", "path to default image")
	mark             = flag.String("mark", "images", "Mark for nginx")
	quality          = flag.Int("q", 0, "image quality after resize")
)

// loadSettings loads settings from settings.json
// and from command-line
func (s *Settings) loadSettings() {

	s.Scheme = "http"
	s.AllowedSizes = nil
	s.AllowedMedia = nil

	//defaults for vips
	s.Options.Crop = true
	s.Options.Enlarge = true
	s.Options.Quality = 80
	s.Options.Extend = vips.EXTEND_WHITE
	s.Options.Interpolator = vips.BILINEAR
	s.Options.Gravity = vips.CENTRE

	var sizes = "[0-9]*x[0-9]*"
	var medias = ""
	var proxyMark = *mark

	s.ListenAddr = *listenAddr

	if *allowedMedia != "" {
		s.AllowedMedia = strings.Split(*allowedMedia, ",")
	}

	if *allowedSizes != "" {
		s.AllowedSizes = strings.Split(*allowedSizes, ",")
	}

	if *dirsToSearch != "" {
		s.Directories = strings.Split(*dirsToSearch, ",")
	}

	s.CacheDir = *cacheDir
	s.Local404Thumb = *local404Thumb

	if *quality != 0 {
		s.Options.Quality = *quality
	}

	if len(s.AllowedSizes) > 0 {
		sizes = strings.Join(s.AllowedSizes, "|")
	}

	if len(s.AllowedMedia) > 0 {
		medias = strings.Join(s.AllowedMedia, "|")
	}

	s.UrlTemplate = fmt.Sprintf(
		"/{mark:%s}/{storage:loc|rem}/{size:%s}/{path:%s.+}", proxyMark, sizes, medias)
}

// makeCachePath generates cache path from resized image
func (s *Settings) makeCachePath() {
	var subPath string
	var cacheImageName string

	pathParts := strings.Split(s.Context.Path, "/")
	lastIndex := len(pathParts) - 1
	imageData := strings.Split(pathParts[lastIndex], ".")
	imageName, imageFormat := imageData[0], strings.ToLower(imageData[1])

	if s.Options.Webp {
		cacheImageName = fmt.Sprintf(
			"%s_%dx%d_webp_.%s", imageName, s.Options.Width, s.Options.Height, imageFormat)
	} else {
		cacheImageName = fmt.Sprintf(
			"%s_%dx%d.%s", imageName, s.Options.Width, s.Options.Height, imageFormat)
	}

	switch s.Context.Storage {
	case "loc":
		subPath = strings.Join(pathParts[:lastIndex], "/")
	case "rem":
		subPath = strings.Join(pathParts[1:lastIndex], "/")
	}
	s.Context.Format = imageFormat
	s.Context.CachePath = fmt.Sprintf(
		"%s/%s/%s", s.CacheDir, subPath, cacheImageName)
}

// getLocalImage fetches original image from file system
func getLocalImage(s *Settings) ([]byte, error) {
	var image []byte
	var filePath string
	var file *os.File
	var err error

	if len(s.Directories) > 0 {
		found := false
		for _, dir := range s.Directories {
			filePath = path.Join("/", dir, s.Context.Path)
			file, err = os.Open(filePath)
			if err == nil {
				found = true
				break
			}
		}
		if !found {
			file, err = os.Open(s.Local404Thumb)
			if err != nil {
				return image, err
			}
		}
	} else {
		file, err = os.Open(path.Join("/", s.Context.Path))
		if err != nil {
			file, err = os.Open(s.Local404Thumb)
			if err != nil {
				return image, err
			}
		}
	}

	info, _ := file.Stat()
	image = make([]byte, info.Size())

	_, err = file.Read(image)
	if err != nil {
		return image, err
	}

	return image, nil
}

// getRemoteImage fetches original image by http url
func getRemoteImage(url string) ([]byte, error) {
	var image []byte

	resp, err := http.Get(url)
	if err != nil {
		return image, err
	}
	defer resp.Body.Close()

	image, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return image, err
	}

	return image, nil
}

// getOrCreateImage check cache path for requested image
// if image doesn't exist - creates it
func getOrCreateImage() []byte {
	sett := settings
	sett.makeCachePath()

	var c *cache.Cache
	var image []byte
	var err error

	if image, err = c.Get(sett.Context.CachePath); err == nil {
		return image
	}

	switch sett.Context.Storage {
	case "loc":
		image, err = getLocalImage(&sett)
		if err != nil {
			log.Println("Can't get orig local file, reason - ", err)
		}

	case "rem":
		imgUrl := fmt.Sprintf("%s://%s", sett.Scheme, sett.Context.Path)
		image, err = getRemoteImage(imgUrl)
		if err != nil {
			log.Println("Can't get orig remote file, reason - ", err)
		}
	}

	if !stringIsExists(sett.Context.Format, supportedFormats) {
		err = c.Set(sett.Context.CachePath, image)
		if err != nil {
			log.Println("Can't set cache, reason - ", err)
		}
		return image
	}

	buf, err := vips.Resize(image, sett.Options)
	if err != nil {
		log.Println("Can't resize image, reason - ", err)
	}

	err = c.Set(sett.Context.CachePath, buf)
	if err != nil {
		log.Println("Can't set cache, reason - ", err)
	}

	return buf
}

func stringIsExists(str string, list []string) bool {
	for _, el := range list {
		if el == str {
			return true
		}
	}
	return false
}

func fetchImage(rw http.ResponseWriter, req *http.Request) {
	acceptedTypes := strings.Split(req.Header["Accept"][0], ",")

	settings.Options.Webp = stringIsExists(WEBP_HEADER, acceptedTypes)
	params := mux.Vars(req)
	sizes := strings.Split(params["size"], "x")

	settings.Context.Storage = params["storage"]
	settings.Context.Path = params["path"]
	settings.Options.Width, _ = strconv.Atoi(sizes[0])
	settings.Options.Height, _ = strconv.Atoi(sizes[1])

	resultImage := getOrCreateImage()

	rw.Write(resultImage)
}

func main() {
	flag.Parse()
	settings.loadSettings()

	r := mux.NewRouter()
	r.HandleFunc(settings.UrlTemplate, fetchImage).Methods("GET")

	log.Printf("ImgWizard started on http://%s", settings.ListenAddr)
	http.ListenAndServe(settings.ListenAddr, r)
}
<commit_msg>net/http custom router instead of gorilla<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
	"strings"

	"github.com/shifr/imgwizard/cache"
	"github.com/shifr/vips"
)

type Route struct {
	pattern *regexp.Regexp
	handler http.Handler
}

type RegexpHandler struct {
	routes []*Route
}

func (h *RegexpHandler) HandleFunc(pattern *regexp.Regexp, handler func(http.ResponseWriter, *http.Request)) {
	h.routes = append(h.routes, &Route{pattern, http.HandlerFunc(handler)})
}

func (h *RegexpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	for _, route := range h.routes {
		if route.pattern.MatchString(r.URL.Path) {
			route.handler.ServeHTTP(w, r)
			return
		}
	}
	http.NotFound(w, r)
}

type Context struct {
	Path      string
	Format    string
	CachePath string
	Storage   string
	Width     int
	Height    int
}

type Settings struct {
	ListenAddr    string
	CacheDir      string
	Scheme        string
	Local404Thumb string
	AllowedSizes  []string
	AllowedMedia  []string
	Directories   []string
	UrlExp        *regexp.Regexp

	Context Context
	Options vips.Options
}

const (
	WEBP_HEADER = "image/webp"
)

var (
	settings         Settings
	supportedFormats = []string{"jpg", "jpeg", "png"}
	listenAddr       = flag.String("l", "127.0.0.1:8070", "Address to listen on")
	allowedMedia     = flag.String("m", "", "comma separated list of allowed media server hosts")
	allowedSizes     = flag.String("s", "", "comma separated list of allowed sizes")
	cacheDir         = flag.String("c", "/tmp/imgwizard", "directory for cached files")
	dirsToSearch     = flag.String("d", "", "comma separated list of directories to search requested file")
	local404Thumb    = flag.String("thumb", "/tmp/404.jpg", "path to default image")
	mark             = flag.String("mark", "images", "Mark for nginx")
	quality          = flag.Int("q", 0, "image quality after resize")
)

// loadSettings loads settings from settings.json
// and from command-line
func (s *Settings) loadSettings() {

	s.Scheme = "http"
	s.AllowedSizes = nil
	s.AllowedMedia = nil

	//defaults for vips
	s.Options.Crop = true
	s.Options.Enlarge = true
	s.Options.Quality = 80
	s.Options.Extend = vips.EXTEND_WHITE
	s.Options.Interpolator = vips.BILINEAR
	s.Options.Gravity = vips.CENTRE

	var sizes = "[0-9]*x[0-9]*"
	var medias = ""
	var proxyMark = *mark

	s.ListenAddr = *listenAddr

	if *allowedMedia != "" {
		s.AllowedMedia = strings.Split(*allowedMedia, ",")
	}

	if *allowedSizes != "" {
		s.AllowedSizes = strings.Split(*allowedSizes, ",")
	}

	if *dirsToSearch != "" {
		s.Directories = strings.Split(*dirsToSearch, ",")
	}

	s.CacheDir = *cacheDir
	s.Local404Thumb = *local404Thumb

	if *quality != 0 {
		s.Options.Quality = *quality
	}

	if len(s.AllowedSizes) > 0 {
		sizes = strings.Join(s.AllowedSizes, "|")
	}

	if len(s.AllowedMedia) > 0 {
		medias = strings.Join(s.AllowedMedia, "|")
	}

	template := fmt.Sprintf(
		"/(?P<mark>%s)/(?P<storage>loc|rem)/(?P<size>%s)/(?P<path>%s.+)", proxyMark, sizes, medias)
	s.UrlExp, _ = regexp.Compile(template)
}

// makeCachePath generates cache path from resized image
func (s *Settings) makeCachePath() {
	var subPath string
	var cacheImageName string

	pathParts := strings.Split(s.Context.Path, "/")
	lastIndex := len(pathParts) - 1
	imageData := strings.Split(pathParts[lastIndex], ".")
	imageName, imageFormat := imageData[0], strings.ToLower(imageData[1])

	if s.Options.Webp {
		cacheImageName = fmt.Sprintf(
			"%s_%dx%d_webp_.%s", imageName, s.Options.Width, s.Options.Height, imageFormat)
	} else {
		cacheImageName = fmt.Sprintf(
			"%s_%dx%d.%s", imageName, s.Options.Width, s.Options.Height, imageFormat)
	}

	switch s.Context.Storage {
	case "loc":
		subPath = strings.Join(pathParts[:lastIndex], "/")
	case "rem":
		subPath = strings.Join(pathParts[1:lastIndex], "/")
	}
	s.Context.Format = imageFormat
	s.Context.CachePath = fmt.Sprintf(
		"%s/%s/%s", s.CacheDir, subPath, cacheImageName)
}

// getLocalImage fetches original image from file system
func getLocalImage(s *Settings) ([]byte, error) {
	var image []byte
	var filePath string
	var file *os.File
	var err error

	if len(s.Directories) > 0 {
		found := false
		for _, dir := range s.Directories {
			filePath = path.Join("/", dir, s.Context.Path)
			file, err = os.Open(filePath)
			if err == nil {
				found = true
				break
			}
		}
		if !found {
			file, err = os.Open(s.Local404Thumb)
			if err != nil {
				return image, err
			}
		}
	} else {
		file, err = os.Open(path.Join("/", s.Context.Path))
		if err != nil {
			file, err = os.Open(s.Local404Thumb)
			if err != nil {
				return image, err
			}
		}
	}

	info, _ := file.Stat()
	image = make([]byte, info.Size())

	_, err = file.Read(image)
	if err != nil {
		return image, err
	}

	return image, nil
}

// getRemoteImage fetches original image by http url
func getRemoteImage(url string) ([]byte, error) {
	var image []byte

	resp, err := http.Get(url)
	if err != nil {
		return image, err
	}
	defer resp.Body.Close()

	image, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return image, err
	}

	return image, nil
}

// getOrCreateImage check cache path for requested image
// if image doesn't exist - creates it
func getOrCreateImage() []byte {
	sett := settings
	sett.makeCachePath()

	var c *cache.Cache
	var image []byte
	var err error

	if image, err = c.Get(sett.Context.CachePath); err == nil {
		return image
	}

	switch sett.Context.Storage {
	case "loc":
		image, err = getLocalImage(&sett)
		if err != nil {
			log.Println("Can't get orig local file, reason - ", err)
		}

	case "rem":
		imgUrl := fmt.Sprintf("%s://%s", sett.Scheme, sett.Context.Path)
		image, err = getRemoteImage(imgUrl)
		if err != nil {
			log.Println("Can't get orig remote file, reason - ", err)
		}
	}

	if !stringIsExists(sett.Context.Format, supportedFormats) {
		err = c.Set(sett.Context.CachePath, image)
		if err != nil {
			log.Println("Can't set cache, reason - ", err)
		}
		return image
	}

	buf, err := vips.Resize(image, sett.Options)
	if err != nil {
		log.Println("Can't resize image, reason - ", err)
	}

	err = c.Set(sett.Context.CachePath, buf)
	if err != nil {
		log.Println("Can't set cache, reason - ", err)
	}

	return buf
}

func stringIsExists(str string, list []string) bool {
	for _, el := range list {
		if el == str {
			return true
		}
	}
	return false
}

func parseVars(req *http.Request) map[string]string {
	params := make(map[string]string)
	match := settings.UrlExp.FindStringSubmatch(req.RequestURI)
	for i, name := range settings.UrlExp.SubexpNames() {
		params[name] = match[i]
	}

	return params
}

func fetchImage(rw http.ResponseWriter, req *http.Request) {
	acceptedTypes := strings.Split(req.Header["Accept"][0], ",")
	params := parseVars(req)
	sizes := strings.Split(params["size"], "x")

	settings.Options.Webp = stringIsExists(WEBP_HEADER, acceptedTypes)
	settings.Context.Storage = params["storage"]
	settings.Context.Path = params["path"]
	settings.Options.Width, _ = strconv.Atoi(sizes[0])
	settings.Options.Height, _ = strconv.Atoi(sizes[1])

	resultImage := getOrCreateImage()

	rw.Header().Set("Content-Length", strconv.Itoa(len(resultImage)))
	rw.Write(resultImage)
}

func main() {
	flag.Parse()
	settings.loadSettings()

	r := new(RegexpHandler)
	r.HandleFunc(settings.UrlExp, fetchImage)

	log.Printf("ImgWizard started on http://%s", settings.ListenAddr)
	http.ListenAndServe(settings.ListenAddr, r)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"regexp"
	"sync"
	"time"

	"github.com/nsqio/go-nsq"
	"github.com/nsqio/nsq/internal/clusterinfo"
	"github.com/nsqio/nsq/internal/http_api"
)

type TopicDiscoverer struct {
	ci       *clusterinfo.ClusterInfo
	topics   map[string]*ConsumerFileLogger
	hupChan  chan os.Signal
	termChan chan os.Signal
	wg       sync.WaitGroup
	cfg      *nsq.Config
}

func newTopicDiscoverer(cfg *nsq.Config,
	hupChan chan os.Signal, termChan chan os.Signal,
	connectTimeout time.Duration, requestTimeout time.Duration) *TopicDiscoverer {
	return &TopicDiscoverer{
		ci:       clusterinfo.New(nil, http_api.NewClient(nil, connectTimeout, requestTimeout)),
		topics:   make(map[string]*ConsumerFileLogger),
		hupChan:  hupChan,
		termChan: termChan,
		cfg:      cfg,
	}
}

func (t *TopicDiscoverer) updateTopics(topics []string, pattern string) {
	for _, topic := range topics {
		if _, ok := t.topics[topic]; ok {
			continue
		}

		if !allowTopicName(pattern, topic) {
			log.Printf("skipping topic %s (doesn't match pattern %s)", topic, pattern)
			continue
		}

		cfl, err := newConsumerFileLogger(topic, t.cfg)
		if err != nil {
			log.Printf("ERROR: couldn't create logger for new topic %s: %s", topic, err)
			continue
		}
		t.topics[topic] = cfl

		t.wg.Add(1)
		go func(cfl *ConsumerFileLogger) {
			cfl.F.router(cfl.C)
			t.wg.Done()
		}(cfl)
	}
}

func (t *TopicDiscoverer) poller(addrs []string, sync bool, pattern string) {
	var ticker <-chan time.Time
	if sync {
		ticker = time.Tick(*topicPollRate)
	}
	for {
		select {
		case <-ticker:
			newTopics, err := t.ci.GetLookupdTopics(addrs)
			if err != nil {
				log.Printf("ERROR: could not retrieve topic list: %s", err)
				continue
			}
			t.updateTopics(newTopics, pattern)
		case <-t.termChan:
			for _, cfl := range t.topics {
				close(cfl.F.termChan)
			}
			break
		case <-t.hupChan:
			for _, cfl := range t.topics {
				cfl.F.hupChan <- true
			}
		}
	}
	t.wg.Wait()
}

func allowTopicName(pattern string, name string) bool {
	if pattern == "" {
		return true
	}
	match, err := regexp.MatchString(pattern, name)
	if err != nil {
		return false
	}
	return match
}
<commit_msg>nsq_to_file: fix term handler<commit_after>package main

import (
	"log"
	"os"
	"regexp"
	"sync"
	"time"

	"github.com/nsqio/go-nsq"
	"github.com/nsqio/nsq/internal/clusterinfo"
	"github.com/nsqio/nsq/internal/http_api"
)

type TopicDiscoverer struct {
	ci       *clusterinfo.ClusterInfo
	topics   map[string]*ConsumerFileLogger
	hupChan  chan os.Signal
	termChan chan os.Signal
	wg       sync.WaitGroup
	cfg      *nsq.Config
}

func newTopicDiscoverer(cfg *nsq.Config,
	hupChan chan os.Signal, termChan chan os.Signal,
	connectTimeout time.Duration, requestTimeout time.Duration) *TopicDiscoverer {
	return &TopicDiscoverer{
		ci:       clusterinfo.New(nil, http_api.NewClient(nil, connectTimeout, requestTimeout)),
		topics:   make(map[string]*ConsumerFileLogger),
		hupChan:  hupChan,
		termChan: termChan,
		cfg:      cfg,
	}
}

func (t *TopicDiscoverer) updateTopics(topics []string, pattern string) {
	for _, topic := range topics {
		if _, ok := t.topics[topic]; ok {
			continue
		}

		if !allowTopicName(pattern, topic) {
			log.Printf("skipping topic %s (doesn't match pattern %s)", topic, pattern)
			continue
		}

		cfl, err := newConsumerFileLogger(topic, t.cfg)
		if err != nil {
			log.Printf("ERROR: couldn't create logger for new topic %s: %s", topic, err)
			continue
		}
		t.topics[topic] = cfl

		t.wg.Add(1)
		go func(cfl *ConsumerFileLogger) {
			cfl.F.router(cfl.C)
			t.wg.Done()
		}(cfl)
	}
}

func (t *TopicDiscoverer) poller(addrs []string, sync bool, pattern string) {
	var ticker <-chan time.Time
	if sync {
		ticker = time.Tick(*topicPollRate)
	}
forloop:
	for {
		select {
		case <-ticker:
			newTopics, err := t.ci.GetLookupdTopics(addrs)
			if err != nil {
				log.Printf("ERROR: could not retrieve topic list: %s", err)
				continue
			}
			t.updateTopics(newTopics, pattern)
		case <-t.termChan:
			for _, cfl := range t.topics {
				close(cfl.F.termChan)
			}
			break forloop
		case <-t.hupChan:
			for _, cfl := range t.topics {
				cfl.F.hupChan <- true
			}
		}
	}
	t.wg.Wait()
}

func allowTopicName(pattern string, name string) bool {
	if pattern == "" {
		return true
	}
	match, err := regexp.MatchString(pattern, name)
	if err != nil {
		return false
	}
	return match
}
<|endoftext|>


<commit_before>// Package cliutil contains methods used across all cli commands
// @todo: get rid of os.Exits and use errors instread
package util

import (
	"encoding/json"
	"fmt"
	"net"
	"os"

	"github.com/micro/cli/v2"
	"github.com/micro/micro/v3/internal/config"
)

const (
	// EnvLocal is a builtin environment, it represents your local `micro server`
	EnvLocal = "local"
	// EnvPlatform is a builtin environment, the One True Micro Live(tm) environment.
	EnvPlatform = "platform"
)

const (
	// localProxyAddress is the default proxy address for environment server
	localProxyAddress = "127.0.0.1:8081"
	// platformProxyAddress is teh default proxy address for environment platform
	platformProxyAddress = "proxy.m3o.com"
)

var (
	// list of services managed
	// TODO: make use server/server list
	services = []string{
		// runtime services
		"config",   // ????
		"network",  // :8085 (peer), :8443 (proxy)
		"runtime",  // :8088
		"registry", // :8000
		"broker",   // :8001
		"store",    // :8002
		"router",   // :8084
		"debug",    // :????
		"proxy",    // :8081
		"api",      // :8080
		"auth",     // :8010
		"web",      // :8082
	}
)

var defaultEnvs = map[string]Env{
	EnvLocal: {
		Name:         EnvLocal,
		ProxyAddress: localProxyAddress,
	},
	EnvPlatform: {
		Name:         EnvPlatform,
		ProxyAddress: platformProxyAddress,
	},
}

func isBuiltinService(command string) bool {
	for _, service := range services {
		if command == service {
			return true
		}
	}
	return false
}

// CLIProxyAddress returns the proxy address which should be set for the client
func CLIProxyAddress(ctx *cli.Context) string {
	// This makes `micro [command name] --help` work without a server
	for _, arg := range os.Args {
		if arg == "--help" || arg == "-h" {
			return ""
		}
	}
	switch ctx.Args().First() {
	case "new", "server", "help", "env":
		return ""
	}

	// fix for "micro service [command]", e.g "micro service auth"
	if ctx.Args().First() == "service" && isBuiltinService(ctx.Args().Get(1)) {
		return ""
	}

	// don't set the proxy address on the proxy
	if ctx.Args().First() == "proxy" {
		return ""
	}

	return GetEnv(ctx).ProxyAddress
}

type Env struct {
	Name         string
	ProxyAddress string
}

func AddEnv(env Env) {
	envs := getEnvs()
	envs[env.Name] = env
	setEnvs(envs)
}

func getEnvs() map[string]Env {
	envsJSON, err := config.Get("envs")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	envs := map[string]Env{}
	if len(envsJSON) > 0 {
		err := json.Unmarshal([]byte(envsJSON), &envs)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}
	for k, v := range defaultEnvs {
		envs[k] = v
	}
	return envs
}

func setEnvs(envs map[string]Env) {
	envsJSON, err := json.Marshal(envs)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	err = config.Set(string(envsJSON), "envs")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

// GetEnv returns the current selected environment
// Does not take
func GetEnv(ctx *cli.Context) Env {
	var envName string
	if len(ctx.String("env")) > 0 {
		envName = ctx.String("env")
	} else {
		env, err := config.Get("env")
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if env == "" {
			env = EnvLocal
		}
		envName = env
	}

	return GetEnvByName(envName)
}

func GetEnvByName(env string) Env {
	envs := getEnvs()

	envir, ok := envs[env]
	if !ok {
		fmt.Println(fmt.Sprintf("Env \"%s\" not found. See `micro env` for available environments.", env))
		os.Exit(1)
	}

	if len(envir.ProxyAddress) == 0 {
		return envir
	}

	// default to :8081 (the proxy port)
	if _, port, _ := net.SplitHostPort(envir.ProxyAddress); len(port) == 0 {
		envir.ProxyAddress = net.JoinHostPort(envir.ProxyAddress, "8081")
	}

	return envir
}

func GetEnvs() []Env {
	envs := getEnvs()
	ret := []Env{defaultEnvs[EnvLocal], defaultEnvs[EnvPlatform]}
	nonDefaults := []Env{}
	for _, env := range envs {
		if _, isDefault := defaultEnvs[env.Name]; !isDefault {
			nonDefaults = append(nonDefaults, env)
		}
	}
	// @todo order nondefault envs alphabetically
	ret = append(ret, nonDefaults...)
	return ret
}

// SetEnv selects an environment to be used.
func SetEnv(envName string) {
	envs := getEnvs()
	_, ok := envs[envName]
	if !ok {
		fmt.Printf("Environment '%v' does not exist\n", envName)
		os.Exit(1)
	}
	config.Set(envName, "env")
}

// DelEnv deletes an env from config
func DelEnv(envName string) {
	envs := getEnvs()
	_, ok := envs[envName]
	if !ok {
		fmt.Printf("Environment '%v' does not exist\n", envName)
		os.Exit(1)
	}
	delete(envs, envName)
	setEnvs(envs)
}

func IsPlatform(ctx *cli.Context) bool {
	return GetEnv(ctx).Name == EnvPlatform
}

type Exec func(*cli.Context, []string) ([]byte, error)

func Print(e Exec) func(*cli.Context) error {
	return func(c *cli.Context) error {
		rsp, err := e(c, c.Args().Slice())
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if len(rsp) > 0 {
			fmt.Printf("%s\n", string(rsp))
		}
		return nil
	}
}
<commit_msg>client/cli: set platform port to 443 (#1160)<commit_after>// Package cliutil contains methods used across all cli commands
// @todo: get rid of os.Exits and use errors instread
package util

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/micro/cli/v2"
	"github.com/micro/micro/v3/internal/config"
)

const (
	// EnvLocal is a builtin environment, it represents your local `micro server`
	EnvLocal = "local"
	// EnvPlatform is a builtin environment, the One True Micro Live(tm) environment.
	EnvPlatform = "platform"
)

const (
	// localProxyAddress is the default proxy address for environment server
	localProxyAddress = "127.0.0.1:8081"
	// platformProxyAddress is teh default proxy address for environment platform
	platformProxyAddress = "proxy.m3o.com:443"
)

var (
	// list of services managed
	// TODO: make use server/server list
	services = []string{
		// runtime services
		"config",   // ????
		"network",  // :8085 (peer), :8443 (proxy)
		"runtime",  // :8088
		"registry", // :8000
		"broker",   // :8001
		"store",    // :8002
		"router",   // :8084
		"debug",    // :????
		"proxy",    // :8081
		"api",      // :8080
		"auth",     // :8010
		"web",      // :8082
	}
)

var defaultEnvs = map[string]Env{
	EnvLocal: {
		Name:         EnvLocal,
		ProxyAddress: localProxyAddress,
	},
	EnvPlatform: {
		Name:         EnvPlatform,
		ProxyAddress: platformProxyAddress,
	},
}

func isBuiltinService(command string) bool {
	for _, service := range services {
		if command == service {
			return true
		}
	}
	return false
}

// CLIProxyAddress returns the proxy address which should be set for the client
func CLIProxyAddress(ctx *cli.Context) string {
	// This makes `micro [command name] --help` work without a server
	for _, arg := range os.Args {
		if arg == "--help" || arg == "-h" {
			return ""
		}
	}
	switch ctx.Args().First() {
	case "new", "server", "help", "env":
		return ""
	}

	// fix for "micro service [command]", e.g "micro service auth"
	if ctx.Args().First() == "service" && isBuiltinService(ctx.Args().Get(1)) {
		return ""
	}

	// don't set the proxy address on the proxy
	if ctx.Args().First() == "proxy" {
		return ""
	}

	return GetEnv(ctx).ProxyAddress
}

type Env struct {
	Name         string
	ProxyAddress string
}

func AddEnv(env Env) {
	envs := getEnvs()
	envs[env.Name] = env
	setEnvs(envs)
}

func getEnvs() map[string]Env {
	envsJSON, err := config.Get("envs")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	envs := map[string]Env{}
	if len(envsJSON) > 0 {
		err := json.Unmarshal([]byte(envsJSON), &envs)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}
	for k, v := range defaultEnvs {
		envs[k] = v
	}
	return envs
}

func setEnvs(envs map[string]Env) {
	envsJSON, err := json.Marshal(envs)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	err = config.Set(string(envsJSON), "envs")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

// GetEnv returns the current selected environment
// Does not take
func GetEnv(ctx *cli.Context) Env {
	var envName string
	if len(ctx.String("env")) > 0 {
		envName = ctx.String("env")
	} else {
		env, err := config.Get("env")
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if env == "" {
			env = EnvLocal
		}
		envName = env
	}

	return GetEnvByName(envName)
}

func GetEnvByName(env string) Env {
	envs := getEnvs()

	envir, ok := envs[env]
	if !ok {
		fmt.Println(fmt.Sprintf("Env \"%s\" not found. See `micro env` for available environments.", env))
		os.Exit(1)
	}
	return envir
}

func GetEnvs() []Env {
	envs := getEnvs()
	ret := []Env{defaultEnvs[EnvLocal], defaultEnvs[EnvPlatform]}
	nonDefaults := []Env{}
	for _, env := range envs {
		if _, isDefault := defaultEnvs[env.Name]; !isDefault {
			nonDefaults = append(nonDefaults, env)
		}
	}
	// @todo order nondefault envs alphabetically
	ret = append(ret, nonDefaults...)
	return ret
}

// SetEnv selects an environment to be used.
func SetEnv(envName string) {
	envs := getEnvs()
	_, ok := envs[envName]
	if !ok {
		fmt.Printf("Environment '%v' does not exist\n", envName)
		os.Exit(1)
	}
	config.Set(envName, "env")
}

// DelEnv deletes an env from config
func DelEnv(envName string) {
	envs := getEnvs()
	_, ok := envs[envName]
	if !ok {
		fmt.Printf("Environment '%v' does not exist\n", envName)
		os.Exit(1)
	}
	delete(envs, envName)
	setEnvs(envs)
}

func IsPlatform(ctx *cli.Context) bool {
	return GetEnv(ctx).Name == EnvPlatform
}

type Exec func(*cli.Context, []string) ([]byte, error)

func Print(e Exec) func(*cli.Context) error {
	return func(c *cli.Context) error {
		rsp, err := e(c, c.Args().Slice())
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		if len(rsp) > 0 {
			fmt.Printf("%s\n", string(rsp))
		}
		return nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/kylelemons/go-gypsy/yaml"
	"github.com/lib/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type Worker struct {
	Db         *sql.DB
	LastRunAt  time.Time
	InsertStmt *sql.Stmt
	UpdateStmt *sql.Stmt
}

var worker Worker

//  command line flags
var (
	environment   = flag.String("environment", "", "Environment to run in, e.g. staging, production")
	config        = flag.String("config", "./config/database.yml", "database configuration file")
	sr_number     = flag.String("sr-number", "", "SR number to fetch")
	backfill      = flag.Bool("backfill", false, "run in reverse and backfill data")
	backfill_date = flag.String("backfill-from", time.Now().Format(time.RFC3339), "date to start backfilling data from. Use RFC3339 format. Default will be the current time.")
)

func init() {
	flag.Parse()

	log.Printf("running in %s environment, configuration file %s", *environment, *config)
	settings := yaml.ConfigFile(*config)

	// setup database connection
	driver, err := settings.Get(fmt.Sprintf("%s.driver", *environment))
	if err != nil {
		log.Fatal("error loading db driver", err)
	}

	connstr, err := settings.Get(fmt.Sprintf("%s.connstr", *environment))
	if err != nil {
		log.Fatal("error loading db connstr", err)
	}

	db, err := sql.Open(driver, connstr)
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}

	log.Printf("database connstr: %s", connstr)

	worker.Db = db
	worker.SetupStmts()
}

func main() {
	defer worker.Db.Close()

	if *sr_number != "" {
		sr := fetchSingleRequest(*sr_number)
		sr.Save()
		return
	}

	start_backfill_from := *backfill_date
	for {
		switch {
		case *backfill:
			requests := backFillRequests(start_backfill_from)
			for _, request := range requests {
				request.Save()
			}

			start_backfill_from = requests[len(requests)-1].Updated_datetime

		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				request.Save()
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

func (w *Worker) SetupStmts() {
	insert, err := worker.Db.Prepare(`INSERT INTO service_requests(service_request_id,
		status, service_name, service_code, agency_responsible,
		address, requested_datetime, updated_datetime, lat, long,
		ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime, notes)
		VALUES ($1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18);`)

	if err != nil {
		log.Fatal("error preparing insert statement ", err)
	}
	w.InsertStmt = insert

	update, err := worker.Db.Prepare(`UPDATE service_requests SET
		status = $2, service_name = $3, service_code = $4, agency_responsible = $5,
		address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10,
		ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15,
		parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17, notes = $18 WHERE service_request_id = $1;`)

	if err != nil {
		log.Fatal("error preparing update statement ", err)
	}
	w.UpdateStmt = update
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (req Open311Request) Save() (persisted bool) {
	// create or update a SR

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return false
	}

	persisted = false

	// find existing record if exists
	var existing_id int
	err := worker.Db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		log.Fatal("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		stmt = worker.InsertStmt
	} else {
		stmt = worker.UpdateStmt
	}

	t := req.ExtractClosedDatetime()
	closed_time := pq.NullTime{Time: t, Valid: !t.IsZero()}
	notes_as_json, err := json.Marshal(req.Notes)
	if err != nil {
		log.Print("error marshaling notes to JSON: ", err)
	}

	_, err = stmt.Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
		notes_as_json,
	)

	if err != nil {
		log.Printf("[error] could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	return persisted
}

func (req Open311Request) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("SR %s closed at: %s", req, parsed_date)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req Open311Request) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func fetchSingleRequest(sr_number string) (request Open311Request) {
	// given an SR, fetch the record
	log.Printf("fetching single SR %s", sr_number)
	open311_api_endpoint := fmt.Sprintf("http://311api.cityofchicago.org/open311/v2/requests/%s.json?extensions=true", sr_number)

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	if err != nil {
		log.Fatal("error fetching from Open311 endpoint", err)
	}
	defer resp.Body.Close()

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	var requests []Open311Request

	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests[0]
}

func fetchRequests() (requests []Open311Request) {
	last_updated_at := time.Now()
	if err := worker.Db.QueryRow("SELECT MAX(updated_datetime) FROM service_requests;").Scan(&last_updated_at); err != nil {
		log.Print("[fetchRequests] error loading most recent SR, will fallback to current time: ", err)
	}

	log.Print("[fetchRequests] most recent SR timestamp ", last_updated_at.Format(time.RFC3339))

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + last_updated_at.Format(time.RFC3339)

	log.Printf("[fetchRequests] fetching from %s", open311_api_endpoint)

	http.DefaultTransport.(*http.Transport).ResponseHeaderTimeout = time.Second * 15

	resp, err := http.Get(open311_api_endpoint)

	if err != nil {
		log.Fatalln("[fetchRequests] error fetching from Open311 endpoint", err)
	}

	defer resp.Body.Close()

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[fetchRequests] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[fetchRequests] error parsing JSON:", err)
	}

	log.Printf("[fetchRequests] received %d requests from Open311", len(requests))

	return requests
}

func backFillRequests(start_from string) (requests []Open311Request) {
	var fetch_from time.Time

	if start_from == "" {
		err := worker.Db.QueryRow("SELECT updated_datetime FROM service_requests ORDER BY updated_datetime ASC LIMIT 1").Scan(&fetch_from)
		if err != nil {
			log.Println("error fetching oldest SR:", err)
		}
		log.Printf("no start_from value provided, so falling back to oldest (by last update) SR in the database: %s", fetch_from)
	} else {
		t, err := time.Parse(time.RFC3339, start_from)
		if err != nil {
			log.Fatal("[backfill] error parsing date to start from", err)
		}
		fetch_from = t
	}

	formatted_date_string_with_tz := fetch_from.Format(time.RFC3339)

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_before=" + formatted_date_string_with_tz

	log.Printf("[backfill] fetching from %s", open311_api_endpoint)
	http.DefaultTransport.(*http.Transport).ResponseHeaderTimeout = time.Second * 15

	resp, err := http.Get(open311_api_endpoint)
	if err != nil {
		log.Fatalln("[backfill] error fetching from Open311 endpoint", err)
	}
	defer resp.Body.Close()

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[backfill] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[backfill] error parsing JSON:", err)
	}

	log.Printf("[backfill] received %d requests from Open311", len(requests))

	return requests
}
<commit_msg>adjust http timeout to 60 seconds<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/kylelemons/go-gypsy/yaml"
	"github.com/lib/pq"
	"io/ioutil"
	"log"
	"net/http"
	"time"
)

const OPEN311_API_URI = "http://311api.cityofchicago.org/open311/v2/requests.json?extensions=true&page_size=500"

type Open311Request struct {
	Lat, Long                                                                                float64
	Ward, Police_district                                                                                int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    string // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type Worker struct {
	Db         *sql.DB
	LastRunAt  time.Time
	InsertStmt *sql.Stmt
	UpdateStmt *sql.Stmt
}

var worker Worker

//  command line flags
var (
	environment   = flag.String("environment", "", "Environment to run in, e.g. staging, production")
	config        = flag.String("config", "./config/database.yml", "database configuration file")
	sr_number     = flag.String("sr-number", "", "SR number to fetch")
	backfill      = flag.Bool("backfill", false, "run in reverse and backfill data")
	backfill_date = flag.String("backfill-from", time.Now().Format(time.RFC3339), "date to start backfilling data from. Use RFC3339 format. Default will be the current time.")
)

func init() {
	flag.Parse()

	log.Printf("running in %s environment, configuration file %s", *environment, *config)
	settings := yaml.ConfigFile(*config)

	// setup database connection
	driver, err := settings.Get(fmt.Sprintf("%s.driver", *environment))
	if err != nil {
		log.Fatal("error loading db driver", err)
	}

	connstr, err := settings.Get(fmt.Sprintf("%s.connstr", *environment))
	if err != nil {
		log.Fatal("error loading db connstr", err)
	}

	db, err := sql.Open(driver, connstr)
	if err != nil {
		log.Fatal("Cannot open database connection", err)
	}

	log.Printf("database connstr: %s", connstr)

	worker.Db = db
	worker.SetupStmts()
}

func main() {
	defer worker.Db.Close()

	if *sr_number != "" {
		sr := fetchSingleRequest(*sr_number)
		sr.Save()
		return
	}

	start_backfill_from := *backfill_date
	for {
		switch {
		case *backfill:
			requests := backFillRequests(start_backfill_from)
			for _, request := range requests {
				request.Save()
			}

			start_backfill_from = requests[len(requests)-1].Updated_datetime

		case time.Since(worker.LastRunAt) > (30 * time.Second):
			// load requests from open311
			for _, request := range fetchRequests() {
				request.Save()
			}
			worker.LastRunAt = time.Now()
		default:
			log.Print("sleeping for 10 seconds")
			time.Sleep(10 * time.Second)
		}
	}
}

func (w *Worker) SetupStmts() {
	insert, err := worker.Db.Prepare(`INSERT INTO service_requests(service_request_id,
		status, service_name, service_code, agency_responsible,
		address, requested_datetime, updated_datetime, lat, long,
		ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime, notes)
		VALUES ($1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18);`)

	if err != nil {
		log.Fatal("error preparing insert statement ", err)
	}
	w.InsertStmt = insert

	update, err := worker.Db.Prepare(`UPDATE service_requests SET
		status = $2, service_name = $3, service_code = $4, agency_responsible = $5,
		address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10,
		ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15,
		parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17, notes = $18 WHERE service_request_id = $1;`)

	if err != nil {
		log.Fatal("error preparing update statement ", err)
	}
	w.UpdateStmt = update
}

func (req Open311Request) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (req Open311Request) Save() (persisted bool) {
	// create or update a SR

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return false
	}

	persisted = false

	// find existing record if exists
	var existing_id int
	err := worker.Db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		log.Fatal("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		stmt = worker.InsertStmt
	} else {
		stmt = worker.UpdateStmt
	}

	t := req.ExtractClosedDatetime()
	closed_time := pq.NullTime{Time: t, Valid: !t.IsZero()}
	notes_as_json, err := json.Marshal(req.Notes)
	if err != nil {
		log.Print("error marshaling notes to JSON: ", err)
	}

	_, err = stmt.Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
		notes_as_json,
	)

	if err != nil {
		log.Printf("[error] could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	return persisted
}

func (req Open311Request) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("SR %s closed at: %s", req, parsed_date)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req Open311Request) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func fetchSingleRequest(sr_number string) (request Open311Request) {
	// given an SR, fetch the record
	log.Printf("fetching single SR %s", sr_number)
	open311_api_endpoint := fmt.Sprintf("http://311api.cityofchicago.org/open311/v2/requests/%s.json?extensions=true", sr_number)

	log.Printf("fetching from %s", open311_api_endpoint)
	resp, err := http.Get(open311_api_endpoint)
	if err != nil {
		log.Fatal("error fetching from Open311 endpoint", err)
	}
	defer resp.Body.Close()

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	var requests []Open311Request

	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("error parsing JSON:", err)
	}

	log.Printf("received %d requests from Open311", len(requests))

	return requests[0]
}

func fetchRequests() (requests []Open311Request) {
	last_updated_at := time.Now()
	if err := worker.Db.QueryRow("SELECT MAX(updated_datetime) FROM service_requests;").Scan(&last_updated_at); err != nil {
		log.Print("[fetchRequests] error loading most recent SR, will fallback to current time: ", err)
	}

	log.Print("[fetchRequests] most recent SR timestamp ", last_updated_at.Format(time.RFC3339))

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_after=" + last_updated_at.Format(time.RFC3339)

	log.Printf("[fetchRequests] fetching from %s", open311_api_endpoint)

	http.DefaultTransport.(*http.Transport).ResponseHeaderTimeout = time.Second * 60

	resp, err := http.Get(open311_api_endpoint)

	if err != nil {
		log.Fatalln("[fetchRequests] error fetching from Open311 endpoint", err)
	}

	defer resp.Body.Close()

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[fetchRequests] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[fetchRequests] error parsing JSON:", err)
	}

	log.Printf("[fetchRequests] received %d requests from Open311", len(requests))

	return requests
}

func backFillRequests(start_from string) (requests []Open311Request) {
	var fetch_from time.Time

	if start_from == "" {
		err := worker.Db.QueryRow("SELECT updated_datetime FROM service_requests ORDER BY updated_datetime ASC LIMIT 1").Scan(&fetch_from)
		if err != nil {
			log.Println("error fetching oldest SR:", err)
		}
		log.Printf("no start_from value provided, so falling back to oldest (by last update) SR in the database: %s", fetch_from)
	} else {
		t, err := time.Parse(time.RFC3339, start_from)
		if err != nil {
			log.Fatal("[backfill] error parsing date to start from", err)
		}
		fetch_from = t
	}

	formatted_date_string_with_tz := fetch_from.Format(time.RFC3339)

	// construct the request URI using base params and the proper time
	open311_api_endpoint := OPEN311_API_URI + "&updated_before=" + formatted_date_string_with_tz

	log.Printf("[backfill] fetching from %s", open311_api_endpoint)
	http.DefaultTransport.(*http.Transport).ResponseHeaderTimeout = time.Second * 60

	resp, err := http.Get(open311_api_endpoint)
	if err != nil {
		log.Fatalln("[backfill] error fetching from Open311 endpoint", err)
	}
	defer resp.Body.Close()

	// load response body
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("[backfill] error loading response body", err)
	}

	// parse JSON and load into an array of Open311Request objects
	err = json.Unmarshal(body, &requests)
	if err != nil {
		log.Fatal("[backfill] error parsing JSON:", err)
	}

	log.Printf("[backfill] received %d requests from Open311", len(requests))

	return requests
}
<|endoftext|>


<commit_before>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Graph API support - Fetch, Create, Update, Delete, and Search
// See: https://login.circonus.com/resources/api/calls/graph

package api

import (
	"encoding/json"
	"fmt"
	"net/url"
	"regexp"

	"github.com/circonus-labs/circonus-gometrics/api/config"
)

// GraphAccessKey defines an access key for a graph
type GraphAccessKey struct {
	Active         bool   `json:"active,omitempty"`           // boolean
	Height         uint   `json:"height,omitempty"`           // uint
	Key            string `json:"key,omitempty"`              // string
	Legend         bool   `json:"legend,omitempty"`           // boolean
	LockDate       bool   `json:"lock_date,omitempty"`        // boolean
	LockMode       string `json:"lock_mode,omitempty"`        // string
	LockRangeEnd   uint   `json:"lock_range_end,omitempty"`   // uint
	LockRangeStart uint   `json:"lock_range_start,omitempty"` // uint
	LockShowTimes  bool   `json:"lock_show_times,omitempty"`  // boolean
	LockZoom       string `json:"lock_zoom,omitempty"`        // string
	Nickname       string `json:"nickname,omitempty"`         // string
	Title          bool   `json:"title,omitempty"`            // boolean
	Width          uint   `json:"width,omitempty"`            // uint
	XLabels        bool   `json:"x_labels,omitempty"`         // boolean
	YLabels        bool   `json:"y_labels,omitempty"`         // boolean
}

// GraphComposite defines a composite
type GraphComposite struct {
	Axis          string  `json:"axis,omitempty"`           // string
	Color         string  `json:"color,omitempty"`          // string
	DataFormula   *string `json:"data_formula,omitempty"`   // string or null
	Hidden        bool    `json:"hidden,omitempty"`         // boolean
	LegendFormula *string `json:"legend_formula,omitempty"` // string or null
	Name          string  `json:"name,omitempty"`           // string
	Stack         *uint   `json:"stack,omitempty"`          // uint or null
}

// GraphDatapoint defines a datapoint
type GraphDatapoint struct {
	Alpha         *string     `json:"alpha,omitempty"`          // string
	Axis          string      `json:"axis,omitempty"`           // string
	CAQL          *string     `json:"caql,omitempty"`           // string or null
	CheckID       uint        `json:"check_id,omitempty"`       // uint
	Color         string      `json:"color,omitempty"`          // string
	DataFormula   *string     `json:"data_formula,omitempty"`   // string or null
	Derive        interface{} `json:"derive,omitempty"`         // BUG doc: string, api: string or boolean(for caql statements)
	Hidden        bool        `json:"hidden"`                   // boolean
	LegendFormula *string     `json:"legend_formula,omitempty"` // string or null
	MetricName    string      `json:"metric_name,omitempty"`    // string
	MetricType    string      `json:"metric_type,omitempty"`    // string
	Name          string      `json:"name"`                     // string
	Stack         *uint       `json:"stack"`                    // uint or null
}

// GraphGuide defines a guide
type GraphGuide struct {
	Color         string  `json:"color,omitempty"`          // string
	DataFormula   *string `json:"data_formula,omitempty"`   // string or null
	Hidden        bool    `json:"hidden,omitempty"`         // boolean
	LegendFormula *string `json:"legend_formula,omitempty"` // string or null
	Name          string  `json:"name,omitempty"`           // string
}

// GraphMetricCluster defines a metric cluster
type GraphMetricCluster struct {
	AggregateFunc string  `json:"aggregation_function,omitempty"` // string
	Axis          string  `json:"axis,omitempty"`                 // string
	DataFormula   *string `json:"data_formula,omitempty"`         // string or null
	Hidden        bool    `json:"hidden"`                         // boolean
	LegendFormula *string `json:"legend_formula,omitempty"`       // string or null
	MetricCluster string  `json:"metric_cluster,omitempty"`       // string
	Name          string  `json:"name,omitempty"`                 // string
	Stack         *uint   `json:"stack"`                          // uint or null
}

// OverlayDataOptions defines overlay options for data. Note, each overlay type requires
// a _subset_ of the options. See Graph API documentation (URL above) for details.
type OverlayDataOptions struct {
	Alerts        string `json:"alerts,omitempty"`         // string BUG doc: numeric, api: string
	ArrayOutput   string `json:"array_output,omitempty"`   // string BUG doc: numeric, api: string
	BasePeriod    string `json:"base_period,omitempty"`    // string BUG doc: numeric, api: string
	Delay         string `json:"delay,omitempty"`          // string BUG doc: numeric, api: string
	Extension     string `json:"extension,omitempty"`      // string
	GraphTitle    string `json:"graph_title,omitempty"`    // string
	GraphUUID     string `json:"graph_id,omitempty"`       // string
	InPercent     string `json:"in_percent,omitempty"`     // string BUG doc: boolean, api: string
	Inverse       string `json:"inverse,omitempty"`        // string BUG doc: numeric, api: string
	Method        string `json:"method,omitempty"`         // string
	Model         string `json:"model,omitempty"`          // string
	ModelEnd      string `json:"model_end,omitempty"`      // string
	ModelPeriod   string `json:"model_period,omitempty"`   // string
	ModelRelative string `json:"model_relative,omitempty"` // string BUG doc: numeric, api: string
	Out           string `json:"out,omitempty"`            // string
	Prequel       string `json:"prequel,omitempty"`        // string
	Presets       string `json:"presets,omitempty"`        // string
	Quantiles     string `json:"quantiles,omitempty"`      // string
	SeasonLength  string `json:"season_length,omitempty"`  // string BUG doc: numeric, api: string
	Sensitivity   string `json:"sensitivity,omitempty"`    // string BUG doc: numeric, api: string
	SingleValue   string `json:"single_value,omitempty"`   // string BUG doc: numeric, api: string
	TargetPeriod  string `json:"target_period,omitempty"`  // string
	TimeOffset    string `json:"time_offset,omitempty"`    // string
	TimeShift     string `json:"time_shift,omitempty"`     // string BUG doc: numeric, api: string
	Transform     string `json:"transform,omitempty"`      // string
	Version       string `json:"version,omitempty"`        // string BUG doc: numeric, api: string
	Window        string `json:"window,omitempty"`         // string BUG doc: numeric, api: string
	XShift        string `json:"x_shift,omitempty"`        // string
}

// OverlayUISpecs defines UI specs for overlay
type OverlayUISpecs struct {
	Decouple bool   `json:"decouple,omitempty"` // boolean
	ID       string `json:"id,omitempty"`       // string
	Label    string `json:"label,omitempty"`    // string
	Type     string `json:"type,omitempty"`     // string
	Z        string `json:"z,omitempty"`        // string BUG doc: numeric, api: string
}

// GraphOverlaySet defines overlays for graph
type GraphOverlaySet struct {
	DataOpts OverlayDataOptions `json:"data_opts,omitempty"` // OverlayDataOptions
	ID       string             `json:"id,omitempty"`        // string
	Title    string             `json:"title,omitempty"`     // string
	UISpecs  OverlayUISpecs     `json:"ui_specs,omitempty"`  // OverlayUISpecs
}

// Graph defines a graph. See https://login.circonus.com/resources/api/calls/graph for more information.
type Graph struct {
	AccessKeys     []GraphAccessKey            `json:"access_keys,omitempty"`          // [] len >= 0
	CID            string                      `json:"_cid,omitempty"`                 // string
	Composites     []GraphComposite            `json:"composites,omitempty"`           // [] len >= 0
	Datapoints     []GraphDatapoint            `json:"datapoints,omitempt"`            // [] len >= 0
	Description    string                      `json:"description,omitempty"`          // string
	Guides         []GraphGuide                `json:"guides,omitempty"`               // [] len >= 0
	LineStyle      string                      `json:"line_style,omitempty"`           // string
	LogLeftY       *int                        `json:"logarithmitc_left_y,omitempty"`  // string or null BUG doc: number (not string)
	LogRightY      *int                        `json:"logarithmitc_right_y,omitempty"` // string or null BUG doc: number (not string)
	MaxLeftY       *string                     `json:"max_left_y,omitempty"`           // string or null BUG doc: number (not string)
	MaxRightY      *string                     `json:"max_right_y,omitempty"`          // string or null BUG doc: number (not string)
	MetricClusters []GraphMetricCluster        `json:"metric_clusters,omitempty"`      // [] len >= 0
	MinLeftY       *string                     `json:"min_left_y,omitempty"`           // string or null BUG doc: number (not string)
	MinRightY      *string                     `json:"min_right_y,omitempty"`          // string or null BUG doc: number (not string)
	Notes          *string                     `json:"notes,omitempty"`                // string or null
	OverlaySets    *map[string]GraphOverlaySet `json:"overlay_sets,omitempty"`         // GroupOverLaySets or null
	Style          string                      `json:"style,omitempty"`                // string
	Tags           []string                    `json:"tags,omitempty"`                 // [] len >= 0
	Title          string                      `json:"title,omitempty"`                // string
}

// NewGraph returns a Graph (with defaults, if applicable)
func NewGraph() *Graph {
	return &Graph{}
}

// FetchGraph retrieves graph with passed cid.
func (a *API) FetchGraph(cid CIDType) (*Graph, error) {
	if cid == nil || *cid == "" {
		return nil, fmt.Errorf("Invalid graph CID [none]")
	}

	graphCID := string(*cid)

	matched, err := regexp.MatchString(config.GraphCIDRegex, graphCID)
	if err != nil {
		return nil, err
	}
	if !matched {
		return nil, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	result, err := a.Get(graphCID)
	if err != nil {
		return nil, err
	}
	if a.Debug {
		a.Log.Printf("[DEBUG] fetch graph, received JSON: %s", string(result))
	}

	graph := new(Graph)
	if err := json.Unmarshal(result, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// FetchGraphs retrieves all graphs available to the API Token.
func (a *API) FetchGraphs() (*[]Graph, error) {
	result, err := a.Get(config.GraphPrefix)
	if err != nil {
		return nil, err
	}

	var graphs []Graph
	if err := json.Unmarshal(result, &graphs); err != nil {
		return nil, err
	}

	return &graphs, nil
}

// UpdateGraph updates passed graph.
func (a *API) UpdateGraph(cfg *Graph) (*Graph, error) {
	if cfg == nil {
		return nil, fmt.Errorf("Invalid graph config [nil]")
	}

	graphCID := string(cfg.CID)

	matched, err := regexp.MatchString(config.GraphCIDRegex, graphCID)
	if err != nil {
		return nil, err
	}
	if !matched {
		return nil, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	jsonCfg, err := json.Marshal(cfg)
	if err != nil {
		return nil, err
	}

	if a.Debug {
		a.Log.Printf("[DEBUG] update graph, sending JSON: %s", string(jsonCfg))
	}

	result, err := a.Put(graphCID, jsonCfg)
	if err != nil {
		return nil, err
	}

	graph := &Graph{}
	if err := json.Unmarshal(result, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// CreateGraph creates a new graph.
func (a *API) CreateGraph(cfg *Graph) (*Graph, error) {
	if cfg == nil {
		return nil, fmt.Errorf("Invalid graph config [nil]")
	}

	jsonCfg, err := json.Marshal(cfg)
	if err != nil {
		return nil, err
	}

	if a.Debug {
		a.Log.Printf("[DEBUG] update graph, sending JSON: %s", string(jsonCfg))
	}

	result, err := a.Post(config.GraphPrefix, jsonCfg)
	if err != nil {
		return nil, err
	}

	graph := &Graph{}
	if err := json.Unmarshal(result, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// DeleteGraph deletes passed graph.
func (a *API) DeleteGraph(cfg *Graph) (bool, error) {
	if cfg == nil {
		return false, fmt.Errorf("Invalid graph config [nil]")
	}
	return a.DeleteGraphByCID(CIDType(&cfg.CID))
}

// DeleteGraphByCID deletes graph with passed cid.
func (a *API) DeleteGraphByCID(cid CIDType) (bool, error) {
	if cid == nil || *cid == "" {
		return false, fmt.Errorf("Invalid graph CID [none]")
	}

	graphCID := string(*cid)

	matched, err := regexp.MatchString(config.GraphCIDRegex, graphCID)
	if err != nil {
		return false, err
	}
	if !matched {
		return false, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	_, err = a.Delete(graphCID)
	if err != nil {
		return false, err
	}

	return true, nil
}

// SearchGraphs returns graphs matching the specified search query
// and/or filter. If nil is passed for both parameters all graphs
// will be returned.
func (a *API) SearchGraphs(searchCriteria *SearchQueryType, filterCriteria *SearchFilterType) (*[]Graph, error) {
	q := url.Values{}

	if searchCriteria != nil && *searchCriteria != "" {
		q.Set("search", string(*searchCriteria))
	}

	if filterCriteria != nil && len(*filterCriteria) > 0 {
		for filter, criteria := range *filterCriteria {
			for _, val := range criteria {
				q.Add(filter, val)
			}
		}
	}

	if q.Encode() == "" {
		return a.FetchGraphs()
	}

	reqURL := url.URL{
		Path:     config.GraphPrefix,
		RawQuery: q.Encode(),
	}

	result, err := a.Get(reqURL.String())
	if err != nil {
		return nil, fmt.Errorf("[ERROR] API call error %+v", err)
	}

	var graphs []Graph
	if err := json.Unmarshal(result, &graphs); err != nil {
		return nil, err
	}

	return &graphs, nil
}
<commit_msg>Two for one: fix the spelling of the attribute in the API and properly encode the integer as a string.<commit_after>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Graph API support - Fetch, Create, Update, Delete, and Search
// See: https://login.circonus.com/resources/api/calls/graph

package api

import (
	"encoding/json"
	"fmt"
	"net/url"
	"regexp"

	"github.com/circonus-labs/circonus-gometrics/api/config"
)

// GraphAccessKey defines an access key for a graph
type GraphAccessKey struct {
	Active         bool   `json:"active,omitempty"`           // boolean
	Height         uint   `json:"height,omitempty"`           // uint
	Key            string `json:"key,omitempty"`              // string
	Legend         bool   `json:"legend,omitempty"`           // boolean
	LockDate       bool   `json:"lock_date,omitempty"`        // boolean
	LockMode       string `json:"lock_mode,omitempty"`        // string
	LockRangeEnd   uint   `json:"lock_range_end,omitempty"`   // uint
	LockRangeStart uint   `json:"lock_range_start,omitempty"` // uint
	LockShowTimes  bool   `json:"lock_show_times,omitempty"`  // boolean
	LockZoom       string `json:"lock_zoom,omitempty"`        // string
	Nickname       string `json:"nickname,omitempty"`         // string
	Title          bool   `json:"title,omitempty"`            // boolean
	Width          uint   `json:"width,omitempty"`            // uint
	XLabels        bool   `json:"x_labels,omitempty"`         // boolean
	YLabels        bool   `json:"y_labels,omitempty"`         // boolean
}

// GraphComposite defines a composite
type GraphComposite struct {
	Axis          string  `json:"axis,omitempty"`           // string
	Color         string  `json:"color,omitempty"`          // string
	DataFormula   *string `json:"data_formula,omitempty"`   // string or null
	Hidden        bool    `json:"hidden,omitempty"`         // boolean
	LegendFormula *string `json:"legend_formula,omitempty"` // string or null
	Name          string  `json:"name,omitempty"`           // string
	Stack         *uint   `json:"stack,omitempty"`          // uint or null
}

// GraphDatapoint defines a datapoint
type GraphDatapoint struct {
	Alpha         *string     `json:"alpha,omitempty"`          // string
	Axis          string      `json:"axis,omitempty"`           // string
	CAQL          *string     `json:"caql,omitempty"`           // string or null
	CheckID       uint        `json:"check_id,omitempty"`       // uint
	Color         string      `json:"color,omitempty"`          // string
	DataFormula   *string     `json:"data_formula,omitempty"`   // string or null
	Derive        interface{} `json:"derive,omitempty"`         // BUG doc: string, api: string or boolean(for caql statements)
	Hidden        bool        `json:"hidden"`                   // boolean
	LegendFormula *string     `json:"legend_formula,omitempty"` // string or null
	MetricName    string      `json:"metric_name,omitempty"`    // string
	MetricType    string      `json:"metric_type,omitempty"`    // string
	Name          string      `json:"name"`                     // string
	Stack         *uint       `json:"stack"`                    // uint or null
}

// GraphGuide defines a guide
type GraphGuide struct {
	Color         string  `json:"color,omitempty"`          // string
	DataFormula   *string `json:"data_formula,omitempty"`   // string or null
	Hidden        bool    `json:"hidden,omitempty"`         // boolean
	LegendFormula *string `json:"legend_formula,omitempty"` // string or null
	Name          string  `json:"name,omitempty"`           // string
}

// GraphMetricCluster defines a metric cluster
type GraphMetricCluster struct {
	AggregateFunc string  `json:"aggregation_function,omitempty"` // string
	Axis          string  `json:"axis,omitempty"`                 // string
	DataFormula   *string `json:"data_formula,omitempty"`         // string or null
	Hidden        bool    `json:"hidden"`                         // boolean
	LegendFormula *string `json:"legend_formula,omitempty"`       // string or null
	MetricCluster string  `json:"metric_cluster,omitempty"`       // string
	Name          string  `json:"name,omitempty"`                 // string
	Stack         *uint   `json:"stack"`                          // uint or null
}

// OverlayDataOptions defines overlay options for data. Note, each overlay type requires
// a _subset_ of the options. See Graph API documentation (URL above) for details.
type OverlayDataOptions struct {
	Alerts        string `json:"alerts,omitempty"`         // string BUG doc: numeric, api: string
	ArrayOutput   string `json:"array_output,omitempty"`   // string BUG doc: numeric, api: string
	BasePeriod    string `json:"base_period,omitempty"`    // string BUG doc: numeric, api: string
	Delay         string `json:"delay,omitempty"`          // string BUG doc: numeric, api: string
	Extension     string `json:"extension,omitempty"`      // string
	GraphTitle    string `json:"graph_title,omitempty"`    // string
	GraphUUID     string `json:"graph_id,omitempty"`       // string
	InPercent     string `json:"in_percent,omitempty"`     // string BUG doc: boolean, api: string
	Inverse       string `json:"inverse,omitempty"`        // string BUG doc: numeric, api: string
	Method        string `json:"method,omitempty"`         // string
	Model         string `json:"model,omitempty"`          // string
	ModelEnd      string `json:"model_end,omitempty"`      // string
	ModelPeriod   string `json:"model_period,omitempty"`   // string
	ModelRelative string `json:"model_relative,omitempty"` // string BUG doc: numeric, api: string
	Out           string `json:"out,omitempty"`            // string
	Prequel       string `json:"prequel,omitempty"`        // string
	Presets       string `json:"presets,omitempty"`        // string
	Quantiles     string `json:"quantiles,omitempty"`      // string
	SeasonLength  string `json:"season_length,omitempty"`  // string BUG doc: numeric, api: string
	Sensitivity   string `json:"sensitivity,omitempty"`    // string BUG doc: numeric, api: string
	SingleValue   string `json:"single_value,omitempty"`   // string BUG doc: numeric, api: string
	TargetPeriod  string `json:"target_period,omitempty"`  // string
	TimeOffset    string `json:"time_offset,omitempty"`    // string
	TimeShift     string `json:"time_shift,omitempty"`     // string BUG doc: numeric, api: string
	Transform     string `json:"transform,omitempty"`      // string
	Version       string `json:"version,omitempty"`        // string BUG doc: numeric, api: string
	Window        string `json:"window,omitempty"`         // string BUG doc: numeric, api: string
	XShift        string `json:"x_shift,omitempty"`        // string
}

// OverlayUISpecs defines UI specs for overlay
type OverlayUISpecs struct {
	Decouple bool   `json:"decouple,omitempty"` // boolean
	ID       string `json:"id,omitempty"`       // string
	Label    string `json:"label,omitempty"`    // string
	Type     string `json:"type,omitempty"`     // string
	Z        string `json:"z,omitempty"`        // string BUG doc: numeric, api: string
}

// GraphOverlaySet defines overlays for graph
type GraphOverlaySet struct {
	DataOpts OverlayDataOptions `json:"data_opts,omitempty"` // OverlayDataOptions
	ID       string             `json:"id,omitempty"`        // string
	Title    string             `json:"title,omitempty"`     // string
	UISpecs  OverlayUISpecs     `json:"ui_specs,omitempty"`  // OverlayUISpecs
}

// Graph defines a graph. See https://login.circonus.com/resources/api/calls/graph for more information.
type Graph struct {
	AccessKeys     []GraphAccessKey            `json:"access_keys,omitempty"`      // [] len >= 0
	CID            string                      `json:"_cid,omitempty"`             // string
	Composites     []GraphComposite            `json:"composites,omitempty"`       // [] len >= 0
	Datapoints     []GraphDatapoint            `json:"datapoints,omitempt"`        // [] len >= 0
	Description    string                      `json:"description,omitempty"`      // string
	Guides         []GraphGuide                `json:"guides,omitempty"`           // [] len >= 0
	LineStyle      string                      `json:"line_style,omitempty"`       // string
	LogLeftY       *int                        `json:"logarithmic_left_y,string"`  // string or null BUG doc: number (not string)
	LogRightY      *int                        `json:"logarithmic_right_y,string"` // string or null BUG doc: number (not string)
	MaxLeftY       *string                     `json:"max_left_y,omitempty"`       // string or null BUG doc: number (not string)
	MaxRightY      *string                     `json:"max_right_y,omitempty"`      // string or null BUG doc: number (not string)
	MetricClusters []GraphMetricCluster        `json:"metric_clusters,omitempty"`  // [] len >= 0
	MinLeftY       *string                     `json:"min_left_y,omitempty"`       // string or null BUG doc: number (not string)
	MinRightY      *string                     `json:"min_right_y,omitempty"`      // string or null BUG doc: number (not string)
	Notes          *string                     `json:"notes,omitempty"`            // string or null
	OverlaySets    *map[string]GraphOverlaySet `json:"overlay_sets,omitempty"`     // GroupOverLaySets or null
	Style          string                      `json:"style,omitempty"`            // string
	Tags           []string                    `json:"tags,omitempty"`             // [] len >= 0
	Title          string                      `json:"title,omitempty"`            // string
}

// NewGraph returns a Graph (with defaults, if applicable)
func NewGraph() *Graph {
	return &Graph{}
}

// FetchGraph retrieves graph with passed cid.
func (a *API) FetchGraph(cid CIDType) (*Graph, error) {
	if cid == nil || *cid == "" {
		return nil, fmt.Errorf("Invalid graph CID [none]")
	}

	graphCID := string(*cid)

	matched, err := regexp.MatchString(config.GraphCIDRegex, graphCID)
	if err != nil {
		return nil, err
	}
	if !matched {
		return nil, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	result, err := a.Get(graphCID)
	if err != nil {
		return nil, err
	}
	if a.Debug {
		a.Log.Printf("[DEBUG] fetch graph, received JSON: %s", string(result))
	}

	graph := new(Graph)
	if err := json.Unmarshal(result, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// FetchGraphs retrieves all graphs available to the API Token.
func (a *API) FetchGraphs() (*[]Graph, error) {
	result, err := a.Get(config.GraphPrefix)
	if err != nil {
		return nil, err
	}

	var graphs []Graph
	if err := json.Unmarshal(result, &graphs); err != nil {
		return nil, err
	}

	return &graphs, nil
}

// UpdateGraph updates passed graph.
func (a *API) UpdateGraph(cfg *Graph) (*Graph, error) {
	if cfg == nil {
		return nil, fmt.Errorf("Invalid graph config [nil]")
	}

	graphCID := string(cfg.CID)

	matched, err := regexp.MatchString(config.GraphCIDRegex, graphCID)
	if err != nil {
		return nil, err
	}
	if !matched {
		return nil, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	jsonCfg, err := json.Marshal(cfg)
	if err != nil {
		return nil, err
	}

	if a.Debug {
		a.Log.Printf("[DEBUG] update graph, sending JSON: %s", string(jsonCfg))
	}

	result, err := a.Put(graphCID, jsonCfg)
	if err != nil {
		return nil, err
	}

	graph := &Graph{}
	if err := json.Unmarshal(result, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// CreateGraph creates a new graph.
func (a *API) CreateGraph(cfg *Graph) (*Graph, error) {
	if cfg == nil {
		return nil, fmt.Errorf("Invalid graph config [nil]")
	}

	jsonCfg, err := json.Marshal(cfg)
	if err != nil {
		return nil, err
	}

	if a.Debug {
		a.Log.Printf("[DEBUG] update graph, sending JSON: %s", string(jsonCfg))
	}

	result, err := a.Post(config.GraphPrefix, jsonCfg)
	if err != nil {
		return nil, err
	}

	graph := &Graph{}
	if err := json.Unmarshal(result, graph); err != nil {
		return nil, err
	}

	return graph, nil
}

// DeleteGraph deletes passed graph.
func (a *API) DeleteGraph(cfg *Graph) (bool, error) {
	if cfg == nil {
		return false, fmt.Errorf("Invalid graph config [nil]")
	}
	return a.DeleteGraphByCID(CIDType(&cfg.CID))
}

// DeleteGraphByCID deletes graph with passed cid.
func (a *API) DeleteGraphByCID(cid CIDType) (bool, error) {
	if cid == nil || *cid == "" {
		return false, fmt.Errorf("Invalid graph CID [none]")
	}

	graphCID := string(*cid)

	matched, err := regexp.MatchString(config.GraphCIDRegex, graphCID)
	if err != nil {
		return false, err
	}
	if !matched {
		return false, fmt.Errorf("Invalid graph CID [%s]", graphCID)
	}

	_, err = a.Delete(graphCID)
	if err != nil {
		return false, err
	}

	return true, nil
}

// SearchGraphs returns graphs matching the specified search query
// and/or filter. If nil is passed for both parameters all graphs
// will be returned.
func (a *API) SearchGraphs(searchCriteria *SearchQueryType, filterCriteria *SearchFilterType) (*[]Graph, error) {
	q := url.Values{}

	if searchCriteria != nil && *searchCriteria != "" {
		q.Set("search", string(*searchCriteria))
	}

	if filterCriteria != nil && len(*filterCriteria) > 0 {
		for filter, criteria := range *filterCriteria {
			for _, val := range criteria {
				q.Add(filter, val)
			}
		}
	}

	if q.Encode() == "" {
		return a.FetchGraphs()
	}

	reqURL := url.URL{
		Path:     config.GraphPrefix,
		RawQuery: q.Encode(),
	}

	result, err := a.Get(reqURL.String())
	if err != nil {
		return nil, fmt.Errorf("[ERROR] API call error %+v", err)
	}

	var graphs []Graph
	if err := json.Unmarshal(result, &graphs); err != nil {
		return nil, err
	}

	return &graphs, nil
}
<|endoftext|>


<commit_before><commit_msg>cluster: remove client auth field<commit_after><|endoftext|>


<commit_before><commit_msg>cmd/xgettext-go: 新的翻译资源提取工具<commit_after><|endoftext|>