<commit_before>// Copyright 2022 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cache

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"

	"golang.org/x/sys/unix"

	"go.chromium.org/luci/common/errors"
)

var cachedMajorVersion int
var cachedMajorVersionOnce sync.Once

func mustGetDarwinMajorVersion() int {
	cachedMajorVersionOnce.Do(func() {
		var utsname unix.Utsname
		if err := unix.Uname(&utsname); err != nil {
			panic(fmt.Sprintf("failed to call uname(): %v", err))
		}

		release := string(utsname.Release[:])
		vers := strings.Split(release, ".")
		if len(vers) < 2 {
			panic(fmt.Sprintf("unexpected release from uname: %s", string(release)))
		}

		var err error
		cachedMajorVersion, err = strconv.Atoi(vers[0])
		if err != nil {
			panic(fmt.Sprintf("failed to parse version %s: %v", vers[0], err))
		}
	})
	return cachedMajorVersion
}

func makeHardLinkOrClone(src, dst string) error {
	// Hardlinked executables don't work well with dyld.
	// Use clonefile instead on macOS 12 (Darwin 21) or newer to workaround that.
	// ref: https://crbug.com/1296318#c54
	if mustGetDarwinMajorVersion() >= 21 {
		if err := unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW|unix.CLONE_NOOWNERCOPY); err != nil {
			return errors.Annotate(err, "failed to call clonefile(%s, %s, CLONE_NOFOLLOW|CLONE_NOOWNERCOPY)", src, dst).Err()
		}
		return nil
	}
	if err := os.Link(src, dst); err != nil {
		return errors.Annotate(err, "failed to call os.Link(%s, %s)", src, dst).Err()
	}
	return nil
}
<commit_msg>cache: comment out clonefile usage<commit_after>// Copyright 2022 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cache

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"

	"golang.org/x/sys/unix"

	"go.chromium.org/luci/common/errors"
)

var cachedMajorVersion int
var cachedMajorVersionOnce sync.Once

func mustGetDarwinMajorVersion() int {
	cachedMajorVersionOnce.Do(func() {
		var utsname unix.Utsname
		if err := unix.Uname(&utsname); err != nil {
			panic(fmt.Sprintf("failed to call uname(): %v", err))
		}

		release := string(utsname.Release[:])
		vers := strings.Split(release, ".")
		if len(vers) < 2 {
			panic(fmt.Sprintf("unexpected release from uname: %s", string(release)))
		}

		var err error
		cachedMajorVersion, err = strconv.Atoi(vers[0])
		if err != nil {
			panic(fmt.Sprintf("failed to parse version %s: %v", vers[0], err))
		}
	})
	return cachedMajorVersion
}

func makeHardLinkOrClone(src, dst string) error {
	// Hardlinked executables don't work well with dyld.
	// Use clonefile instead on macOS 12 (Darwin 21) or newer to workaround that.
	// ref: https://crbug.com/1296318#c54
	// TODO(https://crbug.com/1296318#c68): uncomment here.
	// if mustGetDarwinMajorVersion() >= 21 {
	// 	if err := unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW|unix.CLONE_NOOWNERCOPY); err != nil {
	// 		return errors.Annotate(err, "failed to call clonefile(%s, %s, CLONE_NOFOLLOW|CLONE_NOOWNERCOPY)", src, dst).Err()
	// 	}
	// 	return nil
	// }

	if err := os.Link(src, dst); err != nil {
		return errors.Annotate(err, "failed to call os.Link(%s, %s)", src, dst).Err()
	}
	return nil
}
<|endoftext|>


<commit_before>package sliceops

import "math"

// Returns the minimum value in the slice and the index of
// the minimum value. If the input slice is empty, zero is returned
// as the minimum value and -1 is returned as the index.
// Use: val,ind := sliceops.Min(slice)
func Min(s []float64) (min float64, ind int) {
	if len(s) == 0 {
		return min, -1 // Ind is -1 to make clear it's not the zeroth index.
	}
	min = s[0]
	ind = 0
	for i, val := range s {
		if val < min {
			min = val
			ind = i
		}
	}
	return min, ind
}

// Returns the maximum value in the slice and the location of
// the maximum value. If the input slice is empty, zero is returned
// as the minimum value and -1 is returned as the index.
// Use: val,ind := sliceops.Max(slice)
func Max(s []float64) (max float64, ind int) {
	if len(s) == 0 {
		return max, -1 // Ind is -1 to make clear it's not the zeroth index.
	}
	max = s[0]
	ind = 0
	for i, val := range s {
		if val > max {
			max = val
			ind = i
		}
	}
	return max, ind
}

// Returns the sum of the elements of the slice
func Sum(s []float64) (sum float64) {
	for _, val := range s {
		sum += val
	}
	return
}

// Returns the product of the elements of the slice
// Returns 1 if the input has length zero
func Prod(s []float64) (prod float64) {
	prod = 1
	for _, val := range s {
		prod *= val
	}
	return prod
}

// Returns true if all of the slices have equal length,
// and false otherwise.
// Special case: Returns true if there are no input slices
func HasEqLen(slices ...[]float64) bool {
	if len(slices) == 0 {
		return true
	}
	l := len(slices[0])
	for i := 1; i < len(slices); i++ {
		if len(slices[i]) != l {
			return false
		}
	}
	return true
}

// Returns false if |s1[i] - s2[i]| > tol for any i.
// Assumes that the slices are of equal length. If this
// is in doubt it should be checked with HasEqLen
func Eq(s1, s2 []float64, tol float64) bool {
	for i, val := range s1 {
		if math.Abs(s2[i]-val) > tol {
			return false
		}
	}
	return true
}

// Finds the cumulative sum of the first i elements in
// s and puts them in place into the ith element of the
// receiver. If the receiver is nil a new slice is created
func CumSum(receiver, s []float64) []float64 {
	if receiver == nil {
		receiver = make([]float64, len(s))
	}
	if len(s) == 0 {
		return receiver[:0]
	}
	receiver[0] = s[0]
	for i := 1; i < len(s); i++ {
		receiver[i] = receiver[i-1] + s[i]
	}
	return receiver
}

// Finds the cumulative product of the first i elements in
// s and puts them in place into the ith element of the
// receiver. If the receiver is nil a new slice is created
func CumProd(receiver, s []float64) []float64 {
	if receiver == nil {
		receiver = make([]float64, len(s))
	}
	if len(s) == 0 {
		return receiver[:0]
	}
	receiver[0] = s[0]
	for i := 1; i < len(s); i++ {
		receiver[i] = receiver[i-1] * s[i]
	}
	return receiver
}

// Returns the element-wise sum of the last n slices
// and puts them in place into the first argument. If the
// receiver is nil, a new slice of floats is created.
// For computational efficiency, it is assumed that all of
// the variadic arguments have the same length. If this is
// in doubt, EqLengths can be called. If no slices are input,
// the receiver is unchanged.
func ElemSum(receiver []float64, slices ...[]float64) []float64 {
	if len(slices) == 0 {
		return receiver
	}
	if receiver == nil {
		receiver = make([]float64, len(slices[0]))
	}
	for i, val := range slices[0] {
		receiver[i] = val
	}
	for i := 1; i < len(slices); i++ {
		for j, val := range slices[i] {
			receiver[j] += val
		}
	}
	return receiver
}

// Returns the L norm of the slice S.
// Special cases:
// L = math.Inf(1) gives the maximum value
// Does not deal with the Zero norm. See Zero norm instead
func Norm(s []float64, L float64) (norm float64) {
	// Should this complain if L is not positive?
	// Should this be done in log space for better numerical stability?
	//	would be more cost
	//	maybe only if L is high?
	if math.IsInf(L, 1) {
		norm, _ = Max(s)
		return norm
	}
	for _, val := range s {
		norm += L * math.Pow(math.Abs(val), L)
	}
	return math.Pow(norm, 1/L)
}
<commit_msg>Added LogSumExp<commit_after>package sliceops

import "math"

// Returns the minimum value in the slice and the index of
// the minimum value. If the input slice is empty, zero is returned
// as the minimum value and -1 is returned as the index.
// Use: val,ind := sliceops.Min(slice)
func Min(s []float64) (min float64, ind int) {
	if len(s) == 0 {
		return min, -1 // Ind is -1 to make clear it's not the zeroth index.
	}
	min = s[0]
	ind = 0
	for i, val := range s {
		if val < min {
			min = val
			ind = i
		}
	}
	return min, ind
}

// Returns the maximum value in the slice and the location of
// the maximum value. If the input slice is empty, zero is returned
// as the minimum value and -1 is returned as the index.
// Use: val,ind := sliceops.Max(slice)
func Max(s []float64) (max float64, ind int) {
	if len(s) == 0 {
		return max, -1 // Ind is -1 to make clear it's not the zeroth index.
	}
	max = s[0]
	ind = 0
	for i, val := range s {
		if val > max {
			max = val
			ind = i
		}
	}
	return max, ind
}

// Returns the sum of the elements of the slice
func Sum(s []float64) (sum float64) {
	for _, val := range s {
		sum += val
	}
	return
}

// Returns the product of the elements of the slice
// Returns 1 if the input has length zero
func Prod(s []float64) (prod float64) {
	prod = 1
	for _, val := range s {
		prod *= val
	}
	return prod
}

// Returns true if all of the slices have equal length,
// and false otherwise.
// Special case: Returns true if there are no input slices
func HasEqLen(slices ...[]float64) bool {
	if len(slices) == 0 {
		return true
	}
	l := len(slices[0])
	for i := 1; i < len(slices); i++ {
		if len(slices[i]) != l {
			return false
		}
	}
	return true
}

// Returns false if |s1[i] - s2[i]| > tol for any i.
// Assumes that the slices are of equal length. If this
// is in doubt it should be checked with HasEqLen
func Eq(s1, s2 []float64, tol float64) bool {
	for i, val := range s1 {
		if math.Abs(s2[i]-val) > tol {
			return false
		}
	}
	return true
}

// Finds the cumulative sum of the first i elements in
// s and puts them in place into the ith element of the
// receiver. If the receiver is nil a new slice is created
func CumSum(receiver, s []float64) []float64 {
	if receiver == nil {
		receiver = make([]float64, len(s))
	}
	if len(s) == 0 {
		return receiver[:0]
	}
	receiver[0] = s[0]
	for i := 1; i < len(s); i++ {
		receiver[i] = receiver[i-1] + s[i]
	}
	return receiver
}

// Finds the cumulative product of the first i elements in
// s and puts them in place into the ith element of the
// receiver. If the receiver is nil a new slice is created
func CumProd(receiver, s []float64) []float64 {
	if receiver == nil {
		receiver = make([]float64, len(s))
	}
	if len(s) == 0 {
		return receiver[:0]
	}
	receiver[0] = s[0]
	for i := 1; i < len(s); i++ {
		receiver[i] = receiver[i-1] * s[i]
	}
	return receiver
}

// Returns the element-wise sum of the last n slices
// and puts them in place into the first argument. If the
// receiver is nil, a new slice of floats is created.
// For computational efficiency, it is assumed that all of
// the variadic arguments have the same length. If this is
// in doubt, EqLengths can be called. If no slices are input,
// the receiver is unchanged.
func ElemSum(receiver []float64, slices ...[]float64) []float64 {
	if len(slices) == 0 {
		return receiver
	}
	if receiver == nil {
		receiver = make([]float64, len(slices[0]))
	}
	for i, val := range slices[0] {
		receiver[i] = val
	}
	for i := 1; i < len(slices); i++ {
		for j, val := range slices[i] {
			receiver[j] += val
		}
	}
	return receiver
}

// Returns the L norm of the slice S.
// Special cases:
// L = math.Inf(1) gives the maximum value
// Does not deal with the Zero norm. See Zero norm instead
func Norm(s []float64, L float64) (norm float64) {
	// Should this complain if L is not positive?
	// Should this be done in log space for better numerical stability?
	//	would be more cost
	//	maybe only if L is high?
	if math.IsInf(L, 1) {
		norm, _ = Max(s)
		return norm
	}
	for _, val := range s {
		norm += L * math.Pow(math.Abs(val), L)
	}
	return math.Pow(norm, 1/L)
}

// Adds a constant to all of the values in s
func AddConst(s []float64, c float64) {
	for i := range s {
		s[i] += c
	}
}

// Multiplies every element in s by a constant
func MulConst(s []float64, c float64) {
	for i := range s {
		s[i] *= c
	}
}

// Returns the log of the sum of the exponentials of the values in s
func LogSumExp(s []float64) (logsumexp float64) {
	// Want to do this in a numerically stable way which avoids
	// overflow and underflow
	// TODO: Add in special case for two values

	// First, find the maximum value in the slice.
	minval, _ := Max(s)
	if math.IsInf(minval, 0) {
		// If it's infinity eitherway, the logsumexp will be infinity as well
		// returning now avoids NaNs
		return minval
	}
	// Subtract off the largest value, so the largest value in
	// the new slice is 0
	AddConst(s, -minval)
	defer AddConst(s, minval) // make sure we add it back on at the end

	// compute the sumexp part
	for _, val := range s {
		logsumexp += math.Exp(val)
	}
	// Take the log and add back on the constant taken out
	logsumexp = math.Log(logsumexp) + minval
	return logsumexp
}
<|endoftext|>


<commit_before>package shorty

import (
	"errors"
	"github.com/golang/glog"
	"github.com/qorio/omni/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

func (this OnOff) IsOn() bool {
	return strings.ToLower(string(this)) == "on"
}

func (this *RoutingRule) Validate() (err error) {
	if len(this.Destination) > 0 {
		if _, err = url.Parse(this.Destination); err != nil {
			return
		}
	}
	matching := 0
	if c, err := regexp.Compile(string(this.MatchPlatform)); err != nil {
		return errors.New("Bad platform regex " + string(this.MatchPlatform))
	} else if c != nil {
		matching++
	}
	if c, err := regexp.Compile(string(this.MatchOS)); err != nil {
		return errors.New("Bad os regex " + string(this.MatchOS))
	} else if c != nil {
		matching++
	}
	if c, err := regexp.Compile(string(this.MatchMake)); err != nil {
		return errors.New("Bad make regex " + string(this.MatchMake))
	} else if c != nil {
		matching++
	}
	if c, err := regexp.Compile(string(this.MatchBrowser)); err != nil {
		return errors.New("Bad browser regex " + string(this.MatchBrowser))
	} else if c != nil {
		matching++
	}
	// Must have 1 or more matching regexp
	if matching == 0 {
		err = errors.New("bad-routing-rule:no matching regexp")
		return
	}
	return
}

// TODO - precompile the regexs and store them in the Routing rule
func (this *RoutingRule) Match(service Shorty, ua *http.UserAgent, origin *http.RequestOrigin, cookies http.Cookies) bool {
	// use bit mask to match
	var actual, expect int = 0, 0

	if len(string(this.MatchPlatform)) > 0 {
		expect |= 1 << 0
		if matches, _ := regexp.MatchString(string(this.MatchPlatform), ua.Platform); matches {
			actual |= 1 << 0
		}
	}
	if len(string(this.MatchOS)) > 0 {
		expect |= 1 << 1
		if matches, _ := regexp.MatchString(string(this.MatchOS), ua.OS); matches {
			actual |= 1 << 1
		}
	}
	if len(string(this.MatchMake)) > 0 {
		expect |= 1 << 2
		if matches, _ := regexp.MatchString(string(this.MatchMake), ua.Make); matches {
			actual |= 1 << 2
		}
	}
	if len(string(this.MatchBrowser)) > 0 {
		expect |= 1 << 3
		if matches, _ := regexp.MatchString(string(this.MatchBrowser), ua.Browser); matches {
			actual |= 1 << 3
		}
	}
	if this.MatchMobile.IsOn() {
		expect |= 1 << 4
		if ua.Mobile {
			actual |= 1 << 4
		}
	}
	if len(string(this.MatchReferrer)) > 0 {
		expect |= 1 << 5
		if matches, _ := regexp.MatchString(string(this.MatchReferrer), origin.Referrer); matches {
			actual |= 1 << 5
		}
	}
	if this.MatchNoAppOpenInXDays.IsOn() && this.AppUrlScheme != "" {
		expect |= 1 << 6
		uuid, _ := cookies.GetPlainString(uuidCookieKey)
		appOpen, found, err := service.FindAppOpen(UrlScheme(this.AppUrlScheme), UUID(uuid))
		glog.Infoln("checking app-open", uuid, this.AppUrlScheme, found, appOpen, err)
		if !found || float64(time.Now().Unix()-appOpen.Timestamp) >= this.AppOpenTTLDays*24.*60.*60. {
			actual |= 1 << 6
		}
	}
	// By the time we get here, we have done a match all
	return actual == expect && expect > 0
}
<commit_msg>log in rule matching, part2<commit_after>package shorty

import (
	"errors"
	"github.com/golang/glog"
	"github.com/qorio/omni/http"
	"net/url"
	"regexp"
	"strings"
	"time"
)

func (this OnOff) IsOn() bool {
	return strings.ToLower(string(this)) == "on"
}

func (this *RoutingRule) Validate() (err error) {
	if len(this.Destination) > 0 {
		if _, err = url.Parse(this.Destination); err != nil {
			return
		}
	}
	matching := 0
	if c, err := regexp.Compile(string(this.MatchPlatform)); err != nil {
		return errors.New("Bad platform regex " + string(this.MatchPlatform))
	} else if c != nil {
		matching++
	}
	if c, err := regexp.Compile(string(this.MatchOS)); err != nil {
		return errors.New("Bad os regex " + string(this.MatchOS))
	} else if c != nil {
		matching++
	}
	if c, err := regexp.Compile(string(this.MatchMake)); err != nil {
		return errors.New("Bad make regex " + string(this.MatchMake))
	} else if c != nil {
		matching++
	}
	if c, err := regexp.Compile(string(this.MatchBrowser)); err != nil {
		return errors.New("Bad browser regex " + string(this.MatchBrowser))
	} else if c != nil {
		matching++
	}
	// Must have 1 or more matching regexp
	if matching == 0 {
		err = errors.New("bad-routing-rule:no matching regexp")
		return
	}
	return
}

// TODO - precompile the regexs and store them in the Routing rule
func (this *RoutingRule) Match(service Shorty, ua *http.UserAgent, origin *http.RequestOrigin, cookies http.Cookies) bool {
	// use bit mask to match
	var actual, expect int = 0, 0

	if len(string(this.MatchPlatform)) > 0 {
		expect |= 1 << 0
		if matches, _ := regexp.MatchString(string(this.MatchPlatform), ua.Platform); matches {
			actual |= 1 << 0
		}
	}
	if len(string(this.MatchOS)) > 0 {
		expect |= 1 << 1
		if matches, _ := regexp.MatchString(string(this.MatchOS), ua.OS); matches {
			actual |= 1 << 1
		}
		glog.Infoln("matching expect=", expect, "actual=", actual)
	}
	if len(string(this.MatchMake)) > 0 {
		expect |= 1 << 2
		if matches, _ := regexp.MatchString(string(this.MatchMake), ua.Make); matches {
			actual |= 1 << 2
		}
		glog.Infoln("matching expect=", expect, "actual=", actual)
	}
	if len(string(this.MatchBrowser)) > 0 {
		expect |= 1 << 3
		if matches, _ := regexp.MatchString(string(this.MatchBrowser), ua.Browser); matches {
			actual |= 1 << 3
		}
		glog.Infoln("matching expect=", expect, "actual=", actual)
	}
	if this.MatchMobile.IsOn() {
		expect |= 1 << 4
		if ua.Mobile {
			actual |= 1 << 4
		}
		glog.Infoln("matching expect=", expect, "actual=", actual)
	}
	if len(string(this.MatchReferrer)) > 0 {
		expect |= 1 << 5
		if matches, _ := regexp.MatchString(string(this.MatchReferrer), origin.Referrer); matches {
			actual |= 1 << 5
		}
		glog.Infoln("matching expect=", expect, "actual=", actual)
	}
	if this.MatchNoAppOpenInXDays.IsOn() && this.AppUrlScheme != "" {
		expect |= 1 << 6
		uuid, _ := cookies.GetPlainString(uuidCookieKey)
		appOpen, found, err := service.FindAppOpen(UrlScheme(this.AppUrlScheme), UUID(uuid))
		glog.Infoln("checking app-open", uuid, this.AppUrlScheme, found, appOpen, err)
		if !found || float64(time.Now().Unix()-appOpen.Timestamp) >= this.AppOpenTTLDays*24.*60.*60. {
			actual |= 1 << 6
		}
		glog.Infoln("matching expect=", expect, "actual=", actual)
	}
	// By the time we get here, we have done a match all
	return actual == expect && expect > 0
}
<|endoftext|>


<commit_before>package elasticsearch

import (
	"fmt"
	"strings"

	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	appslisters "k8s.io/client-go/listers/apps/v1beta1"
	"k8s.io/client-go/tools/record"

	v1alpha1 "github.com/jetstack-experimental/navigator/pkg/apis/navigator/v1alpha1"
)

// statefulElasticsearchClusterNodePoolControl manages the lifecycle of a
// stateful node pool. It can be used to create, update and delete pools.
//
// This is an implementation of the ElasticsearchClusterNodePoolControl interface
// as defined in interfaces.go.
type statefulElasticsearchClusterNodePoolControl struct {
	kubeClient        *kubernetes.Clientset
	statefulSetLister appslisters.StatefulSetLister

	recorder record.EventRecorder
}

var _ ElasticsearchClusterNodePoolControl = &statefulElasticsearchClusterNodePoolControl{}

func NewStatefulElasticsearchClusterNodePoolControl(
	kubeClient *kubernetes.Clientset,
	statefulSetLister appslisters.StatefulSetLister,
	recorder record.EventRecorder,
) ElasticsearchClusterNodePoolControl {
	return &statefulElasticsearchClusterNodePoolControl{
		kubeClient:        kubeClient,
		statefulSetLister: statefulSetLister,
		recorder:          recorder,
	}
}

func (e *statefulElasticsearchClusterNodePoolControl) CreateElasticsearchClusterNodePool(c v1alpha1.ElasticsearchCluster, np v1alpha1.ElasticsearchClusterNodePool) error {
	ss, err := nodePoolStatefulSet(c, np)

	if err != nil {
		e.recordNodePoolEvent("create", c, np, err)
		return fmt.Errorf("error generating statefulset manifest: %s", err.Error())
	}

	ss, err = e.kubeClient.Apps().StatefulSets(c.Namespace).Create(ss)

	if err != nil {
		e.recordNodePoolEvent("create", c, np, err)
		return fmt.Errorf("error creating statefulset: %s", err.Error())
	}

	e.recordNodePoolEvent("create", c, np, err)
	return nil
}

func (e *statefulElasticsearchClusterNodePoolControl) UpdateElasticsearchClusterNodePool(c v1alpha1.ElasticsearchCluster, np v1alpha1.ElasticsearchClusterNodePool) error {
	ss, err := nodePoolStatefulSet(c, np)

	if err != nil {
		e.recordNodePoolEvent("update", c, np, err)
		return fmt.Errorf("error generating statefulset manifest: %s", err.Error())
	}

	ss, err = e.kubeClient.Apps().StatefulSets(c.Namespace).Update(ss)

	if err != nil {
		e.recordNodePoolEvent("update", c, np, err)
		return fmt.Errorf("error updating statefulset: %s", err.Error())
	}

	e.recordNodePoolEvent("update", c, np, err)
	return nil
}

func (e *statefulElasticsearchClusterNodePoolControl) DeleteElasticsearchClusterNodePool(c v1alpha1.ElasticsearchCluster, np v1alpha1.ElasticsearchClusterNodePool) error {
	ss, err := nodePoolStatefulSet(c, np)

	if err != nil {
		e.recordNodePoolEvent("delete", c, np, err)
		return fmt.Errorf("error generating statefulset manifest: %s", err.Error())
	}

	err = e.kubeClient.Apps().StatefulSets(c.Namespace).Delete(ss.Name, &metav1.DeleteOptions{OrphanDependents: &falseVar})

	if err != nil {
		e.recordNodePoolEvent("delete", c, np, err)
		return fmt.Errorf("error deleting statefulset: %s", err.Error())
	}

	e.recordNodePoolEvent("delete", c, np, err)
	return nil
}

// recordNodePoolEvent records an event for verb applied to a NodePool in an ElasticsearchCluster. If err is nil the generated event will
// have a reason of v1.EventTypeNormal. If err is not nil the generated event will have a reason of v1.EventTypeWarning.
func (e *statefulElasticsearchClusterNodePoolControl) recordNodePoolEvent(verb string, cluster v1alpha1.ElasticsearchCluster, pool v1alpha1.ElasticsearchClusterNodePool, err error) {
	if err == nil {
		reason := fmt.Sprintf("Successful%s", strings.Title(verb))
		message := fmt.Sprintf("%s StatefulNodePool %s in ElasticsearchCluster %s successful",
			strings.ToLower(verb), pool.Name, cluster.Name)
		e.recorder.Event(&cluster, apiv1.EventTypeNormal, reason, message)
	} else {
		reason := fmt.Sprintf("Failed%s", strings.Title(verb))
		message := fmt.Sprintf("%s StatefulNodePool %s in ElasticsearchCluster %s failed error: %s",
			strings.ToLower(verb), pool.Name, cluster.Name, err)
		e.recorder.Event(&cluster, apiv1.EventTypeWarning, reason, message)
	}
}
<commit_msg>Use apps v1beta1 in Kubernetes clientset<commit_after>package elasticsearch

import (
	"fmt"
	"strings"

	apiv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	appslisters "k8s.io/client-go/listers/apps/v1beta1"
	"k8s.io/client-go/tools/record"

	v1alpha1 "github.com/jetstack-experimental/navigator/pkg/apis/navigator/v1alpha1"
)

// statefulElasticsearchClusterNodePoolControl manages the lifecycle of a
// stateful node pool. It can be used to create, update and delete pools.
//
// This is an implementation of the ElasticsearchClusterNodePoolControl interface
// as defined in interfaces.go.
type statefulElasticsearchClusterNodePoolControl struct {
	kubeClient        *kubernetes.Clientset
	statefulSetLister appslisters.StatefulSetLister

	recorder record.EventRecorder
}

var _ ElasticsearchClusterNodePoolControl = &statefulElasticsearchClusterNodePoolControl{}

func NewStatefulElasticsearchClusterNodePoolControl(
	kubeClient *kubernetes.Clientset,
	statefulSetLister appslisters.StatefulSetLister,
	recorder record.EventRecorder,
) ElasticsearchClusterNodePoolControl {
	return &statefulElasticsearchClusterNodePoolControl{
		kubeClient:        kubeClient,
		statefulSetLister: statefulSetLister,
		recorder:          recorder,
	}
}

func (e *statefulElasticsearchClusterNodePoolControl) CreateElasticsearchClusterNodePool(c v1alpha1.ElasticsearchCluster, np v1alpha1.ElasticsearchClusterNodePool) error {
	ss, err := nodePoolStatefulSet(c, np)

	if err != nil {
		e.recordNodePoolEvent("create", c, np, err)
		return fmt.Errorf("error generating statefulset manifest: %s", err.Error())
	}

	ss, err = e.kubeClient.AppsV1beta1().StatefulSets(c.Namespace).Create(ss)

	if err != nil {
		e.recordNodePoolEvent("create", c, np, err)
		return fmt.Errorf("error creating statefulset: %s", err.Error())
	}

	e.recordNodePoolEvent("create", c, np, err)
	return nil
}

func (e *statefulElasticsearchClusterNodePoolControl) UpdateElasticsearchClusterNodePool(c v1alpha1.ElasticsearchCluster, np v1alpha1.ElasticsearchClusterNodePool) error {
	ss, err := nodePoolStatefulSet(c, np)

	if err != nil {
		e.recordNodePoolEvent("update", c, np, err)
		return fmt.Errorf("error generating statefulset manifest: %s", err.Error())
	}

	ss, err = e.kubeClient.AppsV1beta1().StatefulSets(c.Namespace).Update(ss)

	if err != nil {
		e.recordNodePoolEvent("update", c, np, err)
		return fmt.Errorf("error updating statefulset: %s", err.Error())
	}

	e.recordNodePoolEvent("update", c, np, err)
	return nil
}

func (e *statefulElasticsearchClusterNodePoolControl) DeleteElasticsearchClusterNodePool(c v1alpha1.ElasticsearchCluster, np v1alpha1.ElasticsearchClusterNodePool) error {
	ss, err := nodePoolStatefulSet(c, np)

	if err != nil {
		e.recordNodePoolEvent("delete", c, np, err)
		return fmt.Errorf("error generating statefulset manifest: %s", err.Error())
	}

	err = e.kubeClient.AppsV1beta1().StatefulSets(c.Namespace).Delete(ss.Name, &metav1.DeleteOptions{OrphanDependents: &falseVar})

	if err != nil {
		e.recordNodePoolEvent("delete", c, np, err)
		return fmt.Errorf("error deleting statefulset: %s", err.Error())
	}

	e.recordNodePoolEvent("delete", c, np, err)
	return nil
}

// recordNodePoolEvent records an event for verb applied to a NodePool in an ElasticsearchCluster. If err is nil the generated event will
// have a reason of v1.EventTypeNormal. If err is not nil the generated event will have a reason of v1.EventTypeWarning.
func (e *statefulElasticsearchClusterNodePoolControl) recordNodePoolEvent(verb string, cluster v1alpha1.ElasticsearchCluster, pool v1alpha1.ElasticsearchClusterNodePool, err error) {
	if err == nil {
		reason := fmt.Sprintf("Successful%s", strings.Title(verb))
		message := fmt.Sprintf("%s StatefulNodePool %s in ElasticsearchCluster %s successful",
			strings.ToLower(verb), pool.Name, cluster.Name)
		e.recorder.Event(&cluster, apiv1.EventTypeNormal, reason, message)
	} else {
		reason := fmt.Sprintf("Failed%s", strings.Title(verb))
		message := fmt.Sprintf("%s StatefulNodePool %s in ElasticsearchCluster %s failed error: %s",
			strings.ToLower(verb), pool.Name, cluster.Name, err)
		e.recorder.Event(&cluster, apiv1.EventTypeWarning, reason, message)
	}
}
<|endoftext|>


<commit_before>package middleware

import (
	"bufio"
	"compress/flate"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"sync"
)

var defaultCompressibleContentTypes = []string{
	"text/html",
	"text/css",
	"text/plain",
	"text/javascript",
	"application/javascript",
	"application/x-javascript",
	"application/json",
	"application/atom+xml",
	"application/rss+xml",
	"image/svg+xml",
}

// Compress is a middleware that compresses response
// body of a given content types to a data format based
// on Accept-Encoding request header. It uses a given
// compression level.
//
// NOTE: make sure to set the Content-Type header on your response
// otherwise this middleware will not compress the response body. For ex, in
// your handler you should set w.Header().Set("Content-Type", http.DetectContentType(yourBody))
// or set it manually.
//
// Passing a compression level of 5 is sensible value
func Compress(level int, types ...string) func(next http.Handler) http.Handler {
	compressor := NewCompressor(level, types...)
	return compressor.Handler
}

// Compressor represents a set of encoding configurations.
type Compressor struct {
	// The mapping of encoder names to encoder functions.
	encoders map[string]EncoderFunc
	// The mapping of pooled encoders to pools.
	pooledEncoders map[string]*sync.Pool
	// The set of content types allowed to be compressed.
	allowedTypes     map[string]struct{}
	allowedWildcards map[string]struct{}
	// The list of encoders in order of decreasing precedence.
	encodingPrecedence []string
	level              int // The compression level.
}

// NewCompressor creates a new Compressor that will handle encoding responses.
//
// The level should be one of the ones defined in the flate package.
// The types are the content types that are allowed to be compressed.
func NewCompressor(level int, types ...string) *Compressor {
	// If types are provided, set those as the allowed types. If none are
	// provided, use the default list.
	allowedTypes := make(map[string]struct{})
	allowedWildcards := make(map[string]struct{})
	if len(types) > 0 {
		for _, t := range types {
			if strings.Contains(strings.TrimSuffix(t, "/*"), "*") {
				panic(fmt.Sprintf("middleware/compress: Unsupported content-type wildcard pattern '%s'. Only '/*' supported", t))
			}
			if strings.HasSuffix(t, "/*") {
				allowedWildcards[strings.TrimSuffix(t, "/*")] = struct{}{}
			} else {
				allowedTypes[t] = struct{}{}
			}
		}
	} else {
		for _, t := range defaultCompressibleContentTypes {
			allowedTypes[t] = struct{}{}
		}
	}

	c := &Compressor{
		level:            level,
		encoders:         make(map[string]EncoderFunc),
		pooledEncoders:   make(map[string]*sync.Pool),
		allowedTypes:     allowedTypes,
		allowedWildcards: allowedWildcards,
	}

	// Set the default encoders.  The precedence order uses the reverse
	// ordering that the encoders were added. This means adding new encoders
	// will move them to the front of the order.
	//
	// TODO:
	// lzma: Opera.
	// sdch: Chrome, Android. Gzip output + dictionary header.
	// br:   Brotli, see https://github.com/go-chi/chi/pull/326

	// HTTP 1.1 "deflate" (RFC 2616) stands for DEFLATE data (RFC 1951)
	// wrapped with zlib (RFC 1950). The zlib wrapper uses Adler-32
	// checksum compared to CRC-32 used in "gzip" and thus is faster.
	//
	// But.. some old browsers (MSIE, Safari 5.1) incorrectly expect
	// raw DEFLATE data only, without the mentioned zlib wrapper.
	// Because of this major confusion, most modern browsers try it
	// both ways, first looking for zlib headers.
	// Quote by Mark Adler: http://stackoverflow.com/a/9186091/385548
	//
	// The list of browsers having problems is quite big, see:
	// http://zoompf.com/blog/2012/02/lose-the-wait-http-compression
	// https://web.archive.org/web/20120321182910/http://www.vervestudios.co/projects/compression-tests/results
	//
	// That's why we prefer gzip over deflate. It's just more reliable
	// and not significantly slower than gzip.
	c.SetEncoder("deflate", encoderDeflate)

	// TODO: Exception for old MSIE browsers that can't handle non-HTML?
	// https://zoompf.com/blog/2012/02/lose-the-wait-http-compression
	c.SetEncoder("gzip", encoderGzip)

	// NOTE: Not implemented, intentionally:
	// case "compress": // LZW. Deprecated.
	// case "bzip2":    // Too slow on-the-fly.
	// case "zopfli":   // Too slow on-the-fly.
	// case "xz":       // Too slow on-the-fly.
	return c
}

// SetEncoder can be used to set the implementation of a compression algorithm.
//
// The encoding should be a standardised identifier. See:
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding
//
// For example, add the Brotli algortithm:
//
//  import brotli_enc "gopkg.in/kothar/brotli-go.v0/enc"
//
//  compressor := middleware.NewCompressor(5, "text/html")
//  compressor.SetEncoder("br", func(w http.ResponseWriter, level int) io.Writer {
//    params := brotli_enc.NewBrotliParams()
//    params.SetQuality(level)
//    return brotli_enc.NewBrotliWriter(params, w)
//  })
func (c *Compressor) SetEncoder(encoding string, fn EncoderFunc) {
	encoding = strings.ToLower(encoding)
	if encoding == "" {
		panic("the encoding can not be empty")
	}
	if fn == nil {
		panic("attempted to set a nil encoder function")
	}

	// If we are adding a new encoder that is already registered, we have to
	// clear that one out first.
	if _, ok := c.pooledEncoders[encoding]; ok {
		delete(c.pooledEncoders, encoding)
	}
	if _, ok := c.encoders[encoding]; ok {
		delete(c.encoders, encoding)
	}

	// If the encoder supports Resetting (IoReseterWriter), then it can be pooled.
	encoder := fn(ioutil.Discard, c.level)
	if encoder != nil {
		if _, ok := encoder.(ioResetterWriter); ok {
			pool := &sync.Pool{
				New: func() interface{} {
					return fn(ioutil.Discard, c.level)
				},
			}
			c.pooledEncoders[encoding] = pool
		}
	}
	// If the encoder is not in the pooledEncoders, add it to the normal encoders.
	if _, ok := c.pooledEncoders[encoding]; !ok {
		c.encoders[encoding] = fn
	}

	for i, v := range c.encodingPrecedence {
		if v == encoding {
			c.encodingPrecedence = append(c.encodingPrecedence[:i], c.encodingPrecedence[i+1:]...)
		}
	}

	c.encodingPrecedence = append([]string{encoding}, c.encodingPrecedence...)
}

// Handler returns a new middleware that will compress the response based on the
// current Compressor.
func (c *Compressor) Handler(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		encoder, encoding, cleanup := c.selectEncoder(r.Header, w)

		cw := &compressResponseWriter{
			ResponseWriter:   w,
			w:                w,
			contentTypes:     c.allowedTypes,
			contentWildcards: c.allowedWildcards,
			encoding:         encoding,
			compressable:     false, // determined in post-handler
		}
		if encoder != nil {
			cw.w = encoder
		}
		// Re-add the encoder to the pool if applicable.
		defer cleanup()
		defer cw.Close()

		next.ServeHTTP(cw, r)
	})
}

// selectEncoder returns the encoder, the name of the encoder, and a closer function.
func (c *Compressor) selectEncoder(h http.Header, w io.Writer) (io.Writer, string, func()) {
	header := h.Get("Accept-Encoding")

	// Parse the names of all accepted algorithms from the header.
	accepted := strings.Split(strings.ToLower(header), ",")

	// Find supported encoder by accepted list by precedence
	for _, name := range c.encodingPrecedence {
		if matchAcceptEncoding(accepted, name) {
			if pool, ok := c.pooledEncoders[name]; ok {
				encoder := pool.Get().(ioResetterWriter)
				cleanup := func() {
					pool.Put(encoder)
				}
				encoder.Reset(w)
				return encoder, name, cleanup

			}
			if fn, ok := c.encoders[name]; ok {
				return fn(w, c.level), name, func() {}
			}
		}

	}

	// No encoder found to match the accepted encoding
	return nil, "", func() {}
}

func matchAcceptEncoding(accepted []string, encoding string) bool {
	for _, v := range accepted {
		if strings.Contains(v, encoding) {
			return true
		}
	}
	return false
}

// An EncoderFunc is a function that wraps the provided io.Writer with a
// streaming compression algorithm and returns it.
//
// In case of failure, the function should return nil.
type EncoderFunc func(w io.Writer, level int) io.Writer

// Interface for types that allow resetting io.Writers.
type ioResetterWriter interface {
	io.Writer
	Reset(w io.Writer)
}

type compressResponseWriter struct {
	http.ResponseWriter

	// The streaming encoder writer to be used if there is one. Otherwise,
	// this is just the normal writer.
	w                io.Writer
	contentTypes     map[string]struct{}
	contentWildcards map[string]struct{}
	encoding         string
	wroteHeader      bool
	compressable     bool
}

func (cw *compressResponseWriter) isCompressable() bool {
	// Parse the first part of the Content-Type response header.
	contentType := cw.Header().Get("Content-Type")
	if idx := strings.Index(contentType, ";"); idx >= 0 {
		contentType = contentType[0:idx]
	}

	// Is the content type compressable?
	if _, ok := cw.contentTypes[contentType]; ok {
		return true
	}
	if idx := strings.Index(contentType, "/"); idx > 0 {
		contentType = contentType[0:idx]
		_, ok := cw.contentWildcards[contentType]
		return ok
	}
	return false
}

func (cw *compressResponseWriter) WriteHeader(code int) {
	if cw.wroteHeader {
		cw.ResponseWriter.WriteHeader(code) // Allow multiple calls to propagate.
		return
	}
	cw.wroteHeader = true
	defer cw.ResponseWriter.WriteHeader(code)

	// Already compressed data?
	if cw.Header().Get("Content-Encoding") != "" {
		return
	}

	if !cw.isCompressable() {
		cw.compressable = false
		return
	}

	if cw.encoding != "" {
		cw.compressable = true
		cw.Header().Set("Content-Encoding", cw.encoding)
		cw.Header().Set("Vary", "Accept-Encoding")

		// The content-length after compression is unknown
		cw.Header().Del("Content-Length")
	}
}

func (cw *compressResponseWriter) Write(p []byte) (int, error) {
	if !cw.wroteHeader {
		cw.WriteHeader(http.StatusOK)
	}

	return cw.writer().Write(p)
}

func (cw *compressResponseWriter) writer() io.Writer {
	if cw.compressable {
		return cw.w
	} else {
		return cw.ResponseWriter
	}
}

type compressFlusher interface {
	Flush() error
}

func (cw *compressResponseWriter) Flush() {
	if f, ok := cw.writer().(http.Flusher); ok {
		f.Flush()
	}
	// If the underlying writer has a compression flush signature,
	// call this Flush() method instead
	if f, ok := cw.writer().(compressFlusher); ok {
		f.Flush()

		// Also flush the underlying response writer
		if f, ok := cw.ResponseWriter.(http.Flusher); ok {
			f.Flush()
		}
	}
}

func (cw *compressResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	if hj, ok := cw.writer().(http.Hijacker); ok {
		return hj.Hijack()
	}
	return nil, nil, errors.New("chi/middleware: http.Hijacker is unavailable on the writer")
}

func (cw *compressResponseWriter) Push(target string, opts *http.PushOptions) error {
	if ps, ok := cw.writer().(http.Pusher); ok {
		return ps.Push(target, opts)
	}
	return errors.New("chi/middleware: http.Pusher is unavailable on the writer")
}

func (cw *compressResponseWriter) Close() error {
	if c, ok := cw.writer().(io.WriteCloser); ok {
		return c.Close()
	}
	return errors.New("chi/middleware: io.WriteCloser is unavailable on the writer")
}

func encoderGzip(w io.Writer, level int) io.Writer {
	gw, err := gzip.NewWriterLevel(w, level)
	if err != nil {
		return nil
	}
	return gw
}

func encoderDeflate(w io.Writer, level int) io.Writer {
	dw, err := flate.NewWriter(w, level)
	if err != nil {
		return nil
	}
	return dw
}
<commit_msg>correction (#675)<commit_after>package middleware

import (
	"bufio"
	"compress/flate"
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"sync"
)

var defaultCompressibleContentTypes = []string{
	"text/html",
	"text/css",
	"text/plain",
	"text/javascript",
	"application/javascript",
	"application/x-javascript",
	"application/json",
	"application/atom+xml",
	"application/rss+xml",
	"image/svg+xml",
}

// Compress is a middleware that compresses response
// body of a given content types to a data format based
// on Accept-Encoding request header. It uses a given
// compression level.
//
// NOTE: make sure to set the Content-Type header on your response
// otherwise this middleware will not compress the response body. For ex, in
// your handler you should set w.Header().Set("Content-Type", http.DetectContentType(yourBody))
// or set it manually.
//
// Passing a compression level of 5 is sensible value
func Compress(level int, types ...string) func(next http.Handler) http.Handler {
	compressor := NewCompressor(level, types...)
	return compressor.Handler
}

// Compressor represents a set of encoding configurations.
type Compressor struct {
	// The mapping of encoder names to encoder functions.
	encoders map[string]EncoderFunc
	// The mapping of pooled encoders to pools.
	pooledEncoders map[string]*sync.Pool
	// The set of content types allowed to be compressed.
	allowedTypes     map[string]struct{}
	allowedWildcards map[string]struct{}
	// The list of encoders in order of decreasing precedence.
	encodingPrecedence []string
	level              int // The compression level.
}

// NewCompressor creates a new Compressor that will handle encoding responses.
//
// The level should be one of the ones defined in the flate package.
// The types are the content types that are allowed to be compressed.
func NewCompressor(level int, types ...string) *Compressor {
	// If types are provided, set those as the allowed types. If none are
	// provided, use the default list.
	allowedTypes := make(map[string]struct{})
	allowedWildcards := make(map[string]struct{})
	if len(types) > 0 {
		for _, t := range types {
			if strings.Contains(strings.TrimSuffix(t, "/*"), "*") {
				panic(fmt.Sprintf("middleware/compress: Unsupported content-type wildcard pattern '%s'. Only '/*' supported", t))
			}
			if strings.HasSuffix(t, "/*") {
				allowedWildcards[strings.TrimSuffix(t, "/*")] = struct{}{}
			} else {
				allowedTypes[t] = struct{}{}
			}
		}
	} else {
		for _, t := range defaultCompressibleContentTypes {
			allowedTypes[t] = struct{}{}
		}
	}

	c := &Compressor{
		level:            level,
		encoders:         make(map[string]EncoderFunc),
		pooledEncoders:   make(map[string]*sync.Pool),
		allowedTypes:     allowedTypes,
		allowedWildcards: allowedWildcards,
	}

	// Set the default encoders.  The precedence order uses the reverse
	// ordering that the encoders were added. This means adding new encoders
	// will move them to the front of the order.
	//
	// TODO:
	// lzma: Opera.
	// sdch: Chrome, Android. Gzip output + dictionary header.
	// br:   Brotli, see https://github.com/go-chi/chi/pull/326

	// HTTP 1.1 "deflate" (RFC 2616) stands for DEFLATE data (RFC 1951)
	// wrapped with zlib (RFC 1950). The zlib wrapper uses Adler-32
	// checksum compared to CRC-32 used in "gzip" and thus is faster.
	//
	// But.. some old browsers (MSIE, Safari 5.1) incorrectly expect
	// raw DEFLATE data only, without the mentioned zlib wrapper.
	// Because of this major confusion, most modern browsers try it
	// both ways, first looking for zlib headers.
	// Quote by Mark Adler: http://stackoverflow.com/a/9186091/385548
	//
	// The list of browsers having problems is quite big, see:
	// http://zoompf.com/blog/2012/02/lose-the-wait-http-compression
	// https://web.archive.org/web/20120321182910/http://www.vervestudios.co/projects/compression-tests/results
	//
	// That's why we prefer gzip over deflate. It's just more reliable
	// and not significantly slower than deflate.
	c.SetEncoder("deflate", encoderDeflate)

	// TODO: Exception for old MSIE browsers that can't handle non-HTML?
	// https://zoompf.com/blog/2012/02/lose-the-wait-http-compression
	c.SetEncoder("gzip", encoderGzip)

	// NOTE: Not implemented, intentionally:
	// case "compress": // LZW. Deprecated.
	// case "bzip2":    // Too slow on-the-fly.
	// case "zopfli":   // Too slow on-the-fly.
	// case "xz":       // Too slow on-the-fly.
	return c
}

// SetEncoder can be used to set the implementation of a compression algorithm.
//
// The encoding should be a standardised identifier. See:
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding
//
// For example, add the Brotli algortithm:
//
//  import brotli_enc "gopkg.in/kothar/brotli-go.v0/enc"
//
//  compressor := middleware.NewCompressor(5, "text/html")
//  compressor.SetEncoder("br", func(w http.ResponseWriter, level int) io.Writer {
//    params := brotli_enc.NewBrotliParams()
//    params.SetQuality(level)
//    return brotli_enc.NewBrotliWriter(params, w)
//  })
func (c *Compressor) SetEncoder(encoding string, fn EncoderFunc) {
	encoding = strings.ToLower(encoding)
	if encoding == "" {
		panic("the encoding can not be empty")
	}
	if fn == nil {
		panic("attempted to set a nil encoder function")
	}

	// If we are adding a new encoder that is already registered, we have to
	// clear that one out first.
	if _, ok := c.pooledEncoders[encoding]; ok {
		delete(c.pooledEncoders, encoding)
	}
	if _, ok := c.encoders[encoding]; ok {
		delete(c.encoders, encoding)
	}

	// If the encoder supports Resetting (IoReseterWriter), then it can be pooled.
	encoder := fn(ioutil.Discard, c.level)
	if encoder != nil {
		if _, ok := encoder.(ioResetterWriter); ok {
			pool := &sync.Pool{
				New: func() interface{} {
					return fn(ioutil.Discard, c.level)
				},
			}
			c.pooledEncoders[encoding] = pool
		}
	}
	// If the encoder is not in the pooledEncoders, add it to the normal encoders.
	if _, ok := c.pooledEncoders[encoding]; !ok {
		c.encoders[encoding] = fn
	}

	for i, v := range c.encodingPrecedence {
		if v == encoding {
			c.encodingPrecedence = append(c.encodingPrecedence[:i], c.encodingPrecedence[i+1:]...)
		}
	}

	c.encodingPrecedence = append([]string{encoding}, c.encodingPrecedence...)
}

// Handler returns a new middleware that will compress the response based on the
// current Compressor.
func (c *Compressor) Handler(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		encoder, encoding, cleanup := c.selectEncoder(r.Header, w)

		cw := &compressResponseWriter{
			ResponseWriter:   w,
			w:                w,
			contentTypes:     c.allowedTypes,
			contentWildcards: c.allowedWildcards,
			encoding:         encoding,
			compressable:     false, // determined in post-handler
		}
		if encoder != nil {
			cw.w = encoder
		}
		// Re-add the encoder to the pool if applicable.
		defer cleanup()
		defer cw.Close()

		next.ServeHTTP(cw, r)
	})
}

// selectEncoder returns the encoder, the name of the encoder, and a closer function.
func (c *Compressor) selectEncoder(h http.Header, w io.Writer) (io.Writer, string, func()) {
	header := h.Get("Accept-Encoding")

	// Parse the names of all accepted algorithms from the header.
	accepted := strings.Split(strings.ToLower(header), ",")

	// Find supported encoder by accepted list by precedence
	for _, name := range c.encodingPrecedence {
		if matchAcceptEncoding(accepted, name) {
			if pool, ok := c.pooledEncoders[name]; ok {
				encoder := pool.Get().(ioResetterWriter)
				cleanup := func() {
					pool.Put(encoder)
				}
				encoder.Reset(w)
				return encoder, name, cleanup

			}
			if fn, ok := c.encoders[name]; ok {
				return fn(w, c.level), name, func() {}
			}
		}

	}

	// No encoder found to match the accepted encoding
	return nil, "", func() {}
}

func matchAcceptEncoding(accepted []string, encoding string) bool {
	for _, v := range accepted {
		if strings.Contains(v, encoding) {
			return true
		}
	}
	return false
}

// An EncoderFunc is a function that wraps the provided io.Writer with a
// streaming compression algorithm and returns it.
//
// In case of failure, the function should return nil.
type EncoderFunc func(w io.Writer, level int) io.Writer

// Interface for types that allow resetting io.Writers.
type ioResetterWriter interface {
	io.Writer
	Reset(w io.Writer)
}

type compressResponseWriter struct {
	http.ResponseWriter

	// The streaming encoder writer to be used if there is one. Otherwise,
	// this is just the normal writer.
	w                io.Writer
	contentTypes     map[string]struct{}
	contentWildcards map[string]struct{}
	encoding         string
	wroteHeader      bool
	compressable     bool
}

func (cw *compressResponseWriter) isCompressable() bool {
	// Parse the first part of the Content-Type response header.
	contentType := cw.Header().Get("Content-Type")
	if idx := strings.Index(contentType, ";"); idx >= 0 {
		contentType = contentType[0:idx]
	}

	// Is the content type compressable?
	if _, ok := cw.contentTypes[contentType]; ok {
		return true
	}
	if idx := strings.Index(contentType, "/"); idx > 0 {
		contentType = contentType[0:idx]
		_, ok := cw.contentWildcards[contentType]
		return ok
	}
	return false
}

func (cw *compressResponseWriter) WriteHeader(code int) {
	if cw.wroteHeader {
		cw.ResponseWriter.WriteHeader(code) // Allow multiple calls to propagate.
		return
	}
	cw.wroteHeader = true
	defer cw.ResponseWriter.WriteHeader(code)

	// Already compressed data?
	if cw.Header().Get("Content-Encoding") != "" {
		return
	}

	if !cw.isCompressable() {
		cw.compressable = false
		return
	}

	if cw.encoding != "" {
		cw.compressable = true
		cw.Header().Set("Content-Encoding", cw.encoding)
		cw.Header().Set("Vary", "Accept-Encoding")

		// The content-length after compression is unknown
		cw.Header().Del("Content-Length")
	}
}

func (cw *compressResponseWriter) Write(p []byte) (int, error) {
	if !cw.wroteHeader {
		cw.WriteHeader(http.StatusOK)
	}

	return cw.writer().Write(p)
}

func (cw *compressResponseWriter) writer() io.Writer {
	if cw.compressable {
		return cw.w
	} else {
		return cw.ResponseWriter
	}
}

type compressFlusher interface {
	Flush() error
}

func (cw *compressResponseWriter) Flush() {
	if f, ok := cw.writer().(http.Flusher); ok {
		f.Flush()
	}
	// If the underlying writer has a compression flush signature,
	// call this Flush() method instead
	if f, ok := cw.writer().(compressFlusher); ok {
		f.Flush()

		// Also flush the underlying response writer
		if f, ok := cw.ResponseWriter.(http.Flusher); ok {
			f.Flush()
		}
	}
}

func (cw *compressResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
	if hj, ok := cw.writer().(http.Hijacker); ok {
		return hj.Hijack()
	}
	return nil, nil, errors.New("chi/middleware: http.Hijacker is unavailable on the writer")
}

func (cw *compressResponseWriter) Push(target string, opts *http.PushOptions) error {
	if ps, ok := cw.writer().(http.Pusher); ok {
		return ps.Push(target, opts)
	}
	return errors.New("chi/middleware: http.Pusher is unavailable on the writer")
}

func (cw *compressResponseWriter) Close() error {
	if c, ok := cw.writer().(io.WriteCloser); ok {
		return c.Close()
	}
	return errors.New("chi/middleware: io.WriteCloser is unavailable on the writer")
}

func encoderGzip(w io.Writer, level int) io.Writer {
	gw, err := gzip.NewWriterLevel(w, level)
	if err != nil {
		return nil
	}
	return gw
}

func encoderDeflate(w io.Writer, level int) io.Writer {
	dw, err := flate.NewWriter(w, level)
	if err != nil {
		return nil
	}
	return dw
}
<|endoftext|>


<commit_before>package middleware

import (
	"expvar"
	"fmt"
	"net/http"
	"net/http/pprof"

	"github.com/pressly/chi"
)

// Profiler is a convenient subrouter used for mounting net/http/pprof. ie.
//
//  func MyService() http.Handler {
//    r := chi.NewRouter()
//    // ..middlewares
//    r.Mount("/debug", middleware.Profiler())
//    // ..routes
//    return r
//  }
func Profiler() http.Handler {
	r := chi.NewRouter()
	r.Use(NoCache)

	r.Get("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, r.RequestURI+"/pprof/", 301)
	})
	r.HandleFunc("/pprof", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, r.RequestURI+"/", 301)
	})

	r.HandleFunc("/pprof/", pprof.Index)
	r.HandleFunc("/pprof/cmdline", pprof.Cmdline)
	r.HandleFunc("/pprof/profile", pprof.Profile)
	r.HandleFunc("/pprof/symbol", pprof.Symbol)
	r.Handle("/pprof/block", pprof.Handler("block"))
	r.Handle("/pprof/heap", pprof.Handler("heap"))
	r.Handle("/pprof/goroutine", pprof.Handler("goroutine"))
	r.Handle("/pprof/threadcreate", pprof.Handler("threadcreate"))
	r.HandleFunc("/vars", expVars)

	return r
}

// Replicated from expvar.go as not public.
func expVars(w http.ResponseWriter, r *http.Request) {
	first := true
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, "{\n")
	expvar.Do(func(kv expvar.KeyValue) {
		if !first {
			fmt.Fprintf(w, ",\n")
		}
		first = false
		fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
	})
	fmt.Fprintf(w, "\n}\n")
}
<commit_msg>Add trace handler to the profiler<commit_after>package middleware

import (
	"expvar"
	"fmt"
	"net/http"
	"net/http/pprof"

	"github.com/pressly/chi"
)

// Profiler is a convenient subrouter used for mounting net/http/pprof. ie.
//
//  func MyService() http.Handler {
//    r := chi.NewRouter()
//    // ..middlewares
//    r.Mount("/debug", middleware.Profiler())
//    // ..routes
//    return r
//  }
func Profiler() http.Handler {
	r := chi.NewRouter()
	r.Use(NoCache)

	r.Get("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, r.RequestURI+"/pprof/", 301)
	})
	r.HandleFunc("/pprof", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, r.RequestURI+"/", 301)
	})

	r.HandleFunc("/pprof/", pprof.Index)
	r.HandleFunc("/pprof/cmdline", pprof.Cmdline)
	r.HandleFunc("/pprof/profile", pprof.Profile)
	r.HandleFunc("/pprof/symbol", pprof.Symbol)
	r.HandleFunc("/pprof/trace", pprof.Trace)
	r.Handle("/pprof/block", pprof.Handler("block"))
	r.Handle("/pprof/heap", pprof.Handler("heap"))
	r.Handle("/pprof/goroutine", pprof.Handler("goroutine"))
	r.Handle("/pprof/threadcreate", pprof.Handler("threadcreate"))
	r.HandleFunc("/vars", expVars)

	return r
}

// Replicated from expvar.go as not public.
func expVars(w http.ResponseWriter, r *http.Request) {
	first := true
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, "{\n")
	expvar.Do(func(kv expvar.KeyValue) {
		if !first {
			fmt.Fprintf(w, ",\n")
		}
		first = false
		fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
	})
	fmt.Fprintf(w, "\n}\n")
}
<|endoftext|>


<commit_before>package backend

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/influxdata/flux"
	"github.com/influxdata/flux/lang"
	"github.com/influxdata/influxdb"
	platform "github.com/influxdata/influxdb"
	"github.com/influxdata/influxdb/models"
	"github.com/influxdata/influxdb/query"
	"github.com/influxdata/influxdb/storage"
	"github.com/influxdata/influxdb/tsdb"
	"go.uber.org/zap"
)

const (
	runIDField        = "runID"
	scheduledForField = "scheduledFor"
	startedAtField    = "startedAt"
	finishedAtField   = "finishedAt"
	requestedAtField  = "requestedAt"
	logField          = "logs"

	taskIDTag = "taskID"
	statusTag = "status"

	// Fixed system bucket ID for task and run logs.
	taskSystemBucketID platform.ID = 10
)

// NewAnalyticalStorage creates a new analytical store with access to the necessary systems for storing data and to act as a middleware
func NewAnalyticalStorage(logger *zap.Logger, ts influxdb.TaskService, tcs TaskControlService, pw storage.PointsWriter, qs query.QueryService) *AnalyticalStorage {
	return &AnalyticalStorage{
		logger:             logger,
		TaskService:        ts,
		TaskControlService: tcs,
		pw:                 pw,
		qs:                 qs,
	}
}

type AnalyticalStorage struct {
	influxdb.TaskService
	TaskControlService

	pw     storage.PointsWriter
	qs     query.QueryService
	logger *zap.Logger
}

func (as *AnalyticalStorage) FinishRun(ctx context.Context, taskID, runID influxdb.ID) (*influxdb.Run, error) {
	run, err := as.TaskControlService.FinishRun(ctx, taskID, runID)
	if run != nil && run.ID.String() != "" {
		task, err := as.TaskService.FindTaskByID(ctx, run.TaskID)
		if err != nil {
			return run, err
		}

		tags := models.Tags{
			models.NewTag([]byte(taskIDTag), []byte(run.TaskID.String())),
			models.NewTag([]byte(statusTag), []byte(run.Status)),
		}

		// log an error if we have incomplete data on finish
		if !run.ID.Valid() ||
			run.ScheduledFor == "" ||
			run.StartedAt == "" ||
			run.FinishedAt == "" ||
			run.Status == "" {
			as.logger.Error("Run missing critical fields", zap.String("run", fmt.Sprintf("%+v", run)), zap.String("runID", run.ID.String()))
		}

		fields := map[string]interface{}{}
		fields[runIDField] = run.ID.String()
		fields[startedAtField] = run.StartedAt
		fields[finishedAtField] = run.FinishedAt
		fields[scheduledForField] = run.ScheduledFor
		if run.RequestedAt != "" {
			fields[requestedAtField] = run.RequestedAt
		}

		startedAt, err := run.StartedAtTime()
		if err != nil {
			startedAt = time.Now()
		}

		logBytes, err := json.Marshal(run.Log)
		if err != nil {
			return run, err
		}
		fields[logField] = string(logBytes)

		point, err := models.NewPoint("runs", tags, fields, startedAt)
		if err != nil {
			return run, err
		}

		// use the tsdb explode points to convert to the new style.
		// We could split this on our own but its quite possible this could change.
		points, err := tsdb.ExplodePoints(task.OrganizationID, taskSystemBucketID, models.Points{point})
		if err != nil {
			return run, err
		}
		return run, as.pw.WritePoints(ctx, points)
	}
	return run, err
}

// FindLogs returns logs for a run.
// First attempt to use the TaskService, then append additional analytical's logs to the list
func (as *AnalyticalStorage) FindLogs(ctx context.Context, filter influxdb.LogFilter) ([]*influxdb.Log, int, error) {
	var logs []*influxdb.Log
	if filter.Run != nil {
		run, err := as.FindRunByID(ctx, filter.Task, *filter.Run)
		if err != nil {
			return nil, 0, err
		}
		for i := 0; i < len(run.Log); i++ {
			logs = append(logs, &run.Log[i])
		}
		return logs, len(logs), nil
	}

	// add historical logs to the transactional logs.
	runs, n, err := as.FindRuns(ctx, influxdb.RunFilter{Task: filter.Task})
	if err != nil {
		return nil, 0, err
	}

	for _, run := range runs {
		for i := 0; i < len(run.Log); i++ {
			logs = append(logs, &run.Log[i])
		}
	}

	return logs, n, err
}

// FindRuns returns a list of runs that match a filter and the total count of returned runs.
// First attempt to use the TaskService, then append additional analytical's runs to the list
func (as *AnalyticalStorage) FindRuns(ctx context.Context, filter influxdb.RunFilter) ([]*influxdb.Run, int, error) {
	if filter.Limit == 0 {
		filter.Limit = influxdb.TaskDefaultPageSize
	}

	if filter.Limit < 0 || filter.Limit > influxdb.TaskMaxPageSize {
		return nil, 0, influxdb.ErrOutOfBoundsLimit
	}

	runs, n, err := as.TaskService.FindRuns(ctx, filter)
	if err != nil {
		return runs, n, err
	}

	// if we reached the limit lets stop here
	if len(runs) >= filter.Limit {
		return runs, n, err
	}

	task, err := as.TaskService.FindTaskByID(ctx, filter.Task)
	if err != nil {
		return runs, n, err
	}

	filterPart := ""
	if filter.After != nil {
		filterPart = fmt.Sprintf(`|> filter(fn: (r) => r.runID > %q)`, filter.After.String())
	}

	// the data will be stored for 7 days in the system bucket so pulling 14d's is sufficient.
	runsScript := fmt.Sprintf(`from(bucketID: "000000000000000a")
	  |> range(start: -14d)
	  |> filter(fn: (r) => r._measurement == "runs" and r.taskID == %q)
	  %s
	  |> group(columns: ["taskID", "status"])
	  |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
	  |> limit(n:%d)
	  |> sort(columns:["scheduledFor"], desc: true)

	  `, filter.Task.String(), filterPart, filter.Limit-len(runs))

	// At this point we are behind authorization
	// so we are faking a read only permission to the org's system bucket
	runSystemBucketID := taskSystemBucketID
	runAuth := &influxdb.Authorization{
		ID:    taskSystemBucketID,
		OrgID: task.OrganizationID,
		Permissions: []influxdb.Permission{
			influxdb.Permission{
				Action: influxdb.ReadAction,
				Resource: influxdb.Resource{
					Type:  influxdb.BucketsResourceType,
					OrgID: &task.OrganizationID,
					ID:    &runSystemBucketID,
				},
			},
		},
	}
	request := &query.Request{Authorization: runAuth, OrganizationID: task.OrganizationID, Compiler: lang.FluxCompiler{Query: runsScript}}

	ittr, err := as.qs.Query(ctx, request)
	if err != nil {
		return nil, 0, err
	}
	defer ittr.Release()

	re := &runReader{logger: as.logger.With(zap.String("component", "run-reader"), zap.String("taskID", filter.Task.String()))}
	for ittr.More() {
		err := ittr.Next().Tables().Do(re.readTable)
		if err != nil {
			return runs, n, err
		}
	}

	if err := ittr.Err(); err != nil {
		return nil, 0, fmt.Errorf("unexpected internal error while decoding run response: %v", err)
	}

	runs = append(runs, re.runs...)

	return runs, n, err
}

// FindRunByID returns a single run.
// First see if it is in the existing TaskService. If not pull it from analytical storage.
func (as *AnalyticalStorage) FindRunByID(ctx context.Context, taskID, runID influxdb.ID) (*influxdb.Run, error) {
	// check the taskService to see if the run is on its list
	run, err := as.TaskService.FindRunByID(ctx, taskID, runID)
	if err != nil {
		if err, ok := err.(*influxdb.Error); !ok || err.Msg != "run not found" {
			return run, err
		}
	}
	if run != nil {
		return run, err
	}

	task, err := as.TaskService.FindTaskByID(ctx, taskID)
	if err != nil {
		return run, err
	}

	// the data will be stored for 7 days in the system bucket so pulling 14d's is sufficient.
	findRunScript := fmt.Sprintf(`from(bucketID: "000000000000000a")
	|> range(start: -14d)
	|> filter(fn: (r) => r._measurement == "runs" and r.taskID == %q)
	|> group(columns: ["taskID", "status"])
	|> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
	|> filter(fn: (r) => r.runID == %q)
	  `, taskID.String(), runID.String())

	// At this point we are behind authorization
	// so we are faking a read only permission to the org's system bucket
	runSystemBucketID := taskSystemBucketID
	runAuth := &influxdb.Authorization{
		ID:    taskSystemBucketID,
		OrgID: task.OrganizationID,
		Permissions: []influxdb.Permission{
			influxdb.Permission{
				Action: influxdb.ReadAction,
				Resource: influxdb.Resource{
					Type:  influxdb.BucketsResourceType,
					OrgID: &task.OrganizationID,
					ID:    &runSystemBucketID,
				},
			},
		},
	}
	request := &query.Request{Authorization: runAuth, OrganizationID: task.OrganizationID, Compiler: lang.FluxCompiler{Query: findRunScript}}

	ittr, err := as.qs.Query(ctx, request)
	if err != nil {
		return nil, err
	}
	defer ittr.Release()

	re := &runReader{}
	for ittr.More() {
		err := ittr.Next().Tables().Do(re.readTable)
		if err != nil {
			return nil, err
		}
	}

	if err := ittr.Err(); err != nil {
		return nil, fmt.Errorf("unexpected internal error while decoding run response: %v", err)
	}

	if len(re.runs) == 0 {
		return nil, platform.ErrRunNotFound

	}

	if len(re.runs) != 1 {
		return nil, &influxdb.Error{
			Msg:  "found multiple runs with id " + runID.String(),
			Code: influxdb.EInternal,
		}
	}

	return re.runs[0], err
}

func (as *AnalyticalStorage) RetryRun(ctx context.Context, taskID, runID influxdb.ID) (*influxdb.Run, error) {
	run, err := as.TaskService.RetryRun(ctx, taskID, runID)
	if err != nil {
		if err, ok := err.(*influxdb.Error); !ok || err.Msg != "run not found" {
			return run, err
		}
	}

	if run != nil {
		return run, err
	}

	// try finding the run (in our system or underlieing)
	run, err = as.FindRunByID(ctx, taskID, runID)
	if err != nil {
		return run, err
	}

	sf, err := run.ScheduledForTime()
	if err != nil {
		return run, err
	}

	return as.ForceRun(ctx, taskID, sf.Unix())
}

type runReader struct {
	runs   []*influxdb.Run
	logger *zap.Logger
}

func (re *runReader) readTable(tbl flux.Table) error {
	return tbl.Do(re.readRuns)
}

func (re *runReader) readRuns(cr flux.ColReader) error {
	for i := 0; i < cr.Len(); i++ {
		var r influxdb.Run
		for j, col := range cr.Cols() {
			switch col.Label {
			case runIDField:
				if cr.Strings(j).ValueString(i) != "" {
					id, err := influxdb.IDFromString(cr.Strings(j).ValueString(i))
					if err != nil {
						re.logger.Info("failed to parse runID", zap.Error(err))
						continue
					}
					r.ID = *id
				}
			case taskIDTag:
				if cr.Strings(j).ValueString(i) != "" {
					id, err := influxdb.IDFromString(cr.Strings(j).ValueString(i))
					if err != nil {
						re.logger.Info("failed to parse taskID", zap.Error(err))
						continue
					}
					r.TaskID = *id
				}
			case startedAtField:
				r.StartedAt = cr.Strings(j).ValueString(i)
			case requestedAtField:
				r.RequestedAt = cr.Strings(j).ValueString(i)
			case scheduledForField:
				r.ScheduledFor = cr.Strings(j).ValueString(i)
			case statusTag:
				r.Status = cr.Strings(j).ValueString(i)
			case finishedAtField:
				r.FinishedAt = cr.Strings(j).ValueString(i)
			case logField:
				logBytes := bytes.TrimSpace(cr.Strings(j).Value(i))
				if len(logBytes) != 0 {
					err := json.Unmarshal(logBytes, &r.Log)
					if err != nil {
						re.logger.Info("failed to parse log data", zap.Error(err), zap.ByteString("log_bytes", logBytes))
					}
				}
			}
		}

		// if we dont have a full enough data set we fail here.
		if r.ID.Valid() {
			re.runs = append(re.runs, &r)
		}

	}

	return nil
}
<commit_msg>fix(task): Fix inconsistancies in run query (#14661)<commit_after>package backend

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"time"

	"github.com/influxdata/flux"
	"github.com/influxdata/flux/lang"
	"github.com/influxdata/influxdb"
	platform "github.com/influxdata/influxdb"
	"github.com/influxdata/influxdb/models"
	"github.com/influxdata/influxdb/query"
	"github.com/influxdata/influxdb/storage"
	"github.com/influxdata/influxdb/tsdb"
	"go.uber.org/zap"
)

const (
	runIDField        = "runID"
	scheduledForField = "scheduledFor"
	startedAtField    = "startedAt"
	finishedAtField   = "finishedAt"
	requestedAtField  = "requestedAt"
	logField          = "logs"

	taskIDTag = "taskID"
	statusTag = "status"

	// Fixed system bucket ID for task and run logs.
	taskSystemBucketID platform.ID = 10
)

// NewAnalyticalStorage creates a new analytical store with access to the necessary systems for storing data and to act as a middleware
func NewAnalyticalStorage(logger *zap.Logger, ts influxdb.TaskService, tcs TaskControlService, pw storage.PointsWriter, qs query.QueryService) *AnalyticalStorage {
	return &AnalyticalStorage{
		logger:             logger,
		TaskService:        ts,
		TaskControlService: tcs,
		pw:                 pw,
		qs:                 qs,
	}
}

type AnalyticalStorage struct {
	influxdb.TaskService
	TaskControlService

	pw     storage.PointsWriter
	qs     query.QueryService
	logger *zap.Logger
}

func (as *AnalyticalStorage) FinishRun(ctx context.Context, taskID, runID influxdb.ID) (*influxdb.Run, error) {
	run, err := as.TaskControlService.FinishRun(ctx, taskID, runID)
	if run != nil && run.ID.String() != "" {
		task, err := as.TaskService.FindTaskByID(ctx, run.TaskID)
		if err != nil {
			return run, err
		}

		tags := models.Tags{
			models.NewTag([]byte(taskIDTag), []byte(run.TaskID.String())),
			models.NewTag([]byte(statusTag), []byte(run.Status)),
		}

		// log an error if we have incomplete data on finish
		if !run.ID.Valid() ||
			run.ScheduledFor == "" ||
			run.StartedAt == "" ||
			run.FinishedAt == "" ||
			run.Status == "" {
			as.logger.Error("Run missing critical fields", zap.String("run", fmt.Sprintf("%+v", run)), zap.String("runID", run.ID.String()))
		}

		fields := map[string]interface{}{}
		fields[runIDField] = run.ID.String()
		fields[startedAtField] = run.StartedAt
		fields[finishedAtField] = run.FinishedAt
		fields[scheduledForField] = run.ScheduledFor
		if run.RequestedAt != "" {
			fields[requestedAtField] = run.RequestedAt
		}

		startedAt, err := run.StartedAtTime()
		if err != nil {
			startedAt = time.Now()
		}

		logBytes, err := json.Marshal(run.Log)
		if err != nil {
			return run, err
		}
		fields[logField] = string(logBytes)

		point, err := models.NewPoint("runs", tags, fields, startedAt)
		if err != nil {
			return run, err
		}

		// use the tsdb explode points to convert to the new style.
		// We could split this on our own but its quite possible this could change.
		points, err := tsdb.ExplodePoints(task.OrganizationID, taskSystemBucketID, models.Points{point})
		if err != nil {
			return run, err
		}
		return run, as.pw.WritePoints(ctx, points)
	}
	return run, err
}

// FindLogs returns logs for a run.
// First attempt to use the TaskService, then append additional analytical's logs to the list
func (as *AnalyticalStorage) FindLogs(ctx context.Context, filter influxdb.LogFilter) ([]*influxdb.Log, int, error) {
	var logs []*influxdb.Log
	if filter.Run != nil {
		run, err := as.FindRunByID(ctx, filter.Task, *filter.Run)
		if err != nil {
			return nil, 0, err
		}
		for i := 0; i < len(run.Log); i++ {
			logs = append(logs, &run.Log[i])
		}
		return logs, len(logs), nil
	}

	// add historical logs to the transactional logs.
	runs, n, err := as.FindRuns(ctx, influxdb.RunFilter{Task: filter.Task})
	if err != nil {
		return nil, 0, err
	}

	for _, run := range runs {
		for i := 0; i < len(run.Log); i++ {
			logs = append(logs, &run.Log[i])
		}
	}

	return logs, n, err
}

// FindRuns returns a list of runs that match a filter and the total count of returned runs.
// First attempt to use the TaskService, then append additional analytical's runs to the list
func (as *AnalyticalStorage) FindRuns(ctx context.Context, filter influxdb.RunFilter) ([]*influxdb.Run, int, error) {
	if filter.Limit == 0 {
		filter.Limit = influxdb.TaskDefaultPageSize
	}

	if filter.Limit < 0 || filter.Limit > influxdb.TaskMaxPageSize {
		return nil, 0, influxdb.ErrOutOfBoundsLimit
	}

	runs, n, err := as.TaskService.FindRuns(ctx, filter)
	if err != nil {
		return runs, n, err
	}

	// if we reached the limit lets stop here
	if len(runs) >= filter.Limit {
		return runs, n, err
	}

	task, err := as.TaskService.FindTaskByID(ctx, filter.Task)
	if err != nil {
		return runs, n, err
	}

	filterPart := ""
	if filter.After != nil {
		filterPart = fmt.Sprintf(`|> filter(fn: (r) => r.runID > %q)`, filter.After.String())
	}

	// the data will be stored for 7 days in the system bucket so pulling 14d's is sufficient.
	runsScript := fmt.Sprintf(`from(bucketID: "000000000000000a")
	  |> range(start: -14d)
	  |> filter(fn: (r) => r._field != "status")
	  |> filter(fn: (r) => r._measurement == "runs" and r.taskID == %q)
	  %s
	  |> group(columns: ["taskID", "status"])
	  |> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
	  |> sort(columns:["scheduledFor"], desc: true)
	  |> limit(n:%d)

	  `, filter.Task.String(), filterPart, filter.Limit-len(runs))

	// At this point we are behind authorization
	// so we are faking a read only permission to the org's system bucket
	runSystemBucketID := taskSystemBucketID
	runAuth := &influxdb.Authorization{
		ID:    taskSystemBucketID,
		OrgID: task.OrganizationID,
		Permissions: []influxdb.Permission{
			influxdb.Permission{
				Action: influxdb.ReadAction,
				Resource: influxdb.Resource{
					Type:  influxdb.BucketsResourceType,
					OrgID: &task.OrganizationID,
					ID:    &runSystemBucketID,
				},
			},
		},
	}
	request := &query.Request{Authorization: runAuth, OrganizationID: task.OrganizationID, Compiler: lang.FluxCompiler{Query: runsScript}}

	ittr, err := as.qs.Query(ctx, request)
	if err != nil {
		return nil, 0, err
	}
	defer ittr.Release()

	re := &runReader{logger: as.logger.With(zap.String("component", "run-reader"), zap.String("taskID", filter.Task.String()))}
	for ittr.More() {
		err := ittr.Next().Tables().Do(re.readTable)
		if err != nil {
			return runs, n, err
		}
	}

	if err := ittr.Err(); err != nil {
		return nil, 0, fmt.Errorf("unexpected internal error while decoding run response: %v", err)
	}

	runs = append(runs, re.runs...)

	return runs, n, err
}

// FindRunByID returns a single run.
// First see if it is in the existing TaskService. If not pull it from analytical storage.
func (as *AnalyticalStorage) FindRunByID(ctx context.Context, taskID, runID influxdb.ID) (*influxdb.Run, error) {
	// check the taskService to see if the run is on its list
	run, err := as.TaskService.FindRunByID(ctx, taskID, runID)
	if err != nil {
		if err, ok := err.(*influxdb.Error); !ok || err.Msg != "run not found" {
			return run, err
		}
	}
	if run != nil {
		return run, err
	}

	task, err := as.TaskService.FindTaskByID(ctx, taskID)
	if err != nil {
		return run, err
	}

	// the data will be stored for 7 days in the system bucket so pulling 14d's is sufficient.
	findRunScript := fmt.Sprintf(`from(bucketID: "000000000000000a")
	|> range(start: -14d)
	|> filter(fn: (r) => r._field != "status")
	|> filter(fn: (r) => r._measurement == "runs" and r.taskID == %q)
	|> group(columns: ["taskID", "status"])
	|> pivot(rowKey:["_time"], columnKey: ["_field"], valueColumn: "_value")
	|> filter(fn: (r) => r.runID == %q)
	  `, taskID.String(), runID.String())

	// At this point we are behind authorization
	// so we are faking a read only permission to the org's system bucket
	runSystemBucketID := taskSystemBucketID
	runAuth := &influxdb.Authorization{
		ID:    taskSystemBucketID,
		OrgID: task.OrganizationID,
		Permissions: []influxdb.Permission{
			influxdb.Permission{
				Action: influxdb.ReadAction,
				Resource: influxdb.Resource{
					Type:  influxdb.BucketsResourceType,
					OrgID: &task.OrganizationID,
					ID:    &runSystemBucketID,
				},
			},
		},
	}
	request := &query.Request{Authorization: runAuth, OrganizationID: task.OrganizationID, Compiler: lang.FluxCompiler{Query: findRunScript}}

	ittr, err := as.qs.Query(ctx, request)
	if err != nil {
		return nil, err
	}
	defer ittr.Release()

	re := &runReader{}
	for ittr.More() {
		err := ittr.Next().Tables().Do(re.readTable)
		if err != nil {
			return nil, err
		}
	}

	if err := ittr.Err(); err != nil {
		return nil, fmt.Errorf("unexpected internal error while decoding run response: %v", err)
	}

	if len(re.runs) == 0 {
		return nil, platform.ErrRunNotFound

	}

	if len(re.runs) != 1 {
		return nil, &influxdb.Error{
			Msg:  "found multiple runs with id " + runID.String(),
			Code: influxdb.EInternal,
		}
	}

	return re.runs[0], err
}

func (as *AnalyticalStorage) RetryRun(ctx context.Context, taskID, runID influxdb.ID) (*influxdb.Run, error) {
	run, err := as.TaskService.RetryRun(ctx, taskID, runID)
	if err != nil {
		if err, ok := err.(*influxdb.Error); !ok || err.Msg != "run not found" {
			return run, err
		}
	}

	if run != nil {
		return run, err
	}

	// try finding the run (in our system or underlieing)
	run, err = as.FindRunByID(ctx, taskID, runID)
	if err != nil {
		return run, err
	}

	sf, err := run.ScheduledForTime()
	if err != nil {
		return run, err
	}

	return as.ForceRun(ctx, taskID, sf.Unix())
}

type runReader struct {
	runs   []*influxdb.Run
	logger *zap.Logger
}

func (re *runReader) readTable(tbl flux.Table) error {
	return tbl.Do(re.readRuns)
}

func (re *runReader) readRuns(cr flux.ColReader) error {
	for i := 0; i < cr.Len(); i++ {
		var r influxdb.Run
		for j, col := range cr.Cols() {
			switch col.Label {
			case runIDField:
				if cr.Strings(j).ValueString(i) != "" {
					id, err := influxdb.IDFromString(cr.Strings(j).ValueString(i))
					if err != nil {
						re.logger.Info("failed to parse runID", zap.Error(err))
						continue
					}
					r.ID = *id
				}
			case taskIDTag:
				if cr.Strings(j).ValueString(i) != "" {
					id, err := influxdb.IDFromString(cr.Strings(j).ValueString(i))
					if err != nil {
						re.logger.Info("failed to parse taskID", zap.Error(err))
						continue
					}
					r.TaskID = *id
				}
			case startedAtField:
				r.StartedAt = cr.Strings(j).ValueString(i)
			case requestedAtField:
				r.RequestedAt = cr.Strings(j).ValueString(i)
			case scheduledForField:
				r.ScheduledFor = cr.Strings(j).ValueString(i)
			case statusTag:
				r.Status = cr.Strings(j).ValueString(i)
			case finishedAtField:
				r.FinishedAt = cr.Strings(j).ValueString(i)
			case logField:
				logBytes := bytes.TrimSpace(cr.Strings(j).Value(i))
				if len(logBytes) != 0 {
					err := json.Unmarshal(logBytes, &r.Log)
					if err != nil {
						re.logger.Info("failed to parse log data", zap.Error(err), zap.ByteString("log_bytes", logBytes))
					}
				}
			}
		}

		// if we dont have a full enough data set we fail here.
		if r.ID.Valid() {
			re.runs = append(re.runs, &r)
		}

	}

	return nil
}
<|endoftext|>


<commit_before>package api

import (
	"cf"
	"cf/configuration"
	"cf/net"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	testapi "testhelpers/api"
	testnet "testhelpers/net"
	"testing"
)

func TestCreateServiceBinding(t *testing.T) {
	req := testapi.NewCloudControllerTestRequest(testnet.TestRequest{
		Method:   "POST",
		Path:     "/v2/service_bindings",
		Matcher:  testnet.RequestBodyMatcher(`{"app_guid":"my-app-guid","service_instance_guid":"my-service-instance-guid"}`),
		Response: testnet.TestResponse{Status: http.StatusCreated},
	})

	ts, handler, repo := createServiceBindingRepo(t, req)
	defer ts.Close()

	apiResponse := repo.Create("my-service-instance-guid", "my-app-guid")
	assert.True(t, handler.AllRequestsCalled())
	assert.False(t, apiResponse.IsNotSuccessful())
}

func TestCreateServiceBindingIfError(t *testing.T) {
	req := testapi.NewCloudControllerTestRequest(testnet.TestRequest{
		Method:  "POST",
		Path:    "/v2/service_bindings",
		Matcher: testnet.RequestBodyMatcher(`{"app_guid":"my-app-guid","service_instance_guid":"my-service-instance-guid"}`),
		Response: testnet.TestResponse{
			Status: http.StatusBadRequest,
			Body:   `{"code":90003,"description":"The app space binding to service is taken: 7b959018-110a-4913-ac0a-d663e613cdea 346bf237-7eef-41a7-b892-68fb08068f09"}`,
		},
	})

	ts, handler, repo := createServiceBindingRepo(t, req)
	defer ts.Close()

	apiResponse := repo.Create("my-service-instance-guid", "my-app-guid")

	assert.True(t, handler.AllRequestsCalled())
	assert.True(t, apiResponse.IsNotSuccessful())
	assert.Equal(t, apiResponse.ErrorCode, "90003")
}

var deleteBindingReq = testapi.NewCloudControllerTestRequest(testnet.TestRequest{
	Method:   "DELETE",
	Path:     "/v2/service_bindings/service-binding-2-guid",
	Response: testnet.TestResponse{Status: http.StatusOK},
})

func TestDeleteServiceBinding(t *testing.T) {
	ts, handler, repo := createServiceBindingRepo(t, deleteBindingReq)
	defer ts.Close()

	serviceInstance := cf.ServiceInstance{}
	serviceInstance.Guid = "my-service-instance-guid"

	binding := cf.ServiceBindingFields{}
	binding.Url = "/v2/service_bindings/service-binding-1-guid"
	binding.AppGuid = "app-1-guid"
	binding2 := cf.ServiceBindingFields{}
	binding2.Url = "/v2/service_bindings/service-binding-2-guid"
	binding2.AppGuid = "app-2-guid"
	serviceInstance.ServiceBindings = []cf.ServiceBindingFields{binding, binding2}

	found, apiResponse := repo.Delete(serviceInstance, "app-2-guid")

	assert.True(t, handler.AllRequestsCalled())
	assert.False(t, apiResponse.IsNotSuccessful())
	assert.True(t, found)
}

func TestDeleteServiceBindingWhenBindingDoesNotExist(t *testing.T) {
	ts, handler, repo := createServiceBindingRepo(t, deleteBindingReq)
	defer ts.Close()

	serviceInstance := cf.ServiceInstance{}
	serviceInstance.Guid = "my-service-instance-guid"

	found, apiResponse := repo.Delete(serviceInstance, "app-2-guid")

	assert.False(t, handler.AllRequestsCalled())
	assert.False(t, apiResponse.IsNotSuccessful())
	assert.False(t, found)
}

func createServiceBindingRepo(t *testing.T, req testnet.TestRequest) (ts *httptest.Server, handler *testnet.TestHandler, repo ServiceBindingRepository) {
	ts, handler = testnet.NewTLSServer(t, []testnet.TestRequest{req})
	space := cf.SpaceFields{}
	space.Guid = "my-space-guid"
	config := &configuration.Configuration{
		AccessToken: "BEARER my_access_token",
		SpaceFields: space,
		Target:      ts.URL,
	}

	gateway := net.NewCloudControllerGateway()
	repo = NewCloudControllerServiceBindingRepository(config, gateway)
	return
}
<commit_msg>Remove warning in test about requests not being called<commit_after>package api

import (
	"cf"
	"cf/configuration"
	"cf/net"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	testapi "testhelpers/api"
	testnet "testhelpers/net"
	"testing"
)

func TestCreateServiceBinding(t *testing.T) {
	req := testapi.NewCloudControllerTestRequest(testnet.TestRequest{
		Method:   "POST",
		Path:     "/v2/service_bindings",
		Matcher:  testnet.RequestBodyMatcher(`{"app_guid":"my-app-guid","service_instance_guid":"my-service-instance-guid"}`),
		Response: testnet.TestResponse{Status: http.StatusCreated},
	})

	ts, handler, repo := createServiceBindingRepo(t, []testnet.TestRequest{req})
	defer ts.Close()

	apiResponse := repo.Create("my-service-instance-guid", "my-app-guid")
	assert.True(t, handler.AllRequestsCalled())
	assert.False(t, apiResponse.IsNotSuccessful())
}

func TestCreateServiceBindingIfError(t *testing.T) {
	req := testapi.NewCloudControllerTestRequest(testnet.TestRequest{
		Method:  "POST",
		Path:    "/v2/service_bindings",
		Matcher: testnet.RequestBodyMatcher(`{"app_guid":"my-app-guid","service_instance_guid":"my-service-instance-guid"}`),
		Response: testnet.TestResponse{
			Status: http.StatusBadRequest,
			Body:   `{"code":90003,"description":"The app space binding to service is taken: 7b959018-110a-4913-ac0a-d663e613cdea 346bf237-7eef-41a7-b892-68fb08068f09"}`,
		},
	})

	ts, handler, repo := createServiceBindingRepo(t, []testnet.TestRequest{req})
	defer ts.Close()

	apiResponse := repo.Create("my-service-instance-guid", "my-app-guid")

	assert.True(t, handler.AllRequestsCalled())
	assert.True(t, apiResponse.IsNotSuccessful())
	assert.Equal(t, apiResponse.ErrorCode, "90003")
}

var deleteBindingReq = testapi.NewCloudControllerTestRequest(testnet.TestRequest{
	Method:   "DELETE",
	Path:     "/v2/service_bindings/service-binding-2-guid",
	Response: testnet.TestResponse{Status: http.StatusOK},
})

func TestDeleteServiceBinding(t *testing.T) {
	ts, handler, repo := createServiceBindingRepo(t, []testnet.TestRequest{deleteBindingReq})
	defer ts.Close()

	serviceInstance := cf.ServiceInstance{}
	serviceInstance.Guid = "my-service-instance-guid"

	binding := cf.ServiceBindingFields{}
	binding.Url = "/v2/service_bindings/service-binding-1-guid"
	binding.AppGuid = "app-1-guid"
	binding2 := cf.ServiceBindingFields{}
	binding2.Url = "/v2/service_bindings/service-binding-2-guid"
	binding2.AppGuid = "app-2-guid"
	serviceInstance.ServiceBindings = []cf.ServiceBindingFields{binding, binding2}

	found, apiResponse := repo.Delete(serviceInstance, "app-2-guid")

	assert.True(t, handler.AllRequestsCalled())
	assert.False(t, apiResponse.IsNotSuccessful())
	assert.True(t, found)
}

func TestDeleteServiceBindingWhenBindingDoesNotExist(t *testing.T) {
	ts, handler, repo := createServiceBindingRepo(t, []testnet.TestRequest{})
	defer ts.Close()

	serviceInstance := cf.ServiceInstance{}
	serviceInstance.Guid = "my-service-instance-guid"

	found, apiResponse := repo.Delete(serviceInstance, "app-2-guid")

	assert.Equal(t, handler.CallCount, 0)
	assert.False(t, apiResponse.IsNotSuccessful())
	assert.False(t, found)
}

func createServiceBindingRepo(t *testing.T, requests []testnet.TestRequest) (ts *httptest.Server, handler *testnet.TestHandler, repo ServiceBindingRepository) {
	ts, handler = testnet.NewTLSServer(t, requests)
	space := cf.SpaceFields{}
	space.Guid = "my-space-guid"
	config := &configuration.Configuration{
		AccessToken: "BEARER my_access_token",
		SpaceFields: space,
		Target:      ts.URL,
	}

	gateway := net.NewCloudControllerGateway()
	repo = NewCloudControllerServiceBindingRepository(config, gateway)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

func index(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

	courses := courses{}

	if err := c.db.C("courses").Find(bson.M{}).All(&courses); err != nil {
		return http.StatusNotFound, err
	}

	mj, _ := json.Marshal(courses)

	w.Header().Set("Content-Type", "application/json;charset=UTF-8")
	fmt.Fprint(w, string(mj))

	return http.StatusOK, nil
}

func _display(w http.ResponseWriter, o io.Reader) {
	w.Header().Set("Content-Type", "application/json;charset=UTF-8")
	buf := &bytes.Buffer{}
	buf.ReadFrom(o)
	fmt.Fprint(w, buf.String())
}

func _parseBody(b io.ReadCloser, t interface{}) {
	buf := &bytes.Buffer{}
	buf.ReadFrom(b)
	json.Unmarshal(buf.Bytes(), t)
}

func _list(db *mgo.Database, t interface{}, c string) (io.Reader, int, error) {
	if err := db.C(c).Find(bson.M{}).All(t); err != nil {
		return nil, http.StatusInternalServerError, err
	}
	mj, err := json.Marshal(t)
	if err != nil {
		return nil, http.StatusInternalServerError, err
	}
	return bytes.NewReader(mj), http.StatusOK, nil
}

func _create(db *mgo.Database, t interface{}, c string) (io.Reader, int, error) {
	if err := db.C(c).Insert(); err != nil {
		return nil, http.StatusInternalServerError, err
	}
	mj, _ := json.Marshal(t)
	return bytes.NewReader(mj), http.StatusOK, nil
}

func _delete(db *mgo.Database, id bson.ObjectId, c string) (int, error) {
	if err := db.C(c).RemoveId(id); err != nil {
		return http.StatusInternalServerError, err
	}
	return http.StatusOK, nil
}

// Subject

func listSubjects(c *context, w http.ResponseWriter, r *http.Request) (int, error) {
	o, s, err := _list(c.db, &subjects{}, "subjects")
	if err != nil {
		return s, err
	}
	_display(w, o)
	return s, nil
}

// func getSubject(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

func createSubject(c *context, w http.ResponseWriter, r *http.Request) (int, error) {
	t := &subject{ID: bson.NewObjectId()}
	_parseBody(r.Body, &t)
	o, s, err := _create(c.db, t, "subjects")
	if err != nil {
		return s, err
	}
	_display(w, o)
	return s, nil
}

// func editSubject(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

func deleteSubject(c *context, w http.ResponseWriter, r *http.Request) (int, error) {
	t := &subject{}
	_parseBody(r.Body, &t)
	return _delete(c.db, t.ID, "subjects")
}

// // Topic

// func listTopics(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func getTopic(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func editTopic(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func createTopic(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func deleteTopic(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// // Course

// func listCourses(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func getCourse(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func editCourse(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func createCourse(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func deleteCourse(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// // User

// func listUsers(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func getUser(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func editUser(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func createUser(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func deleteUser(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }
<commit_msg>Fix create item<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"

	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

func index(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

	courses := courses{}

	if err := c.db.C("courses").Find(bson.M{}).All(&courses); err != nil {
		return http.StatusNotFound, err
	}

	mj, _ := json.Marshal(courses)

	w.Header().Set("Content-Type", "application/json;charset=UTF-8")
	fmt.Fprint(w, string(mj))

	return http.StatusOK, nil
}

func _display(w http.ResponseWriter, o io.Reader) {
	w.Header().Set("Content-Type", "application/json;charset=UTF-8")
	buf := &bytes.Buffer{}
	buf.ReadFrom(o)
	fmt.Fprint(w, buf.String())
}

func _parseBody(b io.ReadCloser, t interface{}) {
	buf := &bytes.Buffer{}
	buf.ReadFrom(b)
	json.Unmarshal(buf.Bytes(), t)
}

func _list(db *mgo.Database, t interface{}, c string) (io.Reader, int, error) {
	if err := db.C(c).Find(bson.M{}).All(t); err != nil {
		return nil, http.StatusInternalServerError, err
	}
	mj, err := json.Marshal(t)
	if err != nil {
		return nil, http.StatusInternalServerError, err
	}
	return bytes.NewReader(mj), http.StatusOK, nil
}

func _create(db *mgo.Database, t interface{}, c string) (io.Reader, int, error) {
	if err := db.C(c).Insert(t); err != nil {
		return nil, http.StatusInternalServerError, err
	}
	mj, _ := json.Marshal(t)
	return bytes.NewReader(mj), http.StatusOK, nil
}

func _delete(db *mgo.Database, id bson.ObjectId, c string) (int, error) {
	if err := db.C(c).RemoveId(id); err != nil {
		return http.StatusInternalServerError, err
	}
	return http.StatusOK, nil
}

// Subject

func listSubjects(c *context, w http.ResponseWriter, r *http.Request) (int, error) {
	o, s, err := _list(c.db, &subjects{}, "subjects")
	if err != nil {
		return s, err
	}
	_display(w, o)
	return s, nil
}

// func getSubject(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

func createSubject(c *context, w http.ResponseWriter, r *http.Request) (int, error) {
	t := &subject{ID: bson.NewObjectId()}
	_parseBody(r.Body, &t)
	o, s, err := _create(c.db, t, "subjects")
	if err != nil {
		return s, err
	}
	_display(w, o)
	return s, nil
}

// func editSubject(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

func deleteSubject(c *context, w http.ResponseWriter, r *http.Request) (int, error) {
	t := &subject{}
	_parseBody(r.Body, &t)
	return _delete(c.db, t.ID, "subjects")
}

// // Topic

// func listTopics(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func getTopic(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func editTopic(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func createTopic(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func deleteTopic(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// // Course

// func listCourses(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func getCourse(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func editCourse(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func createCourse(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func deleteCourse(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// // User

// func listUsers(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func getUser(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func editUser(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func createUser(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }

// func deleteUser(c *context, w http.ResponseWriter, r *http.Request) (int, error) {

// }
<|endoftext|>