<commit_before>/*
                       WARNING WARNING WARNING

                Attention all potential contributors

   This testfile is not in the best state. We've been slowly transitioning
   from the built in "testing" package to using Ginkgo. As you can see, we've
   changed the format, but a lot of the setup, test body, descriptions, etc
   are either hardcoded, completely lacking, or misleading.

   For example:

   Describe("Testing with ginkgo"...)      // This is not a great description
   It("TestDoesSoemthing"...)              // This is a horrible description

   Describe("create-user command"...       // Describe the actual object under test
   It("creates a user when provided ..."   // this is more descriptive

   For good examples of writing Ginkgo tests for the cli, refer to

   src/github.com/cloudfoundry/cli/cf/commands/application/delete_app_test.go
   src/github.com/cloudfoundry/cli/cf/terminal/ui_test.go
   src/github.com/cloudfoundry/loggregator_consumer/consumer_test.go
*/

package space_test

import (
	. "github.com/cloudfoundry/cli/cf/commands/space"
	"github.com/cloudfoundry/cli/cf/models"
	testcmd "github.com/cloudfoundry/cli/testhelpers/commands"
	testconfig "github.com/cloudfoundry/cli/testhelpers/configuration"
	testreq "github.com/cloudfoundry/cli/testhelpers/requirements"
	testterm "github.com/cloudfoundry/cli/testhelpers/terminal"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	. "github.com/cloudfoundry/cli/testhelpers/matchers"
)

func callShowSpace(args []string, requirementsFactory *testreq.FakeReqFactory) (ui *testterm.FakeUI) {
	ui = new(testterm.FakeUI)
	config := testconfig.NewRepositoryWithDefaults()
	cmd := NewShowSpace(ui, config)
	testcmd.RunCommand(cmd, args, requirementsFactory)
	return
}

var _ = Describe("Testing with ginkgo", func() {
	It("TestShowSpaceRequirements", func() {
		args := []string{"my-space"}

		requirementsFactory := &testreq.FakeReqFactory{LoginSuccess: false, TargetedOrgSuccess: true}
		callShowSpace(args, requirementsFactory)
		Expect(testcmd.CommandDidPassRequirements).To(BeFalse())

		requirementsFactory = &testreq.FakeReqFactory{LoginSuccess: true, TargetedOrgSuccess: false}
		callShowSpace(args, requirementsFactory)
		Expect(testcmd.CommandDidPassRequirements).To(BeFalse())

		requirementsFactory = &testreq.FakeReqFactory{LoginSuccess: true, TargetedOrgSuccess: true}
		callShowSpace(args, requirementsFactory)
		Expect(testcmd.CommandDidPassRequirements).To(BeTrue())
	})

	It("TestShowSpaceInfoSuccess", func() {
		org := models.OrganizationFields{}
		org.Name = "my-org"

		app := models.ApplicationFields{}
		app.Name = "app1"
		app.Guid = "app1-guid"
		apps := []models.ApplicationFields{app}

		domain := models.DomainFields{}
		domain.Name = "domain1"
		domain.Guid = "domain1-guid"
		domains := []models.DomainFields{domain}

		serviceInstance := models.ServiceInstanceFields{}
		serviceInstance.Name = "service1"
		serviceInstance.Guid = "service1-guid"
		services := []models.ServiceInstanceFields{serviceInstance}

		space := models.Space{}
		space.Name = "space1"
		space.Organization = org
		space.Applications = apps
		space.Domains = domains
		space.ServiceInstances = services

		requirementsFactory := &testreq.FakeReqFactory{LoginSuccess: true, TargetedOrgSuccess: true, Space: space}
		ui := callShowSpace([]string{"space1"}, requirementsFactory)
		Expect(ui.Outputs).To(ContainSubstrings(
			[]string{"Getting info for space", "space1", "my-org", "my-user"},
			[]string{"OK"},
			[]string{"space1"},
			[]string{"Org", "my-org"},
			[]string{"Apps", "app1"},
			[]string{"Domains", "domain1"},
			[]string{"Services", "service1"},
		))
	})
})
<commit_msg>cleanup space command test<commit_after>package space_test

import (
	. "github.com/cloudfoundry/cli/cf/commands/space"
	"github.com/cloudfoundry/cli/cf/configuration"
	"github.com/cloudfoundry/cli/cf/models"
	testcmd "github.com/cloudfoundry/cli/testhelpers/commands"
	testconfig "github.com/cloudfoundry/cli/testhelpers/configuration"
	testreq "github.com/cloudfoundry/cli/testhelpers/requirements"
	testterm "github.com/cloudfoundry/cli/testhelpers/terminal"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	. "github.com/cloudfoundry/cli/testhelpers/matchers"
)

var _ = Describe("space command", func() {
	var (
		ui                  *testterm.FakeUI
		requirementsFactory *testreq.FakeReqFactory
		configRepo          configuration.ReadWriter
	)

	BeforeEach(func() {
		configRepo = testconfig.NewRepositoryWithDefaults()

		ui = &testterm.FakeUI{}
		requirementsFactory = &testreq.FakeReqFactory{}
	})

	runCommand := func(args ...string) {
		testcmd.RunCommand(NewShowSpace(ui, configRepo), args, requirementsFactory)
	}

	Describe("requirements", func() {
		It("fails when not logged in", func() {
			requirementsFactory.TargetedOrgSuccess = true
			runCommand("some-space")
			Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
		})

		It("fails when an org is not targeted", func() {
			requirementsFactory.LoginSuccess = true
			runCommand("some-space")
			Expect(testcmd.CommandDidPassRequirements).To(BeFalse())
		})
	})

	Context("when logged in and an org is targeted", func() {
		BeforeEach(func() {
			org := models.OrganizationFields{}
			org.Name = "my-org"

			app := models.ApplicationFields{}
			app.Name = "app1"
			app.Guid = "app1-guid"
			apps := []models.ApplicationFields{app}

			domain := models.DomainFields{}
			domain.Name = "domain1"
			domain.Guid = "domain1-guid"
			domains := []models.DomainFields{domain}

			serviceInstance := models.ServiceInstanceFields{}
			serviceInstance.Name = "service1"
			serviceInstance.Guid = "service1-guid"
			services := []models.ServiceInstanceFields{serviceInstance}

			space := models.Space{}
			space.Name = "whose-space-is-it-anyway"
			space.Organization = org
			space.Applications = apps
			space.Domains = domains
			space.ServiceInstances = services

			requirementsFactory.LoginSuccess = true
			requirementsFactory.TargetedOrgSuccess = true
			requirementsFactory.Space = space
		})

		It("shows information about the given space", func() {
			runCommand("whose-space-is-it-anyway")
			Expect(ui.Outputs).To(ContainSubstrings(
				[]string{"Getting info for space", "whose-space-is-it-anyway", "my-org", "my-user"},
				[]string{"OK"},
				[]string{"whose-space-is-it-anyway"},
				[]string{"Org", "my-org"},
				[]string{"Apps", "app1"},
				[]string{"Domains", "domain1"},
				[]string{"Services", "service1"},
			))
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2009 The GoMatrix Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package matrix

import "runtime"

func (A *DenseMatrix) Plus(B MatrixRO) (Matrix, error) {
	C := A.Copy()
	err := C.Add(B)
	return C, err
}
func (A *DenseMatrix) PlusDense(B *DenseMatrix) (*DenseMatrix, error) {
	C := A.Copy()
	err := C.AddDense(B)
	return C, err
}

func (A *DenseMatrix) Minus(B MatrixRO) (Matrix, error) {
	C := A.Copy()
	err := C.Subtract(B)
	return C, err
}

func (A *DenseMatrix) MinusDense(B *DenseMatrix) (*DenseMatrix, error) {
	C := A.Copy()
	err := C.SubtractDense(B)
	return C, err
}

func (A *DenseMatrix) Add(B MatrixRO) error {
	if A.cols != B.Cols() || A.rows != B.Rows() {
		return ErrorDimensionMismatch
	}

	for i := 0; i < A.rows; i++ {
		index := i * A.step
		for j := 0; j < A.cols; j++ {
			A.elements[index] += B.Get(i, j)
			index++
		}
	}

	return nil
}

func (A *DenseMatrix) AddDense(B *DenseMatrix) error {
	if A.cols != B.cols || A.rows != B.rows {
		return ErrorDimensionMismatch
	}

	for i := 0; i < A.rows; i++ {
		for j := 0; j < A.cols; j++ {
			A.elements[i*A.step+j] += B.elements[i*B.step+j]
		}
	}

	return nil
}

func (A *DenseMatrix) Subtract(B MatrixRO) error {
	if Bd, ok := B.(*DenseMatrix); ok {
		return A.SubtractDense(Bd)
	}

	if A.cols != B.Cols() || A.rows != B.Rows() {
		return ErrorDimensionMismatch
	}

	for i := 0; i < A.rows; i++ {
		index := i * A.step
		for j := 0; j < A.cols; j++ {
			A.elements[index] -= B.Get(i, j)
			index++
		}
	}

	return nil
}

func (A *DenseMatrix) SubtractDense(B *DenseMatrix) error {

	if A.cols != B.cols || A.rows != B.rows {
		return ErrorDimensionMismatch
	}

	for i := 0; i < A.rows; i++ {
		indexA := i * A.step
		indexB := i * B.step

		for j := 0; j < A.cols; j++ {
			A.elements[indexA] -= B.elements[indexB]
			indexA++
			indexB++
		}
	}

	return nil
}

func (A *DenseMatrix) Times(B MatrixRO) (Matrix, error) {

	if Bd, ok := B.(*DenseMatrix); ok {
		return A.TimesDense(Bd)
	}

	if A.cols != B.Rows() {
		return nil, ErrorDimensionMismatch
	}
	C := Zeros(A.rows, B.Cols())

	for i := 0; i < A.rows; i++ {
		for j := 0; j < B.Cols(); j++ {
			sum := float64(0)
			for k := 0; k < A.cols; k++ {
				sum += A.elements[i*A.step+k] * B.Get(k, j)
			}
			C.elements[i*C.step+j] = sum
		}
	}

	return C, nil
}

type parJob struct {
	start, finish int
}

func parTimes1(A, B, C *DenseMatrix) {
	C = Zeros(A.rows, B.cols)

	mp := runtime.GOMAXPROCS(0)

	jobChan := make(chan box, 1+mp)

	go func() {
		rowCount := A.rows / mp
		for startRow := 0; startRow < A.rows; startRow += rowCount {
			start := startRow
			finish := startRow + rowCount
			if finish >= A.rows {
				finish = A.rows
			}
			jobChan <- parJob{start: start, finish: finish}
		}
		close(jobChan)
	}()

	wait := parFor(jobChan, func(iBox box) {
		job := iBox.(parJob)
		for i := job.start; i < job.finish; i++ {
			sums := C.elements[i*C.step : (i+1)*C.step]
			for k := 0; k < A.cols; k++ {
				for j := 0; j < B.cols; j++ {
					sums[j] += A.elements[i*A.step+k] * B.elements[k*B.step+j]
				}
			}
		}
	})
	wait()

	return
}

//this is an adaptation of code from a go-nuts post made by Dmitriy Vyukov
func parTimes2(A, B, C *DenseMatrix) {
	const threshold = 8

	currentGoroutineCount := 1
	maxGoroutines := runtime.GOMAXPROCS(0) + 2

	var aux func(sync chan bool, A, B, C *DenseMatrix, rs, re, cs, ce, ks, ke int)
	aux = func(sync chan bool, A, B, C *DenseMatrix, rs, re, cs, ce, ks, ke int) {
		dr := re - rs
		dc := ce - cs
		dk := ke - ks
		switch {
		case currentGoroutineCount < maxGoroutines && dr >= dc && dr >= dk && dr >= threshold:
			sync0 := make(chan bool, 1)
			rm := (rs + re) / 2
			currentGoroutineCount++
			go aux(sync0, A, B, C, rs, rm, cs, ce, ks, ke)
			aux(nil, A, B, C, rm, re, cs, ce, ks, ke)
			<-sync0
			currentGoroutineCount--
		case currentGoroutineCount < maxGoroutines && dc >= dk && dc >= dr && dc >= threshold:
			sync0 := make(chan bool, 1)
			cm := (cs + ce) / 2
			currentGoroutineCount++
			go aux(sync0, A, B, C, rs, re, cs, cm, ks, ke)
			aux(nil, A, B, C, rs, re, cm, ce, ks, ke)
			<-sync0
			currentGoroutineCount--
		case currentGoroutineCount < maxGoroutines && dk >= dc && dk >= dr && dk >= threshold:
			km := (ks + ke) / 2
			aux(nil, A, B, C, rs, re, cs, ce, ks, km)
			aux(nil, A, B, C, rs, re, cs, ce, km, ke)
		default:
			for row := rs; row < re; row++ {
				sums := C.elements[row*C.step : (row+1)*C.step]
				for k := ks; k < ke; k++ {
					for col := cs; col < ce; col++ {
						sums[col] += A.elements[row*A.step+k] * B.elements[k*B.step+col]
					}
				}
			}
		}
		if sync != nil {
			sync <- true
		}
	}

	aux(nil, A, B, C, 0, A.rows, 0, B.cols, 0, A.cols)

	return
}

var (
	WhichParMethod  = 2
	WhichSyncMethod = 1
)

func (A *DenseMatrix) TimesDense(B *DenseMatrix) (C *DenseMatrix, err error) {
	C = Zeros(A.rows, B.cols)
	err = A.TimesDenseFill(B, C)
	return
}
func (A *DenseMatrix) TimesDenseFill(B, C *DenseMatrix) (err error) {
	if C.rows != A.rows || C.cols != B.cols || A.cols != B.rows {
		err = ErrorDimensionMismatch
		return
	}
	if runtime.GOMAXPROCS(0) > 1 {
		switch WhichParMethod {
		case 1:
			parTimes1(A, B, C)
		case 2:
			parTimes2(A, B, C)
		}
	} else {
		switch {
		case A.cols > 100 && WhichSyncMethod == 2:
			transposeTimes(A, B, C)
		default:
			for i := 0; i < A.rows; i++ {
				sums := C.elements[i*C.step : (i+1)*C.step]
				for k, a := range A.elements[i*A.step : i*A.step + A.cols] {
					for j, b := range B.elements[k*B.step : k * B.step + B.cols] {
						sums[j] += a * b
					}
				}
			}
		}
	}

	return
}

func transposeTimes(A, B, C *DenseMatrix) {
	Bt := B.Transpose()

	Bcols := Bt.Arrays()

	for i := 0; i < A.rows; i++ {
		Arow := A.elements[i*A.step : i*A.step+A.cols]
		for j := 0; j < B.cols; j++ {
			Bcol := Bcols[j]
			for k := range Arow {
				C.elements[i*C.step+j] += Arow[k] * Bcol[k]
			}
		}
	}

	return
}

func (A *DenseMatrix) ElementMult(B MatrixRO) (Matrix, error) {
	C := A.Copy()
	err := C.ScaleMatrix(B)
	return C, err
}

func (A *DenseMatrix) ElementMultDense(B *DenseMatrix) (*DenseMatrix, error) {
	C := A.Copy()
	err := C.ScaleMatrixDense(B)
	return C, err
}

func (A *DenseMatrix) Scale(f float64) {
	for i := 0; i < A.rows; i++ {
		index := i * A.step
		for j := 0; j < A.cols; j++ {
			A.elements[index] *= f
			index++
		}
	}
}

func (A *DenseMatrix) ScaleMatrix(B MatrixRO) error {
	if Bd, ok := B.(*DenseMatrix); ok {
		return A.ScaleMatrixDense(Bd)
	}

	if A.rows != B.Rows() || A.cols != B.Cols() {
		return ErrorDimensionMismatch
	}
	for i := 0; i < A.rows; i++ {
		indexA := i * A.step
		for j := 0; j < A.cols; j++ {
			A.elements[indexA] *= B.Get(i, j)
			indexA++
		}
	}
	return nil
}

func (A *DenseMatrix) ScaleMatrixDense(B *DenseMatrix) error {
	if A.rows != B.rows || A.cols != B.cols {
		return ErrorDimensionMismatch
	}
	for i := 0; i < A.rows; i++ {
		indexA := i * A.step
		indexB := i * B.step
		for j := 0; j < A.cols; j++ {
			A.elements[indexA] *= B.elements[indexB]
			indexA++
			indexB++
		}
	}
	return nil
}
<commit_msg>When 'WhichParMethod' is 0, don't do any concurrency.<commit_after>// Copyright 2009 The GoMatrix Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package matrix

import "runtime"

func (A *DenseMatrix) Plus(B MatrixRO) (Matrix, error) {
	C := A.Copy()
	err := C.Add(B)
	return C, err
}
func (A *DenseMatrix) PlusDense(B *DenseMatrix) (*DenseMatrix, error) {
	C := A.Copy()
	err := C.AddDense(B)
	return C, err
}

func (A *DenseMatrix) Minus(B MatrixRO) (Matrix, error) {
	C := A.Copy()
	err := C.Subtract(B)
	return C, err
}

func (A *DenseMatrix) MinusDense(B *DenseMatrix) (*DenseMatrix, error) {
	C := A.Copy()
	err := C.SubtractDense(B)
	return C, err
}

func (A *DenseMatrix) Add(B MatrixRO) error {
	if A.cols != B.Cols() || A.rows != B.Rows() {
		return ErrorDimensionMismatch
	}

	for i := 0; i < A.rows; i++ {
		index := i * A.step
		for j := 0; j < A.cols; j++ {
			A.elements[index] += B.Get(i, j)
			index++
		}
	}

	return nil
}

func (A *DenseMatrix) AddDense(B *DenseMatrix) error {
	if A.cols != B.cols || A.rows != B.rows {
		return ErrorDimensionMismatch
	}

	for i := 0; i < A.rows; i++ {
		for j := 0; j < A.cols; j++ {
			A.elements[i*A.step+j] += B.elements[i*B.step+j]
		}
	}

	return nil
}

func (A *DenseMatrix) Subtract(B MatrixRO) error {
	if Bd, ok := B.(*DenseMatrix); ok {
		return A.SubtractDense(Bd)
	}

	if A.cols != B.Cols() || A.rows != B.Rows() {
		return ErrorDimensionMismatch
	}

	for i := 0; i < A.rows; i++ {
		index := i * A.step
		for j := 0; j < A.cols; j++ {
			A.elements[index] -= B.Get(i, j)
			index++
		}
	}

	return nil
}

func (A *DenseMatrix) SubtractDense(B *DenseMatrix) error {

	if A.cols != B.cols || A.rows != B.rows {
		return ErrorDimensionMismatch
	}

	for i := 0; i < A.rows; i++ {
		indexA := i * A.step
		indexB := i * B.step

		for j := 0; j < A.cols; j++ {
			A.elements[indexA] -= B.elements[indexB]
			indexA++
			indexB++
		}
	}

	return nil
}

func (A *DenseMatrix) Times(B MatrixRO) (Matrix, error) {

	if Bd, ok := B.(*DenseMatrix); ok {
		return A.TimesDense(Bd)
	}

	if A.cols != B.Rows() {
		return nil, ErrorDimensionMismatch
	}
	C := Zeros(A.rows, B.Cols())

	for i := 0; i < A.rows; i++ {
		for j := 0; j < B.Cols(); j++ {
			sum := float64(0)
			for k := 0; k < A.cols; k++ {
				sum += A.elements[i*A.step+k] * B.Get(k, j)
			}
			C.elements[i*C.step+j] = sum
		}
	}

	return C, nil
}

type parJob struct {
	start, finish int
}

func parTimes1(A, B, C *DenseMatrix) {
	C = Zeros(A.rows, B.cols)

	mp := runtime.GOMAXPROCS(0)

	jobChan := make(chan box, 1+mp)

	go func() {
		rowCount := A.rows / mp
		for startRow := 0; startRow < A.rows; startRow += rowCount {
			start := startRow
			finish := startRow + rowCount
			if finish >= A.rows {
				finish = A.rows
			}
			jobChan <- parJob{start: start, finish: finish}
		}
		close(jobChan)
	}()

	wait := parFor(jobChan, func(iBox box) {
		job := iBox.(parJob)
		for i := job.start; i < job.finish; i++ {
			sums := C.elements[i*C.step : (i+1)*C.step]
			for k := 0; k < A.cols; k++ {
				for j := 0; j < B.cols; j++ {
					sums[j] += A.elements[i*A.step+k] * B.elements[k*B.step+j]
				}
			}
		}
	})
	wait()

	return
}

//this is an adaptation of code from a go-nuts post made by Dmitriy Vyukov
func parTimes2(A, B, C *DenseMatrix) {
	const threshold = 8

	currentGoroutineCount := 1
	maxGoroutines := runtime.GOMAXPROCS(0) + 2

	var aux func(sync chan bool, A, B, C *DenseMatrix, rs, re, cs, ce, ks, ke int)
	aux = func(sync chan bool, A, B, C *DenseMatrix, rs, re, cs, ce, ks, ke int) {
		dr := re - rs
		dc := ce - cs
		dk := ke - ks
		switch {
		case currentGoroutineCount < maxGoroutines && dr >= dc && dr >= dk && dr >= threshold:
			sync0 := make(chan bool, 1)
			rm := (rs + re) / 2
			currentGoroutineCount++
			go aux(sync0, A, B, C, rs, rm, cs, ce, ks, ke)
			aux(nil, A, B, C, rm, re, cs, ce, ks, ke)
			<-sync0
			currentGoroutineCount--
		case currentGoroutineCount < maxGoroutines && dc >= dk && dc >= dr && dc >= threshold:
			sync0 := make(chan bool, 1)
			cm := (cs + ce) / 2
			currentGoroutineCount++
			go aux(sync0, A, B, C, rs, re, cs, cm, ks, ke)
			aux(nil, A, B, C, rs, re, cm, ce, ks, ke)
			<-sync0
			currentGoroutineCount--
		case currentGoroutineCount < maxGoroutines && dk >= dc && dk >= dr && dk >= threshold:
			km := (ks + ke) / 2
			aux(nil, A, B, C, rs, re, cs, ce, ks, km)
			aux(nil, A, B, C, rs, re, cs, ce, km, ke)
		default:
			for row := rs; row < re; row++ {
				sums := C.elements[row*C.step : (row+1)*C.step]
				for k := ks; k < ke; k++ {
					for col := cs; col < ce; col++ {
						sums[col] += A.elements[row*A.step+k] * B.elements[k*B.step+col]
					}
				}
			}
		}
		if sync != nil {
			sync <- true
		}
	}

	aux(nil, A, B, C, 0, A.rows, 0, B.cols, 0, A.cols)

	return
}

var (
	WhichParMethod  = 2
	WhichSyncMethod = 1
)

func (A *DenseMatrix) TimesDense(B *DenseMatrix) (C *DenseMatrix, err error) {
	C = Zeros(A.rows, B.cols)
	err = A.TimesDenseFill(B, C)
	return
}
func (A *DenseMatrix) TimesDenseFill(B, C *DenseMatrix) (err error) {
	if C.rows != A.rows || C.cols != B.cols || A.cols != B.rows {
		err = ErrorDimensionMismatch
		return
	}
	if WhichParMethod > 0 && runtime.GOMAXPROCS(0) > 1 {
		switch WhichParMethod {
		case 1:
			parTimes1(A, B, C)
		case 2:
			parTimes2(A, B, C)
		}
	} else {
		switch {
		case A.cols > 100 && WhichSyncMethod == 2:
			transposeTimes(A, B, C)
		default:
			for i := 0; i < A.rows; i++ {
				sums := C.elements[i*C.step : (i+1)*C.step]
				for k, a := range A.elements[i*A.step : i*A.step + A.cols] {
					for j, b := range B.elements[k*B.step : k * B.step + B.cols] {
						sums[j] += a * b
					}
				}
			}
		}
	}

	return
}

func transposeTimes(A, B, C *DenseMatrix) {
	Bt := B.Transpose()

	Bcols := Bt.Arrays()

	for i := 0; i < A.rows; i++ {
		Arow := A.elements[i*A.step : i*A.step+A.cols]
		for j := 0; j < B.cols; j++ {
			Bcol := Bcols[j]
			for k := range Arow {
				C.elements[i*C.step+j] += Arow[k] * Bcol[k]
			}
		}
	}

	return
}

func (A *DenseMatrix) ElementMult(B MatrixRO) (Matrix, error) {
	C := A.Copy()
	err := C.ScaleMatrix(B)
	return C, err
}

func (A *DenseMatrix) ElementMultDense(B *DenseMatrix) (*DenseMatrix, error) {
	C := A.Copy()
	err := C.ScaleMatrixDense(B)
	return C, err
}

func (A *DenseMatrix) Scale(f float64) {
	for i := 0; i < A.rows; i++ {
		index := i * A.step
		for j := 0; j < A.cols; j++ {
			A.elements[index] *= f
			index++
		}
	}
}

func (A *DenseMatrix) ScaleMatrix(B MatrixRO) error {
	if Bd, ok := B.(*DenseMatrix); ok {
		return A.ScaleMatrixDense(Bd)
	}

	if A.rows != B.Rows() || A.cols != B.Cols() {
		return ErrorDimensionMismatch
	}
	for i := 0; i < A.rows; i++ {
		indexA := i * A.step
		for j := 0; j < A.cols; j++ {
			A.elements[indexA] *= B.Get(i, j)
			indexA++
		}
	}
	return nil
}

func (A *DenseMatrix) ScaleMatrixDense(B *DenseMatrix) error {
	if A.rows != B.rows || A.cols != B.cols {
		return ErrorDimensionMismatch
	}
	for i := 0; i < A.rows; i++ {
		indexA := i * A.step
		indexB := i * B.step
		for j := 0; j < A.cols; j++ {
			A.elements[indexA] *= B.elements[indexB]
			indexA++
			indexB++
		}
	}
	return nil
}
<|endoftext|>


<commit_before>//  Copyright 2018 Google LLC
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at

//        https://www.apache.org/licenses/LICENSE-2.0

//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//	limitations under the License.

// Package grpcutil contains a set of common libraries used across minions and overlord.
// We really need to keep this to a minimum and avoid the kitchen-sink effect.
package grpcutil

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"log"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// GetSslServerCreds reads the SSL keys and certs and generates the required options for a GRPC server.
// If the keys and certs have both been left blank, it will return nil. If only one is set it will return
// an error. If a CA certificate is provided, it will also set up client authentication using it.
func GetSslServerCreds(certPath string, keyPath string, caCertPath string) (grpc.ServerOption, error) {
	if certPath == "" && keyPath == "" {
		log.Println("no certificate and key set")
		return nil, nil
	}
	if certPath == "" || keyPath == "" {
		return nil, errors.New("please specify both an SSL key and certificate")
	}
	// Create the TLS credentials
	certificate, err := tls.LoadX509KeyPair(certPath, keyPath)
	if err != nil {
		return nil, err
	}

	var creds credentials.TransportCredentials
	if caCertPath == "" {
		log.Println("no CA set, all clients will be able to connect")
		creds = credentials.NewTLS(&tls.Config{
			Certificates: []tls.Certificate{certificate},
		})
	} else {
		cas := x509.NewCertPool()
		ca, err := ioutil.ReadFile(caCertPath)
		if err != nil {
			return nil, err
		}
		if ok := cas.AppendCertsFromPEM(ca); !ok {
			return nil, errors.New("failed while creating CA pool for client verification. Check the CA cert")
		}

		log.Println("CA set and configured, enforcing client authentication")
		creds = credentials.NewTLS(&tls.Config{
			Certificates: []tls.Certificate{certificate},
			ClientAuth:   tls.RequireAndVerifyClientCert,
			ClientCAs:    cas,
		})
	}
	return grpc.Creds(creds), nil
}

// GetSslClientOptions returns dial options by looking at SSL related
// flags - essentially, either ingests a CA certificate to validate
// the server or just gives up validating anything at all.
func GetSslClientOptions(serverAddress string, caCertPath string) (grpc.DialOption, error) {
	if caCertPath == "" {
		log.Println("WARNING: no CA specified. We will NOT check server's SSL certs")
		return grpc.WithInsecure(), nil
	}
	// NOTE: this is where we'd put client certs if we ever want to.
	cas := x509.NewCertPool()
	ca, err := ioutil.ReadFile(caCertPath)
	if err != nil {
		return nil, fmt.Errorf("could not read ca certificate: %s", err)
	}
	if ok := cas.AppendCertsFromPEM(ca); !ok {
		return nil, errors.New("failed to append ca certs")
	}
	log.Println("CA loaded, will check server SSL certs. That's good.")
	creds := credentials.NewTLS(&tls.Config{
		RootCAs:    cas,
		ServerName: serverAddress,
	})
	return grpc.WithTransportCredentials(creds), nil
}
<commit_msg>Drop the else branch - more idiomatic golang for everybody<commit_after>//  Copyright 2018 Google LLC
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at

//        https://www.apache.org/licenses/LICENSE-2.0

//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//	limitations under the License.

// Package grpcutil contains a set of common libraries used across minions and overlord.
// We really need to keep this to a minimum and avoid the kitchen-sink effect.
package grpcutil

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"log"

	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// GetSslServerCreds reads the SSL keys and certs and generates the required options for a GRPC server.
// If the keys and certs have both been left blank, it will return nil. If only one is set it will return
// an error. If a CA certificate is provided, it will also set up client authentication using it.
func GetSslServerCreds(certPath string, keyPath string, caCertPath string) (grpc.ServerOption, error) {
	if certPath == "" && keyPath == "" {
		log.Println("no certificate and key set")
		return nil, nil
	}
	if certPath == "" || keyPath == "" {
		return nil, errors.New("please specify both an SSL key and certificate")
	}
	// Create the TLS credentials
	certificate, err := tls.LoadX509KeyPair(certPath, keyPath)
	if err != nil {
		return nil, err
	}

	var creds credentials.TransportCredentials
	if caCertPath == "" {
		log.Println("no CA set, all clients will be able to connect")
		creds = credentials.NewTLS(&tls.Config{
			Certificates: []tls.Certificate{certificate},
		})
		return grpc.Creds(creds) nil
	}
	cas := x509.NewCertPool()
	ca, err := ioutil.ReadFile(caCertPath)
	if err != nil {
		return nil, err
	}
	if ok := cas.AppendCertsFromPEM(ca); !ok {
		return nil, errors.New("failed while creating CA pool for client verification. Check the CA cert")
	}

	log.Println("CA set and configured, enforcing client authentication")
	creds = credentials.NewTLS(&tls.Config{
		Certificates: []tls.Certificate{certificate},
		ClientAuth:   tls.RequireAndVerifyClientCert,
		ClientCAs:    cas,
	})
	return grpc.Creds(creds), nil
}

// GetSslClientOptions returns dial options by looking at SSL related
// flags - essentially, either ingests a CA certificate to validate
// the server or just gives up validating anything at all.
func GetSslClientOptions(serverAddress string, caCertPath string) (grpc.DialOption, error) {
	if caCertPath == "" {
		log.Println("WARNING: no CA specified. We will NOT check server's SSL certs")
		return grpc.WithInsecure(), nil
	}
	// NOTE: this is where we'd put client certs if we ever want to.
	cas := x509.NewCertPool()
	ca, err := ioutil.ReadFile(caCertPath)
	if err != nil {
		return nil, fmt.Errorf("could not read ca certificate: %s", err)
	}
	if ok := cas.AppendCertsFromPEM(ca); !ok {
		return nil, errors.New("failed to append ca certs")
	}
	log.Println("CA loaded, will check server SSL certs. That's good.")
	creds := credentials.NewTLS(&tls.Config{
		RootCAs:    cas,
		ServerName: serverAddress,
	})
	return grpc.WithTransportCredentials(creds), nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Package frontend implements GAE web server for luci-cron service.
//
// Due to the way classic GAE imports work, this package can not have
// subpackages (or at least subpackages referenced via absolute import path).
// We can't use relative imports because luci-go will then become unbuildable
// by regular (non GAE) toolset.
//
// See https://groups.google.com/forum/#!topic/google-appengine-go/dNhqV6PBqVc.
package frontend

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"

	"golang.org/x/net/context"

	"github.com/luci/gae/impl/prod"
	"github.com/luci/gae/service/info"
	"github.com/luci/gae/service/taskqueue"

	"github.com/luci/luci-go/appengine/gaeauth"
	"github.com/luci/luci-go/appengine/gaelogger"
	cfgmemory "github.com/luci/luci-go/common/config/impl/memory"
	cfgremote "github.com/luci/luci-go/common/config/impl/remote"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"

	cat "github.com/luci/luci-go/appengine/cmd/cron/catalog"
	eng "github.com/luci/luci-go/appengine/cmd/cron/engine"

	"github.com/luci/luci-go/appengine/cmd/cron/task"
	"github.com/luci/luci-go/appengine/cmd/cron/task/noop"
	"github.com/luci/luci-go/appengine/cmd/cron/task/urlfetch"
)

//// Global state. See init().

var (
	catalog cat.Catalog
	engine  eng.Engine

	// Known kinds of tasks.
	managers = []task.Manager{
		&noop.TaskManager{},
		&urlfetch.TaskManager{},
	}
)

const (
	// configServiceURL is URL of luci-config service.
	// TODO(vadimsh): Make it configurable.
	configServiceURL = "https://luci-config.appspot.com"

	// configServiceTimeout is deadline for luci-config url fetch calls.
	configServiceTimeout = 150 * time.Second
)

//// Helpers.

type handler func(c *requestContext)

type requestContext struct {
	context.Context

	w http.ResponseWriter
	r *http.Request
}

// fail writes error message to the log and the response and sets status code.
func (c *requestContext) fail(code int, msg string, args ...interface{}) {
	body := fmt.Sprintf(msg, args...)
	logging.Errorf(c, "HTTP %d: %s", code, body)
	http.Error(c.w, body, code)
}

// err sets status to 500 on transient errors or 202 on fatal ones. Returning
// status code in range [200–299] is the only way to tell Task Queues to stop
// retrying the task.
func (c *requestContext) err(e error, msg string, args ...interface{}) {
	code := 500
	if !errors.IsTransient(e) {
		code = 202
	}
	args = append(args, e)
	c.fail(code, msg+" - %s", args...)
}

// ok sets status to 200 and puts "OK" in response.
func (c *requestContext) ok() {
	c.w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	c.w.WriteHeader(200)
	fmt.Fprintln(c.w, "OK")
}

///

var globalInit sync.Once
var isProdGAE = true

func makeProdRequestContext(w http.ResponseWriter, r *http.Request) *requestContext {
	c := prod.UseRequest(r)
	c = gaelogger.Use(c)
	c = gaeauth.Use(c, nil, nil)

	// Use fake config data on dev server for simplicity.
	if info.Get(c).IsDevAppServer() {
		c = cfgmemory.Use(c, devServerConfigs())
	} else {
		c = cfgremote.Use(c, configServiceURL+"/_ah/api/config/v1/")
	}

	rc := &requestContext{c, w, r}

	// One time initialization for stuff that needs active GAE context.
	globalInit.Do(func() { initializeGlobalState(rc) })
	return rc
}

func initializeGlobalState(rc *requestContext) {
	// Dev app server doesn't preserve the state of task queues across restarts,
	// need to reset datastore state accordingly, otherwise everything gets stuck.
	if info.Get(rc.Context).IsDevAppServer() {
		isProdGAE = false
		if err := engine.ResetAllJobsOnDevServer(rc.Context); err != nil {
			logging.Errorf(rc.Context, "Failed to reset jobs: %s", err)
		}
	}
}

func gaeHandler(h handler) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		h(makeProdRequestContext(w, r))
	}
}

func cronHandler(h handler) http.HandlerFunc {
	return gaeHandler(func(c *requestContext) {
		if c.r.Header.Get("X-AppEngine-Cron") != "true" && isProdGAE {
			c.fail(403, "Only internal cron jobs can do this")
		} else {
			h(c)
		}
	})
}

func taskQueueHandler(queue string, h handler) http.HandlerFunc {
	return gaeHandler(func(c *requestContext) {
		got := c.r.Header.Get("X-AppEngine-QueueName")
		if got != queue && isProdGAE {
			c.fail(403, "Only internal queue %q can call this, got %q", queue, got)
		} else {
			h(c)
		}
	})
}

//// Routes.

func init() {
	// Setup global singletons.
	catalog = cat.NewCatalog()
	for _, m := range managers {
		if err := catalog.RegisterTaskManager(m); err != nil {
			panic(err)
		}
	}
	engine = eng.NewEngine(eng.Config{
		Catalog:              catalog,
		TimersQueuePath:      "/internal/tasks/timers",
		TimersQueueName:      "timers",
		InvocationsQueuePath: "/internal/tasks/invocations",
		InvocationsQueueName: "invocations",
	})

	// Setup HTTP routes.
	registerFrontendHandlers(http.DefaultServeMux)
	registerBackendHandlers(http.DefaultServeMux)
}

func registerFrontendHandlers(mux *http.ServeMux) {
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" {
			http.NotFound(w, r)
			return
		}
		fmt.Fprint(w, "Hi there!")
	})
	// To call initializeGlobalState on devserver sooner than later.
	mux.HandleFunc("/_ah/warmup", gaeHandler(func(c *requestContext) { c.ok() }))
}

func registerBackendHandlers(mux *http.ServeMux) {
	mux.HandleFunc("/internal/cron/read-config", cronHandler(readConfigCron))
	mux.HandleFunc(
		"/internal/tasks/read-project-config",
		taskQueueHandler("read-project-config", readProjectConfigTask))
	mux.HandleFunc("/internal/tasks/timers", taskQueueHandler("timers", actionTask))
	mux.HandleFunc("/internal/tasks/invocations", taskQueueHandler("invocations", actionTask))
}

//// Actual handlers.

// readConfigCron grabs a list of projects from the catalog and datastore and
// dispatches task queue tasks to update each project's cron jobs.
func readConfigCron(c *requestContext) {
	projectsToVisit := map[string]bool{}

	// Visit all projects in the catalog.
	ctx, _ := context.WithTimeout(c.Context, configServiceTimeout)
	projects, err := catalog.GetAllProjects(ctx)
	if err != nil {
		c.err(err, "Failed to grab a list of project IDs from catalog")
		return
	}
	for _, id := range projects {
		projectsToVisit[id] = true
	}

	// Also visit all registered projects that do not show up in the catalog
	// listing anymore. It will unregister all crons belonging to them.
	existing, err := engine.GetAllProjects(c.Context)
	if err != nil {
		c.err(err, "Failed to grab a list of project IDs from datastore")
		return
	}
	for _, id := range existing {
		projectsToVisit[id] = true
	}

	// Handle each project in its own task to avoid "bad" projects (e.g. ones with
	// lots of crons) to slow down "good" ones.
	tasks := make([]*taskqueue.Task, 0, len(projectsToVisit))
	for projectID := range projectsToVisit {
		tasks = append(tasks, &taskqueue.Task{
			Path: "/internal/tasks/read-project-config?projectID=" + url.QueryEscape(projectID),
		})
	}
	tq := taskqueue.Get(c)
	if err = tq.AddMulti(tasks, "read-project-config"); err != nil {
		c.err(errors.WrapTransient(err), "Failed to add tasks to task queue")
	} else {
		c.ok()
	}
}

// readProjectConfigTask grabs a list of cron jobs in a project from catalog,
// updates all changed cron jobs, adds new ones, disables old ones.
func readProjectConfigTask(c *requestContext) {
	projectID := c.r.URL.Query().Get("projectID")
	if projectID == "" {
		// Return 202 to avoid retry, it is fatal error.
		c.fail(202, "Missing projectID query attribute")
		return
	}
	ctx, _ := context.WithTimeout(c.Context, configServiceTimeout)
	jobs, err := catalog.GetProjectJobs(ctx, projectID)
	if err != nil {
		c.err(err, "Failed to query for a list of jobs")
		return
	}
	if err = engine.UpdateProjectJobs(c.Context, projectID, jobs); err != nil {
		c.err(err, "Failed to update some cron jobs")
		return
	}
	c.ok()
}

// actionTask is used to route actions emitted by cron job state transitions
// back into Engine (see enqueueActions).
func actionTask(c *requestContext) {
	body, err := ioutil.ReadAll(c.r.Body)
	if err != nil {
		c.fail(500, "Failed to read request body: %s", err)
		return
	}
	count, _ := strconv.Atoi(c.r.Header.Get("X-AppEngine-TaskExecutionCount"))
	err = engine.ExecuteSerializedAction(c.Context, body, count)
	if err != nil {
		c.err(err, "Error when executing the action")
		return
	}
	c.ok()
}
<commit_msg>cron: Use appengine/middleware library.<commit_after>// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Package frontend implements GAE web server for luci-cron service.
//
// Due to the way classic GAE imports work, this package can not have
// subpackages (or at least subpackages referenced via absolute import path).
// We can't use relative imports because luci-go will then become unbuildable
// by regular (non GAE) toolset.
//
// See https://groups.google.com/forum/#!topic/google-appengine-go/dNhqV6PBqVc.
package frontend

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"

	"github.com/julienschmidt/httprouter"
	"golang.org/x/net/context"

	"github.com/luci/gae/service/info"
	"github.com/luci/gae/service/taskqueue"

	"github.com/luci/luci-go/appengine/gaeauth"
	"github.com/luci/luci-go/appengine/middleware"

	cfgmemory "github.com/luci/luci-go/common/config/impl/memory"
	cfgremote "github.com/luci/luci-go/common/config/impl/remote"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"

	cat "github.com/luci/luci-go/appengine/cmd/cron/catalog"
	eng "github.com/luci/luci-go/appengine/cmd/cron/engine"

	"github.com/luci/luci-go/appengine/cmd/cron/task"
	"github.com/luci/luci-go/appengine/cmd/cron/task/noop"
	"github.com/luci/luci-go/appengine/cmd/cron/task/urlfetch"
)

//// Global state. See init().

var (
	catalog cat.Catalog
	engine  eng.Engine

	// Known kinds of tasks.
	managers = []task.Manager{
		&noop.TaskManager{},
		&urlfetch.TaskManager{},
	}
)

const (
	// configServiceURL is URL of luci-config service.
	// TODO(vadimsh): Make it configurable.
	configServiceURL = "https://luci-config.appspot.com"

	// configServiceTimeout is deadline for luci-config url fetch calls.
	configServiceTimeout = 150 * time.Second
)

//// Helpers.

type handler func(c *requestContext)

type requestContext struct {
	context.Context

	w http.ResponseWriter
	r *http.Request
	p httprouter.Params
}

// fail writes error message to the log and the response and sets status code.
func (c *requestContext) fail(code int, msg string, args ...interface{}) {
	body := fmt.Sprintf(msg, args...)
	logging.Errorf(c, "HTTP %d: %s", code, body)
	http.Error(c.w, body, code)
}

// err sets status to 500 on transient errors or 202 on fatal ones. Returning
// status code in range [200–299] is the only way to tell Task Queues to stop
// retrying the task.
func (c *requestContext) err(e error, msg string, args ...interface{}) {
	code := 500
	if !errors.IsTransient(e) {
		code = 202
	}
	args = append(args, e)
	c.fail(code, msg+" - %s", args...)
}

// ok sets status to 200 and puts "OK" in response.
func (c *requestContext) ok() {
	c.w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	c.w.WriteHeader(200)
	fmt.Fprintln(c.w, "OK")
}

///

var globalInit sync.Once
var isProdGAE = true

func initializeGlobalState(rc *requestContext) {
	// Dev app server doesn't preserve the state of task queues across restarts,
	// need to reset datastore state accordingly, otherwise everything gets stuck.
	if info.Get(rc.Context).IsDevAppServer() {
		isProdGAE = false
		if err := engine.ResetAllJobsOnDevServer(rc.Context); err != nil {
			logging.Errorf(rc.Context, "Failed to reset jobs: %s", err)
		}
	}
}

// wrap converts the handler to format accepted by middleware lib. It also adds
// context initialization code.
func wrap(h handler) middleware.Handler {
	return func(c context.Context, w http.ResponseWriter, r *http.Request, p httprouter.Params) {
		c = gaeauth.Use(c, nil, nil)

		// Use fake config data on dev server for simplicity.
		if info.Get(c).IsDevAppServer() {
			c = cfgmemory.Use(c, devServerConfigs())
		} else {
			c = cfgremote.Use(c, configServiceURL+"/_ah/api/config/v1/")
		}

		rc := &requestContext{c, w, r, p}

		// One time initialization for stuff that needs active GAE context.
		globalInit.Do(func() { initializeGlobalState(rc) })

		h(rc)
	}
}

// publicHandler returns handler for publicly accessible routes.
func publicHandler(h handler) httprouter.Handle {
	return middleware.BaseProd(wrap(h))
}

// cronHandler returns handler intended for cron jobs.
func cronHandler(h handler) httprouter.Handle {
	return middleware.BaseProd(middleware.RequireCron(wrap(h)))
}

// taskQueueHandler returns handler intended for task queue calls.
func taskQueueHandler(name string, h handler) httprouter.Handle {
	return middleware.BaseProd(middleware.RequireTaskQueue(name, wrap(h)))
}

//// Routes.

func init() {
	// Setup global singletons.
	catalog = cat.NewCatalog()
	for _, m := range managers {
		if err := catalog.RegisterTaskManager(m); err != nil {
			panic(err)
		}
	}
	engine = eng.NewEngine(eng.Config{
		Catalog:              catalog,
		TimersQueuePath:      "/internal/tasks/timers",
		TimersQueueName:      "timers",
		InvocationsQueuePath: "/internal/tasks/invocations",
		InvocationsQueueName: "invocations",
	})

	// Setup HTTP routes.
	router := httprouter.New()
	registerFrontendHandlers(router)
	registerBackendHandlers(router)
	http.DefaultServeMux.Handle("/", router)
}

func registerFrontendHandlers(router *httprouter.Router) {
	router.GET("/", publicHandler(indexPage))
	router.GET("/_ah/warmup", publicHandler(warmupHandler))
}

func registerBackendHandlers(router *httprouter.Router) {
	router.GET("/internal/cron/read-config", cronHandler(readConfigCron))
	router.POST("/internal/tasks/read-project-config", taskQueueHandler("read-project-config", readProjectConfigTask))
	router.POST("/internal/tasks/timers", taskQueueHandler("timers", actionTask))
	router.POST("/internal/tasks/invocations", taskQueueHandler("invocations", actionTask))
}

//// Frontend handlers.

func indexPage(rc *requestContext) {
	fmt.Fprint(rc.w, "Hi there!")
}

func warmupHandler(rc *requestContext) {
	rc.ok()
}

//// Backend handlers.

// readConfigCron grabs a list of projects from the catalog and datastore and
// dispatches task queue tasks to update each project's cron jobs.
func readConfigCron(c *requestContext) {
	projectsToVisit := map[string]bool{}

	// Visit all projects in the catalog.
	ctx, _ := context.WithTimeout(c.Context, configServiceTimeout)
	projects, err := catalog.GetAllProjects(ctx)
	if err != nil {
		c.err(err, "Failed to grab a list of project IDs from catalog")
		return
	}
	for _, id := range projects {
		projectsToVisit[id] = true
	}

	// Also visit all registered projects that do not show up in the catalog
	// listing anymore. It will unregister all crons belonging to them.
	existing, err := engine.GetAllProjects(c.Context)
	if err != nil {
		c.err(err, "Failed to grab a list of project IDs from datastore")
		return
	}
	for _, id := range existing {
		projectsToVisit[id] = true
	}

	// Handle each project in its own task to avoid "bad" projects (e.g. ones with
	// lots of crons) to slow down "good" ones.
	tasks := make([]*taskqueue.Task, 0, len(projectsToVisit))
	for projectID := range projectsToVisit {
		tasks = append(tasks, &taskqueue.Task{
			Path: "/internal/tasks/read-project-config?projectID=" + url.QueryEscape(projectID),
		})
	}
	tq := taskqueue.Get(c)
	if err = tq.AddMulti(tasks, "read-project-config"); err != nil {
		c.err(errors.WrapTransient(err), "Failed to add tasks to task queue")
	} else {
		c.ok()
	}
}

// readProjectConfigTask grabs a list of cron jobs in a project from catalog,
// updates all changed cron jobs, adds new ones, disables old ones.
func readProjectConfigTask(c *requestContext) {
	projectID := c.r.URL.Query().Get("projectID")
	if projectID == "" {
		// Return 202 to avoid retry, it is fatal error.
		c.fail(202, "Missing projectID query attribute")
		return
	}
	ctx, _ := context.WithTimeout(c.Context, configServiceTimeout)
	jobs, err := catalog.GetProjectJobs(ctx, projectID)
	if err != nil {
		c.err(err, "Failed to query for a list of jobs")
		return
	}
	if err = engine.UpdateProjectJobs(c.Context, projectID, jobs); err != nil {
		c.err(err, "Failed to update some cron jobs")
		return
	}
	c.ok()
}

// actionTask is used to route actions emitted by cron job state transitions
// back into Engine (see enqueueActions).
func actionTask(c *requestContext) {
	body, err := ioutil.ReadAll(c.r.Body)
	if err != nil {
		c.fail(500, "Failed to read request body: %s", err)
		return
	}
	count, _ := strconv.Atoi(c.r.Header.Get("X-AppEngine-TaskExecutionCount"))
	err = engine.ExecuteSerializedAction(c.Context, body, count)
	if err != nil {
		c.err(err, "Error when executing the action")
		return
	}
	c.ok()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/version"

	log "github.com/lxc/lxd/shared/log15"
)

type instanceType struct {
	// Amount of CPUs (can be a fraction)
	CPU float32 `yaml:"cpu"`

	// Amount of memory in GB
	Memory float32 `yaml:"mem"`
}

var instanceTypes map[string]map[string]*instanceType

func instanceSaveCache() error {
	if instanceTypes == nil {
		return nil
	}

	data, err := yaml.Marshal(&instanceTypes)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(shared.CachePath("instance_types.yaml"), data, 0600)
	if err != nil {
		return err
	}

	return nil
}

func instanceLoadCache() error {
	if !shared.PathExists(shared.CachePath("instance_types.yaml")) {
		return nil
	}

	content, err := ioutil.ReadFile(shared.CachePath("instance_types.yaml"))
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(content, &instanceTypes)
	if err != nil {
		return err
	}

	return nil
}

func instanceRefreshTypesTask(d *Daemon) (task.Func, task.Schedule) {
	// This is basically a check of whether we're on Go >= 1.8 and
	// http.Request has cancellation support. If that's the case, it will
	// be used internally by instanceRefreshTypes to terminate gracefully,
	// otherwise we'll wrap instanceRefreshTypes in a goroutine and force
	// returning in case the context expires.
	_, hasCancellationSupport := interface{}(&http.Request{}).(util.ContextAwareRequest)
	f := func(ctx context.Context) {
		opRun := func(op *operation) error {
			if hasCancellationSupport {
				return instanceRefreshTypes(ctx, d)
			}

			ch := make(chan error)
			go func() {
				ch <- instanceRefreshTypes(ctx, d)
			}()
			select {
			case <-ctx.Done():
				return nil
			case err := <-ch:
				return err
			}
		}

		op, err := operationCreate(d.cluster, "", operationClassTask, db.OperationInstanceTypesUpdate, nil, nil, opRun, nil, nil)
		if err != nil {
			logger.Error("Failed to start instance types update operation", log.Ctx{"err": err})
			return
		}

		logger.Info("Updating instance types")
		_, err = op.Run()
		if err != nil {
			logger.Error("Failed to update instance types", log.Ctx{"err": err})
		}
		logger.Infof("Done updating instance types")
	}

	return f, task.Daily()
}

func instanceRefreshTypes(ctx context.Context, d *Daemon) error {
	// Attempt to download the new definitions
	downloadParse := func(filename string, target interface{}) error {
		url := fmt.Sprintf("https://images.linuxcontainers.org/meta/instance-types/%s", filename)

		httpClient, err := util.HTTPClient("", d.proxy)
		if err != nil {
			return err
		}

		httpReq, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return err
		}

		httpReq.Header.Set("User-Agent", version.UserAgent)

		cancelableRequest, ok := interface{}(httpReq).(util.ContextAwareRequest)
		if ok {
			httpReq = cancelableRequest.WithContext(ctx)
		}

		resp, err := httpClient.Do(httpReq)
		if err != nil {
			return err
		}
		if ctx.Err() != nil {
			return ctx.Err()
		}
		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			return fmt.Errorf("Failed to get %s", url)
		}

		content, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}

		err = yaml.Unmarshal(content, target)
		if err != nil {
			return err
		}

		return nil
	}

	// Set an initial value from the cache
	if instanceTypes == nil {
		instanceLoadCache()
	}

	// Get the list of instance type sources
	sources := map[string]string{}
	err := downloadParse(".yaml", &sources)
	if err != nil {
		if err != ctx.Err() {
			logger.Warnf("Failed to update instance types: %v", err)
		}
		return err
	}

	// Parse the individual files
	newInstanceTypes := map[string]map[string]*instanceType{}
	for name, filename := range sources {
		types := map[string]*instanceType{}
		err = downloadParse(filename, &types)
		if err != nil {
			logger.Warnf("Failed to update instance types: %v", err)
			return err
		}

		newInstanceTypes[name] = types
	}

	// Update the global map
	instanceTypes = newInstanceTypes

	// And save in the cache
	err = instanceSaveCache()
	if err != nil {
		logger.Warnf("Failed to update instance types cache: %v", err)
		return err
	}

	return nil
}

func instanceParseType(value string) (map[string]string, error) {
	sourceName := ""
	sourceType := ""
	fields := strings.SplitN(value, ":", 2)

	// Check if the name of the source was provided
	if len(fields) != 2 {
		sourceType = value
	} else {
		sourceName = fields[0]
		sourceType = fields[1]
	}

	// If not, lets go look for a match
	if instanceTypes != nil && sourceName == "" {
		for name, types := range instanceTypes {
			_, ok := types[sourceType]
			if ok {
				if sourceName != "" {
					return nil, fmt.Errorf("Ambiguous instance type provided: %s", value)
				}

				sourceName = name
			}
		}
	}

	// Check if we have a limit for the provided value
	limits, ok := instanceTypes[sourceName][sourceType]
	if !ok {
		// Check if it's maybe just a resource limit
		if sourceName == "" && value != "" {
			newLimits := instanceType{}
			fields := strings.Split(value, "-")
			for _, field := range fields {
				if len(field) < 2 || (field[0] != 'c' && field[0] != 'm') {
					return nil, fmt.Errorf("Bad instance type: %s", value)
				}

				value, err := strconv.ParseFloat(field[1:], 32)
				if err != nil {
					return nil, err
				}

				if field[0] == 'c' {
					newLimits.CPU = float32(value)
				} else if field[0] == 'm' {
					newLimits.Memory = float32(value)
				}
			}

			limits = &newLimits
		}

		if limits == nil {
			return nil, fmt.Errorf("Provided instance type doesn't exist: %s", value)
		}
	}
	out := map[string]string{}

	// Handle CPU
	if limits.CPU > 0 {
		cpuCores := int(limits.CPU)
		if float32(cpuCores) < limits.CPU {
			cpuCores++
		}
		cpuTime := int(limits.CPU / float32(cpuCores) * 100.0)

		out["limits.cpu"] = fmt.Sprintf("%d", cpuCores)
		if cpuTime < 100 {
			out["limits.cpu.allowance"] = fmt.Sprintf("%d%%", cpuTime)
		}
	}

	// Handle memory
	if limits.Memory > 0 {
		rawLimit := int64(limits.Memory * 1024)
		out["limits.memory"] = fmt.Sprintf("%dMB", rawLimit)
	}

	return out, nil
}
<commit_msg>lxd/instances: Improve errors<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/task"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/version"

	log "github.com/lxc/lxd/shared/log15"
)

type instanceType struct {
	// Amount of CPUs (can be a fraction)
	CPU float32 `yaml:"cpu"`

	// Amount of memory in GB
	Memory float32 `yaml:"mem"`
}

var instanceTypes map[string]map[string]*instanceType

func instanceSaveCache() error {
	if instanceTypes == nil {
		return nil
	}

	data, err := yaml.Marshal(&instanceTypes)
	if err != nil {
		return err
	}

	err = ioutil.WriteFile(shared.CachePath("instance_types.yaml"), data, 0600)
	if err != nil {
		return err
	}

	return nil
}

func instanceLoadCache() error {
	if !shared.PathExists(shared.CachePath("instance_types.yaml")) {
		return nil
	}

	content, err := ioutil.ReadFile(shared.CachePath("instance_types.yaml"))
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(content, &instanceTypes)
	if err != nil {
		return err
	}

	return nil
}

func instanceRefreshTypesTask(d *Daemon) (task.Func, task.Schedule) {
	// This is basically a check of whether we're on Go >= 1.8 and
	// http.Request has cancellation support. If that's the case, it will
	// be used internally by instanceRefreshTypes to terminate gracefully,
	// otherwise we'll wrap instanceRefreshTypes in a goroutine and force
	// returning in case the context expires.
	_, hasCancellationSupport := interface{}(&http.Request{}).(util.ContextAwareRequest)
	f := func(ctx context.Context) {
		opRun := func(op *operation) error {
			if hasCancellationSupport {
				return instanceRefreshTypes(ctx, d)
			}

			ch := make(chan error)
			go func() {
				ch <- instanceRefreshTypes(ctx, d)
			}()
			select {
			case <-ctx.Done():
				return nil
			case err := <-ch:
				return err
			}
		}

		op, err := operationCreate(d.cluster, "", operationClassTask, db.OperationInstanceTypesUpdate, nil, nil, opRun, nil, nil)
		if err != nil {
			logger.Error("Failed to start instance types update operation", log.Ctx{"err": err})
			return
		}

		logger.Info("Updating instance types")
		_, err = op.Run()
		if err != nil {
			logger.Error("Failed to update instance types", log.Ctx{"err": err})
		}
		logger.Infof("Done updating instance types")
	}

	return f, task.Daily()
}

func instanceRefreshTypes(ctx context.Context, d *Daemon) error {
	// Attempt to download the new definitions
	downloadParse := func(filename string, target interface{}) error {
		url := fmt.Sprintf("https://images.linuxcontainers.org/meta/instance-types/%s", filename)

		httpClient, err := util.HTTPClient("", d.proxy)
		if err != nil {
			return err
		}

		httpReq, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return err
		}

		httpReq.Header.Set("User-Agent", version.UserAgent)

		cancelableRequest, ok := interface{}(httpReq).(util.ContextAwareRequest)
		if ok {
			httpReq = cancelableRequest.WithContext(ctx)
		}

		resp, err := httpClient.Do(httpReq)
		if err != nil {
			return err
		}
		if ctx.Err() != nil {
			return ctx.Err()
		}
		defer resp.Body.Close()

		if resp.StatusCode != http.StatusOK {
			return fmt.Errorf("Failed to get %s", url)
		}

		content, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}

		err = yaml.Unmarshal(content, target)
		if err != nil {
			return err
		}

		return nil
	}

	// Set an initial value from the cache
	if instanceTypes == nil {
		instanceLoadCache()
	}

	// Get the list of instance type sources
	sources := map[string]string{}
	err := downloadParse(".yaml", &sources)
	if err != nil {
		if err != ctx.Err() {
			logger.Warnf("Failed to update instance types: %v", err)
		}
		return err
	}

	// Parse the individual files
	newInstanceTypes := map[string]map[string]*instanceType{}
	for name, filename := range sources {
		types := map[string]*instanceType{}
		err = downloadParse(filename, &types)
		if err != nil {
			logger.Warnf("Failed to update instance types: %v", err)
			return err
		}

		newInstanceTypes[name] = types
	}

	// Update the global map
	instanceTypes = newInstanceTypes

	// And save in the cache
	err = instanceSaveCache()
	if err != nil {
		logger.Warnf("Failed to update instance types cache: %v", err)
		return err
	}

	return nil
}

func instanceParseType(value string) (map[string]string, error) {
	sourceName := ""
	sourceType := ""
	fields := strings.SplitN(value, ":", 2)

	// Check if the name of the source was provided
	if len(fields) != 2 {
		sourceType = value
	} else {
		sourceName = fields[0]
		sourceType = fields[1]
	}

	// If not, lets go look for a match
	if instanceTypes != nil && sourceName == "" {
		for name, types := range instanceTypes {
			_, ok := types[sourceType]
			if ok {
				if sourceName != "" {
					return nil, fmt.Errorf("Ambiguous instance type provided: %s", value)
				}

				sourceName = name
			}
		}
	}

	// Check if we have a limit for the provided value
	limits, ok := instanceTypes[sourceName][sourceType]
	if !ok {
		// Check if it's maybe just a resource limit
		if sourceName == "" && value != "" {
			newLimits := instanceType{}
			fields := strings.Split(value, "-")
			for _, field := range fields {
				if len(field) < 2 || (field[0] != 'c' && field[0] != 'm') {
					return nil, fmt.Errorf("Provided instance type doesn't exist: %s", value)
				}

				floatValue, err := strconv.ParseFloat(field[1:], 32)
				if err != nil {
					return nil, fmt.Errorf("Bad custom instance type: %s", value)
				}

				if field[0] == 'c' {
					newLimits.CPU = float32(floatValue)
				} else if field[0] == 'm' {
					newLimits.Memory = float32(floatValue)
				}
			}

			limits = &newLimits
		}

		if limits == nil {
			return nil, fmt.Errorf("Provided instance type doesn't exist: %s", value)
		}
	}
	out := map[string]string{}

	// Handle CPU
	if limits.CPU > 0 {
		cpuCores := int(limits.CPU)
		if float32(cpuCores) < limits.CPU {
			cpuCores++
		}
		cpuTime := int(limits.CPU / float32(cpuCores) * 100.0)

		out["limits.cpu"] = fmt.Sprintf("%d", cpuCores)
		if cpuTime < 100 {
			out["limits.cpu.allowance"] = fmt.Sprintf("%d%%", cpuTime)
		}
	}

	// Handle memory
	if limits.Memory > 0 {
		rawLimit := int64(limits.Memory * 1024)
		out["limits.memory"] = fmt.Sprintf("%dMB", rawLimit)
	}

	return out, nil
}
<|endoftext|>


<commit_before>package testdata

import "testing"

func Test_name_Name(t *testing.T) {
	type args struct {
		n string
	}
	tests := []struct {
		name string
		n    name
		args args
		want string
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := tt.n.Name(tt.args.n); got != tt.want {
				t.Errorf("name.Name() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestName_Name1(t *testing.T) {
	type fields struct {
		Name string
	}
	type args struct {
		n string
	}
	tests := []struct {
		name   string
		fields fields
		args   args
		want   string
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			n := &Name{
				Name: tt.fields.Name,
			}
			if got := n.Name1(tt.args.n); got != tt.want {
				t.Errorf("Name.Name1() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestName_Name2(t *testing.T) {
	type fields struct {
		Name string
	}
	type args struct {
		name string
	}
	tests := []struct {
		name   string
		fields fields
		args   args
		want   string
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			n := &Name{
				Name: tt.fields.Name,
			}
			if got := n.Name2(tt.args.name); got != tt.want {
				t.Errorf("Name.Name2() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestName_Name3(t *testing.T) {
	type fields struct {
		Name string
	}
	type args struct {
		nn string
	}
	tests := []struct {
		name     string
		fields   fields
		args     args
		wantName string
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			n := &Name{
				Name: tt.fields.Name,
			}
			if gotName := n.Name3(tt.args.nn); gotName != tt.wantName {
				t.Errorf("Name.Name3() = %v, want %v", gotName, tt.wantName)
			}
		})
	}
}<commit_msg>Missing newline in golden.<commit_after>package testdata

import "testing"

func Test_name_Name(t *testing.T) {
	type args struct {
		n string
	}
	tests := []struct {
		name string
		n    name
		args args
		want string
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := tt.n.Name(tt.args.n); got != tt.want {
				t.Errorf("name.Name() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestName_Name1(t *testing.T) {
	type fields struct {
		Name string
	}
	type args struct {
		n string
	}
	tests := []struct {
		name   string
		fields fields
		args   args
		want   string
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			n := &Name{
				Name: tt.fields.Name,
			}
			if got := n.Name1(tt.args.n); got != tt.want {
				t.Errorf("Name.Name1() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestName_Name2(t *testing.T) {
	type fields struct {
		Name string
	}
	type args struct {
		name string
	}
	tests := []struct {
		name   string
		fields fields
		args   args
		want   string
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			n := &Name{
				Name: tt.fields.Name,
			}
			if got := n.Name2(tt.args.name); got != tt.want {
				t.Errorf("Name.Name2() = %v, want %v", got, tt.want)
			}
		})
	}
}

func TestName_Name3(t *testing.T) {
	type fields struct {
		Name string
	}
	type args struct {
		nn string
	}
	tests := []struct {
		name     string
		fields   fields
		args     args
		wantName string
	}{
	// TODO: Add test cases.
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			n := &Name{
				Name: tt.fields.Name,
			}
			if gotName := n.Name3(tt.args.nn); gotName != tt.wantName {
				t.Errorf("Name.Name3() = %v, want %v", gotName, tt.wantName)
			}
		})
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testutil

import (
	"context"

	sppb "google.golang.org/genproto/googleapis/spanner/v1"
	"google.golang.org/grpc"
)

// FuncMock overloads some of MockCloudSpannerClient's methods with pluggable
// functions.
//
// Note: if you overload a method, you're in charge of making sure
// MockCloudSpannerClient.ReceivedRequests receives the request appropriately.
type FuncMock struct {
	CommitFn           func(ctx context.Context, r *sppb.CommitRequest, opts ...grpc.CallOption) (*sppb.CommitResponse, error)
	BeginTransactionFn func(ctx context.Context, r *sppb.BeginTransactionRequest, opts ...grpc.CallOption) (*sppb.Transaction, error)
	GetSessionFn       func(ctx context.Context, r *sppb.GetSessionRequest, opts ...grpc.CallOption) (*sppb.Session, error)
	CreateSessionFn    func(ctx context.Context, r *sppb.CreateSessionRequest, opts ...grpc.CallOption) (*sppb.Session, error)
	*MockCloudSpannerClient
}

func (s FuncMock) Commit(ctx context.Context, r *sppb.CommitRequest, opts ...grpc.CallOption) (*sppb.CommitResponse, error) {
	if s.CommitFn == nil {
		return s.MockCloudSpannerClient.Commit(ctx, r, opts...)
	}
	return s.CommitFn(ctx, r, opts...)
}

func (s FuncMock) BeginTransaction(ctx context.Context, r *sppb.BeginTransactionRequest, opts ...grpc.CallOption) (*sppb.Transaction, error) {
	if s.BeginTransactionFn == nil {
		return s.MockCloudSpannerClient.BeginTransaction(ctx, r, opts...)
	}
	return s.BeginTransactionFn(ctx, r, opts...)
}

func (s *FuncMock) GetSession(ctx context.Context, r *sppb.GetSessionRequest, opts ...grpc.CallOption) (*sppb.Session, error) {
	if s.GetSessionFn == nil {
		return s.MockCloudSpannerClient.GetSession(ctx, r, opts...)
	}
	return s.GetSessionFn(ctx, r, opts...)
}

func (s *FuncMock) CreateSession(c context.Context, r *sppb.CreateSessionRequest, opts ...grpc.CallOption) (*sppb.Session, error) {
	if s.CreateSessionFn == nil {
		return s.MockCloudSpannerClient.CreateSession(c, r, opts...)
	}
	return s.CreateSessionFn(c, r, opts...)
}
<commit_msg>spanner/internal/testutil: delete FuncMock<commit_after><|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package constants

var (
	KubeadmImages = map[string]map[string]string{
		"v1.22": {
			"k8s.gcr.io/coredns/coredns":         "v1.8.4",
			"k8s.gcr.io/etcd":                    "3.5.0-0",
			"k8s.gcr.io/kube-apiserver":          "v1.22.1",
			"k8s.gcr.io/kube-controller-manager": "v1.22.1",
			"k8s.gcr.io/kube-proxy":              "v1.22.1",
			"k8s.gcr.io/kube-scheduler":          "v1.22.1",
			"k8s.gcr.io/pause":                   "3.5",
		},
		"v1.21": {
			"k8s.gcr.io/coredns/coredns":         "v1.8.0",
			"k8s.gcr.io/etcd":                    "3.4.13-0",
			"k8s.gcr.io/kube-apiserver":          "v1.21.3",
			"k8s.gcr.io/kube-controller-manager": "v1.21.3",
			"k8s.gcr.io/kube-proxy":              "v1.21.3",
			"k8s.gcr.io/kube-scheduler":          "v1.21.3",
			"k8s.gcr.io/pause":                   "3.4.1",
		},
		"v1.20": {
			"k8s.gcr.io/coredns":                 "1.7.0",
			"k8s.gcr.io/etcd":                    "3.4.13-0",
			"k8s.gcr.io/kube-apiserver":          "v1.20.9",
			"k8s.gcr.io/kube-controller-manager": "v1.20.9",
			"k8s.gcr.io/kube-proxy":              "v1.20.9",
			"k8s.gcr.io/kube-scheduler":          "v1.20.9",
			"k8s.gcr.io/pause":                   "3.2",
		},
		"v1.19": {
			"k8s.gcr.io/coredns":                 "1.7.0",
			"k8s.gcr.io/etcd":                    "3.4.9-1",
			"k8s.gcr.io/kube-apiserver":          "v1.19.13",
			"k8s.gcr.io/kube-controller-manager": "v1.19.13",
			"k8s.gcr.io/kube-proxy":              "v1.19.13",
			"k8s.gcr.io/kube-scheduler":          "v1.19.13",
			"k8s.gcr.io/pause":                   "3.2",
		},
		"v1.18": {
			"k8s.gcr.io/coredns":                 "1.6.7",
			"k8s.gcr.io/etcd":                    "3.4.3-0",
			"k8s.gcr.io/kube-apiserver":          "v1.18.20",
			"k8s.gcr.io/kube-controller-manager": "v1.18.20",
			"k8s.gcr.io/kube-proxy":              "v1.18.20",
			"k8s.gcr.io/kube-scheduler":          "v1.18.20",
			"k8s.gcr.io/pause":                   "3.2",
		},
		"v1.17": {
			"k8s.gcr.io/coredns":                 "1.6.5",
			"k8s.gcr.io/etcd":                    "3.4.3-0",
			"k8s.gcr.io/kube-apiserver":          "v1.17.17",
			"k8s.gcr.io/kube-controller-manager": "v1.17.17",
			"k8s.gcr.io/kube-proxy":              "v1.17.17",
			"k8s.gcr.io/kube-scheduler":          "v1.17.17",
			"k8s.gcr.io/pause":                   "3.1",
		},
		"v1.16": {
			"k8s.gcr.io/coredns":                 "1.6.2",
			"k8s.gcr.io/etcd":                    "3.3.15-0",
			"k8s.gcr.io/kube-apiserver":          "v1.16.15",
			"k8s.gcr.io/kube-controller-manager": "v1.16.15",
			"k8s.gcr.io/kube-proxy":              "v1.16.15",
			"k8s.gcr.io/kube-scheduler":          "v1.16.15",
			"k8s.gcr.io/pause":                   "3.1",
		},
		"v1.15": {
			"k8s.gcr.io/coredns":                 "1.3.1",
			"k8s.gcr.io/etcd":                    "3.3.10",
			"k8s.gcr.io/kube-apiserver":          "v1.15.12",
			"k8s.gcr.io/kube-controller-manager": "v1.15.12",
			"k8s.gcr.io/kube-proxy":              "v1.15.12",
			"k8s.gcr.io/kube-scheduler":          "v1.15.12",
			"k8s.gcr.io/pause":                   "3.1",
		},

		"v1.14": {
			"k8s.gcr.io/coredns":                 "1.3.1",
			"k8s.gcr.io/etcd":                    "3.3.10",
			"k8s.gcr.io/kube-apiserver":          "v1.14.10",
			"k8s.gcr.io/kube-controller-manager": "v1.14.10",
			"k8s.gcr.io/kube-proxy":              "v1.14.10",
			"k8s.gcr.io/kube-scheduler":          "v1.14.10",
			"k8s.gcr.io/pause":                   "3.1",
		},
		"v1.13": {
			"k8s.gcr.io/coredns":                 "1.2.6",
			"k8s.gcr.io/etcd":                    "3.2.24",
			"k8s.gcr.io/kube-apiserver":          "v1.13.12",
			"k8s.gcr.io/kube-controller-manager": "v1.13.12",
			"k8s.gcr.io/kube-proxy":              "v1.13.12",
			"k8s.gcr.io/kube-scheduler":          "v1.13.12",
			"k8s.gcr.io/pause":                   "3.1",
		},
		"v1.12": {
			"k8s.gcr.io/coredns":                 "1.2.2",
			"k8s.gcr.io/etcd":                    "3.2.24",
			"k8s.gcr.io/kube-apiserver":          "v1.22.1",
			"k8s.gcr.io/kube-controller-manager": "v1.22.1",
			"k8s.gcr.io/kube-proxy":              "v1.22.1",
			"k8s.gcr.io/kube-scheduler":          "v1.22.1",
			"k8s.gcr.io/pause":                   "3.1",
		},
		"v1.11": {
			"k8s.gcr.io/coredns":                       "1.1.3",
			"k8s.gcr.io/etcd-amd64":                    "3.2.18",
			"k8s.gcr.io/kube-apiserver-amd64":          "v1.11.10",
			"k8s.gcr.io/kube-controller-manager-amd64": "v1.11.10",
			"k8s.gcr.io/kube-proxy-amd64":              "v1.11.10",
			"k8s.gcr.io/kube-scheduler-amd64":          "v1.11.10",
			"k8s.gcr.io/pause-amd64":                   "3.1",
		},
	}
)
<commit_msg>updated kubeadm-constants image version<commit_after>/*
Copyright 2021 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package constants

var (
	KubeadmImages = map[string]map[string]string{
		"v1.22": {
			"k8s.gcr.io/coredns/coredns":         "v1.8.4",
			"k8s.gcr.io/etcd":                    "3.5.0-0",
			"k8s.gcr.io/kube-apiserver":          "v1.22.2",
			"k8s.gcr.io/kube-controller-manager": "v1.22.2",
			"k8s.gcr.io/kube-proxy":              "v1.22.2",
			"k8s.gcr.io/kube-scheduler":          "v1.22.2",
			"k8s.gcr.io/pause":                   "3.5",
		},
		"v1.21": {
			"k8s.gcr.io/coredns/coredns":         "v1.8.0",
			"k8s.gcr.io/etcd":                    "3.4.13-0",
			"k8s.gcr.io/kube-apiserver":          "v1.21.3",
			"k8s.gcr.io/kube-controller-manager": "v1.21.3",
			"k8s.gcr.io/kube-proxy":              "v1.21.3",
			"k8s.gcr.io/kube-scheduler":          "v1.21.3",
			"k8s.gcr.io/pause":                   "3.4.1",
		},
		"v1.20": {
			"k8s.gcr.io/coredns":                 "1.7.0",
			"k8s.gcr.io/etcd":                    "3.4.13-0",
			"k8s.gcr.io/kube-apiserver":          "v1.20.9",
			"k8s.gcr.io/kube-controller-manager": "v1.20.9",
			"k8s.gcr.io/kube-proxy":              "v1.20.9",
			"k8s.gcr.io/kube-scheduler":          "v1.20.9",
			"k8s.gcr.io/pause":                   "3.2",
		},
		"v1.19": {
			"k8s.gcr.io/coredns":                 "1.7.0",
			"k8s.gcr.io/etcd":                    "3.4.9-1",
			"k8s.gcr.io/kube-apiserver":          "v1.19.13",
			"k8s.gcr.io/kube-controller-manager": "v1.19.13",
			"k8s.gcr.io/kube-proxy":              "v1.19.13",
			"k8s.gcr.io/kube-scheduler":          "v1.19.13",
			"k8s.gcr.io/pause":                   "3.2",
		},
		"v1.18": {
			"k8s.gcr.io/coredns":                 "1.6.7",
			"k8s.gcr.io/etcd":                    "3.4.3-0",
			"k8s.gcr.io/kube-apiserver":          "v1.18.20",
			"k8s.gcr.io/kube-controller-manager": "v1.18.20",
			"k8s.gcr.io/kube-proxy":              "v1.18.20",
			"k8s.gcr.io/kube-scheduler":          "v1.18.20",
			"k8s.gcr.io/pause":                   "3.2",
		},
		"v1.17": {
			"k8s.gcr.io/coredns":                 "1.6.5",
			"k8s.gcr.io/etcd":                    "3.4.3-0",
			"k8s.gcr.io/kube-apiserver":          "v1.17.17",
			"k8s.gcr.io/kube-controller-manager": "v1.17.17",
			"k8s.gcr.io/kube-proxy":              "v1.17.17",
			"k8s.gcr.io/kube-scheduler":          "v1.17.17",
			"k8s.gcr.io/pause":                   "3.1",
		},
		"v1.16": {
			"k8s.gcr.io/coredns":                 "1.6.2",
			"k8s.gcr.io/etcd":                    "3.3.15-0",
			"k8s.gcr.io/kube-apiserver":          "v1.16.15",
			"k8s.gcr.io/kube-controller-manager": "v1.16.15",
			"k8s.gcr.io/kube-proxy":              "v1.16.15",
			"k8s.gcr.io/kube-scheduler":          "v1.16.15",
			"k8s.gcr.io/pause":                   "3.1",
		},
		"v1.15": {
			"k8s.gcr.io/coredns":                 "1.3.1",
			"k8s.gcr.io/etcd":                    "3.3.10",
			"k8s.gcr.io/kube-apiserver":          "v1.15.12",
			"k8s.gcr.io/kube-controller-manager": "v1.15.12",
			"k8s.gcr.io/kube-proxy":              "v1.15.12",
			"k8s.gcr.io/kube-scheduler":          "v1.15.12",
			"k8s.gcr.io/pause":                   "3.1",
		},

		"v1.14": {
			"k8s.gcr.io/coredns":                 "1.3.1",
			"k8s.gcr.io/etcd":                    "3.3.10",
			"k8s.gcr.io/kube-apiserver":          "v1.14.10",
			"k8s.gcr.io/kube-controller-manager": "v1.14.10",
			"k8s.gcr.io/kube-proxy":              "v1.14.10",
			"k8s.gcr.io/kube-scheduler":          "v1.14.10",
			"k8s.gcr.io/pause":                   "3.1",
		},
		"v1.13": {
			"k8s.gcr.io/coredns":                 "1.2.6",
			"k8s.gcr.io/etcd":                    "3.2.24",
			"k8s.gcr.io/kube-apiserver":          "v1.13.12",
			"k8s.gcr.io/kube-controller-manager": "v1.13.12",
			"k8s.gcr.io/kube-proxy":              "v1.13.12",
			"k8s.gcr.io/kube-scheduler":          "v1.13.12",
			"k8s.gcr.io/pause":                   "3.1",
		},
		"v1.12": {
			"k8s.gcr.io/coredns":                 "1.2.2",
			"k8s.gcr.io/etcd":                    "3.2.24",
			"k8s.gcr.io/kube-apiserver":          "v1.22.1",
			"k8s.gcr.io/kube-controller-manager": "v1.22.1",
			"k8s.gcr.io/kube-proxy":              "v1.22.1",
			"k8s.gcr.io/kube-scheduler":          "v1.22.1",
			"k8s.gcr.io/pause":                   "3.1",
		},
		"v1.11": {
			"k8s.gcr.io/coredns":                       "1.1.3",
			"k8s.gcr.io/etcd-amd64":                    "3.2.18",
			"k8s.gcr.io/kube-apiserver-amd64":          "v1.11.10",
			"k8s.gcr.io/kube-controller-manager-amd64": "v1.11.10",
			"k8s.gcr.io/kube-proxy-amd64":              "v1.11.10",
			"k8s.gcr.io/kube-scheduler-amd64":          "v1.11.10",
			"k8s.gcr.io/pause-amd64":                   "3.1",
		},
	}
)
<|endoftext|>


<commit_before>package main

import (
	"os"

	log "github.com/cihub/seelog"
	"github.com/hnakamur/commango/modules/directory"
	"github.com/hnakamur/commango/modules/file"
	"github.com/hnakamur/commango/modules/redhat/group"
	"github.com/hnakamur/commango/modules/redhat/service"
	"github.com/hnakamur/commango/modules/redhat/user"
	"github.com/hnakamur/commango/modules/redhat/yum"
	"github.com/hnakamur/commango/modules/shell"
	"github.com/hnakamur/commango/modules/template"
	"github.com/hnakamur/commango/task"
)

func configLogger() {
	config := `
<seelog type="sync">
	<outputs>
		<filter levels="trace">
			<console formatid="skipped"/>
		</filter>
		<filter levels="debug">
			<console formatid="unchanged"/>
		</filter>
		<filter levels="info">
			<console formatid="plain"/>
		</filter>
		<filter levels="warn,error,critical">
			<console formatid="error"/>
		</filter>
	</outputs>
	<formats>
		<format id="error" format="%EscM(31)%Msg%EscM(0)%n"/>
		<format id="plain" format="%Msg%n"/>
		<format id="unchanged" format="%EscM(32)%Msg%EscM(0)%n"/>
		<format id="skipped" format="%EscM(34)%Msg%EscM(0)%n"/>
	</formats>
</seelog>`

	logger, err := log.LoggerFromConfigAsBytes([]byte(config))
	if err != nil {
		panic(err)
	}
	log.ReplaceLogger(logger)
}

const NTP_CONF_TEMPLATE = `# For more information about this file, see the man pages
# ntp.conf(5), ntp_acc(5), ntp_auth(5), ntp_clock(5), ntp_misc(5), ntp_mon(5).

driftfile /var/lib/ntp/drift

# Permit time synchronization with our time source, but do not
# permit the source to query or modify the service on this system.
restrict default kod nomodify notrap nopeer noquery
restrict -6 default kod nomodify notrap nopeer noquery

# Permit all access over the loopback interface.  This could
# be tightened as well, but to do so would effect some of
# the administrative functions.
restrict 127.0.0.1
restrict -6 ::1

# Hosts on local network are less restricted.
#restrict 192.168.1.0 mask 255.255.255.0 nomodify notrap

# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
{{range .ntp_servers}}{{/*
*/}}server {{.}}
{{end}}
#broadcast 192.168.1.255 autokey    # broadcast server
#broadcastclient            # broadcast client
#broadcast 224.0.1.1 autokey        # multicast server
#multicastclient 224.0.1.1      # multicast client
#manycastserver 239.255.254.254     # manycast server
#manycastclient 239.255.254.254 autokey # manycast client

# Undisciplined Local Clock. This is a fake driver intended for backup
# and when no outside source of synchronized time is available.
#server 127.127.1.0 # local clock
#fudge  127.127.1.0 stratum 10

# Enable public key cryptography.
#crypto

includefile /etc/ntp/crypto/pw

# Key file containing the keys and key identifiers used when operating
# with symmetric key cryptography.
keys /etc/ntp/keys

# Specify the key identifiers which are trusted.
#trustedkey 4 8 42

# Specify the key identifier to use with the ntpdc utility.
#requestkey 8

# Specify the key identifier to use with the ntpq utility.
#controlkey 8

# Enable writing of statistics records.
#statistics clockstats cryptostats loopstats peerstats
`

func main() {
	configLogger()

	queue := task.NewTaskQueue()
	queue.Add(
		&shell.Shell{
			Command: "echo hostname=`hostname`",
		},
		&shell.Shell{
			Chdir:   "/tmp",
			Command: "pwd",
		},
		&directory.Directory{
			State: directory.Present,
			Path:  "/tmp/foo/bar",
            Owner: "root",
            Group: "root",
			Mode:  0755,
		},
		&file.Chmod{
			Path:      "/tmp/foo",
			Mode:      0750,
			Recursive: true,
		},
		&template.Template{
			Path:    "/tmp/foo/bar/baz.conf",
			Content: NTP_CONF_TEMPLATE,
			Data: map[string]interface{}{
				"ntp_servers": []string{
					"ntp.nict.jp",
					"ntp.jst.mfeed.ad.jp",
					"ntp.ring.gr.jp",
				},
			},
			Mode: 0644,
		},
		&user.User{
			State: user.Present,
			Name:  "foo",
			Uid:   user.AUTO_UID,
		},
		&group.Group{
			State: group.Present,
			Name:  "bar",
			Gid:   group.AUTO_GID,
		},
		&yum.Yum{
			State: yum.Installed,
			Name:  "ntp",
		},
		&service.Service{
			State:            service.STARTED,
			Name:             "ntpd",
			AutoStartEnabled: true,
		},
	)
	err := queue.RunLoop()
	if err != nil {
		os.Exit(1)
	}
}
<commit_msg>Fix indent.<commit_after>package main

import (
	"os"

	log "github.com/cihub/seelog"
	"github.com/hnakamur/commango/modules/directory"
	"github.com/hnakamur/commango/modules/file"
	"github.com/hnakamur/commango/modules/redhat/group"
	"github.com/hnakamur/commango/modules/redhat/service"
	"github.com/hnakamur/commango/modules/redhat/user"
	"github.com/hnakamur/commango/modules/redhat/yum"
	"github.com/hnakamur/commango/modules/shell"
	"github.com/hnakamur/commango/modules/template"
	"github.com/hnakamur/commango/task"
)

func configLogger() {
	config := `
<seelog type="sync">
	<outputs>
		<filter levels="trace">
			<console formatid="skipped"/>
		</filter>
		<filter levels="debug">
			<console formatid="unchanged"/>
		</filter>
		<filter levels="info">
			<console formatid="plain"/>
		</filter>
		<filter levels="warn,error,critical">
			<console formatid="error"/>
		</filter>
	</outputs>
	<formats>
		<format id="error" format="%EscM(31)%Msg%EscM(0)%n"/>
		<format id="plain" format="%Msg%n"/>
		<format id="unchanged" format="%EscM(32)%Msg%EscM(0)%n"/>
		<format id="skipped" format="%EscM(34)%Msg%EscM(0)%n"/>
	</formats>
</seelog>`

	logger, err := log.LoggerFromConfigAsBytes([]byte(config))
	if err != nil {
		panic(err)
	}
	log.ReplaceLogger(logger)
}

const NTP_CONF_TEMPLATE = `# For more information about this file, see the man pages
# ntp.conf(5), ntp_acc(5), ntp_auth(5), ntp_clock(5), ntp_misc(5), ntp_mon(5).

driftfile /var/lib/ntp/drift

# Permit time synchronization with our time source, but do not
# permit the source to query or modify the service on this system.
restrict default kod nomodify notrap nopeer noquery
restrict -6 default kod nomodify notrap nopeer noquery

# Permit all access over the loopback interface.  This could
# be tightened as well, but to do so would effect some of
# the administrative functions.
restrict 127.0.0.1
restrict -6 ::1

# Hosts on local network are less restricted.
#restrict 192.168.1.0 mask 255.255.255.0 nomodify notrap

# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
{{range .ntp_servers}}{{/*
*/}}server {{.}}
{{end}}
#broadcast 192.168.1.255 autokey    # broadcast server
#broadcastclient            # broadcast client
#broadcast 224.0.1.1 autokey        # multicast server
#multicastclient 224.0.1.1      # multicast client
#manycastserver 239.255.254.254     # manycast server
#manycastclient 239.255.254.254 autokey # manycast client

# Undisciplined Local Clock. This is a fake driver intended for backup
# and when no outside source of synchronized time is available.
#server 127.127.1.0 # local clock
#fudge  127.127.1.0 stratum 10

# Enable public key cryptography.
#crypto

includefile /etc/ntp/crypto/pw

# Key file containing the keys and key identifiers used when operating
# with symmetric key cryptography.
keys /etc/ntp/keys

# Specify the key identifiers which are trusted.
#trustedkey 4 8 42

# Specify the key identifier to use with the ntpdc utility.
#requestkey 8

# Specify the key identifier to use with the ntpq utility.
#controlkey 8

# Enable writing of statistics records.
#statistics clockstats cryptostats loopstats peerstats
`

func main() {
	configLogger()

	queue := task.NewTaskQueue()
	queue.Add(
		&shell.Shell{
			Command: "echo hostname=`hostname`",
		},
		&shell.Shell{
			Chdir:   "/tmp",
			Command: "pwd",
		},
		&directory.Directory{
			State: directory.Present,
			Path:  "/tmp/foo/bar",
			Owner: "root",
			Group: "root",
			Mode:  0755,
		},
		&file.Chmod{
			Path:      "/tmp/foo",
			Mode:      0750,
			Recursive: true,
		},
		&template.Template{
			Path:    "/tmp/foo/bar/baz.conf",
			Content: NTP_CONF_TEMPLATE,
			Data: map[string]interface{}{
				"ntp_servers": []string{
					"ntp.nict.jp",
					"ntp.jst.mfeed.ad.jp",
					"ntp.ring.gr.jp",
				},
			},
			Mode: 0644,
		},
		&user.User{
			State: user.Present,
			Name:  "foo",
			Uid:   user.AUTO_UID,
		},
		&group.Group{
			State: group.Present,
			Name:  "bar",
			Gid:   group.AUTO_GID,
		},
		&yum.Yum{
			State: yum.Installed,
			Name:  "ntp",
		},
		&service.Service{
			State:            service.STARTED,
			Name:             "ntpd",
			AutoStartEnabled: true,
		},
	)
	err := queue.RunLoop()
	if err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package proxy

import (
	"github.com/coyove/goflyway/pkg/logg"
	"github.com/coyove/goflyway/pkg/lookup"
	"github.com/coyove/goflyway/pkg/lru"
	"github.com/coyove/tcpmux"

	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strconv"
	"strings"
	"time"
)

type ServerConfig struct {
	Throttling    int64
	ThrottlingMax int64
	DisableUDP    bool
	ProxyPassAddr string

	Users map[string]UserConfig

	*Cipher
}

// for multi-users server, not implemented yet
type UserConfig struct {
	Auth          string
	Throttling    int64
	ThrottlingMax int64
}

type ProxyUpstream struct {
	tp            *http.Transport
	rp            http.Handler
	blacklist     *lru.Cache
	trustedTokens map[string]bool
	rkeyHeader    string

	Localaddr string

	*ServerConfig
}

func (proxy *ProxyUpstream) auth(auth string) bool {
	if _, existed := proxy.Users[auth]; existed {
		// we don't have multi-user mode currently
		return true
	}

	return false
}

func (proxy *ProxyUpstream) getIOConfig(auth string) IOConfig {
	var ioc IOConfig
	if proxy.Throttling > 0 {
		ioc.Bucket = NewTokenBucket(proxy.Throttling, proxy.ThrottlingMax)
	}
	return ioc
}

func (proxy *ProxyUpstream) Write(w http.ResponseWriter, key, p []byte, code int) (n int, err error) {
	if ctr := proxy.Cipher.getCipherStream(key); ctr != nil {
		ctr.XorBuffer(p)
	}

	w.WriteHeader(code)
	return w.Write(p)
}

func (proxy *ProxyUpstream) hijack(w http.ResponseWriter) net.Conn {
	hij, ok := w.(http.Hijacker)
	if !ok {
		logg.E("webserver doesn't support hijacking")
		return nil
	}

	conn, _, err := hij.Hijack()
	if err != nil {
		logg.E("hijacking: ", err.Error())
		return nil
	}

	return conn
}

func (proxy *ProxyUpstream) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	replySomething := func() {
		if proxy.rp == nil {
			round := proxy.Rand.Intn(32) + 32
			buf := make([]byte, 2048)
			for r := 0; r < round; r++ {
				ln := proxy.Rand.Intn(1024) + 1024

				for i := 0; i < ln; i++ {
					buf[i] = byte(proxy.Rand.Intn(256))
				}

				w.Write(buf[:ln])
				time.Sleep(time.Duration(proxy.Rand.Intn(100)) * time.Millisecond)
			}
		} else {
			proxy.rp.ServeHTTP(w, r)
		}
	}

	addr, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		logg.W("unknown address: ", r.RemoteAddr)
		replySomething()
		return
	}

	rkey := r.Header.Get(proxy.rkeyHeader)
	options, rkeybuf, authbuf := proxy.Cipher.ReverseIV(rkey)

	if rkeybuf == nil {
		logg.D("cannot find header, check your client's key, from: ", addr)
		proxy.blacklist.Add(addr, nil)
		replySomething()
		return
	}

	var auth string
	if proxy.Users != nil {
		if authbuf == nil || string(authbuf) == "" || !proxy.auth(string(authbuf)) {
			logg.W("user auth failed, from: ", addr)
			return
		}

		auth = string(authbuf)
	}

	if options == 0 {
		r := isTrustedToken("unlock", rkeybuf)

		if r == -1 {
			logg.W("someone is using an old token: ", addr)
			proxy.blacklist.Add(addr, nil)
			replySomething()
			return
		}

		if r == 1 {
			proxy.blacklist.Remove(addr)
			logg.L("unlock request accepted from: ", addr)
			return
		}
	}

	if h, _ := proxy.blacklist.GetHits(addr); h > invalidRequestRetry {
		logg.D("repeated access using invalid key from: ", addr)
		// replySomething()
		// return
	}

	if (options & doDNS) > 0 {
		host := string(rkeybuf)
		ip, err := lookup.LookupIPv4(host)
		if err != nil {
			logg.W(err)
			ip = "127.0.0.1"
		}

		logg.D("dns: ", host, " ", ip)
		w.Header().Add(dnsRespHeader, ip)
		w.WriteHeader(200)

	} else if options.IsSet(doConnect) {
		host := proxy.Cipher.DecryptDecompress(stripURI(r.RequestURI), rkeybuf...)
		if host == "" {
			logg.W("we had a valid rkey, but invalid host, from: ", addr)
			replySomething()
			return
		}

		logg.D("CONNECT ", host)
		downstreamConn := proxy.hijack(w)
		if downstreamConn == nil {
			return
		}

		ioc := proxy.getIOConfig(auth)
		ioc.Partial = options.IsSet(doPartial)

		var targetSiteConn net.Conn
		var err error

		if options.IsSet(doUDPRelay) {
			if proxy.DisableUDP {
				logg.W("client is trying to send UDP data but we disabled it")
				downstreamConn.Close()
				return
			}

			uaddr, _ := net.ResolveUDPAddr("udp", host)

			var rconn *net.UDPConn
			rconn, err = net.DialUDP("udp", nil, uaddr)
			targetSiteConn = &udpBridgeConn{
				UDPConn: rconn,
				udpSrc:  uaddr,
			}
			// rconn.Write([]byte{6, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 98, 97, 105, 100, 117, 3, 99, 111, 109, 0, 0, 1, 0, 1})
		} else {
			targetSiteConn, err = net.Dial("tcp", host)
		}

		if err != nil {
			logg.E(err)
			downstreamConn.Close()
			return
		}

		var p string
		if options.IsSet(doWebSocket) {
			ioc.WSCtrl = wsServer
			p = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: upgrade\r\nSec-WebSocket-Accept: " + (rkey + rkey)[4:32] + "\r\n\r\n"
		} else {
			p = "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nDate: " + time.Now().UTC().Format(time.RFC1123) + "\r\n\r\n"
		}

		downstreamConn.Write([]byte(p))
		go proxy.Cipher.IO.Bridge(downstreamConn, targetSiteConn, rkeybuf, ioc)
	} else if options.IsSet(doForward) {
		if !proxy.decryptRequest(r, rkeybuf) {
			replySomething()
			return
		}

		logg.D(r.Method, " ", r.URL.String())

		r.Header.Del(proxy.rkeyHeader)
		resp, err := proxy.tp.RoundTrip(r)
		if err != nil {
			logg.E("HTTP forward: ", r.URL, ", ", err)
			proxy.Write(w, rkeybuf, []byte(err.Error()), http.StatusInternalServerError)
			return
		}

		if resp.StatusCode >= 400 {
			logg.D("[", resp.Status, "] - ", r.URL)
		}

		copyHeaders(w.Header(), resp.Header, proxy.Cipher, true, rkeybuf)
		w.WriteHeader(resp.StatusCode)

		if nr, err := proxy.Cipher.IO.Copy(w, resp.Body, rkeybuf, proxy.getIOConfig(auth)); err != nil {
			logg.E("copy ", nr, " bytes: ", err)
		}

		tryClose(resp.Body)
	} else {
		proxy.blacklist.Add(addr, nil)
		replySomething()
	}
}

func (proxy *ProxyUpstream) Start() error {
	ln, err := tcpmux.Listen(proxy.Localaddr, true)
	if err != nil {
		return err
	}

	proxy.Cipher.IO.Ob = ln.(*tcpmux.ListenPool)
	return http.Serve(ln, proxy)
}

func NewServer(addr string, config *ServerConfig) *ProxyUpstream {
	proxy := &ProxyUpstream{
		tp: &http.Transport{TLSClientConfig: tlsSkip},

		ServerConfig:  config,
		blacklist:     lru.NewCache(128),
		trustedTokens: make(map[string]bool),
		rkeyHeader:    "X-" + config.Cipher.Alias,
	}

	tcpmux.Version = checksum1b([]byte(config.Cipher.Alias)) | 0x80

	if config.ProxyPassAddr != "" {
		if strings.HasPrefix(config.ProxyPassAddr, "http") {
			u, err := url.Parse(config.ProxyPassAddr)
			if err != nil {
				logg.F(err)
				return nil
			}

			proxy.rp = httputil.NewSingleHostReverseProxy(u)
		} else {
			proxy.rp = http.FileServer(http.Dir(config.ProxyPassAddr))
		}
	}

	if port, lerr := strconv.Atoi(addr); lerr == nil {
		addr = (&net.TCPAddr{IP: net.IPv4zero, Port: port}).String()
	}

	proxy.Localaddr = addr
	return proxy
}
<commit_msg>404 page<commit_after>package proxy

import (
	"github.com/coyove/goflyway/pkg/logg"
	"github.com/coyove/goflyway/pkg/lookup"
	"github.com/coyove/goflyway/pkg/lru"
	"github.com/coyove/tcpmux"

	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"strconv"
	"strings"
	"time"
)

type ServerConfig struct {
	Throttling    int64
	ThrottlingMax int64
	DisableUDP    bool
	ProxyPassAddr string

	Users map[string]UserConfig

	*Cipher
}

// for multi-users server, not implemented yet
type UserConfig struct {
	Auth          string
	Throttling    int64
	ThrottlingMax int64
}

type ProxyUpstream struct {
	tp            *http.Transport
	rp            http.Handler
	blacklist     *lru.Cache
	trustedTokens map[string]bool
	rkeyHeader    string

	Localaddr string

	*ServerConfig
}

func (proxy *ProxyUpstream) auth(auth string) bool {
	if _, existed := proxy.Users[auth]; existed {
		// we don't have multi-user mode currently
		return true
	}

	return false
}

func (proxy *ProxyUpstream) getIOConfig(auth string) IOConfig {
	var ioc IOConfig
	if proxy.Throttling > 0 {
		ioc.Bucket = NewTokenBucket(proxy.Throttling, proxy.ThrottlingMax)
	}
	return ioc
}

func (proxy *ProxyUpstream) Write(w http.ResponseWriter, key, p []byte, code int) (n int, err error) {
	if ctr := proxy.Cipher.getCipherStream(key); ctr != nil {
		ctr.XorBuffer(p)
	}

	w.WriteHeader(code)
	return w.Write(p)
}

func (proxy *ProxyUpstream) hijack(w http.ResponseWriter) net.Conn {
	hij, ok := w.(http.Hijacker)
	if !ok {
		logg.E("webserver doesn't support hijacking")
		return nil
	}

	conn, _, err := hij.Hijack()
	if err != nil {
		logg.E("hijacking: ", err.Error())
		return nil
	}

	return conn
}

func (proxy *ProxyUpstream) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	replySomething := func() {
		if proxy.rp == nil {
			w.WriteHeader(404)
			w.Write([]byte(`
<title>404 Not Found</title>

404 Not Found

nginx

`))
		} else {
			proxy.rp.ServeHTTP(w, r)
		}
	}

	addr, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		logg.W("unknown address: ", r.RemoteAddr)
		replySomething()
		return
	}

	rkey := r.Header.Get(proxy.rkeyHeader)
	options, rkeybuf, authbuf := proxy.Cipher.ReverseIV(rkey)

	if rkeybuf == nil {
		logg.D("cannot find header, check your client's key, from: ", addr)
		proxy.blacklist.Add(addr, nil)
		replySomething()
		return
	}

	var auth string
	if proxy.Users != nil {
		if authbuf == nil || string(authbuf) == "" || !proxy.auth(string(authbuf)) {
			logg.W("user auth failed, from: ", addr)
			return
		}

		auth = string(authbuf)
	}

	if options == 0 {
		r := isTrustedToken("unlock", rkeybuf)

		if r == -1 {
			logg.W("someone is using an old token: ", addr)
			proxy.blacklist.Add(addr, nil)
			replySomething()
			return
		}

		if r == 1 {
			proxy.blacklist.Remove(addr)
			logg.L("unlock request accepted from: ", addr)
			return
		}
	}

	if h, _ := proxy.blacklist.GetHits(addr); h > invalidRequestRetry {
		logg.D("repeated access using invalid key from: ", addr)
		// replySomething()
		// return
	}

	if (options & doDNS) > 0 {
		host := string(rkeybuf)
		ip, err := lookup.LookupIPv4(host)
		if err != nil {
			logg.W(err)
			ip = "127.0.0.1"
		}

		logg.D("dns: ", host, " ", ip)
		w.Header().Add(dnsRespHeader, ip)
		w.WriteHeader(200)

	} else if options.IsSet(doConnect) {
		host := proxy.Cipher.DecryptDecompress(stripURI(r.RequestURI), rkeybuf...)
		if host == "" {
			logg.W("we had a valid rkey, but invalid host, from: ", addr)
			replySomething()
			return
		}

		logg.D("CONNECT ", host)
		downstreamConn := proxy.hijack(w)
		if downstreamConn == nil {
			return
		}

		ioc := proxy.getIOConfig(auth)
		ioc.Partial = options.IsSet(doPartial)

		var targetSiteConn net.Conn
		var err error

		if options.IsSet(doUDPRelay) {
			if proxy.DisableUDP {
				logg.W("client is trying to send UDP data but we disabled it")
				downstreamConn.Close()
				return
			}

			uaddr, _ := net.ResolveUDPAddr("udp", host)

			var rconn *net.UDPConn
			rconn, err = net.DialUDP("udp", nil, uaddr)
			targetSiteConn = &udpBridgeConn{
				UDPConn: rconn,
				udpSrc:  uaddr,
			}
			// rconn.Write([]byte{6, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 98, 97, 105, 100, 117, 3, 99, 111, 109, 0, 0, 1, 0, 1})
		} else {
			targetSiteConn, err = net.Dial("tcp", host)
		}

		if err != nil {
			logg.E(err)
			downstreamConn.Close()
			return
		}

		var p string
		if options.IsSet(doWebSocket) {
			ioc.WSCtrl = wsServer
			p = "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: upgrade\r\nSec-WebSocket-Accept: " + (rkey + rkey)[4:32] + "\r\n\r\n"
		} else {
			p = "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nDate: " + time.Now().UTC().Format(time.RFC1123) + "\r\n\r\n"
		}

		downstreamConn.Write([]byte(p))
		go proxy.Cipher.IO.Bridge(downstreamConn, targetSiteConn, rkeybuf, ioc)
	} else if options.IsSet(doForward) {
		if !proxy.decryptRequest(r, rkeybuf) {
			replySomething()
			return
		}

		logg.D(r.Method, " ", r.URL.String())

		r.Header.Del(proxy.rkeyHeader)
		resp, err := proxy.tp.RoundTrip(r)
		if err != nil {
			logg.E("HTTP forward: ", r.URL, ", ", err)
			proxy.Write(w, rkeybuf, []byte(err.Error()), http.StatusInternalServerError)
			return
		}

		if resp.StatusCode >= 400 {
			logg.D("[", resp.Status, "] - ", r.URL)
		}

		copyHeaders(w.Header(), resp.Header, proxy.Cipher, true, rkeybuf)
		w.WriteHeader(resp.StatusCode)

		if nr, err := proxy.Cipher.IO.Copy(w, resp.Body, rkeybuf, proxy.getIOConfig(auth)); err != nil {
			logg.E("copy ", nr, " bytes: ", err)
		}

		tryClose(resp.Body)
	} else {
		proxy.blacklist.Add(addr, nil)
		replySomething()
	}
}

func (proxy *ProxyUpstream) Start() error {
	ln, err := tcpmux.Listen(proxy.Localaddr, true)
	if err != nil {
		return err
	}

	proxy.Cipher.IO.Ob = ln.(*tcpmux.ListenPool)
	return http.Serve(ln, proxy)
}

func NewServer(addr string, config *ServerConfig) *ProxyUpstream {
	proxy := &ProxyUpstream{
		tp: &http.Transport{TLSClientConfig: tlsSkip},

		ServerConfig:  config,
		blacklist:     lru.NewCache(128),
		trustedTokens: make(map[string]bool),
		rkeyHeader:    "X-" + config.Cipher.Alias,
	}

	tcpmux.Version = checksum1b([]byte(config.Cipher.Alias)) | 0x80

	if config.ProxyPassAddr != "" {
		if strings.HasPrefix(config.ProxyPassAddr, "http") {
			u, err := url.Parse(config.ProxyPassAddr)
			if err != nil {
				logg.F(err)
				return nil
			}

			proxy.rp = httputil.NewSingleHostReverseProxy(u)
		} else {
			proxy.rp = http.FileServer(http.Dir(config.ProxyPassAddr))
		}
	}

	if port, lerr := strconv.Atoi(addr); lerr == nil {
		addr = (&net.TCPAddr{IP: net.IPv4zero, Port: port}).String()
	}

	proxy.Localaddr = addr
	return proxy
}
<|endoftext|>