<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements a parser test harness. The files in the testdata
// directory are parsed and the errors reported are compared against the
// error messages expected in the test files. The test files must end in
// .src rather than .go so that they are not disturbed by gofmt runs.
//
// Expected errors are indicated in the test files by putting a comment
// of the form /* ERROR "rx" */ immediately following an offending token.
// The harness will verify that an error matching the regular expression
// rx is reported at that source position.
//
// For instance, the following test file indicates that a "not declared"
// error should be reported for the undeclared variable x:
//
//	package p
//	func f() {
//		_ = x /* ERROR "not declared" */ + 1
//	}

package parser

import (
	"go/scanner"
	"go/token"
	"io/ioutil"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
)

const testdata = "testdata"

var fsetErrs *token.FileSet

// getFile assumes that each filename occurs at most once
func getFile(filename string) (file *token.File) {
	fsetErrs.Iterate(func(f *token.File) bool {
		if f.Name() == filename {
			if file != nil {
				panic(filename + " used multiple times")
			}
			file = f
		}
		return true
	})
	return file
}

func getPos(filename string, offset int) token.Pos {
	if f := getFile(filename); f != nil {
		return f.Pos(offset)
	}
	return token.NoPos
}

// ERROR comments must be of the form /* ERROR "rx" */ and rx is
// a regular expression that matches the expected error message.
// The special form /* ERROR HERE "rx" */ must be used for error
// messages that appear immediately after a token, rather than at
// a token's position.
//
var errRx = regexp.MustCompile(`^/\* *ERROR *(HERE)? *"([^"]*)" *\*/$`)

// expectedErrors collects the regular expressions of ERROR comments found
// in files and returns them as a map of error positions to error messages.
//
func expectedErrors(t *testing.T, filename string, src []byte) map[token.Pos]string {
	errors := make(map[token.Pos]string)

	var s scanner.Scanner
	// file was parsed already - do not add it again to the file
	// set otherwise the position information returned here will
	// not match the position information collected by the parser
	s.Init(getFile(filename), src, nil, scanner.ScanComments)
	var prev token.Pos // position of last non-comment, non-semicolon token
	var here token.Pos // position immediately after the token at position prev

	for {
		pos, tok, lit := s.Scan()
		switch tok {
		case token.EOF:
			return errors
		case token.COMMENT:
			s := errRx.FindStringSubmatch(lit)
			if len(s) == 3 {
				pos := prev
				if s[1] == "HERE" {
					pos = here
				}
				errors[pos] = string(s[2])
			}
		default:
			prev = pos
			var l int // token length
			if tok.IsLiteral() {
				l = len(lit)
			} else {
				l = len(tok.String())
			}
			here = prev + token.Pos(l)
		}
	}
}

// compareErrors compares the map of expected error messages with the list
// of found errors and reports discrepancies.
//
func compareErrors(t *testing.T, expected map[token.Pos]string, found scanner.ErrorList) {
	for _, error := range found {
		// error.Pos is a token.Position, but we want
		// a token.Pos so we can do a map lookup
		pos := getPos(error.Pos.Filename, error.Pos.Offset)
		if msg, found := expected[pos]; found {
			// we expect a message at pos; check if it matches
			rx, err := regexp.Compile(msg)
			if err != nil {
				t.Errorf("%s: %v", error.Pos, err)
				continue
			}
			if match := rx.MatchString(error.Msg); !match {
				t.Errorf("%s: %q does not match %q", error.Pos, error.Msg, msg)
				continue
			}
			// we have a match - eliminate this error
			delete(expected, pos)
		} else {
			// To keep in mind when analyzing failed test output:
			// If the same error position occurs multiple times in errors,
			// this message will be triggered (because the first error at
			// the position removes this position from the expected errors).
			t.Errorf("%s: unexpected error: %s", error.Pos, error.Msg)
		}
	}

	// there should be no expected errors left
	if len(expected) > 0 {
		t.Errorf("%d errors not reported:", len(expected))
		for pos, msg := range expected {
			t.Errorf("%s: %s\n", fsetErrs.Position(pos), msg)
		}
	}
}

func checkErrors(t *testing.T, filename string, input interface{}) {
	src, err := readSource(filename, input)
	if err != nil {
		t.Error(err)
		return
	}

	_, err = ParseFile(fsetErrs, filename, src, DeclarationErrors|AllErrors)
	found, ok := err.(scanner.ErrorList)
	if err != nil && !ok {
		t.Error(err)
		return
	}
	found.RemoveMultiples()

	// we are expecting the following errors
	// (collect these after parsing a file so that it is found in the file set)
	expected := expectedErrors(t, filename, src)

	// verify errors returned by the parser
	compareErrors(t, expected, found)
}

func TestErrors(t *testing.T) {
	fsetErrs = token.NewFileSet()
	list, err := ioutil.ReadDir(testdata)
	if err != nil {
		t.Fatal(err)
	}
	for _, fi := range list {
		name := fi.Name()
		if !fi.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".src") {
			checkErrors(t, filepath.Join(testdata, name), nil)
		}
	}
}
<commit_msg>go/parser: initialize file set even in only some tests are run<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file implements a parser test harness. The files in the testdata
// directory are parsed and the errors reported are compared against the
// error messages expected in the test files. The test files must end in
// .src rather than .go so that they are not disturbed by gofmt runs.
//
// Expected errors are indicated in the test files by putting a comment
// of the form /* ERROR "rx" */ immediately following an offending token.
// The harness will verify that an error matching the regular expression
// rx is reported at that source position.
//
// For instance, the following test file indicates that a "not declared"
// error should be reported for the undeclared variable x:
//
//	package p
//	func f() {
//		_ = x /* ERROR "not declared" */ + 1
//	}

package parser

import (
	"go/scanner"
	"go/token"
	"io/ioutil"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
)

const testdata = "testdata"

var fsetErrs = token.NewFileSet()

// getFile assumes that each filename occurs at most once
func getFile(filename string) (file *token.File) {
	fsetErrs.Iterate(func(f *token.File) bool {
		if f.Name() == filename {
			if file != nil {
				panic(filename + " used multiple times")
			}
			file = f
		}
		return true
	})
	return file
}

func getPos(filename string, offset int) token.Pos {
	if f := getFile(filename); f != nil {
		return f.Pos(offset)
	}
	return token.NoPos
}

// ERROR comments must be of the form /* ERROR "rx" */ and rx is
// a regular expression that matches the expected error message.
// The special form /* ERROR HERE "rx" */ must be used for error
// messages that appear immediately after a token, rather than at
// a token's position.
//
var errRx = regexp.MustCompile(`^/\* *ERROR *(HERE)? *"([^"]*)" *\*/$`)

// expectedErrors collects the regular expressions of ERROR comments found
// in files and returns them as a map of error positions to error messages.
//
func expectedErrors(t *testing.T, filename string, src []byte) map[token.Pos]string {
	errors := make(map[token.Pos]string)

	var s scanner.Scanner
	// file was parsed already - do not add it again to the file
	// set otherwise the position information returned here will
	// not match the position information collected by the parser
	s.Init(getFile(filename), src, nil, scanner.ScanComments)
	var prev token.Pos // position of last non-comment, non-semicolon token
	var here token.Pos // position immediately after the token at position prev

	for {
		pos, tok, lit := s.Scan()
		switch tok {
		case token.EOF:
			return errors
		case token.COMMENT:
			s := errRx.FindStringSubmatch(lit)
			if len(s) == 3 {
				pos := prev
				if s[1] == "HERE" {
					pos = here
				}
				errors[pos] = string(s[2])
			}
		default:
			prev = pos
			var l int // token length
			if tok.IsLiteral() {
				l = len(lit)
			} else {
				l = len(tok.String())
			}
			here = prev + token.Pos(l)
		}
	}
}

// compareErrors compares the map of expected error messages with the list
// of found errors and reports discrepancies.
//
func compareErrors(t *testing.T, expected map[token.Pos]string, found scanner.ErrorList) {
	for _, error := range found {
		// error.Pos is a token.Position, but we want
		// a token.Pos so we can do a map lookup
		pos := getPos(error.Pos.Filename, error.Pos.Offset)
		if msg, found := expected[pos]; found {
			// we expect a message at pos; check if it matches
			rx, err := regexp.Compile(msg)
			if err != nil {
				t.Errorf("%s: %v", error.Pos, err)
				continue
			}
			if match := rx.MatchString(error.Msg); !match {
				t.Errorf("%s: %q does not match %q", error.Pos, error.Msg, msg)
				continue
			}
			// we have a match - eliminate this error
			delete(expected, pos)
		} else {
			// To keep in mind when analyzing failed test output:
			// If the same error position occurs multiple times in errors,
			// this message will be triggered (because the first error at
			// the position removes this position from the expected errors).
			t.Errorf("%s: unexpected error: %s", error.Pos, error.Msg)
		}
	}

	// there should be no expected errors left
	if len(expected) > 0 {
		t.Errorf("%d errors not reported:", len(expected))
		for pos, msg := range expected {
			t.Errorf("%s: %s\n", fsetErrs.Position(pos), msg)
		}
	}
}

func checkErrors(t *testing.T, filename string, input interface{}) {
	src, err := readSource(filename, input)
	if err != nil {
		t.Error(err)
		return
	}

	_, err = ParseFile(fsetErrs, filename, src, DeclarationErrors|AllErrors)
	found, ok := err.(scanner.ErrorList)
	if err != nil && !ok {
		t.Error(err)
		return
	}
	found.RemoveMultiples()

	// we are expecting the following errors
	// (collect these after parsing a file so that it is found in the file set)
	expected := expectedErrors(t, filename, src)

	// verify errors returned by the parser
	compareErrors(t, expected, found)
}

func TestErrors(t *testing.T) {
	list, err := ioutil.ReadDir(testdata)
	if err != nil {
		t.Fatal(err)
	}
	for _, fi := range list {
		name := fi.Name()
		if !fi.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".src") {
			checkErrors(t, filepath.Join(testdata, name), nil)
		}
	}
}
<|endoftext|>


<commit_before>package jobserver

import (
	"fmt"
	"time"

	"github.com/pachyderm/pachyderm/src/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/jobserver/run"
	"github.com/pachyderm/pachyderm/src/pps/persist"
	"go.pedge.io/google-protobuf"
	"go.pedge.io/proto/rpclog"
	"go.pedge.io/proto/time"
	"golang.org/x/net/context"
	kube "k8s.io/kubernetes/pkg/client/unversioned"
)

type apiServer struct {
	protorpclog.Logger
	persistAPIClient persist.APIClient
	client           *kube.Client
}

func newAPIServer(
	persistAPIClient persist.APIClient,
	client *kube.Client,
) *apiServer {
	return &apiServer{
		protorpclog.NewLogger("pachyderm.pps.JobAPI"),
		persistAPIClient,
		client,
	}
}

func (a *apiServer) CreateJob(ctx context.Context, request *pps.CreateJobRequest) (response *pps.Job, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	persistJobInfo := &persist.JobInfo{
		Input:        request.Input,
		OutputParent: request.OutputParent,
	}
	if request.GetTransform() != nil {
		persistJobInfo.Spec = &persist.JobInfo_Transform{
			Transform: request.GetTransform(),
		}
	} else if request.GetPipeline() != nil {
		persistJobInfo.Spec = &persist.JobInfo_PipelineName{
			PipelineName: request.GetPipeline().Name,
		}
	} else {
		return nil, fmt.Errorf("pachyderm.pps.jobserver: both transform and pipeline are not set on %v", request)
	}
	persistJobInfo.JobId = uuid.NewWithoutDashes()
	persistJobInfo.CreatedAt = prototime.TimeToTimestamp(time.Now())
	if err := jobserverrun.StartJob(a.client, persistJobInfo); err != nil {
		return nil, err
	}
	_, err = a.persistAPIClient.CreateJobInfo(ctx, persistJobInfo)
	if err != nil {
		return nil, err
	}
	return &pps.Job{
		Id: persistJobInfo.JobId,
	}, nil
}

func (a *apiServer) InspectJob(ctx context.Context, request *pps.InspectJobRequest) (response *pps.JobInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	persistJobInfo, err := a.persistAPIClient.GetJobInfo(ctx, request.Job)
	if err != nil {
		return nil, err
	}
	return a.persistJobInfoToJobInfo(ctx, persistJobInfo)
}

func (a *apiServer) ListJob(ctx context.Context, request *pps.ListJobRequest) (response *pps.JobInfos, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	var persistJobInfos *persist.JobInfos
	persistJobInfos, err = a.persistAPIClient.ListJobInfos(ctx, request)
	if err != nil {
		return nil, err
	}
	jobInfos := make([]*pps.JobInfo, len(persistJobInfos.JobInfo))
	for i, persistJobInfo := range persistJobInfos.JobInfo {
		jobInfo, err := a.persistJobInfoToJobInfo(ctx, persistJobInfo)
		if err != nil {
			return nil, err
		}
		jobInfos[i] = jobInfo
	}
	return &pps.JobInfos{
		JobInfo: jobInfos,
	}, nil
}

func (a *apiServer) GetJobLogs(request *pps.GetJobLogsRequest, responseServer pps.JobAPI_GetJobLogsServer) (err error) {
	// TODO: filter by output stream
	persistJobLogs, err := a.persistAPIClient.GetJobLogs(context.Background(), request.Job)
	if err != nil {
		return err
	}
	for _, persistJobLog := range persistJobLogs.JobLog {
		if request.OutputStream == pps.OutputStream_OUTPUT_STREAM_ALL || persistJobLog.OutputStream == request.OutputStream {
			if err := responseServer.Send(&google_protobuf.BytesValue{Value: persistJobLog.Value}); err != nil {
				return err
			}
		}
	}
	return nil
}

// TODO: bulk get
func (a *apiServer) persistJobInfoToJobInfo(ctx context.Context, persistJobInfo *persist.JobInfo) (*pps.JobInfo, error) {
	job := &pps.Job{Id: persistJobInfo.JobId}
	persistJobStatuses, err := a.persistAPIClient.GetJobStatuses(ctx, job)
	if err != nil {
		return nil, err
	}
	persistJobOutput, err := a.persistAPIClient.GetJobOutput(ctx, job)
	if err != nil {
		return nil, err
	}
	jobInfo := &pps.JobInfo{
		Job:   job,
		Input: persistJobInfo.Input,
	}
	if persistJobInfo.GetTransform() != nil {
		jobInfo.Spec = &pps.JobInfo_Transform{
			Transform: persistJobInfo.GetTransform(),
		}
	}
	if persistJobInfo.GetPipelineName() != "" {
		jobInfo.Spec = &pps.JobInfo_Pipeline{
			Pipeline: &pps.Pipeline{
				Name: persistJobInfo.GetPipelineName(),
			},
		}
	}
	jobInfo.JobStatus = make([]*pps.JobStatus, len(persistJobStatuses.JobStatus))
	for i, persistJobStatus := range persistJobStatuses.JobStatus {
		jobInfo.JobStatus[i] = &pps.JobStatus{
			Type:      persistJobStatus.Type,
			Timestamp: persistJobStatus.Timestamp,
			Message:   persistJobStatus.Message,
		}
	}
	if persistJobOutput != nil {
		jobInfo.Output = persistJobOutput.Output
	}
	return jobInfo, nil
}
<commit_msg>Move run code into api_server.<commit_after>package jobserver

import (
	"fmt"
	"time"

	"go.pedge.io/google-protobuf"
	"go.pedge.io/proto/rpclog"
	"go.pedge.io/proto/time"
	"golang.org/x/net/context"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apis/extensions"
	kube "k8s.io/kubernetes/pkg/client/unversioned"

	"github.com/pachyderm/pachyderm/src/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/persist"
)

type apiServer struct {
	protorpclog.Logger
	persistAPIClient persist.APIClient
	kubeClient       *kube.Client
}

func newAPIServer(
	persistAPIClient persist.APIClient,
	kubeClient *kube.Client,
) *apiServer {
	return &apiServer{
		protorpclog.NewLogger("pachyderm.pps.JobAPI"),
		persistAPIClient,
		kubeClient,
	}
}

func (a *apiServer) CreateJob(ctx context.Context, request *pps.CreateJobRequest) (response *pps.Job, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	persistJobInfo := &persist.JobInfo{
		Input:        request.Input,
		OutputParent: request.OutputParent,
	}
	if request.GetTransform() != nil {
		persistJobInfo.Spec = &persist.JobInfo_Transform{
			Transform: request.GetTransform(),
		}
	} else if request.GetPipeline() != nil {
		persistJobInfo.Spec = &persist.JobInfo_PipelineName{
			PipelineName: request.GetPipeline().Name,
		}
	} else {
		return nil, fmt.Errorf("pachyderm.pps.jobserver: both transform and pipeline are not set on %v", request)
	}
	persistJobInfo.JobId = uuid.NewWithoutDashes()
	persistJobInfo.CreatedAt = prototime.TimeToTimestamp(time.Now())
	if _, err := a.kubeClient.Jobs(api.NamespaceDefault).Create(job(persistJobInfo)); err != nil {
		return nil, err
	}
	_, err = a.persistAPIClient.CreateJobInfo(ctx, persistJobInfo)
	if err != nil {
		return nil, err
	}
	return &pps.Job{
		Id: persistJobInfo.JobId,
	}, nil
}

func (a *apiServer) InspectJob(ctx context.Context, request *pps.InspectJobRequest) (response *pps.JobInfo, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	persistJobInfo, err := a.persistAPIClient.GetJobInfo(ctx, request.Job)
	if err != nil {
		return nil, err
	}
	return a.persistJobInfoToJobInfo(ctx, persistJobInfo)
}

func (a *apiServer) ListJob(ctx context.Context, request *pps.ListJobRequest) (response *pps.JobInfos, err error) {
	defer func(start time.Time) { a.Log(request, response, err, time.Since(start)) }(time.Now())
	var persistJobInfos *persist.JobInfos
	persistJobInfos, err = a.persistAPIClient.ListJobInfos(ctx, request)
	if err != nil {
		return nil, err
	}
	jobInfos := make([]*pps.JobInfo, len(persistJobInfos.JobInfo))
	for i, persistJobInfo := range persistJobInfos.JobInfo {
		jobInfo, err := a.persistJobInfoToJobInfo(ctx, persistJobInfo)
		if err != nil {
			return nil, err
		}
		jobInfos[i] = jobInfo
	}
	return &pps.JobInfos{
		JobInfo: jobInfos,
	}, nil
}

func (a *apiServer) GetJobLogs(request *pps.GetJobLogsRequest, responseServer pps.JobAPI_GetJobLogsServer) (err error) {
	// TODO: filter by output stream
	persistJobLogs, err := a.persistAPIClient.GetJobLogs(context.Background(), request.Job)
	if err != nil {
		return err
	}
	for _, persistJobLog := range persistJobLogs.JobLog {
		if request.OutputStream == pps.OutputStream_OUTPUT_STREAM_ALL || persistJobLog.OutputStream == request.OutputStream {
			if err := responseServer.Send(&google_protobuf.BytesValue{Value: persistJobLog.Value}); err != nil {
				return err
			}
		}
	}
	return nil
}

// TODO: bulk get
func (a *apiServer) persistJobInfoToJobInfo(ctx context.Context, persistJobInfo *persist.JobInfo) (*pps.JobInfo, error) {
	job := &pps.Job{Id: persistJobInfo.JobId}
	persistJobStatuses, err := a.persistAPIClient.GetJobStatuses(ctx, job)
	if err != nil {
		return nil, err
	}
	persistJobOutput, err := a.persistAPIClient.GetJobOutput(ctx, job)
	if err != nil {
		return nil, err
	}
	jobInfo := &pps.JobInfo{
		Job:   job,
		Input: persistJobInfo.Input,
	}
	if persistJobInfo.GetTransform() != nil {
		jobInfo.Spec = &pps.JobInfo_Transform{
			Transform: persistJobInfo.GetTransform(),
		}
	}
	if persistJobInfo.GetPipelineName() != "" {
		jobInfo.Spec = &pps.JobInfo_Pipeline{
			Pipeline: &pps.Pipeline{
				Name: persistJobInfo.GetPipelineName(),
			},
		}
	}
	jobInfo.JobStatus = make([]*pps.JobStatus, len(persistJobStatuses.JobStatus))
	for i, persistJobStatus := range persistJobStatuses.JobStatus {
		jobInfo.JobStatus[i] = &pps.JobStatus{
			Type:      persistJobStatus.Type,
			Timestamp: persistJobStatus.Timestamp,
			Message:   persistJobStatus.Message,
		}
	}
	if persistJobOutput != nil {
		jobInfo.Output = persistJobOutput.Output
	}
	return jobInfo, nil
}

func job(jobInfo *persist.JobInfo) *extensions.Job {
	app := jobInfo.JobId
	return &extensions.Job{
		TypeMeta: unversioned.TypeMeta{
			Kind:       "Job",
			APIVersion: "v1",
		},
		ObjectMeta: api.ObjectMeta{
			Name: jobInfo.JobId,
			Labels: map[string]string{
				"app": app,
			},
		},
		Spec: extensions.JobSpec{
			Selector: &extensions.PodSelector{
				MatchLabels: map[string]string{
					"app": app,
				},
			},
			Template: api.PodTemplateSpec{
				ObjectMeta: api.ObjectMeta{
					Name: jobInfo.JobId,
					Labels: map[string]string{
						"app": app,
					},
				},
				Spec: api.PodSpec{
					Containers: []api.Container{
						{
							Name:    "user",
							Image:   jobInfo.GetTransform().Image,
							Command: jobInfo.GetTransform().Cmd,
						},
					},
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"context"
	"fmt"
	"time"

	_ "github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/dialers/postgres"
	"github.com/apigee/registry/server/registry/internal/storage/models"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"gorm.io/driver/postgres"
	"gorm.io/driver/sqlite"
	"gorm.io/gorm"
)

// A complete list of entities used by the storage system
// and managed using gorm features.
var entities = []interface{}{
	&models.Project{},
	&models.Api{},
	&models.Version{},
	&models.Spec{},
	&models.SpecRevisionTag{},
	&models.Deployment{},
	&models.DeploymentRevisionTag{},
	&models.Artifact{},
	&models.Blob{},
}

// Client represents a connection to a storage provider.
type Client struct {
	db *gorm.DB
}

// NewClient creates a new database session using the provided driver and data source name.
// Driver must be one of [ sqlite3, postgres, cloudsqlpostgres ]. DSN format varies per database driver.
//
// PostgreSQL DSN Reference: See "Connection Strings" at https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING
// SQLite DSN Reference: See "URI filename examples" at https://www.sqlite.org/c3ref/open.html
func NewClient(ctx context.Context, driver, dsn string) (*Client, error) {
	switch driver {
	case "sqlite3":
		db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
			Logger:      NewGormLogger(ctx),
			PrepareStmt: true,
		})
		if err != nil {
			c := &Client{db: db}
			c.close()
			return nil, grpcErrorForDBError(ctx, err)
		}
		if err := applyConnectionLimits(db); err != nil {
			c := &Client{db: db}
			c.close()
			return nil, grpcErrorForDBError(ctx, err)
		}
		return &Client{db: db}, nil
	case "postgres", "cloudsqlpostgres":
		db, err := gorm.Open(postgres.New(postgres.Config{
			DriverName: driver,
			DSN:        dsn,
		}), &gorm.Config{
			Logger: NewGormLogger(ctx),
		})
		if err != nil {
			c := &Client{db: db}
			c.close()
			return nil, grpcErrorForDBError(ctx, err)
		}
		if err := applyConnectionLimits(db); err != nil {
			c := &Client{db: db}
			c.close()
			return nil, grpcErrorForDBError(ctx, err)
		}
		return &Client{db: db}, nil
	default:
		return nil, fmt.Errorf("unsupported database %s", driver)
	}
}

// Applies limits to concurrent connections.
// Could be made configurable.
func applyConnectionLimits(db *gorm.DB) error {
	sqlDB, err := db.DB()
	if err != nil {
		return err
	}
	sqlDB.SetMaxOpenConns(10)
	sqlDB.SetMaxIdleConns(10)
	sqlDB.SetConnMaxLifetime(60 * time.Second)
	return nil
}

// Close closes a database session.
func (c *Client) Close() {
	c.close()
}

func (c *Client) close() {
	sqlDB, _ := c.db.DB()
	sqlDB.Close()
}

func (c *Client) ensureTable(ctx context.Context, v interface{}) error {
	if !c.db.Migrator().HasTable(v) {
		if err := c.db.Migrator().CreateTable(v); err != nil {
			return grpcErrorForDBError(ctx, err)
		}
	}
	return nil
}

// EnsureTables ensures that all necessary tables exist in the database.
func (c *Client) EnsureTables(ctx context.Context) error {
	for _, entity := range entities {
		if err := c.ensureTable(ctx, entity); err != nil {
			return err
		}
	}
	return nil
}

func (c *Client) Migrate(ctx context.Context) error {
	return grpcErrorForDBError(ctx, c.db.WithContext(ctx).AutoMigrate(entities...))
}

func (c *Client) DatabaseName(ctx context.Context) string {
	return c.db.WithContext(ctx).Name()
}

func (c *Client) TableNames(ctx context.Context) ([]string, error) {
	var tableNames []string
	switch c.db.WithContext(ctx).Name() {
	case "postgres":
		if err := c.db.WithContext(ctx).Table("information_schema.tables").Where("table_schema = ?", "public").Order("table_name").Pluck("table_name", &tableNames).Error; err != nil {
			return nil, grpcErrorForDBError(ctx, err)
		}
	case "sqlite":
		if err := c.db.WithContext(ctx).Table("sqlite_schema").Where("type = 'table' AND name NOT LIKE 'sqlite_%'").Order("name").Pluck("name", &tableNames).Error; err != nil {
			return nil, grpcErrorForDBError(ctx, err)
		}
	default:
		return nil, status.Errorf(codes.Internal, "unsupported database %s", c.db.Name())
	}
	return tableNames, nil
}

func (c *Client) RowCount(ctx context.Context, tableName string) (int64, error) {
	var count int64
	err := c.db.WithContext(ctx).Table(tableName).Count(&count).Error
	return count, grpcErrorForDBError(ctx, err)
}

func (c *Client) Transaction(ctx context.Context, fn func(context.Context, *Client) error) error {
	err := c.db.Transaction(func(tx *gorm.DB) error {
		return fn(ctx, &Client{db: tx})
	})
	return grpcErrorForDBError(ctx, err)
}
<commit_msg>use prepared statements in postgres (#727)<commit_after>// Copyright 2020 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"context"
	"fmt"
	"time"

	_ "github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/dialers/postgres"
	"github.com/apigee/registry/server/registry/internal/storage/models"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"gorm.io/driver/postgres"
	"gorm.io/driver/sqlite"
	"gorm.io/gorm"
)

// A complete list of entities used by the storage system
// and managed using gorm features.
var entities = []interface{}{
	&models.Project{},
	&models.Api{},
	&models.Version{},
	&models.Spec{},
	&models.SpecRevisionTag{},
	&models.Deployment{},
	&models.DeploymentRevisionTag{},
	&models.Artifact{},
	&models.Blob{},
}

// Client represents a connection to a storage provider.
type Client struct {
	db *gorm.DB
}

// NewClient creates a new database session using the provided driver and data source name.
// Driver must be one of [ sqlite3, postgres, cloudsqlpostgres ]. DSN format varies per database driver.
//
// PostgreSQL DSN Reference: See "Connection Strings" at https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING
// SQLite DSN Reference: See "URI filename examples" at https://www.sqlite.org/c3ref/open.html
func NewClient(ctx context.Context, driver, dsn string) (*Client, error) {
	switch driver {
	case "sqlite3":
		db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
			Logger:      NewGormLogger(ctx),
			PrepareStmt: true,
		})
		if err != nil {
			c := &Client{db: db}
			c.close()
			return nil, grpcErrorForDBError(ctx, err)
		}
		if err := applyConnectionLimits(db); err != nil {
			c := &Client{db: db}
			c.close()
			return nil, grpcErrorForDBError(ctx, err)
		}
		return &Client{db: db}, nil
	case "postgres", "cloudsqlpostgres":
		db, err := gorm.Open(postgres.New(postgres.Config{
			DriverName: driver,
			DSN:        dsn,
		}), &gorm.Config{
			Logger:      NewGormLogger(ctx),
			PrepareStmt: true,
		})
		if err != nil {
			c := &Client{db: db}
			c.close()
			return nil, grpcErrorForDBError(ctx, err)
		}
		if err := applyConnectionLimits(db); err != nil {
			c := &Client{db: db}
			c.close()
			return nil, grpcErrorForDBError(ctx, err)
		}
		return &Client{db: db}, nil
	default:
		return nil, fmt.Errorf("unsupported database %s", driver)
	}
}

// Applies limits to concurrent connections.
// Could be made configurable.
func applyConnectionLimits(db *gorm.DB) error {
	sqlDB, err := db.DB()
	if err != nil {
		return err
	}
	sqlDB.SetMaxOpenConns(10)
	sqlDB.SetMaxIdleConns(10)
	sqlDB.SetConnMaxLifetime(60 * time.Second)
	return nil
}

// Close closes a database session.
func (c *Client) Close() {
	c.close()
}

func (c *Client) close() {
	sqlDB, _ := c.db.DB()
	sqlDB.Close()
}

func (c *Client) ensureTable(ctx context.Context, v interface{}) error {
	if !c.db.Migrator().HasTable(v) {
		if err := c.db.Migrator().CreateTable(v); err != nil {
			return grpcErrorForDBError(ctx, err)
		}
	}
	return nil
}

// EnsureTables ensures that all necessary tables exist in the database.
func (c *Client) EnsureTables(ctx context.Context) error {
	for _, entity := range entities {
		if err := c.ensureTable(ctx, entity); err != nil {
			return err
		}
	}
	return nil
}

func (c *Client) Migrate(ctx context.Context) error {
	return grpcErrorForDBError(ctx, c.db.WithContext(ctx).AutoMigrate(entities...))
}

func (c *Client) DatabaseName(ctx context.Context) string {
	return c.db.WithContext(ctx).Name()
}

func (c *Client) TableNames(ctx context.Context) ([]string, error) {
	var tableNames []string
	switch c.db.WithContext(ctx).Name() {
	case "postgres":
		if err := c.db.WithContext(ctx).Table("information_schema.tables").Where("table_schema = ?", "public").Order("table_name").Pluck("table_name", &tableNames).Error; err != nil {
			return nil, grpcErrorForDBError(ctx, err)
		}
	case "sqlite":
		if err := c.db.WithContext(ctx).Table("sqlite_schema").Where("type = 'table' AND name NOT LIKE 'sqlite_%'").Order("name").Pluck("name", &tableNames).Error; err != nil {
			return nil, grpcErrorForDBError(ctx, err)
		}
	default:
		return nil, status.Errorf(codes.Internal, "unsupported database %s", c.db.Name())
	}
	return tableNames, nil
}

func (c *Client) RowCount(ctx context.Context, tableName string) (int64, error) {
	var count int64
	err := c.db.WithContext(ctx).Table(tableName).Count(&count).Error
	return count, grpcErrorForDBError(ctx, err)
}

func (c *Client) Transaction(ctx context.Context, fn func(context.Context, *Client) error) error {
	err := c.db.Transaction(func(tx *gorm.DB) error {
		return fn(ctx, &Client{db: tx})
	})
	return grpcErrorForDBError(ctx, err)
}
<|endoftext|>


<commit_before>// Copyright 2018 The Moov Authors
// Use of this source code is governed by an Apache License
// license that can be found in the LICENSE file.

package test

import (
	"encoding/json"
	"fmt"
	"github.com/moov-io/ach"
	"github.com/moov-io/ach/server"
	"github.com/moov-io/base"
	"log"
	"os"
	"testing"
	"time"
)

// TestCreateJSONFromACHFiles creates JSON from existing ACH Files
func TestCreateJSONFromACHFiles(t *testing.T) {
	files := getFileNames()

	for _, file := range files {
		f, err := os.Open(file.AchFileName)
		if err != nil {
			log.Fatal(err)
		}
		r := ach.NewReader(f)
		achFile, err := r.Read()
		if err != nil {
			fmt.Printf("Issue reading file: %+v \n", err)
		}
		// ensure we have a validated file structure
		if achFile.Validate(); err != nil {
			fmt.Printf("Could not validate entire read file: %v", err)
		}
		// If you trust the file but it's formatting is off building will probably resolve the malformed file.
		if achFile.Create(); err != nil {
			fmt.Printf("Could not build file with read properties: %v", err)
		}

		f.Close()

		fmt.Printf("ACH File: %v \r\n", file.AchFileName)

		// ENR ACH Files does not have BatchHeader.EffectiveEntryDate, so setting this to Today +1 to be included
		// in the JSON File.  For this test after the ACH file is converted to JSON, the test validates the JSON by
		// calling ach.FileFromJSON(bs) and it fails with an empty date time.
		if file.SECCode == "ENR" {
			for _, batch := range achFile.Batches {
				batch.GetHeader().EffectiveEntryDate = base.NewTime(time.Now().AddDate(0, 0, 1))
			}

		}

		bs, err := json.MarshalIndent(achFile, "", " ")
		if err != nil {
			fmt.Println("error:", err)
		}

		fmt.Printf("JSON Output: %v \r\n", string(bs))

		fmt.Printf("Validating JSON byte stream %v created \r\n", file.jsonName)

		// Create ACH File from JSON
		file, err := ach.FileFromJSON(bs)
		if err != nil {
			t.Fatal(err.Error())
		}

		// Validate ACH File
		if file.Validate(); err != nil {
			fmt.Printf("Could not validate file: %v", err)
		}

		repository := server.NewRepositoryInMemory()
		s := server.NewService(repository)

		// Create and store ACH File in repository
		fileID, err := s.CreateFile(&file.Header)
		if err != nil {
			t.Fatal(err.Error())
		}

		// Get the stored ACH File from repository
		getFile, err := s.GetFile(fileID)
		if err == server.ErrNotFound {
			t.Errorf("expected %s received %s w/ error %s", "ErrNotFound", getFile.ID, err)
		}

	}
}

type fileNames struct {
	SECCode     string
	AchFileName string
	jsonName    string
}

func getFileNames() []fileNames {
	fn := []fileNames{
		{"ACK", "../../test/ach-ack-read/ack-read.ach", "ack-credit"},
		{"ADV", "../../test/ach-adv-read/adv-read.ach", "adv-debitForCreditsOriginated"},
		{"ARC", "../../test/ach-arc-read/arc-debit.ach", "arc-debit"},
		{"ATX", "../../test/ach-atx-read/atx-read.ach", "atx-credit"},
		{"BOC", "../../test/ach-boc-read/boc-debit.ach", "boc-debit"},
		{"CCD", "../../test/ach-ccd-read/ccd-debit.ach", "ccd-debit"},
		{"CIE", "../../test/ach-cie-read/cie-credit.ach", "cie-credit"},
		{"CTX", "../../test/ach-ctx-read/ctx-debit.ach", "ctx-debit"},
		{"DNE", "../../test/ach-dne-read/dne-read.ach", "dne-debit"},
		{"ENR", "../../test/ach-enr-read/enr-read.ach", "enr-debit"},
		{"IAT", "../../test/ach-iat-read/iat-credit.ach", "iat-credit"},
		{"MTE", "../../test/ach-mte-read/mte-read.ach", "mte-debit"},
		{"POP", "../../test/ach-pop-read/pop-debit.ach", "pop-debit"},
		{"POS", "../../test/ach-pos-read/pos-debit.ach", "pos-debit"},
		{"PPD", "../../test/ach-ppd-read/ppd-credit.ach", "ppd-credit"},
		{"RCK", "../../test/ach-rck-read/rck-debit.ach", "rck-debit"},
		{"SHR", "../../test/ach-shr-read/shr-debit.ach", "shr-debit"},
		{"TEL", "../../test/ach-tel-read/tel-debit.ach", "tel-debit"},
		{"TRC", "../../test/ach-trc-read/trc-debit.ach", "trc-debit"},
		{"TRX", "../../test/ach-trx-read/trx-debit.ach", "trx-debit"},
		{"WEB", "../../test/ach-web-read/web-credit.ach", "web-credit"},
		{"XCK", "../../test/ach-xck-read/xck-debit.ach", "xck-debit"},
	}
	return fn
}
<commit_msg>minor updates<commit_after>// Copyright 2018 The Moov Authors
// Use of this source code is governed by an Apache License
// license that can be found in the LICENSE file.

package test

import (
	"encoding/json"
	"fmt"
	"github.com/moov-io/ach"
	"github.com/moov-io/ach/server"
	"github.com/moov-io/base"
	"log"
	"os"
	"testing"
	"time"
)

// TestCreateJSONFromACHFiles creates JSON from existing ACH Files
func TestCreateJSONFromACHFiles(t *testing.T) {
	files := getFileNames()

	for _, file := range files {
		f, err := os.Open(file.AchFileName)
		if err != nil {
			log.Fatal(err)
		}
		r := ach.NewReader(f)
		achFile, err := r.Read()
		if err != nil {
			fmt.Printf("Issue reading file: %+v \n", err)
		}
		// ensure we have a validated file structure
		if achFile.Validate(); err != nil {
			fmt.Printf("Could not validate entire read file: %v", err)
		}
		// If you trust the file but it's formatting is off building will probably resolve the malformed file.
		if achFile.Create(); err != nil {
			fmt.Printf("Could not build file with read properties: %v", err)
		}

		f.Close()

		fmt.Printf("ACH File: %v \r\n", file.AchFileName)

		// ENR ACH Files does not have BatchHeader.EffectiveEntryDate, so setting this to Today +1 to be included
		// in the JSON File.  For this test after the ACH file is converted to JSON, the test validates the JSON by
		// calling ach.FileFromJSON(bs) and it fails with an empty date time.
		if file.SECCode == "ENR" {
			for _, batch := range achFile.Batches {
				batch.GetHeader().EffectiveEntryDate = base.NewTime(time.Now().AddDate(0, 0, 1))
			}

		}

		bs, err := json.MarshalIndent(achFile, "", " ")
		if err != nil {
			fmt.Println("error:", err)
		}

		fmt.Printf("JSON Output: %v \r\n", string(bs))

		fmt.Printf("Validating JSON byte stream %v \r\n", file.jsonName)

		// Create ACH File from JSON
		file, err := ach.FileFromJSON(bs)
		if err != nil {
			t.Fatal(err.Error())
		}

		// Validate ACH File
		if file.Validate(); err != nil {
			fmt.Printf("Could not validate file: %v", err)
		}

		repository := server.NewRepositoryInMemory()
		s := server.NewService(repository)

		// Create and store ACH File in repository
		fileID, err := s.CreateFile(&file.Header)
		if err != nil {
			t.Fatal(err.Error())
		}

		// Get the stored ACH File from repository
		getFile, err := s.GetFile(fileID)
		if err == server.ErrNotFound {
			t.Errorf("expected %s received %s w/ error %s", "ErrNotFound", getFile.ID, err)
		}

	}
}

type fileNames struct {
	SECCode     string
	AchFileName string
	jsonName    string
}

func getFileNames() []fileNames {
	fn := []fileNames{
		{"ACK", "../../test/ach-ack-read/ack-read.ach", "ack-credit"},
		{"ADV", "../../test/ach-adv-read/adv-read.ach", "adv-debitForCreditsOriginated"},
		{"ARC", "../../test/ach-arc-read/arc-debit.ach", "arc-debit"},
		{"ATX", "../../test/ach-atx-read/atx-read.ach", "atx-credit"},
		{"BOC", "../../test/ach-boc-read/boc-debit.ach", "boc-debit"},
		{"CCD", "../../test/ach-ccd-read/ccd-debit.ach", "ccd-debit"},
		{"CIE", "../../test/ach-cie-read/cie-credit.ach", "cie-credit"},
		{"CTX", "../../test/ach-ctx-read/ctx-debit.ach", "ctx-debit"},
		{"DNE", "../../test/ach-dne-read/dne-read.ach", "dne-debit"},
		{"ENR", "../../test/ach-enr-read/enr-read.ach", "enr-debit"},
		{"IAT", "../../test/ach-iat-read/iat-credit.ach", "iat-credit"},
		{"MTE", "../../test/ach-mte-read/mte-read.ach", "mte-debit"},
		{"POP", "../../test/ach-pop-read/pop-debit.ach", "pop-debit"},
		{"POS", "../../test/ach-pos-read/pos-debit.ach", "pos-debit"},
		{"PPD", "../../test/ach-ppd-read/ppd-credit.ach", "ppd-credit"},
		{"RCK", "../../test/ach-rck-read/rck-debit.ach", "rck-debit"},
		{"SHR", "../../test/ach-shr-read/shr-debit.ach", "shr-debit"},
		{"TEL", "../../test/ach-tel-read/tel-debit.ach", "tel-debit"},
		{"TRC", "../../test/ach-trc-read/trc-debit.ach", "trc-debit"},
		{"TRX", "../../test/ach-trx-read/trx-debit.ach", "trx-debit"},
		{"WEB", "../../test/ach-web-read/web-credit.ach", "web-credit"},
		{"XCK", "../../test/ach-xck-read/xck-debit.ach", "xck-debit"},
	}
	return fn
}
<|endoftext|>


<commit_before>package goboots

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"encoding/xml"
	"github.com/gabstv/i18ngo"
	"io"
	"log"
	"net/http"
	"reflect"
	"text/template"
)

const (
	outPre          = 0
	outJSON         = 1
	outXML          = 2
	outTemplate     = 3
	outTemplateSolo = 4
	outString       = 5
	outBytes        = 6
)

type In struct {
	R             *http.Request
	W             http.ResponseWriter
	URLParts      []string
	session       *Session
	Content       InContent
	LayoutContent InContent
	App           *App
	Controller    IController
	LangCode      string
	GlobalTitle   string
	closers       []io.Closer
}

func (in *In) closeall() {
	if in.closers == nil {
		return
	}
	for _, v := range in.closers {
		v.Close()
	}
}

type InContent struct {
	vals map[string]interface{}
}

func (c *InContent) init() {
	if c.vals == nil {
		c.vals = make(map[string]interface{})
	}
}

func (c *InContent) Merge(v interface{}) *InContent {
	if v == nil {
		return c
	}
	c.init()
	vtype := reflect.TypeOf(v)
	switch vtype.Kind() {
	case reflect.Map, reflect.Struct:
		// PASS
	case reflect.Ptr:
		if vtype.Elem().Kind() != reflect.Struct {
			return c
		}
	default:
		// tried to merge an invalid type
		return c
	}
	vl := reflect.ValueOf(v)
	if vtype.Kind() == reflect.Ptr {
		vl = vl.Elem()
		vtype = vtype.Elem()
		log.Println(vl.Kind().String())
	}
	if vtype.Kind() == reflect.Map {
		// merge mappy things
		if vtype.Key().Kind() != reflect.String {
			log.Println("MAP KEY IS NOT A STRING!")
			return c
		}
		keys := vl.MapKeys()
		for _, key := range keys {
			val := vl.MapIndex(key)
			c.vals[key.String()] = val.Interface()
		}
	} else {
		// merge structy things
		len0 := vl.NumField()
		for i := 0; i < len0; i++ {
			field := vl.Field(i)
			c.vals[vtype.Field(i).Name] = field.Interface()
		}
	}
	return c
}

func (c *InContent) Get2(key string) (val interface{}, ok bool) {
	c.init()
	val, ok = c.vals[key]
	return
}

func (c *InContent) GetString2(key string) (val string, ok bool) {
	c.init()
	var rval interface{}
	rval, ok = c.vals[key]
	if ok {
		val = rval.(string)
	}
	return
}

func (c *InContent) GetInt2(key string) (val int, ok bool) {
	c.init()
	var rval interface{}
	rval, ok = c.vals[key]
	if ok {
		val, _ = rval.(int)
	}
	return
}

func (c *InContent) Set(key string, val interface{}) *InContent {
	c.init()
	c.vals[key] = val
	return c
}

func (c *InContent) All() map[string]interface{} {
	c.init()
	return c.vals
}

// Translates text to the user language (if available)
func (in *In) T(format string, v ...interface{}) string {
	return i18ngo.TL(in.LangCode, format, v...)
}

func (in *In) Session() *Session {
	if in.session == nil {
		in.session = GetSession(in.W, in.R)
	}
	return in.session
}

func (in *In) OutputTpl(tplPath string) *Out {
	return in.outputTpl(tplPath, "")
}

func (in *In) outputTpl(tplPath, customLayout string) *Out {
	o := &Out{}
	in.LayoutContent.Set("Content", in.OutputSoloTpl(tplPath).String())
	if v, ok := in.LayoutContent.GetString2("Title"); ok {
		in.LayoutContent.Set("Title", in.GlobalTitle+v)
	} else {
		in.LayoutContent.Set("Title", in.GlobalTitle+in.T(in.Controller.GetPageTitle()))
	}
	o.kind = outTemplate
	o.contentObj = in.LayoutContent.Set("Flash", in.Session().Flash.All()).All()
	ln := in.Controller.GetLayoutName()
	if len(ln) == 0 && len(customLayout) == 0 {
		// no layout defined!
		o.kind = outBytes
		var buff bytes.Buffer
		binary.Write(&buff, binary.BigEndian, o.contentObj)
		o.contentBytes = buff.Bytes()
		o.contentObj = nil
		return o
	}
	if len(customLayout) > 0 {
		ln = customLayout
	}
	var layoutTpl *template.Template
	if len(in.App.Config.DefaultLanguage) > 0 {
		layoutTpl = in.App.GetLocalizedLayout(ln, in.W, in.R)
	} else {
		layoutTpl = in.App.GetLayout(ln)
	}
	o.tpl = layoutTpl
	return o
}

func (in *In) OutputSoloTpl(tplPath string) *Out {
	o := &Out{}
	o.kind = outTemplateSolo
	o.contentObj = in.Content.Set("Flash", in.Session().Flash.All()).All()
	var tpl *template.Template
	if len(in.App.Config.DefaultLanguage) > 0 {
		tpl = in.App.GetLocalizedViewTemplate(tplPath, in.W, in.R)
	} else {
		tpl = in.App.GetViewTemplate(tplPath)
	}
	o.tpl = tpl
	return o
}

func (in *In) OutputJSON(jobj interface{}) *Out {
	o := &Out{}
	o.kind = outJSON
	o.contentObj = jobj
	return o
}

func (in *In) OutputXML(xobj interface{}) *Out {
	o := &Out{}
	o.kind = outXML
	o.contentObj = xobj
	return o
}

func (in *In) OutputString(str string) *Out {
	o := &Out{}
	o.kind = outString
	o.contentStr = str
	return o
}

func (in *In) Continue() *Out {
	o := &Out{}
	o.kind = outPre
	return o
}

type Out struct {
	kind         int
	contentObj   interface{}
	contentStr   string
	contentBytes []byte
	tpl          *template.Template
}

func (o *Out) mustb(b []byte, err error) []byte {
	if err != nil {
		panic(err)
		return b
	}
	return b
}

func (o *Out) render(w http.ResponseWriter) {
	switch o.kind {
	case outJSON:
		w.Header().Set("Content-Type", "application/json; charset=utf8")
		w.Write(o.mustb(json.Marshal(o.contentObj)))
	case outXML:
		w.Write(o.mustb(xml.Marshal(o.contentObj)))
	case outTemplateSolo, outTemplate:
		o.tpl.Execute(w, o.contentObj)
	case outString:
		w.Write([]byte(o.contentStr))
	case outBytes:
		w.Write(o.contentBytes)
	}
}

func (o *Out) String() string {
	switch o.kind {
	case outJSON:
		return string(o.mustb(json.Marshal(o.contentObj)))
	case outXML:
		return string(o.mustb(xml.Marshal(o.contentObj)))
	case outTemplateSolo, outTemplate:
		var buffer bytes.Buffer
		o.tpl.Execute(&buffer, o.contentObj)
		return buffer.String()
	case outString:
		return o.contentStr
	case outBytes:
		return string(o.contentBytes)
	}
	return ""
}
<commit_msg>added IsContinue method<commit_after>package goboots

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"encoding/xml"
	"github.com/gabstv/i18ngo"
	"io"
	"log"
	"net/http"
	"reflect"
	"text/template"
)

const (
	outPre          = 0
	outJSON         = 1
	outXML          = 2
	outTemplate     = 3
	outTemplateSolo = 4
	outString       = 5
	outBytes        = 6
)

type In struct {
	R             *http.Request
	W             http.ResponseWriter
	URLParts      []string
	session       *Session
	Content       InContent
	LayoutContent InContent
	App           *App
	Controller    IController
	LangCode      string
	GlobalTitle   string
	closers       []io.Closer
}

func (in *In) closeall() {
	if in.closers == nil {
		return
	}
	for _, v := range in.closers {
		v.Close()
	}
}

type InContent struct {
	vals map[string]interface{}
}

func (c *InContent) init() {
	if c.vals == nil {
		c.vals = make(map[string]interface{})
	}
}

func (c *InContent) Merge(v interface{}) *InContent {
	if v == nil {
		return c
	}
	c.init()
	vtype := reflect.TypeOf(v)
	switch vtype.Kind() {
	case reflect.Map, reflect.Struct:
		// PASS
	case reflect.Ptr:
		if vtype.Elem().Kind() != reflect.Struct {
			return c
		}
	default:
		// tried to merge an invalid type
		return c
	}
	vl := reflect.ValueOf(v)
	if vtype.Kind() == reflect.Ptr {
		vl = vl.Elem()
		vtype = vtype.Elem()
		log.Println(vl.Kind().String())
	}
	if vtype.Kind() == reflect.Map {
		// merge mappy things
		if vtype.Key().Kind() != reflect.String {
			log.Println("MAP KEY IS NOT A STRING!")
			return c
		}
		keys := vl.MapKeys()
		for _, key := range keys {
			val := vl.MapIndex(key)
			c.vals[key.String()] = val.Interface()
		}
	} else {
		// merge structy things
		len0 := vl.NumField()
		for i := 0; i < len0; i++ {
			field := vl.Field(i)
			c.vals[vtype.Field(i).Name] = field.Interface()
		}
	}
	return c
}

func (c *InContent) Get2(key string) (val interface{}, ok bool) {
	c.init()
	val, ok = c.vals[key]
	return
}

func (c *InContent) GetString2(key string) (val string, ok bool) {
	c.init()
	var rval interface{}
	rval, ok = c.vals[key]
	if ok {
		val = rval.(string)
	}
	return
}

func (c *InContent) GetInt2(key string) (val int, ok bool) {
	c.init()
	var rval interface{}
	rval, ok = c.vals[key]
	if ok {
		val, _ = rval.(int)
	}
	return
}

func (c *InContent) Set(key string, val interface{}) *InContent {
	c.init()
	c.vals[key] = val
	return c
}

func (c *InContent) All() map[string]interface{} {
	c.init()
	return c.vals
}

// Translates text to the user language (if available)
func (in *In) T(format string, v ...interface{}) string {
	return i18ngo.TL(in.LangCode, format, v...)
}

func (in *In) Session() *Session {
	if in.session == nil {
		in.session = GetSession(in.W, in.R)
	}
	return in.session
}

func (in *In) OutputTpl(tplPath string) *Out {
	return in.outputTpl(tplPath, "")
}

func (in *In) outputTpl(tplPath, customLayout string) *Out {
	o := &Out{}
	in.LayoutContent.Set("Content", in.OutputSoloTpl(tplPath).String())
	if v, ok := in.LayoutContent.GetString2("Title"); ok {
		in.LayoutContent.Set("Title", in.GlobalTitle+v)
	} else {
		in.LayoutContent.Set("Title", in.GlobalTitle+in.T(in.Controller.GetPageTitle()))
	}
	o.kind = outTemplate
	o.contentObj = in.LayoutContent.Set("Flash", in.Session().Flash.All()).All()
	ln := in.Controller.GetLayoutName()
	if len(ln) == 0 && len(customLayout) == 0 {
		// no layout defined!
		o.kind = outBytes
		var buff bytes.Buffer
		binary.Write(&buff, binary.BigEndian, o.contentObj)
		o.contentBytes = buff.Bytes()
		o.contentObj = nil
		return o
	}
	if len(customLayout) > 0 {
		ln = customLayout
	}
	var layoutTpl *template.Template
	if len(in.App.Config.DefaultLanguage) > 0 {
		layoutTpl = in.App.GetLocalizedLayout(ln, in.W, in.R)
	} else {
		layoutTpl = in.App.GetLayout(ln)
	}
	o.tpl = layoutTpl
	return o
}

func (in *In) OutputSoloTpl(tplPath string) *Out {
	o := &Out{}
	o.kind = outTemplateSolo
	o.contentObj = in.Content.Set("Flash", in.Session().Flash.All()).All()
	var tpl *template.Template
	if len(in.App.Config.DefaultLanguage) > 0 {
		tpl = in.App.GetLocalizedViewTemplate(tplPath, in.W, in.R)
	} else {
		tpl = in.App.GetViewTemplate(tplPath)
	}
	o.tpl = tpl
	return o
}

func (in *In) OutputJSON(jobj interface{}) *Out {
	o := &Out{}
	o.kind = outJSON
	o.contentObj = jobj
	return o
}

func (in *In) OutputXML(xobj interface{}) *Out {
	o := &Out{}
	o.kind = outXML
	o.contentObj = xobj
	return o
}

func (in *In) OutputString(str string) *Out {
	o := &Out{}
	o.kind = outString
	o.contentStr = str
	return o
}

func (in *In) Continue() *Out {
	o := &Out{}
	o.kind = outPre
	return o
}

type Out struct {
	kind         int
	contentObj   interface{}
	contentStr   string
	contentBytes []byte
	tpl          *template.Template
}

func (o *Out) IsContinue() bool {
	return o.kind == outPre
}

func (o *Out) mustb(b []byte, err error) []byte {
	if err != nil {
		panic(err)
		return b
	}
	return b
}

func (o *Out) render(w http.ResponseWriter) {
	switch o.kind {
	case outJSON:
		w.Header().Set("Content-Type", "application/json; charset=utf8")
		w.Write(o.mustb(json.Marshal(o.contentObj)))
	case outXML:
		w.Write(o.mustb(xml.Marshal(o.contentObj)))
	case outTemplateSolo, outTemplate:
		o.tpl.Execute(w, o.contentObj)
	case outString:
		w.Write([]byte(o.contentStr))
	case outBytes:
		w.Write(o.contentBytes)
	}
}

func (o *Out) String() string {
	switch o.kind {
	case outJSON:
		return string(o.mustb(json.Marshal(o.contentObj)))
	case outXML:
		return string(o.mustb(xml.Marshal(o.contentObj)))
	case outTemplateSolo, outTemplate:
		var buffer bytes.Buffer
		o.tpl.Execute(&buffer, o.contentObj)
		return buffer.String()
	case outString:
		return o.contentStr
	case outBytes:
		return string(o.contentBytes)
	}
	return ""
}
<|endoftext|>


<commit_before>/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cleanup

import (
	"flag"
	"regexp"
	"testing"

	"github.com/onsi/ginkgo/v2"

	"k8s.io/klog/v2"
	"k8s.io/klog/v2/ktesting"
	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/framework/internal/output"
	testapiserver "k8s.io/kubernetes/test/utils/apiserver"
)

// The line number of the following code is checked in TestFailureOutput below.
// Be careful when moving it around or changing the import statements above.
// Here are some intentionally blank lines that can be removed to compensate
// for future additional import statements.
//
//
//
//
//
//
//
//
//
//
//
//
//
// This must be line #50.

var _ = ginkgo.Describe("e2e", func() {
	ginkgo.BeforeEach(func() {
		framework.Logf("before")
	})

	f := framework.NewDefaultFramework("test-namespace")

	ginkgo.AfterEach(func() {
		framework.Logf("after")
		if f.ClientSet == nil {
			framework.Fail("Wrong order of cleanup operations: framework.AfterEach already ran and cleared f.ClientSet.")
		}
	})

	ginkgo.It("works", func() {
		// DeferCleanup invokes in first-in-last-out order
		ginkgo.DeferCleanup(func() {
			framework.Logf("cleanup last")
		})
		ginkgo.DeferCleanup(func() {
			framework.Logf("cleanup first")
		})
	})
})

const (
	ginkgoOutput = `[BeforeEach] e2e
  cleanup_test.go:53
INFO: before
[BeforeEach] e2e
  set up framework | framework.go:xxx
STEP: Creating a kubernetes client
INFO: >>> kubeConfig: yyy/kube.config
STEP: Building a namespace api object, basename test-namespace
INFO: Skipping waiting for service account
[It] works
  cleanup_test.go:66
[AfterEach] e2e
  cleanup_test.go:59
INFO: after
[DeferCleanup] e2e
  cleanup_test.go:71
INFO: cleanup first
[DeferCleanup] e2e
  cleanup_test.go:68
INFO: cleanup last
[DeferCleanup] e2e
  dump namespaces | framework.go:xxx
[DeferCleanup] e2e
  tear down framework | framework.go:xxx
STEP: Destroying namespace "test-namespace-zzz" for this suite.
`
)

func TestCleanup(t *testing.T) {
	// The control plane is noisy and randomly logs through klog, for example:
	// E0912 07:08:46.100164   75466 controller.go:254] unable to sync kubernetes service: Endpoints "kubernetes" is invalid: subsets[0].addresses[0].ip: Invalid value: "127.0.0.1": may not be in the loopback range (127.0.0.0/8, ::1/128)
	//
	// By creating a ktesting logger and registering that as global
	// default logger we get the control plane output into the
	// "go test" output in case of a failure (useful for debugging!)
	// while keeping it out of the captured Ginkgo output that
	// the test is comparing below.
	//
	// There are some small drawbacks:
	// - The source code location for control plane log messages
	//   is shown as klog.go because klog does not properly
	//   skip its own helper functions. That's okay, normally
	//   ktesting should not be installed as logging backend like this.
	// - klog.Infof messages are printed with an extra newline.
	logger, _ := ktesting.NewTestContext(t)
	klog.SetLogger(logger)

	apiServer := testapiserver.StartAPITestServer(t)

	// This simulates how test/e2e uses the framework and how users
	// invoke test/e2e.
	framework.RegisterCommonFlags(flag.CommandLine)
	framework.RegisterClusterFlags(flag.CommandLine)
	for flagname, value := range map[string]string{
		"kubeconfig": apiServer.KubeConfigFile,
		// Some features are not supported by the fake cluster.
		"e2e-verify-service-account": "false",
		"allowed-not-ready-nodes":    "-1",
		// This simplifies the text comparison.
		"ginkgo.no-color": "true",
	} {
		if err := flag.Set(flagname, value); err != nil {
			t.Fatalf("set %s: %v", flagname, err)
		}
	}
	framework.AfterReadingAllFlags(&framework.TestContext)
	suiteConfig, reporterConfig := framework.CreateGinkgoConfig()

	expected := output.SuiteResults{
		output.TestResult{
			Name:            "e2e works",
			NormalizeOutput: normalizeOutput,
			Output:          ginkgoOutput,
		},
	}

	output.TestGinkgoOutput(t, expected, suiteConfig, reporterConfig)
}

func normalizeOutput(output string) string {
	for exp, replacement := range map[string]string{
		// Ignore line numbers inside framework source code (likely to change).
		`framework\.go:\d+`: `framework.go:xxx`,
		// Config file name varies for each run.
		`kubeConfig: .*/kube.config`: `kubeConfig: yyy/kube.config`,
		// Random suffix for namespace.
		`test-namespace-\d+`: `test-namespace-zzz`,
	} {
		output = regexp.MustCompile(exp).ReplaceAllString(output, replacement)
	}
	return output
}
<commit_msg>skip etcd unit tests on non fully supported architectures<commit_after>//go:build linux && amd64

/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This test uses etcd that is only fully supported for AMD64 and Linux
// https://etcd.io/docs/v3.5/op-guide/supported-platform/#support-tiers

package cleanup

import (
	"flag"
	"regexp"
	"testing"

	"github.com/onsi/ginkgo/v2"

	"k8s.io/klog/v2"
	"k8s.io/klog/v2/ktesting"
	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/framework/internal/output"
	testapiserver "k8s.io/kubernetes/test/utils/apiserver"
)

// The line number of the following code is checked in TestFailureOutput below.
// Be careful when moving it around or changing the import statements above.
// Here are some intentionally blank lines that can be removed to compensate
// for future additional import statements.
//
//
//
//
//
//
//
//
// This must be line #50.

var _ = ginkgo.Describe("e2e", func() {
	ginkgo.BeforeEach(func() {
		framework.Logf("before")
	})

	f := framework.NewDefaultFramework("test-namespace")

	ginkgo.AfterEach(func() {
		framework.Logf("after")
		if f.ClientSet == nil {
			framework.Fail("Wrong order of cleanup operations: framework.AfterEach already ran and cleared f.ClientSet.")
		}
	})

	ginkgo.It("works", func() {
		// DeferCleanup invokes in first-in-last-out order
		ginkgo.DeferCleanup(func() {
			framework.Logf("cleanup last")
		})
		ginkgo.DeferCleanup(func() {
			framework.Logf("cleanup first")
		})
	})
})

const (
	ginkgoOutput = `[BeforeEach] e2e
  cleanup_test.go:53
INFO: before
[BeforeEach] e2e
  set up framework | framework.go:xxx
STEP: Creating a kubernetes client
INFO: >>> kubeConfig: yyy/kube.config
STEP: Building a namespace api object, basename test-namespace
INFO: Skipping waiting for service account
[It] works
  cleanup_test.go:66
[AfterEach] e2e
  cleanup_test.go:59
INFO: after
[DeferCleanup] e2e
  cleanup_test.go:71
INFO: cleanup first
[DeferCleanup] e2e
  cleanup_test.go:68
INFO: cleanup last
[DeferCleanup] e2e
  dump namespaces | framework.go:xxx
[DeferCleanup] e2e
  tear down framework | framework.go:xxx
STEP: Destroying namespace "test-namespace-zzz" for this suite.
`
)

func TestCleanup(t *testing.T) {
	// The control plane is noisy and randomly logs through klog, for example:
	// E0912 07:08:46.100164   75466 controller.go:254] unable to sync kubernetes service: Endpoints "kubernetes" is invalid: subsets[0].addresses[0].ip: Invalid value: "127.0.0.1": may not be in the loopback range (127.0.0.0/8, ::1/128)
	//
	// By creating a ktesting logger and registering that as global
	// default logger we get the control plane output into the
	// "go test" output in case of a failure (useful for debugging!)
	// while keeping it out of the captured Ginkgo output that
	// the test is comparing below.
	//
	// There are some small drawbacks:
	// - The source code location for control plane log messages
	//   is shown as klog.go because klog does not properly
	//   skip its own helper functions. That's okay, normally
	//   ktesting should not be installed as logging backend like this.
	// - klog.Infof messages are printed with an extra newline.
	logger, _ := ktesting.NewTestContext(t)
	klog.SetLogger(logger)

	apiServer := testapiserver.StartAPITestServer(t)

	// This simulates how test/e2e uses the framework and how users
	// invoke test/e2e.
	framework.RegisterCommonFlags(flag.CommandLine)
	framework.RegisterClusterFlags(flag.CommandLine)
	for flagname, value := range map[string]string{
		"kubeconfig": apiServer.KubeConfigFile,
		// Some features are not supported by the fake cluster.
		"e2e-verify-service-account": "false",
		"allowed-not-ready-nodes":    "-1",
		// This simplifies the text comparison.
		"ginkgo.no-color": "true",
	} {
		if err := flag.Set(flagname, value); err != nil {
			t.Fatalf("set %s: %v", flagname, err)
		}
	}
	framework.AfterReadingAllFlags(&framework.TestContext)
	suiteConfig, reporterConfig := framework.CreateGinkgoConfig()

	expected := output.SuiteResults{
		output.TestResult{
			Name:            "e2e works",
			NormalizeOutput: normalizeOutput,
			Output:          ginkgoOutput,
		},
	}

	output.TestGinkgoOutput(t, expected, suiteConfig, reporterConfig)
}

func normalizeOutput(output string) string {
	for exp, replacement := range map[string]string{
		// Ignore line numbers inside framework source code (likely to change).
		`framework\.go:\d+`: `framework.go:xxx`,
		// Config file name varies for each run.
		`kubeConfig: .*/kube.config`: `kubeConfig: yyy/kube.config`,
		// Random suffix for namespace.
		`test-namespace-\d+`: `test-namespace-zzz`,
	} {
		output = regexp.MustCompile(exp).ReplaceAllString(output, replacement)
	}
	return output
}
<|endoftext|>


<commit_before>package mouryou

import (
	"../math"
	"log"
	"sync"
)

type cluster struct {
	Threshold float64
	Timeout   int64
	LB        loadBalancer
	HVs       []hypervisor
	VMs       []virtualMachine
}

func (c *cluster) init() {
	for _, hv := range c.HVs {
		hv.assignVMs()
		c.VMs = append(c.VMs, hv.VMs...)
	}
}

func (c cluster) operatingRatios() []float64 {
	var wg sync.WaitGroup
	ors := make([]float64, len(c.VMs))

	for i, vm := range c.VMs {
		wg.Add(1)
		go func(i int, vm virtualMachine) {
			ors[i] = vm.operatingRatio()
			wg.Done()
		}(i, vm)
	}
	wg.Wait()

	return ors
}

func (c cluster) avgor() float64 {
	ors := c.operatingRatios()
	log.Printf("%+v\n", ors)
	return math.Average(ors)
}

func (c cluster) Log() {
	// log.Printf("%.5f\n", c.avgor())
	c.avgor()
}
<commit_msg>goroutine内でsync.Mutexを使用<commit_after>package mouryou

import (
	"../math"
	"log"
	"sync"
)

type cluster struct {
	Threshold float64
	Timeout   int64
	LB        loadBalancer
	HVs       []hypervisor
	VMs       []virtualMachine
}

func (c *cluster) init() {
	for _, hv := range c.HVs {
		hv.assignVMs()
		c.VMs = append(c.VMs, hv.VMs...)
	}
}

func (c cluster) operatingRatios() []float64 {
	var wg sync.WaitGroup
	var m sync.Mutex
	ors := make([]float64, len(c.VMs))

	for i, vm := range c.VMs {
		wg.Add(1)
		go func(i int, vm virtualMachine) {
			defer wg.Done()
			or := vm.operatingRatio()

			m.Lock()
			ors[i] = or
			m.Unlock()
		}(i, vm)
	}
	wg.Wait()

	return ors
}

func (c cluster) avgor() float64 {
	ors := c.operatingRatios()
	log.Printf("%+v\n", ors)
	return math.Average(ors)
}

func (c cluster) Log() {
	// log.Printf("%.5f\n", c.avgor())
	c.avgor()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"github.com/elazarl/goproxy"
	"github.com/golang/glog"
	"io"
	"io/ioutil"
	"net/http"
	"sync"
	"time"
)

type EtchProxy struct {
	goproxy.ProxyHttpServer
	Cache *Cache
}

type EtchContextData struct {
	CachedContent *bytes.Buffer
}

func ReqMethodIs(method string) goproxy.ReqConditionFunc {
	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		return req.Method == method
	}
}

func StatusCodeIs(code int) goproxy.RespCondition {
	return goproxy.RespConditionFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) bool {
		if resp == nil {
			return false
		}
		return resp.StatusCode == code
	})
}

type RequestMutex struct {
	sync.Mutex
	resChans map[string][]chan *http.Response
}

var requestMutex = &RequestMutex{resChans: make(map[string][]chan *http.Response)}

func (proxy *EtchProxy) GuardRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
	requestMutex.Lock()
	chans, ok := requestMutex.resChans[req.URL.String()]

	if ok {
		ch := make(chan *http.Response)
		requestMutex.resChans[req.URL.String()] = append(chans, ch)
		requestMutex.Unlock()

		glog.V(3).Infof("[%s] Waiting for ongoing request", req.URL)

		res := <-ch

		glog.V(3).Infof("[%s] Response got from chan: %s", req.URL, res)

		if res != nil {
			return req, res
		}
	} else {
		requestMutex.resChans[req.URL.String()] = make([]chan *http.Response, 0)
		requestMutex.Unlock()
	}

	return req, nil
}

func (proxy *EtchProxy) MakeRangedRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
	cache := proxy.Cache

	if entry := cache.GetEntry(req.URL); entry.FileInfo != nil {
		glog.Infof("%s: found cache entry", req.URL)

		content, err := entry.Content()
		if err != nil {
			glog.Errorf("OnRequest: retrieving cache content: %s", err)
			return req, nil
		}

		// TODO mutex
		cachedContent := bytes.NewBuffer(content)
		req.Header.Add("Range", fmt.Sprintf("bytes=%d-", cachedContent.Len()-1))
		req.Header.Add("If-Modified-Since", entry.FileInfo.ModTime().Format(time.RFC850))

		_, resp, err := proxy.Tr.DetailedRoundTrip(req)
		if err != nil {
			glog.Errorf("OnRequest: executing request: %s", err)
			return req, nil
		}

		if resp.StatusCode == 416 {
			glog.Infof("[%s] Got 416: attempting re-fetch", req.URL)

			// clear cache
			req.Header.Del("Range")
			req.Header.Del("If-Modified-Since")

			_, _resp, err := proxy.Tr.DetailedRoundTrip(req)

			if err != nil {
				glog.Errorf("OnRequest: re-fetch: %s", err)
				return req, nil
			}

			resp = _resp
			cachedContent = new(bytes.Buffer)
		}

		ctx.UserData = &EtchContextData{cachedContent}

		return req, resp
	}

	return req, nil
}

func (proxy *EtchProxy) RestoreCache(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	glog.V(3).Infof("[%s] Headers: %s", ctx.Req.URL, resp.Header)

	if ctx.UserData == nil {
		return resp
	}

	userData := ctx.UserData.(*EtchContextData)

	switch resp.StatusCode {
	case 200:
		// nop

	case 206:
		buf := userData.CachedContent

		// TODO check Content-Range value
		contentRange := resp.Header.Get("Content-Range")
		glog.V(2).Infof("Content-Range: %s", contentRange)

		// 1 バイトだけキャッシュと重複してるはずなので
		// その部分を整合性チェックに使う
		responseBody := bufio.NewReader(resp.Body)
		firstByte, err := responseBody.ReadByte()

		if err != nil {
			glog.Errorf("[%s] Reading response: %s", ctx.Req.URL, err)
			return goproxy.NewResponse(
				ctx.Req, goproxy.ContentTypeText, 500, fmt.Sprintf("Reading response: %s", err))
		}

		if buf.Bytes()[buf.Len()-1] != firstByte {
			// TODO re-attempt
			glog.Errorf("[%s] Cache mismatch", ctx.Req.URL)
			return resp
		}

		io.Copy(buf, responseBody)

		resp.StatusCode = 200
		resp.Header.Del("Content-Range")
		resp.Body = ioutil.NopCloser(buf)

	case 304:
		resp.StatusCode = 200
		resp.Body = ioutil.NopCloser(userData.CachedContent)

	default:
		glog.Errorf("[%s] Unhandled status code: %d", ctx.Req.URL, resp.StatusCode)
	}

	return resp
}

func (proxy *EtchProxy) StoreCache(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	cache := proxy.Cache

	lastModified := time.Now()
	lastModifiedString := resp.Header.Get("Last-Modified")
	if lastModifiedString != "" {
		var err error
		lastModified, err = http.ParseTime(lastModifiedString)
		if err != nil {
			glog.Errorf(`[%s]: Parsing Last-Modified header "%s": %s`, ctx.Req.URL, lastModifiedString, err)
		}
	}

	cacheEntry := cache.GetEntry(ctx.Req.URL)
	if cacheEntry.FileInfo == nil || cacheEntry.FileInfo.ModTime().Before(lastModified) {
		glog.Infof("[%s] update cache: %s", ctx.Req.URL, cacheEntry.FilePath)

		cacheWriter, err := cacheEntry.GetWriter()
		if err != nil {
			glog.Errorf("[%s]: cacheEntry.GetWriter: %s", ctx.Req.URL, err)
			return resp
		}

		buf := new(bytes.Buffer)
		io.Copy(io.MultiWriter(buf, cacheWriter), resp.Body)
		cacheWriter.Close()

		cacheEntry.SetMtime(lastModified)

		resp.Body = ioutil.NopCloser(buf)
	} else {
		glog.V(2).Infof("[%s] Response is not fresher than cache: %s <= %s", ctx.Req.URL, lastModified, cacheEntry.FileInfo.ModTime())
	}

	return resp
}

func (proxy *EtchProxy) UnguardRequest(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	requestMutex.Lock()
	defer requestMutex.Unlock()
	if chans := requestMutex.resChans[ctx.Req.URL.String()]; chans != nil {
		for _, ch := range chans {
			ch <- resp
		}
		delete(requestMutex.resChans, ctx.Req.URL.String())
	}

	return resp
}

func main() {
	cacheDir := flag.String("cache-dir", "cache", "cache directory")
	proxyPort := flag.Int("port", 8080, "proxy port")

	flag.Parse()

	cache := &Cache{*cacheDir}
	proxy := &EtchProxy{*goproxy.NewProxyHttpServer(), cache}

	proxy.OnRequest(ReqMethodIs("GET")).DoFunc(proxy.GuardRequest)
	proxy.OnRequest(ReqMethodIs("GET")).DoFunc(proxy.MakeRangedRequest)
	proxy.OnResponse(ReqMethodIs("GET")).DoFunc(proxy.RestoreCache)
	proxy.OnResponse(goproxy.ContentTypeIs("text/plain"), ReqMethodIs("GET"), StatusCodeIs(200)).DoFunc(proxy.StoreCache)
	proxy.OnResponse().DoFunc(proxy.UnguardRequest)

	proxy.Verbose = true

	glog.Errorf("%s", http.ListenAndServe(fmt.Sprintf(":%d", *proxyPort), proxy))
}
<commit_msg>use constants<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"github.com/elazarl/goproxy"
	"github.com/golang/glog"
	"io"
	"io/ioutil"
	"net/http"
	"sync"
	"time"
)

type EtchProxy struct {
	goproxy.ProxyHttpServer
	Cache *Cache
}

type EtchContextData struct {
	CachedContent *bytes.Buffer
}

func ReqMethodIs(method string) goproxy.ReqConditionFunc {
	return func(req *http.Request, ctx *goproxy.ProxyCtx) bool {
		return req.Method == method
	}
}

func StatusCodeIs(code int) goproxy.RespCondition {
	return goproxy.RespConditionFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) bool {
		if resp == nil {
			return false
		}
		return resp.StatusCode == code
	})
}

type RequestMutex struct {
	sync.Mutex
	resChans map[string][]chan *http.Response
}

var requestMutex = &RequestMutex{resChans: make(map[string][]chan *http.Response)}

func (proxy *EtchProxy) GuardRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
	requestMutex.Lock()
	chans, ok := requestMutex.resChans[req.URL.String()]

	if ok {
		ch := make(chan *http.Response)
		requestMutex.resChans[req.URL.String()] = append(chans, ch)
		requestMutex.Unlock()

		glog.V(3).Infof("[%s] Waiting for ongoing request", req.URL)

		res := <-ch

		glog.V(3).Infof("[%s] Response got from chan: %s", req.URL, res)

		if res != nil {
			return req, res
		}
	} else {
		requestMutex.resChans[req.URL.String()] = make([]chan *http.Response, 0)
		requestMutex.Unlock()
	}

	return req, nil
}

func (proxy *EtchProxy) MakeRangedRequest(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
	cache := proxy.Cache

	if entry := cache.GetEntry(req.URL); entry.FileInfo != nil {
		glog.Infof("%s: found cache entry", req.URL)

		content, err := entry.Content()
		if err != nil {
			glog.Errorf("OnRequest: retrieving cache content: %s", err)
			return req, nil
		}

		// TODO mutex
		cachedContent := bytes.NewBuffer(content)
		req.Header.Add("Range", fmt.Sprintf("bytes=%d-", cachedContent.Len()-1))
		req.Header.Add("If-Modified-Since", entry.FileInfo.ModTime().Format(time.RFC850))

		_, resp, err := proxy.Tr.DetailedRoundTrip(req)
		if err != nil {
			glog.Errorf("OnRequest: executing request: %s", err)
			return req, nil
		}

		if resp.StatusCode == 416 {
			glog.Infof("[%s] Got 416: attempting re-fetch", req.URL)

			// clear cache
			req.Header.Del("Range")
			req.Header.Del("If-Modified-Since")

			_, _resp, err := proxy.Tr.DetailedRoundTrip(req)

			if err != nil {
				glog.Errorf("OnRequest: re-fetch: %s", err)
				return req, nil
			}

			resp = _resp
			cachedContent = new(bytes.Buffer)
		}

		ctx.UserData = &EtchContextData{cachedContent}

		return req, resp
	}

	return req, nil
}

func (proxy *EtchProxy) RestoreCache(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	glog.V(3).Infof("[%s] Headers: %s", ctx.Req.URL, resp.Header)

	if ctx.UserData == nil {
		return resp
	}

	userData := ctx.UserData.(*EtchContextData)

	switch resp.StatusCode {
	case http.StatusOK:
		// nop

	case http.StatusPartialContent:
		buf := userData.CachedContent

		// TODO check Content-Range value
		contentRange := resp.Header.Get("Content-Range")
		glog.V(2).Infof("Content-Range: %s", contentRange)

		// 1 バイトだけキャッシュと重複してるはずなので
		// その部分を整合性チェックに使う
		responseBody := bufio.NewReader(resp.Body)
		firstByte, err := responseBody.ReadByte()

		if err != nil {
			glog.Errorf("[%s] Reading response: %s", ctx.Req.URL, err)
			return goproxy.NewResponse(
				ctx.Req, goproxy.ContentTypeText, http.StatusInternalServerError, fmt.Sprintf("Reading response: %s", err))
		}

		if buf.Bytes()[buf.Len()-1] != firstByte {
			// TODO re-attempt
			glog.Errorf("[%s] Cache mismatch", ctx.Req.URL)
			return resp
		}

		io.Copy(buf, responseBody)

		resp.StatusCode = http.StatusOK
		resp.Header.Del("Content-Range")
		resp.Body = ioutil.NopCloser(buf)

	case http.StatusNotModified:
		resp.StatusCode = http.StatusOK
		resp.Body = ioutil.NopCloser(userData.CachedContent)

	case http.StatusNonAuthoritativeInfo:
		resp.StatusCode = http.StatusGone

	default:
		glog.Errorf("[%s] Unhandled status code: %d", ctx.Req.URL, resp.StatusCode)
	}

	return resp
}

func (proxy *EtchProxy) StoreCache(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	cache := proxy.Cache

	lastModified := time.Now()
	lastModifiedString := resp.Header.Get("Last-Modified")
	if lastModifiedString != "" {
		var err error
		lastModified, err = http.ParseTime(lastModifiedString)
		if err != nil {
			glog.Errorf(`[%s]: Parsing Last-Modified header "%s": %s`, ctx.Req.URL, lastModifiedString, err)
		}
	}

	cacheEntry := cache.GetEntry(ctx.Req.URL)
	if cacheEntry.FileInfo == nil || cacheEntry.FileInfo.ModTime().Before(lastModified) {
		glog.Infof("[%s] update cache: %s", ctx.Req.URL, cacheEntry.FilePath)

		cacheWriter, err := cacheEntry.GetWriter()
		if err != nil {
			glog.Errorf("[%s]: cacheEntry.GetWriter: %s", ctx.Req.URL, err)
			return resp
		}

		buf := new(bytes.Buffer)
		io.Copy(io.MultiWriter(buf, cacheWriter), resp.Body)
		cacheWriter.Close()

		cacheEntry.SetMtime(lastModified)

		resp.Body = ioutil.NopCloser(buf)
	} else {
		glog.V(2).Infof("[%s] Response is not fresher than cache: %s <= %s", ctx.Req.URL, lastModified, cacheEntry.FileInfo.ModTime())
	}

	return resp
}

func (proxy *EtchProxy) UnguardRequest(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
	requestMutex.Lock()
	defer requestMutex.Unlock()
	if chans := requestMutex.resChans[ctx.Req.URL.String()]; chans != nil {
		for _, ch := range chans {
			ch <- resp
		}
		delete(requestMutex.resChans, ctx.Req.URL.String())
	}

	return resp
}

func main() {
	cacheDir := flag.String("cache-dir", "cache", "cache directory")
	proxyPort := flag.Int("port", 8080, "proxy port")

	flag.Parse()

	cache := &Cache{*cacheDir}
	proxy := &EtchProxy{*goproxy.NewProxyHttpServer(), cache}

	proxy.OnRequest(ReqMethodIs("GET")).DoFunc(proxy.GuardRequest)
	proxy.OnRequest(ReqMethodIs("GET")).DoFunc(proxy.MakeRangedRequest)
	proxy.OnResponse(ReqMethodIs("GET")).DoFunc(proxy.RestoreCache)
	proxy.OnResponse(goproxy.ContentTypeIs("text/plain"), ReqMethodIs("GET"), StatusCodeIs(200)).DoFunc(proxy.StoreCache)
	proxy.OnResponse().DoFunc(proxy.UnguardRequest)

	proxy.Verbose = true

	glog.Errorf("%s", http.ListenAndServe(fmt.Sprintf(":%d", *proxyPort), proxy))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

type scope map[string]sexpr

func (v cons) String() string {
	return "<cons>"
}

func doEval(c chan sexpr) {
	for e := range c {
		v := eval(e)
		switch v.(type) {
		case cons:
			fmt.Printf("<cons>\n")
		case sym:
			fmt.Printf("<sym : %s>\n", string(v.(sym)))
		case float64:
			fmt.Printf("%f\n", v.(float64))
		case string:
			fmt.Printf("\"%s\"\n", v.(string))
		default:
			fmt.Printf("nil\n")
		}
	}
}

func isFunction(s sexpr) bool {
	_, ok := s.(func ([]sexpr) sexpr)
	return ok
}

func isSyntax(s sexpr) bool {
	return false
}

// Perform appropriate syntax transformations on the given s-expression. Note
// that some s-expressions that 'should' involve syntax transformations, such
// as (if cond x y) and (lambda ...), don't - they just aren't evaluated as
// normal functions. (TODO make user-defined transformations more flexible to
// add symmetry.)
func transform(e sexpr) sexpr {
	return e // TODO
}

// Evaluates an s-expression, excluding syntax transformations (macros).
func eval(e sexpr) sexpr {
	switch e.(type) {
	case cons: // a function to evaluate
		cons := e.(cons)
		car := eval(cons.car)
		cdr := cons.cdr
		if !isFunction(car) {
			panic("Attempted application on non-function")
		}
		return apply(car.(func([]sexpr) sexpr), cdr)
	case sym:
		return lookup(string(e.(sym)))
	case float64:
		return e
	case string:
		return e
	}
	return Nil
}

func flatten(s sexpr) (ss []sexpr) {
	_, ok := s.(cons)
	for ok {
		ss = append(ss, s.(cons).car)
		s = s.(cons).cdr
		_, ok = s.(cons)
	}
	// TODO what if s isn't nil now?
	return
}

// Applies the given function to an s-expression.
func apply(f func ([]sexpr) sexpr, args sexpr) sexpr {
	return f(flatten(args))
}

// Performs lookup of symbols for values.
func lookup(sym string) sexpr {
	// TODO attempt lookup in reflect

	v, ok := global[sym]
	if ok {
		return v
	}
	panic("undefined")
}
<commit_msg>Bugfix - arbitrary s-expressions can now be eval'd<commit_after>package main

import (
	"fmt"
)

type scope map[string]sexpr

func (v cons) String() string {
	return "<cons>"
}

func doEval(c chan sexpr) {
	for e := range c {
		v := eval(e)
		switch v.(type) {
		case cons:
			fmt.Printf("<cons>\n")
		case sym:
			fmt.Printf("<sym : %s>\n", string(v.(sym)))
		case float64:
			fmt.Printf("%f\n", v.(float64))
		case string:
			fmt.Printf("\"%s\"\n", v.(string))
		default:
			fmt.Printf("nil\n")
		}
	}
}

func isFunction(s sexpr) bool {
	_, ok := s.(func ([]sexpr) sexpr)
	return ok
}

func isSyntax(s sexpr) bool {
	return false
}

// Perform appropriate syntax transformations on the given s-expression. Note
// that some s-expressions that 'should' involve syntax transformations, such
// as (if cond x y) and (lambda ...), don't - they just aren't evaluated as
// normal functions. (TODO make user-defined transformations more flexible to
// add symmetry.)
func transform(e sexpr) sexpr {
	return e // TODO
}

// Evaluates an s-expression, excluding syntax transformations (macros).
func eval(e sexpr) sexpr {
	switch e.(type) {
	case cons: // a function to evaluate
		cons := e.(cons)
		car := eval(cons.car)
		cdr := cons.cdr
		if !isFunction(car) {
			panic("Attempted application on non-function")
		}
		args := flatten(cdr)
		f := car.(func([]sexpr) sexpr)
		// This is a function - evaluate all arguments
		for i, a := range args {
			args[i] = eval(a)
		}
		return f(args)
	case sym:
		return lookup(string(e.(sym)))
	case float64:
		return e
	case string:
		return e
	}
	return Nil
}

func flatten(s sexpr) (ss []sexpr) {
	_, ok := s.(cons)
	for ok {
		ss = append(ss, s.(cons).car)
		s = s.(cons).cdr
		_, ok = s.(cons)
	}
	// TODO what if s isn't nil now?
	return
}

// Performs lookup of symbols for values.
func lookup(sym string) sexpr {
	// TODO attempt lookup in reflect

	v, ok := global[sym]
	if ok {
		return v
	}
	panic("undefined")
}
<|endoftext|>