<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package text

import (
	"bytes"
	"compress/gzip"
	"io"
	"io/ioutil"
	"testing"
	dto "github.com/prometheus/client_model/go"

	"github.com/matttproud/golang_protobuf_extensions/ext"
)

// Benchmarks to show how much penalty text format parsing accually inflicts.
//
// Example results on Linux 3.13.0, Intel(R) Core(TM) i7-4700MQ CPU @ 2.40GHz, go1.4.
//
// BenchmarkParseText          1000           1188535 ns/op          205085 B/op       6135 allocs/op
// BenchmarkParseTextGzip      1000           1376567 ns/op          246224 B/op       6151 allocs/op
// BenchmarkParseProto        10000            172790 ns/op           52258 B/op       1160 allocs/op
// BenchmarkParseProtoGzip     5000            324021 ns/op           94931 B/op       1211 allocs/op
// BenchmarkParseProtoMap     10000            187946 ns/op           58714 B/op       1203 allocs/op
//
// CONCLUSION: The overhead for the map is negligible. Text format needs ~5x more allocations.
// Without compression, it needs ~7x longer, but with compression (the more relevant scenario),
// the difference becomes less relevant, only ~4x.
//
// The test data contains 248 samples.
//
// BenchmarkProcessor002ParseOnly in the extraction package is not quite
// comparable to the benchmarks here, but it gives an idea: JSON parsing is even
// slower than text parsing and needs a comparable amount of allocs.

// BenchmarkParseText benchmarks the parsing of a text-format scrape into metric
// family DTOs.
func BenchmarkParseText(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/text")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		if _, err := parser.TextToMetricFamilies(bytes.NewReader(data)); err != nil {
			b.Fatal(err)
		}
	}
}

// BenchmarkParseTextGzip benchmarks the parsing of a gzipped text-format scrape
// into metric family DTOs.
func BenchmarkParseTextGzip(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/text.gz")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		in, err := gzip.NewReader(bytes.NewReader(data))
		if err != nil {
			b.Fatal(err)
		}
		if _, err := parser.TextToMetricFamilies(in); err != nil {
			b.Fatal(err)
		}
	}
}

// BenchmarkParseProto benchmarks the parsinge of a protobuf-format scrape into
// metric family DTOs. Note that this does not build a map of matric families
// (as the text version does), because it is not required for Prometheus
// ingestion either. (However, it is required for the text-format parsing, as
// the metric family might be sprinkled all over the text, while the
// protobuf-format guarantees bundling at one place.)
func BenchmarkParseProto(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/protobuf")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		family := &dto.MetricFamily{}
		in := bytes.NewReader(data)
		for {
			family.Reset()
			if _, err := ext.ReadDelimited(in, family); err != nil {
				if err == io.EOF {
					break
				}
				b.Fatal(err)
			}
		}
	}
}

// BenchmarkParseProtoGzip is like BenchmarkParseProto above, but parses gzipped
// protobuf format.
func BenchmarkParseProtoGzip(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/protobuf.gz")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		family := &dto.MetricFamily{}
		in, err := gzip.NewReader(bytes.NewReader(data))
		if err != nil {
			b.Fatal(err)
		}
		for {
			family.Reset()
			if _, err := ext.ReadDelimited(in, family); err != nil {
				if err == io.EOF {
					break
				}
				b.Fatal(err)
			}
		}
	}
}

// BenchmarkParseProtoMap is like BenchmarkParseProto but DOES put the parsed
// metric family DTOs into a map. This is not happening during Prometheus
// ingestion. It is just here to measure the overhead of that map creation and
// separate it from the overhead of the text format parsing.
func BenchmarkParseProtoMap(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/protobuf")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		families := map[string]*dto.MetricFamily{}
		in := bytes.NewReader(data)
		for {
			family := &dto.MetricFamily{}
			if _, err := ext.ReadDelimited(in, family); err != nil {
				if err == io.EOF {
					break
				}
				b.Fatal(err)
			}
			families[family.GetName()] = family
		}
	}
}
<commit_msg>Fix typos.<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package text

import (
	"bytes"
	"compress/gzip"
	"io"
	"io/ioutil"
	"testing"
	dto "github.com/prometheus/client_model/go"

	"github.com/matttproud/golang_protobuf_extensions/ext"
)

// Benchmarks to show how much penalty text format parsing actually inflicts.
//
// Example results on Linux 3.13.0, Intel(R) Core(TM) i7-4700MQ CPU @ 2.40GHz, go1.4.
//
// BenchmarkParseText          1000           1188535 ns/op          205085 B/op       6135 allocs/op
// BenchmarkParseTextGzip      1000           1376567 ns/op          246224 B/op       6151 allocs/op
// BenchmarkParseProto        10000            172790 ns/op           52258 B/op       1160 allocs/op
// BenchmarkParseProtoGzip     5000            324021 ns/op           94931 B/op       1211 allocs/op
// BenchmarkParseProtoMap     10000            187946 ns/op           58714 B/op       1203 allocs/op
//
// CONCLUSION: The overhead for the map is negligible. Text format needs ~5x more allocations.
// Without compression, it needs ~7x longer, but with compression (the more relevant scenario),
// the difference becomes less relevant, only ~4x.
//
// The test data contains 248 samples.
//
// BenchmarkProcessor002ParseOnly in the extraction package is not quite
// comparable to the benchmarks here, but it gives an idea: JSON parsing is even
// slower than text parsing and needs a comparable amount of allocs.

// BenchmarkParseText benchmarks the parsing of a text-format scrape into metric
// family DTOs.
func BenchmarkParseText(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/text")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		if _, err := parser.TextToMetricFamilies(bytes.NewReader(data)); err != nil {
			b.Fatal(err)
		}
	}
}

// BenchmarkParseTextGzip benchmarks the parsing of a gzipped text-format scrape
// into metric family DTOs.
func BenchmarkParseTextGzip(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/text.gz")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		in, err := gzip.NewReader(bytes.NewReader(data))
		if err != nil {
			b.Fatal(err)
		}
		if _, err := parser.TextToMetricFamilies(in); err != nil {
			b.Fatal(err)
		}
	}
}

// BenchmarkParseProto benchmarks the parsing of a protobuf-format scrape into
// metric family DTOs. Note that this does not build a map of metric families
// (as the text version does), because it is not required for Prometheus
// ingestion either. (However, it is required for the text-format parsing, as
// the metric family might be sprinkled all over the text, while the
// protobuf-format guarantees bundling at one place.)
func BenchmarkParseProto(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/protobuf")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		family := &dto.MetricFamily{}
		in := bytes.NewReader(data)
		for {
			family.Reset()
			if _, err := ext.ReadDelimited(in, family); err != nil {
				if err == io.EOF {
					break
				}
				b.Fatal(err)
			}
		}
	}
}

// BenchmarkParseProtoGzip is like BenchmarkParseProto above, but parses gzipped
// protobuf format.
func BenchmarkParseProtoGzip(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/protobuf.gz")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		family := &dto.MetricFamily{}
		in, err := gzip.NewReader(bytes.NewReader(data))
		if err != nil {
			b.Fatal(err)
		}
		for {
			family.Reset()
			if _, err := ext.ReadDelimited(in, family); err != nil {
				if err == io.EOF {
					break
				}
				b.Fatal(err)
			}
		}
	}
}

// BenchmarkParseProtoMap is like BenchmarkParseProto but DOES put the parsed
// metric family DTOs into a map. This is not happening during Prometheus
// ingestion. It is just here to measure the overhead of that map creation and
// separate it from the overhead of the text format parsing.
func BenchmarkParseProtoMap(b *testing.B) {
	b.StopTimer()
	data, err := ioutil.ReadFile("testdata/protobuf")
	if err != nil {
		b.Fatal(err)
	}
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		families := map[string]*dto.MetricFamily{}
		in := bytes.NewReader(data)
		for {
			family := &dto.MetricFamily{}
			if _, err := ext.ReadDelimited(in, family); err != nil {
				if err == io.EOF {
					break
				}
				b.Fatal(err)
			}
			families[family.GetName()] = family
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	// "net"
	"net/http"
	"os"
	"time"

	"tileserver/ligneous"
	"tileserver/maptiles"

	log "github.com/cihub/seelog"
)

type Config struct {
	Cache  string            `json:"cache"`
	Engine string            `json:"engine"`
	Layers map[string]string `json:"layers"`
	Port   int               `json:"port"`
}

var (
	config Config
	// engine string
	// port string
	// db_cache string
	config_file   string
	print_version bool
	version       string = "0.1.0"
	//logger seelog.LoggerInterface
)

// func changeState(conn net.Conn, state http.ConnState) {
// 	log.Info(conn)
// 	log.Info(state)
// }

// Serve a single stylesheet via HTTP. Open view_tileserver.html in your browser
// to see the results.
// The created tiles are cached in an sqlite database (MBTiles 1.2 conform) so
// successive access a tile is much faster.
func TileserverWithCaching(engine string, layer_config map[string]string) {
	bind := fmt.Sprintf("0.0.0.0:%v", config.Port)
	if engine == "postgres" {
		t := maptiles.NewTileServerPostgres(config.Cache)
		for i := range layer_config {
			t.AddMapnikLayer(i, layer_config[i])
		}
		log.Info("Connecting to postgres database:")
		log.Info("*** ", config.Cache)
		log.Info(fmt.Sprintf("Magic happens on port %v...", config.Port))
		srv := &http.Server{
			Addr:         bind,
			Handler:      t,
			ReadTimeout:  5 * time.Second,
			WriteTimeout: 10 * time.Second,
		}
		log.Error(srv.ListenAndServe())
		//log.Error(http.ListenAndServe(bind, t))
	} else {
		t := maptiles.NewTileServerSqlite(config.Cache)
		for i := range layer_config {
			t.AddMapnikLayer(i, layer_config[i])
		}
		log.Info("Connecting to sqlite3 database:")
		log.Info("*** ", config.Cache)
		log.Info(fmt.Sprintf("Magic happens on port %v...", config.Port))
		// srv := &http.Server{
		// 	Addr:    bind,
		// 	Handler: t,
		// 	ReadTimeout:  5 * time.Second,
		// 	WriteTimeout: 10 * time.Second,
		// 	// ConnState:    changeState,
		// }
		// log.Error(srv.ListenAndServe())
		log.Error(http.ListenAndServe(bind, t))
	}
}

func init() {
	// TODO: add config file
	// flag.StringVar(&port, "p", "8080", "server port")
	// flag.StringVar(&engine, "e", "sqlite", "database engine [sqlite or postgres]")
	// flag.StringVar(&db_cache, "d", "tilecache.mbtiles", "tile cache database")
	flag.StringVar(&config_file, "c", "", "tile server config")
	flag.BoolVar(&print_version, "v", false, "version")
	flag.Parse()
	// if engine != "sqlite" {
	// 	if engine != "postgres" {
	// 		logger.Fatal("Unsupported database engines")
	// 	}
	// }
	if print_version {
		fmt.Println("TileServer", version)
		os.Exit(1)
	}

	logger, err := ligneous.InitLogger()
	if nil != err {
		fmt.Println("Error starting logging")
		os.Exit(1)
	}
	log.UseLogger(logger)

}

func getConfig() {
	// check if file exists!!!
	if _, err := os.Stat(config_file); err == nil {

		file, err := ioutil.ReadFile(config_file)
		if err != nil {
			panic(err)
		}

		err = json.Unmarshal(file, &config)
		if err != nil {
			fmt.Println("error:", err)
			os.Exit(1)
		}

		if config.Engine != "sqlite" {
			if config.Engine != "postgres" {
				fmt.Println("Unsupported database engine")
				os.Exit(1)
			}
		}

		log.Debug(config)
	} else {
		fmt.Println("Config file not found")
		os.Exit(1)
	}
}

// Before uncommenting the GenerateOSMTiles call make sure you have
// the neccessary OSM sources. Consult OSM wiki for details.
func main() {
	getConfig()
	TileserverWithCaching(config.Engine, config.Layers)
}

// sudo su mapnik
// psql -d mbtiles -U mapnik -W
<commit_msg>timeouts<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	// "net"
	"net/http"
	"os"
	"time"

	"tileserver/ligneous"
	"tileserver/maptiles"

	log "github.com/cihub/seelog"
)

type Config struct {
	Cache  string            `json:"cache"`
	Engine string            `json:"engine"`
	Layers map[string]string `json:"layers"`
	Port   int               `json:"port"`
}

var (
	config Config
	// engine string
	// port string
	// db_cache string
	config_file   string
	print_version bool
	version       string = "0.1.0"
	//logger seelog.LoggerInterface
)

// func changeState(conn net.Conn, state http.ConnState) {
// 	log.Info(conn)
// 	log.Info(state)
// }

// Serve a single stylesheet via HTTP. Open view_tileserver.html in your browser
// to see the results.
// The created tiles are cached in an sqlite database (MBTiles 1.2 conform) so
// successive access a tile is much faster.
func TileserverWithCaching(engine string, layer_config map[string]string) {
	bind := fmt.Sprintf("0.0.0.0:%v", config.Port)
	if engine == "postgres" {
		t := maptiles.NewTileServerPostgres(config.Cache)
		for i := range layer_config {
			t.AddMapnikLayer(i, layer_config[i])
		}
		log.Info("Connecting to postgres database:")
		log.Info("*** ", config.Cache)
		log.Info(fmt.Sprintf("Magic happens on port %v...", config.Port))
		srv := &http.Server{
			Addr:         bind,
			Handler:      t,
			ReadTimeout:  5 * time.Second,
			WriteTimeout: 10 * time.Second,
		}
		log.Error(srv.ListenAndServe())
		//log.Error(http.ListenAndServe(bind, t))
	} else {
		t := maptiles.NewTileServerSqlite(config.Cache)
		for i := range layer_config {
			t.AddMapnikLayer(i, layer_config[i])
		}
		log.Info("Connecting to sqlite3 database:")
		log.Info("*** ", config.Cache)
		log.Info(fmt.Sprintf("Magic happens on port %v...", config.Port))
		srv := &http.Server{
			Addr:         bind,
			Handler:      t,
			ReadTimeout:  5 * time.Second,
			WriteTimeout: 10 * time.Second,
		}
		log.Error(srv.ListenAndServe())
		// log.Error(http.ListenAndServe(bind, t))
	}
}

func init() {
	// TODO: add config file
	// flag.StringVar(&port, "p", "8080", "server port")
	// flag.StringVar(&engine, "e", "sqlite", "database engine [sqlite or postgres]")
	// flag.StringVar(&db_cache, "d", "tilecache.mbtiles", "tile cache database")
	flag.StringVar(&config_file, "c", "", "tile server config")
	flag.BoolVar(&print_version, "v", false, "version")
	flag.Parse()
	// if engine != "sqlite" {
	// 	if engine != "postgres" {
	// 		logger.Fatal("Unsupported database engines")
	// 	}
	// }
	if print_version {
		fmt.Println("TileServer", version)
		os.Exit(1)
	}

	logger, err := ligneous.InitLogger()
	if nil != err {
		fmt.Println("Error starting logging")
		os.Exit(1)
	}
	log.UseLogger(logger)

}

func getConfig() {
	// check if file exists!!!
	if _, err := os.Stat(config_file); err == nil {

		file, err := ioutil.ReadFile(config_file)
		if err != nil {
			panic(err)
		}

		err = json.Unmarshal(file, &config)
		if err != nil {
			fmt.Println("error:", err)
			os.Exit(1)
		}

		if config.Engine != "sqlite" {
			if config.Engine != "postgres" {
				fmt.Println("Unsupported database engine")
				os.Exit(1)
			}
		}

		log.Debug(config)
	} else {
		fmt.Println("Config file not found")
		os.Exit(1)
	}
}

// Before uncommenting the GenerateOSMTiles call make sure you have
// the neccessary OSM sources. Consult OSM wiki for details.
func main() {
	getConfig()
	TileserverWithCaching(config.Engine, config.Layers)
}

// sudo su mapnik
// psql -d mbtiles -U mapnik -W
<|endoftext|>


<commit_before>package tq

import (
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"

	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/httputil"
	"github.com/git-lfs/git-lfs/lfsapi"
	"github.com/git-lfs/git-lfs/progress"
)

const (
	BasicAdapterName = "basic"
)

// Adapter for basic uploads (non resumable)
type basicUploadAdapter struct {
	*adapterBase
}

func (a *basicUploadAdapter) ClearTempStorage() error {
	// Should be empty already but also remove dir
	return os.RemoveAll(a.tempDir())
}

func (a *basicUploadAdapter) tempDir() string {
	// Must be dedicated to this adapter as deleted by ClearTempStorage
	d := filepath.Join(os.TempDir(), "git-lfs-basic-temp")
	if err := os.MkdirAll(d, 0755); err != nil {
		return os.TempDir()
	}
	return d
}

func (a *basicUploadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
	return nil, nil
}
func (a *basicUploadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
}

func (a *basicUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error {
	rel, err := t.Actions.Get("upload")
	if err != nil {
		return err
		// return fmt.Errorf("No upload action for this object.")
	}

	req, err := httputil.NewHttpRequest("PUT", rel.Href, rel.Header)
	if err != nil {
		return err
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}

	if req.Header.Get("Transfer-Encoding") == "chunked" {
		req.TransferEncoding = []string{"chunked"}
	} else {
		req.Header.Set("Content-Length", strconv.FormatInt(t.Size, 10))
	}

	req.ContentLength = t.Size

	f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
	if err != nil {
		return errors.Wrap(err, "basic upload")
	}
	defer f.Close()

	// Ensure progress callbacks made while uploading
	// Wrap callback to give name context
	ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		if cb != nil {
			return cb(t.Name, totalSize, readSoFar, readSinceLast)
		}
		return nil
	}
	var reader io.Reader
	reader = &progress.CallbackReader{
		C:         ccb,
		TotalSize: t.Size,
		Reader:    f,
	}

	// Signal auth was ok on first read; this frees up other workers to start
	if authOkFunc != nil {
		reader = newStartCallbackReader(reader, func(*startCallbackReader) {
			authOkFunc()
		})
	}

	req.Body = ioutil.NopCloser(reader)

	res, err := httputil.DoHttpRequest(config.Config, req, !t.Authenticated)
	if err != nil {
		return errors.NewRetriableError(err)
	}
	httputil.LogTransfer(config.Config, "lfs.data.upload", res)

	// A status code of 403 likely means that an authentication token for the
	// upload has expired. This can be safely retried.
	if res.StatusCode == 403 {
		err = errors.New("http: received status 403")
		return errors.NewRetriableError(err)
	}

	if res.StatusCode > 299 {
		return errors.Wrapf(nil, "Invalid status for %s: %d", httputil.TraceHttpReq(req), res.StatusCode)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	cli := &lfsapi.Client{}
	return verifyUpload(cli, t)
}

// startCallbackReader is a reader wrapper which calls a function as soon as the
// first Read() call is made. This callback is only made once
type startCallbackReader struct {
	r      io.Reader
	cb     func(*startCallbackReader)
	cbDone bool
}

func (s *startCallbackReader) Read(p []byte) (n int, err error) {
	if !s.cbDone && s.cb != nil {
		s.cb(s)
		s.cbDone = true
	}
	return s.r.Read(p)
}
func newStartCallbackReader(r io.Reader, cb func(*startCallbackReader)) *startCallbackReader {
	return &startCallbackReader{r, cb, false}
}

func configureBasicUploadAdapter(m *Manifest) {
	m.RegisterNewAdapterFunc(BasicAdapterName, Upload, func(name string, dir Direction) Adapter {
		switch dir {
		case Upload:
			bu := &basicUploadAdapter{newAdapterBase(name, dir, nil)}
			// self implements impl
			bu.transferImpl = bu
			return bu
		case Download:
			panic("Should never ask this func for basic download")
		}
		return nil
	})
}
<commit_msg>tq: teach basic upload adapter to use lfsapi.Client<commit_after>package tq

import (
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/progress"
)

const (
	BasicAdapterName = "basic"
)

// Adapter for basic uploads (non resumable)
type basicUploadAdapter struct {
	*adapterBase
}

func (a *basicUploadAdapter) ClearTempStorage() error {
	// Should be empty already but also remove dir
	return os.RemoveAll(a.tempDir())
}

func (a *basicUploadAdapter) tempDir() string {
	// Must be dedicated to this adapter as deleted by ClearTempStorage
	d := filepath.Join(os.TempDir(), "git-lfs-basic-temp")
	if err := os.MkdirAll(d, 0755); err != nil {
		return os.TempDir()
	}
	return d
}

func (a *basicUploadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
	return nil, nil
}
func (a *basicUploadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
}

func (a *basicUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error {
	rel, err := t.Actions.Get("upload")
	if err != nil {
		return err
		// return fmt.Errorf("No upload action for this object.")
	}

	req, err := http.NewRequest("PUT", rel.Href, nil)
	if err != nil {
		return err
	}

	for key, value := range rel.Header {
		req.Header.Set(key, value)
	}

	if len(req.Header.Get("Content-Type")) == 0 {
		req.Header.Set("Content-Type", "application/octet-stream")
	}

	if req.Header.Get("Transfer-Encoding") == "chunked" {
		req.TransferEncoding = []string{"chunked"}
	} else {
		req.Header.Set("Content-Length", strconv.FormatInt(t.Size, 10))
	}

	req.ContentLength = t.Size

	f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
	if err != nil {
		return errors.Wrap(err, "basic upload")
	}
	defer f.Close()

	// Ensure progress callbacks made while uploading
	// Wrap callback to give name context
	ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		if cb != nil {
			return cb(t.Name, totalSize, readSoFar, readSinceLast)
		}
		return nil
	}
	var reader io.Reader
	reader = &progress.CallbackReader{
		C:         ccb,
		TotalSize: t.Size,
		Reader:    f,
	}

	// Signal auth was ok on first read; this frees up other workers to start
	if authOkFunc != nil {
		reader = newStartCallbackReader(reader, func(*startCallbackReader) {
			authOkFunc()
		})
	}

	req.Body = ioutil.NopCloser(reader)

	var res *http.Response
	if t.Authenticated {
		res, err = a.apiClient.Do(req)
	} else {
		res, err = a.apiClient.DoWithAuth(a.remote, req)
	}
	if err != nil {
		return errors.NewRetriableError(err)
	}

	a.apiClient.LogResponse("lfs.data.upload", res)

	// A status code of 403 likely means that an authentication token for the
	// upload has expired. This can be safely retried.
	if res.StatusCode == 403 {
		err = errors.New("http: received status 403")
		return errors.NewRetriableError(err)
	}

	if res.StatusCode > 299 {
		return errors.Wrapf(nil, "Invalid status for %s %s: %d",
			req.Method,
			strings.SplitN(req.URL.String(), "?", 2)[0],
			res.StatusCode,
		)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	return verifyUpload(a.apiClient, t)
}

// startCallbackReader is a reader wrapper which calls a function as soon as the
// first Read() call is made. This callback is only made once
type startCallbackReader struct {
	r      io.Reader
	cb     func(*startCallbackReader)
	cbDone bool
}

func (s *startCallbackReader) Read(p []byte) (n int, err error) {
	if !s.cbDone && s.cb != nil {
		s.cb(s)
		s.cbDone = true
	}
	return s.r.Read(p)
}
func newStartCallbackReader(r io.Reader, cb func(*startCallbackReader)) *startCallbackReader {
	return &startCallbackReader{r, cb, false}
}

func configureBasicUploadAdapter(m *Manifest) {
	m.RegisterNewAdapterFunc(BasicAdapterName, Upload, func(name string, dir Direction) Adapter {
		switch dir {
		case Upload:
			bu := &basicUploadAdapter{newAdapterBase(name, dir, nil)}
			// self implements impl
			bu.transferImpl = bu
			return bu
		case Download:
			panic("Should never ask this func for basic download")
		}
		return nil
	})
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"errors"
	"flag"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"time"
)

var defaultPort int

var ptInfo PtServerInfo

// When a connection handler starts, +1 is written to this channel; when it
// ends, -1 is written.
var handlerChan = make(chan int)

func logDebug(format string, v ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", v...)
}

type websocketConn struct {
	Ws         *Websocket
	Base64     bool
	messageBuf []byte
}

func (conn *websocketConn) Read(b []byte) (n int, err error) {
	for len(conn.messageBuf) == 0 {
		var m WebsocketMessage
		m, err = conn.Ws.ReadMessage()
		if err != nil {
			return
		}
		if m.Opcode == 8 {
			err = io.EOF
			return
		}
		if conn.Base64 {
			if m.Opcode != 1 {
				err = errors.New(fmt.Sprintf("got non-text opcode %d with the base64 subprotocol", m.Opcode))
				return
			}
			conn.messageBuf = make([]byte, base64.StdEncoding.DecodedLen(len(m.Payload)))
			var num int
			num, err = base64.StdEncoding.Decode(conn.messageBuf, m.Payload)
			if err != nil {
				return
			}
			conn.messageBuf = conn.messageBuf[:num]
		} else {
			if m.Opcode != 2 {
				err = errors.New(fmt.Sprintf("got non-binary opcode %d with no subprotocol", m.Opcode))
				return
			}
			conn.messageBuf = m.Payload
		}
	}

	n = copy(b, conn.messageBuf)
	conn.messageBuf = conn.messageBuf[n:]

	return
}

func (conn *websocketConn) Write(b []byte) (n int, err error) {
	if conn.Base64 {
		buf := make([]byte, base64.StdEncoding.EncodedLen(len(b)))
		base64.StdEncoding.Encode(buf, b)
		err = conn.Ws.WriteMessage(1, buf)
		if err != nil {
			return
		}
		n = len(b)
	} else {
		err = conn.Ws.WriteMessage(2, b)
		n = len(b)
	}
	return
}

func (conn *websocketConn) Close() (err error) {
	err = conn.Ws.WriteFrame(8, nil)
	if err != nil {
		conn.Ws.Conn.Close()
		return
	}
	err = conn.Ws.Conn.Close()
	return
}

func (conn *websocketConn) LocalAddr() net.Addr {
	return conn.Ws.Conn.LocalAddr()
}

func (conn *websocketConn) RemoteAddr() net.Addr {
	return conn.Ws.Conn.RemoteAddr()
}

func (conn *websocketConn) SetDeadline(t time.Time) error {
	return conn.Ws.Conn.SetDeadline(t)
}

func (conn *websocketConn) SetReadDeadline(t time.Time) error {
	return conn.Ws.Conn.SetReadDeadline(t)
}

func (conn *websocketConn) SetWriteDeadline(t time.Time) error {
	return conn.Ws.Conn.SetWriteDeadline(t)
}

func NewWebsocketConn(ws *Websocket) websocketConn {
	var conn websocketConn
	conn.Ws = ws
	conn.Base64 = (ws.Subprotocol == "base64")
	return conn
}

func proxy(local *net.TCPConn, conn *websocketConn) {
	var wg sync.WaitGroup

	wg.Add(2)

	go func() {
		_, err := io.Copy(conn, local)
		if err != nil {
			logDebug("error copying ORPort to WebSocket: " + err.Error())
		}
		local.CloseRead()
		conn.Close()
		wg.Done()
	}()

	go func() {
		_, err := io.Copy(local, conn)
		if err != nil {
			logDebug("error copying WebSocket to ORPort: " + err.Error())
		}
		local.CloseWrite()
		conn.Close()
		wg.Done()
	}()

	wg.Wait()
}

func websocketHandler(ws *Websocket) {
	conn := NewWebsocketConn(ws)

	handlerChan <- 1
	defer func() {
		handlerChan <- -1
	}()

	s, err := net.DialTCP("tcp", nil, ptInfo.OrAddr)
	if err != nil {
		logDebug("Failed to connect to ORPort: " + err.Error())
		return
	}

	proxy(s, &conn)
}

func startListener(addr *net.TCPAddr) (*net.TCPListener, error) {
	ln, err := net.ListenTCP("tcp", addr)
	if err != nil {
		return nil, err
	}
	go func() {
		var config WebsocketConfig
		config.Subprotocols = []string{"base64"}
		config.MaxMessageSize = 2500
		http.Handle("/", config.Handler(websocketHandler))
		err = http.Serve(ln, nil)
		if err != nil {
			logDebug("http.Serve: " + err.Error())
		}
	}()
	return ln, nil
}

func main() {
	const ptMethodName = "websocket"

	flag.IntVar(&defaultPort, "port", 0, "port to listen on if unspecified by Tor")
	flag.Parse()

	ptInfo = PtServerSetup([]string{ptMethodName})

	listeners := make([]*net.TCPListener, 0)
	for _, bindAddr := range ptInfo.BindAddrs {
		// When tor tells us a port of 0, we are supposed to pick a
		// random port. But we actually want to use the configured port.
		if bindAddr.Addr.Port == 0 {
			bindAddr.Addr.Port = defaultPort
		}

		ln, err := startListener(bindAddr.Addr)
		if err != nil {
			PtSmethodError(bindAddr.MethodName, err.Error())
		}
		PtSmethod(bindAddr.MethodName, ln.Addr())
		listeners = append(listeners, ln)
	}
	PtSmethodsDone()

	var numHandlers int = 0

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)
	var sigint bool = false
	for !sigint {
		select {
		case n := <-handlerChan:
			numHandlers += n
		case <-signalChan:
			logDebug("SIGINT")
			sigint = true
		}
	}

	for _, ln := range listeners {
		ln.Close()
	}

	sigint = false
	for numHandlers != 0 && !sigint {
		select {
		case n := <-handlerChan:
			numHandlers += n
		case <-signalChan:
			logDebug("SIGINT")
			sigint = true
		}
	}
}
<commit_msg>Descope a variable.<commit_after>package main

import (
	"encoding/base64"
	"errors"
	"flag"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"os/signal"
	"sync"
	"time"
)

var ptInfo PtServerInfo

// When a connection handler starts, +1 is written to this channel; when it
// ends, -1 is written.
var handlerChan = make(chan int)

func logDebug(format string, v ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", v...)
}

type websocketConn struct {
	Ws         *Websocket
	Base64     bool
	messageBuf []byte
}

func (conn *websocketConn) Read(b []byte) (n int, err error) {
	for len(conn.messageBuf) == 0 {
		var m WebsocketMessage
		m, err = conn.Ws.ReadMessage()
		if err != nil {
			return
		}
		if m.Opcode == 8 {
			err = io.EOF
			return
		}
		if conn.Base64 {
			if m.Opcode != 1 {
				err = errors.New(fmt.Sprintf("got non-text opcode %d with the base64 subprotocol", m.Opcode))
				return
			}
			conn.messageBuf = make([]byte, base64.StdEncoding.DecodedLen(len(m.Payload)))
			var num int
			num, err = base64.StdEncoding.Decode(conn.messageBuf, m.Payload)
			if err != nil {
				return
			}
			conn.messageBuf = conn.messageBuf[:num]
		} else {
			if m.Opcode != 2 {
				err = errors.New(fmt.Sprintf("got non-binary opcode %d with no subprotocol", m.Opcode))
				return
			}
			conn.messageBuf = m.Payload
		}
	}

	n = copy(b, conn.messageBuf)
	conn.messageBuf = conn.messageBuf[n:]

	return
}

func (conn *websocketConn) Write(b []byte) (n int, err error) {
	if conn.Base64 {
		buf := make([]byte, base64.StdEncoding.EncodedLen(len(b)))
		base64.StdEncoding.Encode(buf, b)
		err = conn.Ws.WriteMessage(1, buf)
		if err != nil {
			return
		}
		n = len(b)
	} else {
		err = conn.Ws.WriteMessage(2, b)
		n = len(b)
	}
	return
}

func (conn *websocketConn) Close() (err error) {
	err = conn.Ws.WriteFrame(8, nil)
	if err != nil {
		conn.Ws.Conn.Close()
		return
	}
	err = conn.Ws.Conn.Close()
	return
}

func (conn *websocketConn) LocalAddr() net.Addr {
	return conn.Ws.Conn.LocalAddr()
}

func (conn *websocketConn) RemoteAddr() net.Addr {
	return conn.Ws.Conn.RemoteAddr()
}

func (conn *websocketConn) SetDeadline(t time.Time) error {
	return conn.Ws.Conn.SetDeadline(t)
}

func (conn *websocketConn) SetReadDeadline(t time.Time) error {
	return conn.Ws.Conn.SetReadDeadline(t)
}

func (conn *websocketConn) SetWriteDeadline(t time.Time) error {
	return conn.Ws.Conn.SetWriteDeadline(t)
}

func NewWebsocketConn(ws *Websocket) websocketConn {
	var conn websocketConn
	conn.Ws = ws
	conn.Base64 = (ws.Subprotocol == "base64")
	return conn
}

func proxy(local *net.TCPConn, conn *websocketConn) {
	var wg sync.WaitGroup

	wg.Add(2)

	go func() {
		_, err := io.Copy(conn, local)
		if err != nil {
			logDebug("error copying ORPort to WebSocket: " + err.Error())
		}
		local.CloseRead()
		conn.Close()
		wg.Done()
	}()

	go func() {
		_, err := io.Copy(local, conn)
		if err != nil {
			logDebug("error copying WebSocket to ORPort: " + err.Error())
		}
		local.CloseWrite()
		conn.Close()
		wg.Done()
	}()

	wg.Wait()
}

func websocketHandler(ws *Websocket) {
	conn := NewWebsocketConn(ws)

	handlerChan <- 1
	defer func() {
		handlerChan <- -1
	}()

	s, err := net.DialTCP("tcp", nil, ptInfo.OrAddr)
	if err != nil {
		logDebug("Failed to connect to ORPort: " + err.Error())
		return
	}

	proxy(s, &conn)
}

func startListener(addr *net.TCPAddr) (*net.TCPListener, error) {
	ln, err := net.ListenTCP("tcp", addr)
	if err != nil {
		return nil, err
	}
	go func() {
		var config WebsocketConfig
		config.Subprotocols = []string{"base64"}
		config.MaxMessageSize = 2500
		http.Handle("/", config.Handler(websocketHandler))
		err = http.Serve(ln, nil)
		if err != nil {
			logDebug("http.Serve: " + err.Error())
		}
	}()
	return ln, nil
}

func main() {
	const ptMethodName = "websocket"
	var defaultPort int

	flag.IntVar(&defaultPort, "port", 0, "port to listen on if unspecified by Tor")
	flag.Parse()

	ptInfo = PtServerSetup([]string{ptMethodName})

	listeners := make([]*net.TCPListener, 0)
	for _, bindAddr := range ptInfo.BindAddrs {
		// When tor tells us a port of 0, we are supposed to pick a
		// random port. But we actually want to use the configured port.
		if bindAddr.Addr.Port == 0 {
			bindAddr.Addr.Port = defaultPort
		}

		ln, err := startListener(bindAddr.Addr)
		if err != nil {
			PtSmethodError(bindAddr.MethodName, err.Error())
		}
		PtSmethod(bindAddr.MethodName, ln.Addr())
		listeners = append(listeners, ln)
	}
	PtSmethodsDone()

	var numHandlers int = 0

	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)
	var sigint bool = false
	for !sigint {
		select {
		case n := <-handlerChan:
			numHandlers += n
		case <-signalChan:
			logDebug("SIGINT")
			sigint = true
		}
	}

	for _, ln := range listeners {
		ln.Close()
	}

	sigint = false
	for numHandlers != 0 && !sigint {
		select {
		case n := <-handlerChan:
			numHandlers += n
		case <-signalChan:
			logDebug("SIGINT")
			sigint = true
		}
	}
}
<|endoftext|>


<commit_before>package s3api

import (
	"context"
	"encoding/xml"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"io"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
)

type ListBucketResultV2 struct {
	XMLName               xml.Name      `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
	Name                  string        `xml:"Name"`
	Prefix                string        `xml:"Prefix"`
	MaxKeys               int           `xml:"MaxKeys"`
	Delimiter             string        `xml:"Delimiter,omitempty"`
	IsTruncated           bool          `xml:"IsTruncated"`
	Contents              []ListEntry   `xml:"Contents,omitempty"`
	CommonPrefixes        []PrefixEntry `xml:"CommonPrefixes,omitempty"`
	ContinuationToken     string        `xml:"ContinuationToken,omitempty"`
	NextContinuationToken string        `xml:"NextContinuationToken,omitempty"`
	KeyCount              int           `xml:"KeyCount"`
	StartAfter            string        `xml:"StartAfter,omitempty"`
}

func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {

	// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html

	// collect parameters
	bucket, _ := getBucketAndObject(r)

	originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())

	if maxKeys < 0 {
		writeErrorResponse(w, s3err.ErrInvalidMaxKeys, r.URL)
		return
	}
	if delimiter != "" && delimiter != "/" {
		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
		return
	}

	marker := continuationToken
	if continuationToken == "" {
		marker = startAfter
	}

	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)

	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}
	responseV2 := &ListBucketResultV2{
		XMLName:               response.XMLName,
		Name:                  response.Name,
		CommonPrefixes:        response.CommonPrefixes,
		Contents:              response.Contents,
		ContinuationToken:     continuationToken,
		Delimiter:             response.Delimiter,
		IsTruncated:           response.IsTruncated,
		KeyCount:              len(response.Contents) + len(response.CommonPrefixes),
		MaxKeys:               response.MaxKeys,
		NextContinuationToken: response.NextMarker,
		Prefix:                response.Prefix,
		StartAfter:            startAfter,
	}

	writeSuccessResponseXML(w, encodeResponse(responseV2))
}

func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {

	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html

	// collect parameters
	bucket, _ := getBucketAndObject(r)

	originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())

	if maxKeys < 0 {
		writeErrorResponse(w, s3err.ErrInvalidMaxKeys, r.URL)
		return
	}
	if delimiter != "" && delimiter != "/" {
		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
		return
	}

	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)

	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	writeSuccessResponseXML(w, encodeResponse(response))
}

func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, marker string, delimiter string) (response ListBucketResult, err error) {
	// convert full path prefix into directory name and prefix for entry name
	reqDir, prefix := filepath.Split(originalPrefix)
	if strings.HasPrefix(reqDir, "/") {
		reqDir = reqDir[1:]
	}
	bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
	reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir)
	if strings.HasSuffix(reqDir, "/") {
		// remove trailing "/"
		reqDir = reqDir[:len(reqDir)-1]
	}

	var contents []ListEntry
	var commonPrefixes []PrefixEntry
	var isTruncated bool
	var doErr error
	var nextMarker string

	// check filer
	err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		_, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) {
			if entry.IsDirectory {
				if delimiter == "/" {
					commonPrefixes = append(commonPrefixes, PrefixEntry{
						Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
					})
				}
			} else {
				storageClass := "STANDARD"
				if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok {
					storageClass = string(v)
				}
				contents = append(contents, ListEntry{
					Key:          fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
					LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
					ETag:         "\"" + filer.ETag(entry) + "\"",
					Size:         int64(filer.FileSize(entry)),
					Owner: CanonicalUser{
						ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
						DisplayName: entry.Attributes.UserName,
					},
					StorageClass: StorageClass(storageClass),
				})
			}
		})
		if doErr != nil {
			return doErr
		}

		if !isTruncated {
			nextMarker = ""
		}

		response = ListBucketResult{
			Name:           bucket,
			Prefix:         originalPrefix,
			Marker:         marker,
			NextMarker:     nextMarker,
			MaxKeys:        maxKeys,
			Delimiter:      delimiter,
			IsTruncated:    isTruncated,
			Contents:       contents,
			CommonPrefixes: commonPrefixes,
		}

		return nil
	})

	return
}

func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
	// invariants
	//   prefix and marker should be under dir, marker may contain "/"
	//   maxKeys should be updated for each recursion

	if prefix == "/" && delimiter == "/" {
		return
	}
	if maxKeys <= 0 {
		return
	}

	if strings.Contains(marker, "/") {
		sepIndex := strings.Index(marker, "/")
		subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
		// println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys)
		subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, eachEntryFn)
		if subErr != nil {
			err = subErr
			return
		}
		isTruncated = isTruncated || subIsTruncated
		maxKeys -= subCounter
		nextMarker = subDir + "/" + subNextMarker
		counter += subCounter
		// finished processing this sub directory
		marker = subDir
	}

	// now marker is also a direct child of dir
	request := &filer_pb.ListEntriesRequest{
		Directory:          dir,
		Prefix:             prefix,
		Limit:              uint32(maxKeys + 1),
		StartFromFileName:  marker,
		InclusiveStartFrom: false,
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	stream, listErr := client.ListEntries(ctx, request)
	if listErr != nil {
		err = fmt.Errorf("list entires %+v: %v", request, listErr)
		return
	}

	for {
		resp, recvErr := stream.Recv()
		if recvErr != nil {
			if recvErr == io.EOF {
				break
			} else {
				err = fmt.Errorf("iterating entires %+v: %v", request, recvErr)
				return
			}
		}
		if counter >= maxKeys {
			isTruncated = true
			return
		}
		entry := resp.Entry
		nextMarker = entry.Name
		if entry.IsDirectory {
			// println("ListEntries", dir, "dir:", entry.Name)
			if entry.Name != ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
				if delimiter != "/" {
					eachEntryFn(dir, entry)
					// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter)
					subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, eachEntryFn)
					if subErr != nil {
						err = fmt.Errorf("doListFilerEntries2: %v", subErr)
						return
					}
					// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated)
					counter += subCounter
					nextMarker = entry.Name + "/" + subNextMarker
					if subIsTruncated {
						isTruncated = true
						return
					}
				} else {
					var isEmpty bool
					if !s3a.option.AllowEmptyFolder {
						if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil {
							glog.Errorf("check empty folder %s: %v", dir, err)
						}
					}
					if !isEmpty {
						eachEntryFn(dir, entry)
						counter++
					}
				}
			}
		} else {
			// println("ListEntries", dir, "file:", entry.Name)
			eachEntryFn(dir, entry)
			counter++
		}
	}
	return
}

func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) {
	prefix = values.Get("prefix")
	token = values.Get("continuation-token")
	startAfter = values.Get("start-after")
	delimiter = values.Get("delimiter")
	if values.Get("max-keys") != "" {
		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
	} else {
		maxkeys = maxObjectListSizeLimit
	}
	fetchOwner = values.Get("fetch-owner") == "true"
	return
}

func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) {
	prefix = values.Get("prefix")
	marker = values.Get("marker")
	delimiter = values.Get("delimiter")
	if values.Get("max-keys") != "" {
		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
	} else {
		maxkeys = maxObjectListSizeLimit
	}
	return
}

func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerClient, parentDir, name string) (isEmpty bool, err error) {
	// println("+ isDirectoryAllEmpty", dir, name)
	glog.V(4).Infof("+ isEmpty %s/%s", parentDir, name)
	defer glog.V(4).Infof("- isEmpty %s/%s %v", parentDir, name, isEmpty)
	var fileCounter int
	var subDirs []string
	currentDir := parentDir + "/" + name
	var startFrom string
	var isExhausted bool
	var foundEntry bool
	for fileCounter == 0 && !isExhausted && err == nil {
		err = filer_pb.SeaweedList(filerClient, currentDir, "", func(entry *filer_pb.Entry, isLast bool) error {
			foundEntry = true
			if entry.IsDirectory {
				subDirs = append(subDirs, entry.Name)
			} else {
				fileCounter++
			}
			startFrom = entry.Name
			isExhausted = isExhausted || isLast
			glog.V(4).Infof("    * %s/%s isLast: %t", currentDir, startFrom, isLast)
			return nil
		}, startFrom, false, 8)
		if !foundEntry {
			break
		}
	}

	if err != nil {
		return false, err
	}

	if fileCounter > 0 {
		return false, nil
	}

	for _, subDir := range subDirs {
		isSubEmpty, subErr := s3a.isDirectoryAllEmpty(filerClient, currentDir, subDir)
		if subErr != nil {
			return false, subErr
		}
		if !isSubEmpty {
			return false, nil
		}
	}

	glog.V(1).Infof("deleting empty folder %s", currentDir)
	if err = doDeleteEntry(filerClient, parentDir, name, true, true); err != nil {
		return
	}

	return true, nil
}
<commit_msg>s3: listing may repeat on the edge<commit_after>package s3api

import (
	"context"
	"encoding/xml"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"io"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
)

type ListBucketResultV2 struct {
	XMLName               xml.Name      `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
	Name                  string        `xml:"Name"`
	Prefix                string        `xml:"Prefix"`
	MaxKeys               int           `xml:"MaxKeys"`
	Delimiter             string        `xml:"Delimiter,omitempty"`
	IsTruncated           bool          `xml:"IsTruncated"`
	Contents              []ListEntry   `xml:"Contents,omitempty"`
	CommonPrefixes        []PrefixEntry `xml:"CommonPrefixes,omitempty"`
	ContinuationToken     string        `xml:"ContinuationToken,omitempty"`
	NextContinuationToken string        `xml:"NextContinuationToken,omitempty"`
	KeyCount              int           `xml:"KeyCount"`
	StartAfter            string        `xml:"StartAfter,omitempty"`
}

func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {

	// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html

	// collect parameters
	bucket, _ := getBucketAndObject(r)

	originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())

	if maxKeys < 0 {
		writeErrorResponse(w, s3err.ErrInvalidMaxKeys, r.URL)
		return
	}
	if delimiter != "" && delimiter != "/" {
		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
		return
	}

	marker := continuationToken
	if continuationToken == "" {
		marker = startAfter
	}

	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)

	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}
	responseV2 := &ListBucketResultV2{
		XMLName:               response.XMLName,
		Name:                  response.Name,
		CommonPrefixes:        response.CommonPrefixes,
		Contents:              response.Contents,
		ContinuationToken:     continuationToken,
		Delimiter:             response.Delimiter,
		IsTruncated:           response.IsTruncated,
		KeyCount:              len(response.Contents) + len(response.CommonPrefixes),
		MaxKeys:               response.MaxKeys,
		NextContinuationToken: response.NextMarker,
		Prefix:                response.Prefix,
		StartAfter:            startAfter,
	}

	writeSuccessResponseXML(w, encodeResponse(responseV2))
}

func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {

	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html

	// collect parameters
	bucket, _ := getBucketAndObject(r)

	originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())

	if maxKeys < 0 {
		writeErrorResponse(w, s3err.ErrInvalidMaxKeys, r.URL)
		return
	}
	if delimiter != "" && delimiter != "/" {
		writeErrorResponse(w, s3err.ErrNotImplemented, r.URL)
		return
	}

	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)

	if err != nil {
		writeErrorResponse(w, s3err.ErrInternalError, r.URL)
		return
	}

	writeSuccessResponseXML(w, encodeResponse(response))
}

func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, marker string, delimiter string) (response ListBucketResult, err error) {
	// convert full path prefix into directory name and prefix for entry name
	reqDir, prefix := filepath.Split(originalPrefix)
	if strings.HasPrefix(reqDir, "/") {
		reqDir = reqDir[1:]
	}
	bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
	reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir)
	if strings.HasSuffix(reqDir, "/") {
		// remove trailing "/"
		reqDir = reqDir[:len(reqDir)-1]
	}

	var contents []ListEntry
	var commonPrefixes []PrefixEntry
	var isTruncated bool
	var doErr error
	var nextMarker string

	// check filer
	err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		_, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) {
			if entry.IsDirectory {
				if delimiter == "/" {
					commonPrefixes = append(commonPrefixes, PrefixEntry{
						Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
					})
				}
			} else {
				storageClass := "STANDARD"
				if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok {
					storageClass = string(v)
				}
				contents = append(contents, ListEntry{
					Key:          fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
					LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
					ETag:         "\"" + filer.ETag(entry) + "\"",
					Size:         int64(filer.FileSize(entry)),
					Owner: CanonicalUser{
						ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
						DisplayName: entry.Attributes.UserName,
					},
					StorageClass: StorageClass(storageClass),
				})
			}
		})
		if doErr != nil {
			return doErr
		}

		if !isTruncated {
			nextMarker = ""
		}

		response = ListBucketResult{
			Name:           bucket,
			Prefix:         originalPrefix,
			Marker:         marker,
			NextMarker:     nextMarker,
			MaxKeys:        maxKeys,
			Delimiter:      delimiter,
			IsTruncated:    isTruncated,
			Contents:       contents,
			CommonPrefixes: commonPrefixes,
		}

		return nil
	})

	return
}

func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
	// invariants
	//   prefix and marker should be under dir, marker may contain "/"
	//   maxKeys should be updated for each recursion

	if prefix == "/" && delimiter == "/" {
		return
	}
	if maxKeys <= 0 {
		return
	}

	if strings.Contains(marker, "/") {
		sepIndex := strings.Index(marker, "/")
		subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
		// println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys)
		subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, eachEntryFn)
		if subErr != nil {
			err = subErr
			return
		}
		isTruncated = isTruncated || subIsTruncated
		maxKeys -= subCounter
		nextMarker = subDir + "/" + subNextMarker
		counter += subCounter
		// finished processing this sub directory
		marker = subDir
	}

	// now marker is also a direct child of dir
	request := &filer_pb.ListEntriesRequest{
		Directory:          dir,
		Prefix:             prefix,
		Limit:              uint32(maxKeys + 1),
		StartFromFileName:  marker,
		InclusiveStartFrom: false,
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	stream, listErr := client.ListEntries(ctx, request)
	if listErr != nil {
		err = fmt.Errorf("list entires %+v: %v", request, listErr)
		return
	}

	for {
		resp, recvErr := stream.Recv()
		if recvErr != nil {
			if recvErr == io.EOF {
				break
			} else {
				err = fmt.Errorf("iterating entires %+v: %v", request, recvErr)
				return
			}
		}
		if counter >= maxKeys + 1 {
			isTruncated = true
			return
		}
		entry := resp.Entry
		nextMarker = entry.Name
		if entry.IsDirectory {
			// println("ListEntries", dir, "dir:", entry.Name)
			if entry.Name != ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
				if delimiter != "/" {
					eachEntryFn(dir, entry)
					// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter)
					subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, eachEntryFn)
					if subErr != nil {
						err = fmt.Errorf("doListFilerEntries2: %v", subErr)
						return
					}
					// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated)
					counter += subCounter
					nextMarker = entry.Name + "/" + subNextMarker
					if subIsTruncated {
						isTruncated = true
						return
					}
				} else {
					var isEmpty bool
					if !s3a.option.AllowEmptyFolder {
						if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil {
							glog.Errorf("check empty folder %s: %v", dir, err)
						}
					}
					if !isEmpty {
						eachEntryFn(dir, entry)
						counter++
					}
				}
			}
		} else {
			// println("ListEntries", dir, "file:", entry.Name)
			eachEntryFn(dir, entry)
			counter++
		}
	}
	return
}

func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) {
	prefix = values.Get("prefix")
	token = values.Get("continuation-token")
	startAfter = values.Get("start-after")
	delimiter = values.Get("delimiter")
	if values.Get("max-keys") != "" {
		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
	} else {
		maxkeys = maxObjectListSizeLimit
	}
	fetchOwner = values.Get("fetch-owner") == "true"
	return
}

func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) {
	prefix = values.Get("prefix")
	marker = values.Get("marker")
	delimiter = values.Get("delimiter")
	if values.Get("max-keys") != "" {
		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
	} else {
		maxkeys = maxObjectListSizeLimit
	}
	return
}

func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerClient, parentDir, name string) (isEmpty bool, err error) {
	// println("+ isDirectoryAllEmpty", dir, name)
	glog.V(4).Infof("+ isEmpty %s/%s", parentDir, name)
	defer glog.V(4).Infof("- isEmpty %s/%s %v", parentDir, name, isEmpty)
	var fileCounter int
	var subDirs []string
	currentDir := parentDir + "/" + name
	var startFrom string
	var isExhausted bool
	var foundEntry bool
	for fileCounter == 0 && !isExhausted && err == nil {
		err = filer_pb.SeaweedList(filerClient, currentDir, "", func(entry *filer_pb.Entry, isLast bool) error {
			foundEntry = true
			if entry.IsDirectory {
				subDirs = append(subDirs, entry.Name)
			} else {
				fileCounter++
			}
			startFrom = entry.Name
			isExhausted = isExhausted || isLast
			glog.V(4).Infof("    * %s/%s isLast: %t", currentDir, startFrom, isLast)
			return nil
		}, startFrom, false, 8)
		if !foundEntry {
			break
		}
	}

	if err != nil {
		return false, err
	}

	if fileCounter > 0 {
		return false, nil
	}

	for _, subDir := range subDirs {
		isSubEmpty, subErr := s3a.isDirectoryAllEmpty(filerClient, currentDir, subDir)
		if subErr != nil {
			return false, subErr
		}
		if !isSubEmpty {
			return false, nil
		}
	}

	glog.V(1).Infof("deleting empty folder %s", currentDir)
	if err = doDeleteEntry(filerClient, parentDir, name, true, true); err != nil {
		return
	}

	return true, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package discoverspaces

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/names"
	"github.com/juju/utils/set"
	"launchpad.net/tomb"

	"github.com/juju/juju/api/discoverspaces"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.discoverspaces")

type discoverspacesWorker struct {
	api  *discoverspaces.API
	tomb tomb.Tomb
}

var invalidChars = regexp.MustCompile("[^0-9a-z-]")
var dashPrefix = regexp.MustCompile("^-*")

func convertSpaceName(name string, existing set.Strings) string {
	// First lower case and replace spaces with dashes.
	name = strings.Replace(name, " ", "-", -1)
	name = strings.ToLower(name)
	// Next replace any character that isn't in the set "-", "a-z", "0-9".
	name = invalidChars.ReplaceAllString(name, "")
	// Next get rid of any dashes at the start as that isn't valid.
	name = dashPrefix.ReplaceAllString(name, "")
	// Special case of when the space name was only dashes or invalid
	// characters!
	if name == "" {
		name = "empty"
	}
	// If this name is in use add a numerical suffix.
	if existing.Contains(name) {
		counter := 2
		for existing.Contains(name + fmt.Sprintf("-%d", counter)) {
			counter += 1
		}
		name = name + fmt.Sprintf("-%d", counter)
	}
	return name
}

// NewWorker returns a worker
func NewWorker(api *discoverspaces.API) worker.Worker {
	dw := &discoverspacesWorker{
		api: api,
	}
	go func() {
		defer dw.tomb.Done()
		dw.tomb.Kill(dw.loop())
	}()
	return dw
}

func (dw *discoverspacesWorker) Kill() {
	dw.tomb.Kill(nil)
}

func (dw *discoverspacesWorker) Wait() error {
	return dw.tomb.Wait()
}

func (dw *discoverspacesWorker) loop() (err error) {
	envCfg, err := dw.api.EnvironConfig()
	if err != nil {
		return err
	}
	environ, err := environs.New(envCfg)
	if err != nil {
		return err
	}
	networkingEnviron, ok := environs.SupportsNetworking(environ)

	if ok {
		// TODO: (mfoord) API should be switched off until this is
		// completed.
		err = dw.handleSubnets(networkingEnviron)
		if err != nil {
			return errors.Trace(err)
		}
	}

	// TODO(mfoord): we'll have a watcher here checking if we need to
	// update the spaces/subnets definition.
	dying := dw.tomb.Dying()
	for {
		select {
		case <-dying:
			return nil
		}
	}
	return nil
}

func (dw *discoverspacesWorker) handleSubnets(env environs.NetworkingEnviron) error {
	ok, err := env.SupportsSpaceDiscovery()
	if err != nil {
		return errors.Trace(err)
	}
	if !ok {
		// Nothing to do.
		return nil
	}
	providerSpaces, err := env.Spaces()
	if err != nil {
		return errors.Trace(err)
	}
	listSpacesResult, err := dw.api.ListSpaces()
	if err != nil {
		return errors.Trace(err)
	}

	stateSubnets, err := dw.api.ListSubnets(params.SubnetsFilters{})
	if err != nil {
		return errors.Trace(err)
	}
	stateSubnetIds := make(set.Strings)
	for _, subnet := range stateSubnets.Results {
		stateSubnetIds.Add(subnet.ProviderId)
	}
	stateSpaceMap := make(map[string]params.ProviderSpace)
	spaceNames := make(set.Strings)
	for _, space := range listSpacesResult.Results {
		stateSpaceMap[space.ProviderId] = space
		spaceNames.Add(space.Name)
	}

	// TODO(mfoord): we need to delete spaces and subnets that no longer
	// exist, so long as they're not in use.
	for _, space := range providerSpaces {
		// Check if the space is already in state, in which case we know
		// its name.
		stateSpace, ok := stateSpaceMap[string(space.ProviderId)]
		var spaceTag names.SpaceTag
		if ok {
			spaceName := stateSpace.Name
			if !names.IsValidSpace(spaceName) {
				// Can only happen if an invalid name is stored
				// in state.
				logger.Errorf("space %q has an invalid name, ignoring", spaceName)
				continue

			}
			spaceTag = names.NewSpaceTag(spaceName)

		} else {
			// The space is new, we need to create a valid name for it
			// in state.
			spaceName := string(space.ProviderId)
			if !names.IsValidSpace(spaceName) {
				// Convert the name into a valid name that isn't already in
				// use.
				spaceName = convertSpaceName(spaceName, spaceNames)
				spaceNames.Add(spaceName)
			}
			spaceTag = names.NewSpaceTag(spaceName)
			// We need to create the space.
			args := params.CreateSpacesParams{
				Spaces: []params.CreateSpaceParams{{
					Public:   false,
					SpaceTag: spaceTag.String(),
				}}}
			result, err := dw.api.CreateSpaces(args)
			if err != nil {
				logger.Errorf("error creating space %v", err)
				return errors.Trace(err)
			}
			if len(result.Results) != 1 {
				return errors.Errorf("unexpected number of results from CreateSpaces, should be 1: %v", result)
			}
			if result.Results[0].Error != nil {
				return errors.Errorf("error from CreateSpaces: %v", result.Results[0].Error)
			}
		}
		// TODO(mfoord): currently no way of removing subnets, or
		// changing the space they're in, so we can only add ones we
		// don't already know about.
		logger.Debugf("Created space %v with %v subnets", spaceTag.String(), len(space.Subnets))
		for _, subnet := range space.Subnets {
			if stateSubnetIds.Contains(string(subnet.ProviderId)) {
				continue
			}
			zones := subnet.AvailabilityZones
			if len(zones) == 0 {
				zones = []string{"default"}
			}
			args := params.AddSubnetsParams{
				Subnets: []params.AddSubnetParams{{
					SubnetProviderId: string(subnet.ProviderId),
					SpaceTag:         spaceTag.String(),
					Zones:            zones,
				}}}
			logger.Tracef("Adding subnet %v", subnet.CIDR)
			result, err := dw.api.AddSubnets(args)
			if err != nil {
				logger.Errorf("invalid creating subnet %v", err)
				return errors.Trace(err)
			}
			if len(result.Results) != 1 {
				return errors.Errorf("unexpected number of results from AddSubnets, should be 1: %v", result)
			}
			if result.Results[0].Error != nil {
				logger.Errorf("error creating subnet %v", result.Results[0].Error)
				return errors.Errorf("error creating subnet %v", result.Results[0].Error)
			}
		}
	}
	return nil
}
<commit_msg>Call convertSpaceName unconditionally to catch duplicates<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package discoverspaces

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/names"
	"github.com/juju/utils/set"
	"launchpad.net/tomb"

	"github.com/juju/juju/api/discoverspaces"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.discoverspaces")

type discoverspacesWorker struct {
	api  *discoverspaces.API
	tomb tomb.Tomb
}

var invalidChars = regexp.MustCompile("[^0-9a-z-]")
var dashPrefix = regexp.MustCompile("^-*")

func convertSpaceName(name string, existing set.Strings) string {
	// First lower case and replace spaces with dashes.
	name = strings.Replace(name, " ", "-", -1)
	name = strings.ToLower(name)
	// Next replace any character that isn't in the set "-", "a-z", "0-9".
	name = invalidChars.ReplaceAllString(name, "")
	// Next get rid of any dashes at the start as that isn't valid.
	name = dashPrefix.ReplaceAllString(name, "")
	// Special case of when the space name was only dashes or invalid
	// characters!
	if name == "" {
		name = "empty"
	}
	// If this name is in use add a numerical suffix.
	if existing.Contains(name) {
		counter := 2
		for existing.Contains(name + fmt.Sprintf("-%d", counter)) {
			counter += 1
		}
		name = name + fmt.Sprintf("-%d", counter)
	}
	return name
}

// NewWorker returns a worker
func NewWorker(api *discoverspaces.API) worker.Worker {
	dw := &discoverspacesWorker{
		api: api,
	}
	go func() {
		defer dw.tomb.Done()
		dw.tomb.Kill(dw.loop())
	}()
	return dw
}

func (dw *discoverspacesWorker) Kill() {
	dw.tomb.Kill(nil)
}

func (dw *discoverspacesWorker) Wait() error {
	return dw.tomb.Wait()
}

func (dw *discoverspacesWorker) loop() (err error) {
	envCfg, err := dw.api.EnvironConfig()
	if err != nil {
		return err
	}
	environ, err := environs.New(envCfg)
	if err != nil {
		return err
	}
	networkingEnviron, ok := environs.SupportsNetworking(environ)

	if ok {
		// TODO: (mfoord) API should be switched off until this is
		// completed.
		err = dw.handleSubnets(networkingEnviron)
		if err != nil {
			return errors.Trace(err)
		}
	}

	// TODO(mfoord): we'll have a watcher here checking if we need to
	// update the spaces/subnets definition.
	dying := dw.tomb.Dying()
	for {
		select {
		case <-dying:
			return nil
		}
	}
	return nil
}

func (dw *discoverspacesWorker) handleSubnets(env environs.NetworkingEnviron) error {
	ok, err := env.SupportsSpaceDiscovery()
	if err != nil {
		return errors.Trace(err)
	}
	if !ok {
		// Nothing to do.
		return nil
	}
	providerSpaces, err := env.Spaces()
	if err != nil {
		return errors.Trace(err)
	}
	listSpacesResult, err := dw.api.ListSpaces()
	if err != nil {
		return errors.Trace(err)
	}

	stateSubnets, err := dw.api.ListSubnets(params.SubnetsFilters{})
	if err != nil {
		return errors.Trace(err)
	}
	stateSubnetIds := make(set.Strings)
	for _, subnet := range stateSubnets.Results {
		stateSubnetIds.Add(subnet.ProviderId)
	}
	stateSpaceMap := make(map[string]params.ProviderSpace)
	spaceNames := make(set.Strings)
	for _, space := range listSpacesResult.Results {
		stateSpaceMap[space.ProviderId] = space
		spaceNames.Add(space.Name)
	}

	// TODO(mfoord): we need to delete spaces and subnets that no longer
	// exist, so long as they're not in use.
	for _, space := range providerSpaces {
		// Check if the space is already in state, in which case we know
		// its name.
		stateSpace, ok := stateSpaceMap[string(space.ProviderId)]
		var spaceTag names.SpaceTag
		if ok {
			spaceName := stateSpace.Name
			if !names.IsValidSpace(spaceName) {
				// Can only happen if an invalid name is stored
				// in state.
				logger.Errorf("space %q has an invalid name, ignoring", spaceName)
				continue

			}
			spaceTag = names.NewSpaceTag(spaceName)

		} else {
			// The space is new, we need to create a valid name for it
			// in state.
			spaceName := string(space.ProviderId)
			// Convert the name into a valid name that isn't already in
			// use.
			spaceName = convertSpaceName(spaceName, spaceNames)
			spaceNames.Add(spaceName)
			spaceTag = names.NewSpaceTag(spaceName)
			// We need to create the space.
			args := params.CreateSpacesParams{
				Spaces: []params.CreateSpaceParams{{
					Public:   false,
					SpaceTag: spaceTag.String(),
				}}}
			result, err := dw.api.CreateSpaces(args)
			if err != nil {
				logger.Errorf("error creating space %v", err)
				return errors.Trace(err)
			}
			if len(result.Results) != 1 {
				return errors.Errorf("unexpected number of results from CreateSpaces, should be 1: %v", result)
			}
			if result.Results[0].Error != nil {
				return errors.Errorf("error from CreateSpaces: %v", result.Results[0].Error)
			}
		}
		// TODO(mfoord): currently no way of removing subnets, or
		// changing the space they're in, so we can only add ones we
		// don't already know about.
		logger.Debugf("Created space %v with %v subnets", spaceTag.String(), len(space.Subnets))
		for _, subnet := range space.Subnets {
			if stateSubnetIds.Contains(string(subnet.ProviderId)) {
				continue
			}
			zones := subnet.AvailabilityZones
			if len(zones) == 0 {
				zones = []string{"default"}
			}
			args := params.AddSubnetsParams{
				Subnets: []params.AddSubnetParams{{
					SubnetProviderId: string(subnet.ProviderId),
					SpaceTag:         spaceTag.String(),
					Zones:            zones,
				}}}
			logger.Tracef("Adding subnet %v", subnet.CIDR)
			result, err := dw.api.AddSubnets(args)
			if err != nil {
				logger.Errorf("invalid creating subnet %v", err)
				return errors.Trace(err)
			}
			if len(result.Results) != 1 {
				return errors.Errorf("unexpected number of results from AddSubnets, should be 1: %v", result)
			}
			if result.Results[0].Error != nil {
				logger.Errorf("error creating subnet %v", result.Results[0].Error)
				return errors.Errorf("error creating subnet %v", result.Results[0].Error)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"

	"hermes/ratings/controller"
	"hermes/ratings/handler"

	"github.com/alecthomas/kingpin"
	"github.com/facebookgo/grace/gracehttp"
	"github.com/labstack/echo"
)

var (
	app            = kingpin.New("hermes", "GCBA product ratings APIs.")
	startCommand   = kingpin.Command("start", "Start an Hermes API.")
	ratingsCommand = startCommand.Command("ratings", "Start the ratings API.")
	ratingsPort    = getRatingsPort()
	noCursor       = "\n\n\033[?25l"
	banner         = `
 _  _ ____ ____ _  _ ____ ____
 |__| |___ |__/ |\/| |___ [__
 |  | |___ |  \ |  | |___ ___] `
)

func main() {
	kingpin.Version("0.0.1")
	fmt.Println("\n", banner, "\n\n")

	switch kingpin.Parse() {
	case "start ratings":
		startRatingsAPI()
	}
}

func startRatingsAPI() {
	routes := map[string]echo.HandlerFunc{
		"OptionsRoot":    controller.OptionsRoot,
		"OptionsRatings": controller.OptionsRatings,
		"PostRatings":    controller.PostRatings}

	handler, castOk := handler.Handler(ratingsPort, routes).(*echo.Echo)

	if !castOk {
		handler.Logger.Fatal("Could not start server")
	}

	fmt.Println("✅  Server started on port", strconv.Itoa(ratingsPort))
	fmt.Print(noCursor)

	handler.Logger.Fatal(gracehttp.Serve(handler.Server))
}

func getRatingsPort() int {
	port, portErr := strconv.Atoi(os.Getenv("API_RATINGS_PORT"))

	if portErr != nil {
		return 5000
	}

	return port
}
<commit_msg>Added a comand to start the statistics API<commit_after>package main

import (
	"fmt"
	"os"
	"strconv"

	"hermes/ratings/controller"
	"hermes/ratings/handler"

	"github.com/alecthomas/kingpin"
	"github.com/facebookgo/grace/gracehttp"
	"github.com/labstack/echo"
)

var (
	app            = kingpin.New("hermes", "GCBA product ratings APIs.")
	startCommand   = kingpin.Command("start", "Start an Hermes API.")
	ratingsCommand = startCommand.Command("ratings", "Start the ratings API.")
	statsCommand   = startCommand.Command("stats", "Start the statistics API.")
	ratingsPort    = getRatingsPort()
	noCursor       = "\n\n\033[?25l"
	banner         = `
 _  _ ____ ____ _  _ ____ ____
 |__| |___ |__/ |\/| |___ [__
 |  | |___ |  \ |  | |___ ___] `
)

func main() {
	kingpin.Version("0.0.1")
	fmt.Println("\n", banner, "\n\n")

	switch kingpin.Parse() {
	case "start ratings":
		startRatingsAPI()
	case "start stats":
		startStatsAPI()
	}
}

func startRatingsAPI() {
	routes := map[string]echo.HandlerFunc{
		"OptionsRoot":    controller.OptionsRoot,
		"OptionsRatings": controller.OptionsRatings,
		"PostRatings":    controller.PostRatings}

	handler, castOk := handler.Handler(ratingsPort, routes).(*echo.Echo)

	if !castOk {
		handler.Logger.Fatal("Could not start server")
	}

	fmt.Println("✅  Server started on port", strconv.Itoa(ratingsPort))
	fmt.Print(noCursor)

	handler.Logger.Fatal(gracehttp.Serve(handler.Server))
}

func startStatsAPI() {
	routes := map[string]echo.HandlerFunc{
		"OptionsRoot":    controller.OptionsRoot,
		"OptionsRatings": controller.OptionsRatings,
		"PostRatings":    controller.PostRatings}

	handler, castOk := handler.Handler(ratingsPort, routes).(*echo.Echo)

	if !castOk {
		handler.Logger.Fatal("Could not start server")
	}

	fmt.Println("✅  Server started on port", strconv.Itoa(ratingsPort))
	fmt.Print(noCursor)

	handler.Logger.Fatal(gracehttp.Serve(handler.Server))
}

func getRatingsPort() int {
	port, portErr := strconv.Atoi(os.Getenv("HERMES_RATINGS_PORT"))

	if portErr != nil {
		return 5000
	}

	return port
}

func getStatsPort() int {
	port, portErr := strconv.Atoi(os.Getenv("HERMES_STATS_PORT"))

	if portErr != nil {
		return 5000
	}

	return port
}
<|endoftext|>


<commit_before>package apiservermsgs

/*
Copyright 2019 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import ()

const PGO_VERSION = "3.5.2-rc5"

// Ok status
const Ok = "ok"
const Error = "error"

// Status ...
type Status struct {
	Code string
	Msg  string
}

type BasicAuthCredentials struct {
	Username     string
	Password     string
	APIServerURL string
}
<commit_msg>update version to 3.5.2 in code<commit_after>package apiservermsgs

/*
Copyright 2019 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import ()

const PGO_VERSION = "3.5.2"

// Ok status
const Ok = "ok"
const Error = "error"

// Status ...
type Status struct {
	Code string
	Msg  string
}

type BasicAuthCredentials struct {
	Username     string
	Password     string
	APIServerURL string
}
<|endoftext|>


<commit_before>package tutum

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"

	"github.com/gorilla/websocket"
)

/*
func ListContainers
Returns : Array of Container objects
*/
func ListContainers() (CListResponse, error) {

	url := "container/"
	request := "GET"
	//Empty Body Request
	body := []byte(`{}`)
	var response CListResponse
	var finalResponse CListResponse

	data, err := TutumCall(url, request, body)
	if err != nil {
		return response, err
	}

	err = json.Unmarshal(data, &response)
	if err != nil {
		return response, err
	}

	finalResponse = response

Loop:
	for {
		if response.Meta.Next != "" {
			var nextResponse CListResponse
			data, err := TutumCall(response.Meta.Next[8:], request, body)
			if err != nil {
				return nextResponse, err
			}
			err = json.Unmarshal(data, &nextResponse)
			if err != nil {
				return nextResponse, err
			}
			finalResponse.Objects = append(finalResponse.Objects, nextResponse.Objects...)
			response = nextResponse

		} else {
			break Loop
		}
	}

	return finalResponse, nil
}

/*
func GetContainer
Argument : uuid
Returns : Container JSON object
*/
func GetContainer(uuid string) (Container, error) {

	url := ""
	if string(uuid[0]) == "/" {
		url = uuid[8:]
	} else {
		url = "container/" + uuid + "/"
	}

	request := "GET"
	//Empty Body Request
	body := []byte(`{}`)
	var response Container

	data, err := TutumCall(url, request, body)
	if err != nil {
		return response, err
	}

	err = json.Unmarshal(data, &response)
	if err != nil {
		return response, err
	}

	return response, nil
}

/*
func GetContainerLogs
Argument : a channel of type string for the output
*/

func (self *Container) Logs(c chan Logs) {

	endpoint := "container/" + self.Uuid + "/logs/?user=" + User + "&token=" + ApiKey
	url := StreamUrl + endpoint

	header := http.Header{}
	header.Add("User-Agent", customUserAgent)

	var Dialer websocket.Dialer
	ws, _, err := Dialer.Dial(url, header)
	if err != nil {
		log.Println(err)
	}

	var msg Logs
	for {
		if err = ws.ReadJSON(&msg); err != nil {
			//Type switches don't work here, so we use a type assertion instead
			_, ok := err.(*websocket.CloseError)
			if ok {
				close(c)
				break
			} else {
				log.Println(err)
			}
		}
		c <- msg
	}
}

/*
func Exec
Arguments : the command to execute, a channel of type string for the output
*/

func (self *Container) Exec(command string, c chan Exec) {
	go self.Run(command, c)
Loop:
	for {
		select {
		case s, open := <-c:
			if open {
				fmt.Printf("%s", s.Output)
			} else {
				break Loop
			}
		}
	}
}

func (self *Container) Run(command string, c chan Exec) {

	endpoint := "container/" + self.Uuid + "/exec/?user=" + User + "&token=" + ApiKey + "&command=" + url.QueryEscape(command)
	url := StreamUrl + endpoint

	header := http.Header{}
	header.Add("User-Agent", customUserAgent)

	var Dialer websocket.Dialer
	ws, _, err := Dialer.Dial(url, header)
	if err != nil {
		log.Println(err)
	}

	var msg Exec
Loop:
	for {
		if err = ws.ReadJSON(&msg); err != nil {
			//Type switches don't work here, so we use a type assertion instead
			_, ok := err.(*websocket.CloseError)
			if ok {
				close(c)
				break Loop
			} else {
				log.Println(err)
			}
		}
		c <- msg
	}
}

/*
func StartContainer
Returns : Error
*/
func (self *Container) Start() error {

	url := "container/" + self.Uuid + "/start/"
	request := "POST"
	//Empty Body Request
	body := []byte(`{}`)
	var response Container

	data, err := TutumCall(url, request, body)
	if err != nil {
		return err
	}

	err = json.Unmarshal(data, &response)
	if err != nil {
		return err
	}

	return nil
}

/*
func StopContainer
Returns : Error
*/
func (self *Container) Stop() error {

	url := "container/" + self.Uuid + "/stop/"
	request := "POST"
	//Empty Body Request
	body := []byte(`{}`)

	_, err := TutumCall(url, request, body)
	if err != nil {
		return err
	}

	return nil
}

/*
func RedeployContainer
Returns : Error
*/
func (self *Container) Redeploy() error {

	url := "container/" + self.Uuid + "/redeploy/"
	request := "POST"
	//Empty Body Request
	body := []byte(`{}`)

	_, err := TutumCall(url, request, body)
	if err != nil {
		return err
	}

	return nil
}

/*
func TerminateContainer
Returns : Error
*/
func (self *Container) Terminate() error {

	url := "container/" + self.Uuid + "/"
	request := "DELETE"
	//Empty Body Request
	body := []byte(`{}`)

	_, err := TutumCall(url, request, body)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Add some doc comments explaining behaviour<commit_after>package tutum

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"

	"github.com/gorilla/websocket"
)

/*
func ListContainers
Returns : Array of Container objects
*/
func ListContainers() (CListResponse, error) {

	url := "container/"
	request := "GET"
	//Empty Body Request
	body := []byte(`{}`)
	var response CListResponse
	var finalResponse CListResponse

	data, err := TutumCall(url, request, body)
	if err != nil {
		return response, err
	}

	err = json.Unmarshal(data, &response)
	if err != nil {
		return response, err
	}

	finalResponse = response

Loop:
	for {
		if response.Meta.Next != "" {
			var nextResponse CListResponse
			data, err := TutumCall(response.Meta.Next[8:], request, body)
			if err != nil {
				return nextResponse, err
			}
			err = json.Unmarshal(data, &nextResponse)
			if err != nil {
				return nextResponse, err
			}
			finalResponse.Objects = append(finalResponse.Objects, nextResponse.Objects...)
			response = nextResponse

		} else {
			break Loop
		}
	}

	return finalResponse, nil
}

/*
func GetContainer
Argument : uuid
Returns : Container JSON object
*/
func GetContainer(uuid string) (Container, error) {

	url := ""
	if string(uuid[0]) == "/" {
		url = uuid[8:]
	} else {
		url = "container/" + uuid + "/"
	}

	request := "GET"
	//Empty Body Request
	body := []byte(`{}`)
	var response Container

	data, err := TutumCall(url, request, body)
	if err != nil {
		return response, err
	}

	err = json.Unmarshal(data, &response)
	if err != nil {
		return response, err
	}

	return response, nil
}

// Logs returns the container's logs through the given channel, wrapped in a Logs struct.
// See https://docs.tutum.co/v2/api/?go#get-the-logs-of-a-container for more info.
func (self *Container) Logs(c chan Logs) {

	endpoint := "container/" + self.Uuid + "/logs/?user=" + User + "&token=" + ApiKey
	url := StreamUrl + endpoint

	header := http.Header{}
	header.Add("User-Agent", customUserAgent)

	var Dialer websocket.Dialer
	ws, _, err := Dialer.Dial(url, header)
	if err != nil {
		log.Println(err)
	}

	var msg Logs
	for {
		if err = ws.ReadJSON(&msg); err != nil {
			//Type switches don't work here, so we use a type assertion instead
			_, ok := err.(*websocket.CloseError)
			if ok {
				close(c)
				break
			} else {
				log.Println(err)
			}
		}
		c <- msg
	}
}

// Exec runs the command in the container, and prints the output to the console.
// When the command exits and the final output has been printed, the function returns.

func (self *Container) Exec(command string, c chan Exec) {
	go self.Run(command, c)
Loop:
	for {
		select {
		case s, open := <-c:
			if open {
				fmt.Printf("%s", s.Output)
			} else {
				break Loop
			}
		}
	}
}

// Run executes the command in the container, and returns the output to the
// channel, wrapped in an Exec struct. When the command exits, the channel closes.
// See https://docs.tutum.co/v2/api/?go#execute-command-inside-a-container for more info.
func (self *Container) Run(command string, c chan Exec) {

	endpoint := "container/" + self.Uuid + "/exec/?user=" + User + "&token=" + ApiKey + "&command=" + url.QueryEscape(command)
	url := StreamUrl + endpoint

	header := http.Header{}
	header.Add("User-Agent", customUserAgent)

	var Dialer websocket.Dialer
	ws, _, err := Dialer.Dial(url, header)
	if err != nil {
		log.Println(err)
	}

	var msg Exec
Loop:
	for {
		if err = ws.ReadJSON(&msg); err != nil {
			//Type switches don't work here, so we use a type assertion instead
			_, ok := err.(*websocket.CloseError)
			if ok {
				close(c)
				break Loop
			} else {
				log.Println(err)
			}
		}
		c <- msg
	}
}

/*
func StartContainer
Returns : Error
*/
func (self *Container) Start() error {

	url := "container/" + self.Uuid + "/start/"
	request := "POST"
	//Empty Body Request
	body := []byte(`{}`)
	var response Container

	data, err := TutumCall(url, request, body)
	if err != nil {
		return err
	}

	err = json.Unmarshal(data, &response)
	if err != nil {
		return err
	}

	return nil
}

/*
func StopContainer
Returns : Error
*/
func (self *Container) Stop() error {

	url := "container/" + self.Uuid + "/stop/"
	request := "POST"
	//Empty Body Request
	body := []byte(`{}`)

	_, err := TutumCall(url, request, body)
	if err != nil {
		return err
	}

	return nil
}

/*
func RedeployContainer
Returns : Error
*/
func (self *Container) Redeploy() error {

	url := "container/" + self.Uuid + "/redeploy/"
	request := "POST"
	//Empty Body Request
	body := []byte(`{}`)

	_, err := TutumCall(url, request, body)
	if err != nil {
		return err
	}

	return nil
}

/*
func TerminateContainer
Returns : Error
*/
func (self *Container) Terminate() error {

	url := "container/" + self.Uuid + "/"
	request := "DELETE"
	//Empty Body Request
	body := []byte(`{}`)

	_, err := TutumCall(url, request, body)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package certificate

import (
	"context"
	"errors"
	"fmt"
	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	restclient "k8s.io/client-go/rest"
	cmdutil "k8s.io/kubectl/pkg/cmd/util"
	"k8s.io/kubectl/pkg/util/i18n"
	"k8s.io/kubectl/pkg/util/templates"
	"time"

	cmclient "github.com/jetstack/cert-manager/pkg/client/clientset/versioned"
)

var (
	long = templates.LongDesc(i18n.T(`
Get details about the current status of a Certificate, including information on related resources like CertificateRequest or Order.`))

	example = templates.Examples(i18n.T(`
# Query status of cert-manager Certificate resource with name my-cert in namespace default
kubectl cert-manager status certificate my-cert --namespace default
`))
)

// Options is a struct to support status certificate command
type Options struct {
	CMClient   cmclient.Interface
	RESTConfig *restclient.Config
	// The Namespace that the Certificate to be renewed resides in.
	// This flag registration is handled by cmdutil.Factory
	Namespace string

	genericclioptions.IOStreams
}

// NewOptions returns initialized Options
func NewOptions(ioStreams genericclioptions.IOStreams) *Options {
	return &Options{
		IOStreams: ioStreams,
	}
}

// NewCmdStatusCert returns a cobra command for status certificate
func NewCmdStatusCert(ioStreams genericclioptions.IOStreams, factory cmdutil.Factory) *cobra.Command {
	o := NewOptions(ioStreams)
	cmd := &cobra.Command{
		Use:     "certificate",
		Short:   "Get details about the current status of a Certificate",
		Long:    long,
		Example: example,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Validate(args))
			cmdutil.CheckErr(o.Complete(factory))
			cmdutil.CheckErr(o.Run(args))
		},
	}
	return cmd
}

// Validate validates the provided options
func (o *Options) Validate(args []string) error {
	if len(args) < 1 {
		return errors.New("the name of the Certificate has to be provided as argument")
	}
	if len(args) > 1 {
		return errors.New("only one argument can be passed in: the name of the Certificate")
	}
	return nil
}

// Complete takes the factory and infers any remaining options.
func (o *Options) Complete(f cmdutil.Factory) error {
	var err error

	o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace()
	if err != nil {
		return err
	}

	o.RESTConfig, err = f.ToRESTConfig()
	if err != nil {
		return err
	}

	o.CMClient, err = cmclient.NewForConfig(o.RESTConfig)
	if err != nil {
		return err
	}

	return nil
}

// Run executes status certificate command
func (o *Options) Run(args []string) error {
	ctx := context.TODO()
	crtName := args[0]

	crt, err := o.CMClient.CertmanagerV1alpha2().Certificates(o.Namespace).Get(ctx, crtName, metav1.GetOptions{})
	if err != nil {
		return fmt.Errorf("error when getting Certificate resource: %v", err)
	}

	fmt.Fprintf(o.Out, fmt.Sprintf("Name: %s\nNamespace: %s\n", crt.Name, crt.Namespace))

	// Get necessary info from Certificate
	// Output one line about each type of Condition that is set.
	conditionMsg := ""
	for _, con := range crt.Status.Conditions {
		conditionMsg += fmt.Sprintf("  %s: %s, Reason: %s, Message: %s\n", con.Type, con.Status, con.Reason, con.Message)
	}
	if conditionMsg == "" {
		conditionMsg = "  No Conditions set\n"
	}
	fmt.Fprintf(o.Out, fmt.Sprintf("Conditions:\n%s", conditionMsg))

	// TODO: Get CR from certificate if exists. I think I can just look for it without caring what condition is
	// What about timing issues? When I query condition it's not ready yet, but then looking for crn it's finished and deleted

	dnsNames := formatStringSlice(crt.Spec.DNSNames)
	fmt.Fprintf(o.Out, fmt.Sprintf("DNS Names:\n%s", dnsNames))

	issuerFormat := `Issuer:
  Name: %s
  Kind: %s`
	fmt.Fprintf(o.Out, fmt.Sprintf(issuerFormat+"\n", crt.Spec.IssuerRef.Name, crt.Spec.IssuerRef.Kind))

	fmt.Fprintf(o.Out, fmt.Sprintf("Secret Name: %s\n", crt.Spec.SecretName))

	fmt.Fprintf(o.Out, fmt.Sprintf("Not Before: %s\n", formatTimeString(crt.Status.NotBefore)))
	fmt.Fprintf(o.Out, fmt.Sprintf("Not After: %s\n", formatTimeString(crt.Status.NotAfter)))
	fmt.Fprintf(o.Out, fmt.Sprintf("Renewal Time: %s\n", formatTimeString(crt.Status.RenewalTime)))

	// TODO: print information about secret
	return nil
}

func formatStringSlice(strings []string) string {
	result := ""
	for _, string := range strings {
		result += "- " + string + "\n"
	}
	return result
}

func formatTimeString(t *metav1.Time) string {
	if t != nil {
		return t.Time.Format(time.RFC3339)
	}
	return "<none>"
}
<commit_msg>Small review changes<commit_after>/*
Copyright 2020 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package certificate

import (
	"context"
	"errors"
	"fmt"
	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	restclient "k8s.io/client-go/rest"
	cmdutil "k8s.io/kubectl/pkg/cmd/util"
	"k8s.io/kubectl/pkg/util/i18n"
	"k8s.io/kubectl/pkg/util/templates"
	"time"

	cmclient "github.com/jetstack/cert-manager/pkg/client/clientset/versioned"
)

var (
	long = templates.LongDesc(i18n.T(`
Get details about the current status of a Certificate.`))

	example = templates.Examples(i18n.T(`
# Query status of cert-manager Certificate resource with name my-cert in namespace default
kubectl cert-manager status certificate my-cert --namespace default
`))
)

// Options is a struct to support status certificate command
type Options struct {
	CMClient   cmclient.Interface
	RESTConfig *restclient.Config
	// The Namespace that the Certificate to be queried about resides in.
	// This flag registration is handled by cmdutil.Factory
	Namespace string

	genericclioptions.IOStreams
}

// NewOptions returns initialized Options
func NewOptions(ioStreams genericclioptions.IOStreams) *Options {
	return &Options{
		IOStreams: ioStreams,
	}
}

// NewCmdStatusCert returns a cobra command for status certificate
func NewCmdStatusCert(ioStreams genericclioptions.IOStreams, factory cmdutil.Factory) *cobra.Command {
	o := NewOptions(ioStreams)
	cmd := &cobra.Command{
		Use:     "certificate",
		Short:   "Get details about the current status of a Certificate",
		Long:    long,
		Example: example,
		Run: func(cmd *cobra.Command, args []string) {
			cmdutil.CheckErr(o.Validate(args))
			cmdutil.CheckErr(o.Complete(factory))
			cmdutil.CheckErr(o.Run(args))
		},
	}
	return cmd
}

// Validate validates the provided options
func (o *Options) Validate(args []string) error {
	if len(args) < 1 {
		return errors.New("the name of the Certificate has to be provided as argument")
	}
	if len(args) > 1 {
		return errors.New("only one argument can be passed in: the name of the Certificate")
	}
	return nil
}

// Complete takes the factory and infers any remaining options.
func (o *Options) Complete(f cmdutil.Factory) error {
	var err error

	o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace()
	if err != nil {
		return err
	}

	o.RESTConfig, err = f.ToRESTConfig()
	if err != nil {
		return err
	}

	o.CMClient, err = cmclient.NewForConfig(o.RESTConfig)
	if err != nil {
		return err
	}

	return nil
}

// Run executes status certificate command
func (o *Options) Run(args []string) error {
	ctx := context.TODO()
	crtName := args[0]

	crt, err := o.CMClient.CertmanagerV1alpha2().Certificates(o.Namespace).Get(ctx, crtName, metav1.GetOptions{})
	if err != nil {
		return fmt.Errorf("error when getting Certificate resource: %v", err)
	}

	fmt.Fprintf(o.Out, fmt.Sprintf("Name: %s\nNamespace: %s\n", crt.Name, crt.Namespace))

	// Get necessary info from Certificate
	// Output one line about each type of Condition that is set.
	conditionMsg := ""
	for _, con := range crt.Status.Conditions {
		conditionMsg += fmt.Sprintf("  %s: %s, Reason: %s, Message: %s\n", con.Type, con.Status, con.Reason, con.Message)
	}
	if conditionMsg == "" {
		conditionMsg = "  No Conditions set\n"
	}
	fmt.Fprintf(o.Out, fmt.Sprintf("Conditions:\n%s", conditionMsg))

	// TODO: Get CR from certificate if exists. I think I can just look for it without caring what condition is
	// What about timing issues? When I query condition it's not ready yet, but then looking for crn it's finished and deleted

	dnsNames := formatStringSlice(crt.Spec.DNSNames)
	fmt.Fprintf(o.Out, fmt.Sprintf("DNS Names:\n%s", dnsNames))

	issuerFormat := `Issuer:
  Name: %s
  Kind: %s`
	fmt.Fprintf(o.Out, fmt.Sprintf(issuerFormat+"\n", crt.Spec.IssuerRef.Name, crt.Spec.IssuerRef.Kind))

	fmt.Fprintf(o.Out, fmt.Sprintf("Secret Name: %s\n", crt.Spec.SecretName))

	fmt.Fprintf(o.Out, fmt.Sprintf("Not Before: %s\n", formatTimeString(crt.Status.NotBefore)))
	fmt.Fprintf(o.Out, fmt.Sprintf("Not After: %s\n", formatTimeString(crt.Status.NotAfter)))
	fmt.Fprintf(o.Out, fmt.Sprintf("Renewal Time: %s\n", formatTimeString(crt.Status.RenewalTime)))

	// TODO: print information about secret
	return nil
}

func formatStringSlice(strings []string) string {
	result := ""
	for _, string := range strings {
		result += "- " + string + "\n"
	}
	return result
}

func formatTimeString(t *metav1.Time) string {
	if t == nil {
		return "<none>"
	}
	return t.Time.Format(time.RFC3339)
}
<|endoftext|>