<commit_before>package service

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mchmarny/go-cmd"
	"io/ioutil"
	"log"
)

// CFClient object
type CFClient struct {
	config *ServiceConfig
}

// NewCFClient creates a new isntance of CFClient
func NewCFClient(c *ServiceConfig) *CFClient {
	return &CFClient{
		config: c,
	}
}

func (c *CFClient) initialize() (*cmd.Command, error) {
	log.Println("initializing...")

	// yep, this is a royal hack, should get this from the env somehow
	pushID := genRandomString(8)
	appDir, err := ioutil.TempDir(c.config.CFEnv.TempDir, pushID)
	if err != nil {
		log.Fatalf("err creating a temp dir: %v", err)
		return nil, err
	}

	// api
	cf := cmd.New("cf")

	// TODO: remove the skip API validation part once real cert deployed
	cf.WithArgs("api", c.config.APIEndpoint, "--skip-ssl-validation").
		WithEnv("CF_HOME", appDir).Exec()
	if cf.Err != nil {
		log.Fatalf("err cmd: %v", cf)
		return cf, cf.Err
	}

	// auth
	cf.WithArgs("auth", c.config.APIUser, c.config.APIPassword).Exec()
	if cf.Err != nil {
		log.Fatalf("err cmd: %v", cf)
		return cf, cf.Err
	}

	return cf, nil
}

func (c *CFClient) provision(ctx *CFServiceContext) error {
	log.Printf("provisioning service: %v", ctx)

	// initialize
	cf, err := c.initialize()
	if err != nil {
		log.Fatalf("err initializing command: %v", err)
		return err
	}

	// target
	cf.WithArgs("target", "-o", ctx.OrgName, "-s", ctx.SpaceName).Exec()
	if cf.Err != nil {
		log.Fatalf("err cmd: %v", cf)
		return cf.Err
	}

	// push
	cf.WithArgs("push", ctx.ServiceName, "-p", c.config.AppSource, "--no-start").Exec()
	if cf.Err != nil {
		log.Printf("err cmd: %v", cf)
		c.deprovision(ctx)
		return cf.Err
	}

	// TODO: Add cleanup of dependencies
	for i, dep := range c.config.Dependencies {
		depName := dep.Name + "-" + ctx.ServiceName
		cf.WithArgs("create-service", dep.Name, dep.Plan, depName).Exec()
		if cf.Err != nil {
			log.Printf("err on dependency[%d]: %s - %v", i, depName, cf)
			return cf.Err
		}

		// bind
		cf.WithArgs("bind-service", ctx.ServiceName, depName).Exec()
		if cf.Err != nil {
			log.Printf("err on bind[%d]: %s > %s - %v", i, ctx.ServiceName, depName, cf)
			return cf.Err
		}

		//TODO: check if we need to restage the app after binding
	}

	// start
	cf.WithArgs("start", ctx.ServiceName).Exec()
	if cf.Err != nil {
		log.Printf("err cmd: %v", cf)
		c.deprovision(ctx)
		return cf.Err
	}

	return nil
}

func (c *CFClient) deprovision(ctx *CFServiceContext) error {
	log.Printf("deprovision service: %v", ctx)

	// initialize
	cf, err := c.initialize()
	if err != nil {
		log.Fatalf("err initializing command: %v", err)
		return err
	}

	// target
	cf.WithArgs("target", "-o", ctx.OrgName, "-s", ctx.SpaceName).Exec()
	if cf.Err != nil {
		log.Fatalf("err cmd: %v", cf)
		return cf.Err
	}

	// delete
	cf.WithArgs("delete", ctx.ServiceName, "-f").Exec()
	if cf.Err != nil {
		log.Printf("err cmd: %v", cf)
		return cf.Err
	}

	// TODO: Does the service have to unbined first
	//       or deleting app will take care of it
	for i, dep := range c.config.Dependencies {
		depName := dep.Name + "-" + ctx.ServiceName
		cf.WithArgs("delete-service", dep.Name, "-f").Exec()
		if cf.Err != nil {
			log.Printf("err on dependency delete[%d]: %s - %v", i, depName, cf)
		}
	}

	return nil
}

func (c *CFClient) runQuery(query string) (string, error) {
	log.Printf("running query: %s", query)
	cf, err := c.initialize()
	if err != nil {
		log.Fatalf("err initializing command: %v", err)
		return "", err
	}
	cf.WithArgs("curl", query).Exec()
	return cf.Out, cf.Err
}

func (c *CFClient) getContext(instanceID string) (*CFServiceContext, error) {
	log.Printf("getting service context for: %s", instanceID)

	t := &CFServiceContext{}
	t.InstanceID = instanceID

	srv, err := c.getService(instanceID)
	if err != nil {
		log.Printf("error getting service: %v", err)
		return nil, err
	}
	t.ServiceName = srv.Name
	t.ServiceURI = srv.URI

	space, err := c.getSpace(srv.SpaceGUID)
	if err != nil {
		log.Printf("error getting space: %v", err)
		return nil, err
	}
	t.SpaceName = space.Name

	org, err := c.getOrg(space.OrgGUID)
	if err != nil {
		log.Printf("error getting org: %v", err)
		return nil, err
	}
	t.OrgName = org.Name

	return t, nil

}

func (c *CFClient) getService(instanceID string) (*CFApp, error) {
	log.Printf("getting service info for: %s", instanceID)
	query := fmt.Sprintf("/v2/service_instances/%s", instanceID)
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	t := &CFAppResource{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("service output: %v", t)
	t.Entity.GUID = t.Meta.GUID
	return &t.Entity, nil
}

func (c *CFClient) getOrg(orgID string) (*CFApp, error) {
	log.Printf("getting org info for: %s", orgID)
	query := fmt.Sprintf("/v2/organizations/%s", orgID)
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	t := &CFAppResource{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("org output: %v", t)
	t.Entity.GUID = t.Meta.GUID
	return &t.Entity, nil
}

func (c *CFClient) getSpace(spaceID string) (*CFSpace, error) {
	log.Printf("getting space info for: %s", spaceID)
	query := fmt.Sprintf("/v2/spaces/%s", spaceID)
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	t := &CFSpaceResource{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("space output: %v", t)
	t.Entity.GUID = t.Meta.GUID
	return &t.Entity, nil
}

func (c *CFClient) getApp(appID string) (*CFApp, error) {
	log.Printf("getting app info for: %s", appID)
	query := fmt.Sprintf("/v2/apps/%s", appID)
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	t := &CFAppResource{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("app output: %v", t)
	t.Entity.GUID = t.Meta.GUID
	return &t.Entity, nil
}

func (c *CFClient) getBinding(bindingID string) (*CFBinding, error) {
	log.Printf("getting service binding for: %s", bindingID)
	query := fmt.Sprintf("/v2/service_bindings/%s", bindingID)
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	t := &CFBindingResource{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("service binding output: %v", t)
	t.Entity.GUID = t.Meta.GUID
	return &t.Entity, nil
}

func (c *CFClient) getApps() (*CFAppsResponce, error) {
	log.Println("getting apps...")
	query := "/v2/apps?results-per-page=100"
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	t := &CFAppsResponce{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("apps output: %v", t)
	return t, nil
}

func (c *CFClient) getServices() (*CFAppsResponce, error) {
	log.Println("getting services...")
	query := "/v2/service_instances?results-per-page=100"
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	t := &CFAppsResponce{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("services output: %v", t)
	return t, nil
}
<commit_msg>log API call reponses<commit_after>package service

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"

	"github.com/mchmarny/go-cmd"
)

// CFClient object
type CFClient struct {
	config *ServiceConfig
}

// NewCFClient creates a new isntance of CFClient
func NewCFClient(c *ServiceConfig) *CFClient {
	return &CFClient{
		config: c,
	}
}

func (c *CFClient) initialize() (*cmd.Command, error) {
	log.Println("initializing...")

	// yep, this is a royal hack, should get this from the env somehow
	pushID := genRandomString(8)
	appDir, err := ioutil.TempDir(c.config.CFEnv.TempDir, pushID)
	if err != nil {
		log.Fatalf("err creating a temp dir: %v", err)
		return nil, err
	}

	// api
	cf := cmd.New("cf")

	// TODO: remove the skip API validation part once real cert deployed
	cf.WithArgs("api", c.config.APIEndpoint, "--skip-ssl-validation").
		WithEnv("CF_HOME", appDir).Exec()
	if cf.Err != nil {
		log.Fatalf("err cmd: %v", cf)
		return cf, cf.Err
	}

	// auth
	cf.WithArgs("auth", c.config.APIUser, c.config.APIPassword).Exec()
	if cf.Err != nil {
		log.Fatalf("err cmd: %v", cf)
		return cf, cf.Err
	}

	return cf, nil
}

func (c *CFClient) provision(ctx *CFServiceContext) error {
	log.Printf("provisioning service: %v", ctx)

	// initialize
	cf, err := c.initialize()
	if err != nil {
		log.Fatalf("err initializing command: %v", err)
		return err
	}

	// target
	cf.WithArgs("target", "-o", ctx.OrgName, "-s", ctx.SpaceName).Exec()
	if cf.Err != nil {
		log.Fatalf("err cmd: %v", cf)
		return cf.Err
	}

	// push
	cf.WithArgs("push", ctx.ServiceName, "-p", c.config.AppSource, "--no-start").Exec()
	if cf.Err != nil {
		log.Printf("err cmd: %v", cf)
		c.deprovision(ctx)
		return cf.Err
	}

	// TODO: Add cleanup of dependencies
	for i, dep := range c.config.Dependencies {
		depName := dep.Name + "-" + ctx.ServiceName
		cf.WithArgs("create-service", dep.Name, dep.Plan, depName).Exec()
		if cf.Err != nil {
			log.Printf("err on dependency[%d]: %s - %v", i, depName, cf)
			return cf.Err
		}

		// bind
		cf.WithArgs("bind-service", ctx.ServiceName, depName).Exec()
		if cf.Err != nil {
			log.Printf("err on bind[%d]: %s > %s - %v", i, ctx.ServiceName, depName, cf)
			return cf.Err
		}

		//TODO: check if we need to restage the app after binding
	}

	// start
	cf.WithArgs("start", ctx.ServiceName).Exec()
	if cf.Err != nil {
		log.Printf("err cmd: %v", cf)
		c.deprovision(ctx)
		return cf.Err
	}

	return nil
}

func (c *CFClient) deprovision(ctx *CFServiceContext) error {
	log.Printf("deprovision service: %v", ctx)

	// initialize
	cf, err := c.initialize()
	if err != nil {
		log.Fatalf("err initializing command: %v", err)
		return err
	}

	// target
	cf.WithArgs("target", "-o", ctx.OrgName, "-s", ctx.SpaceName).Exec()
	if cf.Err != nil {
		log.Fatalf("err cmd: %v", cf)
		return cf.Err
	}

	// delete
	cf.WithArgs("delete", ctx.ServiceName, "-f").Exec()
	if cf.Err != nil {
		log.Printf("err cmd: %v", cf)
		return cf.Err
	}

	// TODO: Does the service have to unbined first
	//       or deleting app will take care of it
	for i, dep := range c.config.Dependencies {
		depName := dep.Name + "-" + ctx.ServiceName
		cf.WithArgs("delete-service", dep.Name, "-f").Exec()
		if cf.Err != nil {
			log.Printf("err on dependency delete[%d]: %s - %v", i, depName, cf)
		}
	}

	return nil
}

func (c *CFClient) runQuery(query string) (string, error) {
	log.Printf("running query: %s", query)
	cf, err := c.initialize()
	if err != nil {
		log.Fatalf("err initializing command: %v", err)
		return "", err
	}
	cf.WithArgs("curl", query).Exec()
	return cf.Out, cf.Err
}

func (c *CFClient) getContext(instanceID string) (*CFServiceContext, error) {
	log.Printf("getting service context for: %s", instanceID)

	t := &CFServiceContext{}
	t.InstanceID = instanceID

	srv, err := c.getService(instanceID)
	if err != nil {
		log.Printf("error getting service: %v", err)
		return nil, err
	}
	t.ServiceName = srv.Name
	t.ServiceURI = srv.URI

	space, err := c.getSpace(srv.SpaceGUID)
	if err != nil {
		log.Printf("error getting space: %v", err)
		return nil, err
	}
	t.SpaceName = space.Name

	org, err := c.getOrg(space.OrgGUID)
	if err != nil {
		log.Printf("error getting org: %v", err)
		return nil, err
	}
	t.OrgName = org.Name

	return t, nil

}

func (c *CFClient) getService(instanceID string) (*CFApp, error) {
	log.Printf("getting service info for: %s", instanceID)
	query := fmt.Sprintf("/v2/service_instances/%s", instanceID)
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}

	// cf-client.go:150: running query: /v2/service_instances/26576e51...
	// cf-client.go:26: initializing...
	// {
	//    "code": 60004,
	//    "description": "The service instance could not be found: 26576e51-8a47-46e3-bd6e-5908287e9935",
	//    "error_code": "CF-ServiceInstanceNotFound"
	// }
	//
	// TODO: map results to a CFError struct to see if an error was returned.
	// FIXME: looks like service instance object doesn't exist when "cf create-service" called
	// TODO: perhaps a background worker to rename service instances later?

	t := &CFAppResource{}
	log.Println(string(resp))
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("service output: %v", t)
	t.Entity.GUID = t.Meta.GUID
	return &t.Entity, nil
}

func (c *CFClient) getOrg(orgID string) (*CFApp, error) {
	log.Printf("getting org info for: %s", orgID)
	query := fmt.Sprintf("/v2/organizations/%s", orgID)
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	log.Println(string(resp))
	t := &CFAppResource{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("org output: %v", t)
	t.Entity.GUID = t.Meta.GUID
	return &t.Entity, nil
}

func (c *CFClient) getSpace(spaceID string) (*CFSpace, error) {
	log.Printf("getting space info for: %s", spaceID)
	query := fmt.Sprintf("/v2/spaces/%s", spaceID)
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	log.Println(string(resp))
	t := &CFSpaceResource{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("space output: %v", t)
	t.Entity.GUID = t.Meta.GUID
	return &t.Entity, nil
}

func (c *CFClient) getApp(appID string) (*CFApp, error) {
	log.Printf("getting app info for: %s", appID)
	query := fmt.Sprintf("/v2/apps/%s", appID)
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	log.Println(string(resp))
	t := &CFAppResource{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("app output: %v", t)
	t.Entity.GUID = t.Meta.GUID
	return &t.Entity, nil
}

func (c *CFClient) getBinding(bindingID string) (*CFBinding, error) {
	log.Printf("getting service binding for: %s", bindingID)
	query := fmt.Sprintf("/v2/service_bindings/%s", bindingID)
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	log.Println(string(resp))
	t := &CFBindingResource{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("service binding output: %v", t)
	t.Entity.GUID = t.Meta.GUID
	return &t.Entity, nil
}

func (c *CFClient) getApps() (*CFAppsResponce, error) {
	log.Println("getting apps...")
	query := "/v2/apps?results-per-page=100"
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	log.Println(string(resp))
	t := &CFAppsResponce{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("apps output: %v", t)
	return t, nil
}

func (c *CFClient) getServices() (*CFAppsResponce, error) {
	log.Println("getting services...")
	query := "/v2/service_instances?results-per-page=100"
	resp, err := c.runQuery(query)
	if err != nil {
		return nil, errors.New("query error")
	}
	log.Println(string(resp))
	t := &CFAppsResponce{}
	err2 := json.Unmarshal([]byte(resp), &t)
	if err2 != nil {
		log.Fatalf("err unmarshaling: %v - %v", err2, resp)
		return nil, errors.New("invalid JSON")
	}
	log.Printf("services output: %v", t)
	return t, nil
}
<|endoftext|>


<commit_before>package services

type Service interface {
	Process(req string) string
  MarshalJSON() ([]byte, error)
  UnmarshalJSON(snap []byte) error
}

func StartService(config string) Service {
  var serv Service
  switch config {
  case "kv-store":
    serv = newStore()
  case "dummy":
    serv = newDummy()
  }
  return serv
}

func GetInteractiveText(config string) string {
	var s string
	switch config {
	case "kv-store":
		s = `
	The following commands are available:
		get [key]: to return the value of a given key
		exists [key]: to test if a given key is present
		update [key] [value]: to set the value of a given key, if key already exists then overwrite
		delete [key]: to remove a key value pair if present
		count: to return the number of keys
		print: to return all key value pairs
	`
	case "dummy":
		s = `
	The following commands are available:
		ping: ping dummy application
	`
	}
	return s
}
<commit_msg>fixing text alignment for interactive console<commit_after>package services

type Service interface {
	Process(req string) string
  MarshalJSON() ([]byte, error)
  UnmarshalJSON(snap []byte) error
}

func StartService(config string) Service {
  var serv Service
  switch config {
  case "kv-store":
    serv = newStore()
  case "dummy":
    serv = newDummy()
  }
  return serv
}

func GetInteractiveText(config string) string {
	var s string
	switch config {
	case "kv-store":
		s =
`The following commands are available:
	get [key]: to return the value of a given key
	exists [key]: to test if a given key is present
	update [key] [value]: to set the value of a given key, if key already exists then overwrite
	delete [key]: to remove a key value pair if present
	count: to return the number of keys
	print: to return all key value pairs
`
	case "dummy":
		s =
`The following commands are available:
		ping: ping dummy application
`
	}
	return s
}
<|endoftext|>


<commit_before>// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"math/rand"
	"sort"
	"testing"

	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/util/testutil"
)

func TestSampleRing(t *testing.T) {
	cases := []struct {
		input []int64
		delta int64
		size  int
	}{
		{
			input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
			delta: 2,
			size:  1,
		},
		{
			input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
			delta: 2,
			size:  2,
		},
		{
			input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
			delta: 7,
			size:  3,
		},
		{
			input: []int64{1, 2, 3, 4, 5, 16, 17, 18, 19, 20},
			delta: 7,
			size:  1,
		},
		{
			input: []int64{1, 2, 3, 4, 6},
			delta: 4,
			size:  4,
		},
	}
	for _, c := range cases {
		r := newSampleRing(c.delta, c.size)

		input := []sample{}
		for _, t := range c.input {
			input = append(input, sample{
				t: t,
				v: float64(rand.Intn(100)),
			})
		}

		for i, s := range input {
			r.add(s.t, s.v)
			buffered := r.samples()

			for _, sold := range input[:i] {
				found := false
				for _, bs := range buffered {
					if bs.t == sold.t && bs.v == sold.v {
						found = true
						break
					}
				}
				if sold.t >= s.t-c.delta && !found {
					t.Fatalf("%d: expected sample %d to be in buffer but was not; buffer %v", i, sold.t, buffered)
				}
				if sold.t < s.t-c.delta && found {
					t.Fatalf("%d: unexpected sample %d in buffer; buffer %v", i, sold.t, buffered)
				}
			}
		}
	}
}

func TestBufferedSeriesIterator(t *testing.T) {
	var it *BufferedSeriesIterator

	bufferEq := func(exp []sample) {
		var b []sample
		bit := it.Buffer()
		for bit.Next() {
			t, v := bit.At()
			b = append(b, sample{t: t, v: v})
		}
		testutil.Equals(t, exp, b, "buffer mismatch")
	}
	sampleEq := func(ets int64, ev float64) {
		ts, v := it.Values()
		testutil.Equals(t, ets, ts, "timestamp mismatch")
		testutil.Equals(t, ev, v, "value mismatch")
	}

	it = NewBufferIterator(newListSeriesIterator([]sample{
		{t: 1, v: 2},
		{t: 2, v: 3},
		{t: 3, v: 4},
		{t: 4, v: 5},
		{t: 5, v: 6},
		{t: 99, v: 8},
		{t: 100, v: 9},
		{t: 101, v: 10},
	}), 2)

	testutil.Assert(t, it.Seek(-123), "seek failed")
	sampleEq(1, 2)
	bufferEq(nil)

	testutil.Assert(t, it.Next(), "next failed")
	sampleEq(2, 3)
	bufferEq([]sample{{t: 1, v: 2}})

	testutil.Assert(t, it.Next(), "next failed")
	testutil.Assert(t, it.Next(), "next failed")
	testutil.Assert(t, it.Next(), "next failed")
	sampleEq(5, 6)
	bufferEq([]sample{{t: 2, v: 3}, {t: 3, v: 4}, {t: 4, v: 5}})

	testutil.Assert(t, it.Seek(5), "seek failed")
	sampleEq(5, 6)
	bufferEq([]sample{{t: 2, v: 3}, {t: 3, v: 4}, {t: 4, v: 5}})

	testutil.Assert(t, it.Seek(101), "seek failed")
	sampleEq(101, 10)
	bufferEq([]sample{{t: 99, v: 8}, {t: 100, v: 9}})

	testutil.Assert(t, !it.Next(), "next succeeded unexpectedly")
}

// At() should not be called once Next() returns false.
func TestBufferedSeriesIteratorNoBadAt(t *testing.T) {
	done := false

	m := &mockSeriesIterator{
		seek: func(int64) bool { return false },
		at: func() (int64, float64) {
			testutil.Assert(t, !done, "unexpectedly done")
			done = true
			return 0, 0
		},
		next: func() bool { return !done },
		err:  func() error { return nil },
	}

	it := NewBufferIterator(m, 60)
	it.Next()
	it.Next()
}

func BenchmarkBufferedSeriesIterator(b *testing.B) {
	// Simulate a 5 minute rate.
	it := NewBufferIterator(newFakeSeriesIterator(int64(b.N), 30), 5*60)

	b.SetBytes(int64(b.N * 16))
	b.ReportAllocs()
	b.ResetTimer()

	for it.Next() {
		// scan everything
	}
	testutil.Ok(b, it.Err())
}

type mockSeriesIterator struct {
	seek func(int64) bool
	at   func() (int64, float64)
	next func() bool
	err  func() error
}

func (m *mockSeriesIterator) Seek(t int64) bool    { return m.seek(t) }
func (m *mockSeriesIterator) At() (int64, float64) { return m.at() }
func (m *mockSeriesIterator) Next() bool           { return m.next() }
func (m *mockSeriesIterator) Err() error           { return m.err() }

type mockSeries struct {
	labels   func() labels.Labels
	iterator func() SeriesIterator
}

func newMockSeries(lset labels.Labels, samples []sample) Series {
	return &mockSeries{
		labels: func() labels.Labels {
			return lset
		},
		iterator: func() SeriesIterator {
			return newListSeriesIterator(samples)
		},
	}
}

func (m *mockSeries) Labels() labels.Labels    { return m.labels() }
func (m *mockSeries) Iterator() SeriesIterator { return m.iterator() }

type listSeriesIterator struct {
	list []sample
	idx  int
}

func newListSeriesIterator(list []sample) *listSeriesIterator {
	return &listSeriesIterator{list: list, idx: -1}
}

func (it *listSeriesIterator) At() (int64, float64) {
	s := it.list[it.idx]
	return s.t, s.v
}

func (it *listSeriesIterator) Next() bool {
	it.idx++
	return it.idx < len(it.list)
}

func (it *listSeriesIterator) Seek(t int64) bool {
	if it.idx == -1 {
		it.idx = 0
	}
	// Do binary search between current position and end.
	it.idx = sort.Search(len(it.list)-it.idx, func(i int) bool {
		s := it.list[i+it.idx]
		return s.t >= t
	})

	return it.idx < len(it.list)
}

func (it *listSeriesIterator) Err() error {
	return nil
}

type fakeSeriesIterator struct {
	nsamples int64
	step     int64
	idx      int64
}

func newFakeSeriesIterator(nsamples, step int64) *fakeSeriesIterator {
	return &fakeSeriesIterator{nsamples: nsamples, step: step, idx: -1}
}

func (it *fakeSeriesIterator) At() (int64, float64) {
	return it.idx * it.step, 123 // value doesn't matter
}

func (it *fakeSeriesIterator) Next() bool {
	it.idx++
	return it.idx < it.nsamples
}

func (it *fakeSeriesIterator) Seek(t int64) bool {
	it.idx = t / it.step
	return it.idx < it.nsamples
}

func (it *fakeSeriesIterator) Err() error {
	return nil
}
<commit_msg>Replaced t.Fatalf() with testutil.Assert() in buffer_test.go (#6084)<commit_after>// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"math/rand"
	"sort"
	"testing"

	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/util/testutil"
)

func TestSampleRing(t *testing.T) {
	cases := []struct {
		input []int64
		delta int64
		size  int
	}{
		{
			input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
			delta: 2,
			size:  1,
		},
		{
			input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
			delta: 2,
			size:  2,
		},
		{
			input: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
			delta: 7,
			size:  3,
		},
		{
			input: []int64{1, 2, 3, 4, 5, 16, 17, 18, 19, 20},
			delta: 7,
			size:  1,
		},
		{
			input: []int64{1, 2, 3, 4, 6},
			delta: 4,
			size:  4,
		},
	}
	for _, c := range cases {
		r := newSampleRing(c.delta, c.size)

		input := []sample{}
		for _, t := range c.input {
			input = append(input, sample{
				t: t,
				v: float64(rand.Intn(100)),
			})
		}

		for i, s := range input {
			r.add(s.t, s.v)
			buffered := r.samples()

			for _, sold := range input[:i] {
				found := false
				for _, bs := range buffered {
					if bs.t == sold.t && bs.v == sold.v {
						found = true
						break
					}
				}

				if found {
					testutil.Assert(t, sold.t >= s.t-c.delta, "%d: unexpected sample %d in buffer; buffer %v", i, sold.t, buffered)
				} else {
					testutil.Assert(t, sold.t < s.t-c.delta, "%d: expected sample %d to be in buffer but was not; buffer %v", i, sold.t, buffered)
				}
			}
		}
	}
}

func TestBufferedSeriesIterator(t *testing.T) {
	var it *BufferedSeriesIterator

	bufferEq := func(exp []sample) {
		var b []sample
		bit := it.Buffer()
		for bit.Next() {
			t, v := bit.At()
			b = append(b, sample{t: t, v: v})
		}
		testutil.Equals(t, exp, b, "buffer mismatch")
	}
	sampleEq := func(ets int64, ev float64) {
		ts, v := it.Values()
		testutil.Equals(t, ets, ts, "timestamp mismatch")
		testutil.Equals(t, ev, v, "value mismatch")
	}

	it = NewBufferIterator(newListSeriesIterator([]sample{
		{t: 1, v: 2},
		{t: 2, v: 3},
		{t: 3, v: 4},
		{t: 4, v: 5},
		{t: 5, v: 6},
		{t: 99, v: 8},
		{t: 100, v: 9},
		{t: 101, v: 10},
	}), 2)

	testutil.Assert(t, it.Seek(-123), "seek failed")
	sampleEq(1, 2)
	bufferEq(nil)

	testutil.Assert(t, it.Next(), "next failed")
	sampleEq(2, 3)
	bufferEq([]sample{{t: 1, v: 2}})

	testutil.Assert(t, it.Next(), "next failed")
	testutil.Assert(t, it.Next(), "next failed")
	testutil.Assert(t, it.Next(), "next failed")
	sampleEq(5, 6)
	bufferEq([]sample{{t: 2, v: 3}, {t: 3, v: 4}, {t: 4, v: 5}})

	testutil.Assert(t, it.Seek(5), "seek failed")
	sampleEq(5, 6)
	bufferEq([]sample{{t: 2, v: 3}, {t: 3, v: 4}, {t: 4, v: 5}})

	testutil.Assert(t, it.Seek(101), "seek failed")
	sampleEq(101, 10)
	bufferEq([]sample{{t: 99, v: 8}, {t: 100, v: 9}})

	testutil.Assert(t, !it.Next(), "next succeeded unexpectedly")
}

// At() should not be called once Next() returns false.
func TestBufferedSeriesIteratorNoBadAt(t *testing.T) {
	done := false

	m := &mockSeriesIterator{
		seek: func(int64) bool { return false },
		at: func() (int64, float64) {
			testutil.Assert(t, !done, "unexpectedly done")
			done = true
			return 0, 0
		},
		next: func() bool { return !done },
		err:  func() error { return nil },
	}

	it := NewBufferIterator(m, 60)
	it.Next()
	it.Next()
}

func BenchmarkBufferedSeriesIterator(b *testing.B) {
	// Simulate a 5 minute rate.
	it := NewBufferIterator(newFakeSeriesIterator(int64(b.N), 30), 5*60)

	b.SetBytes(int64(b.N * 16))
	b.ReportAllocs()
	b.ResetTimer()

	for it.Next() {
		// scan everything
	}
	testutil.Ok(b, it.Err())
}

type mockSeriesIterator struct {
	seek func(int64) bool
	at   func() (int64, float64)
	next func() bool
	err  func() error
}

func (m *mockSeriesIterator) Seek(t int64) bool    { return m.seek(t) }
func (m *mockSeriesIterator) At() (int64, float64) { return m.at() }
func (m *mockSeriesIterator) Next() bool           { return m.next() }
func (m *mockSeriesIterator) Err() error           { return m.err() }

type mockSeries struct {
	labels   func() labels.Labels
	iterator func() SeriesIterator
}

func newMockSeries(lset labels.Labels, samples []sample) Series {
	return &mockSeries{
		labels: func() labels.Labels {
			return lset
		},
		iterator: func() SeriesIterator {
			return newListSeriesIterator(samples)
		},
	}
}

func (m *mockSeries) Labels() labels.Labels    { return m.labels() }
func (m *mockSeries) Iterator() SeriesIterator { return m.iterator() }

type listSeriesIterator struct {
	list []sample
	idx  int
}

func newListSeriesIterator(list []sample) *listSeriesIterator {
	return &listSeriesIterator{list: list, idx: -1}
}

func (it *listSeriesIterator) At() (int64, float64) {
	s := it.list[it.idx]
	return s.t, s.v
}

func (it *listSeriesIterator) Next() bool {
	it.idx++
	return it.idx < len(it.list)
}

func (it *listSeriesIterator) Seek(t int64) bool {
	if it.idx == -1 {
		it.idx = 0
	}
	// Do binary search between current position and end.
	it.idx = sort.Search(len(it.list)-it.idx, func(i int) bool {
		s := it.list[i+it.idx]
		return s.t >= t
	})

	return it.idx < len(it.list)
}

func (it *listSeriesIterator) Err() error {
	return nil
}

type fakeSeriesIterator struct {
	nsamples int64
	step     int64
	idx      int64
}

func newFakeSeriesIterator(nsamples, step int64) *fakeSeriesIterator {
	return &fakeSeriesIterator{nsamples: nsamples, step: step, idx: -1}
}

func (it *fakeSeriesIterator) At() (int64, float64) {
	return it.idx * it.step, 123 // value doesn't matter
}

func (it *fakeSeriesIterator) Next() bool {
	it.idx++
	return it.idx < it.nsamples
}

func (it *fakeSeriesIterator) Seek(t int64) bool {
	it.idx = t / it.step
	return it.idx < it.nsamples
}

func (it *fakeSeriesIterator) Err() error {
	return nil
}
<|endoftext|>


<commit_before>package redis

import (
    "strconv"
    "sync"
    "time"
    "github.com/BluePecker/JwtAuth/storage"
    "github.com/go-redis/redis"
    "crypto/md5"
    "encoding/hex"
    "fmt"
)

type Redis struct {
    create time.Time
    mu     sync.RWMutex
    client *redis.Client
}

func (R *Redis) Initializer(opt storage.Option) error {
    Database, _ := strconv.Atoi(opt.Database)
    R.client = redis.NewClient(&redis.Options{
        Network: "tcp",
        Addr: fmt.Sprintf("%s:%d", opt.Host, opt.Port),
        PoolSize: opt.PoolSize,
        DB: Database,
        MaxRetries: opt.MaxRetries,
    })
    err := R.client.Ping().Err()
    if err != nil {
        defer R.client.Close()
    }
    return err
}

func (R *Redis) TTL(key string) float64 {
    R.mu.RLock()
    defer R.mu.RUnlock()
    return R.client.TTL(R.md5Key(key)).Val().Seconds()
}

func (R *Redis) Read(key string) (interface{}, error) {
    R.mu.RLock()
    defer R.mu.RUnlock()
    status := R.client.Get(R.md5Key(key))
    return status.Val(), status.Err()
}

func (R *Redis) ReadInt(key string) (int, error) {
    R.mu.RLock()
    defer R.mu.RUnlock()
    status := R.client.Get(R.md5Key(key))
    if status.Err() != nil {
        return 0, status.Err()
    }
    return strconv.Atoi(status.Val())
}

func (R *Redis) ReadString(key string) (string, error) {
    R.mu.RLock()
    defer R.mu.RUnlock()
    status := R.client.Get(R.md5Key(key))
    if status.Err() != nil {
        return "", status.Err()
    }
    return status.Val(), nil
}

func (R *Redis) Upgrade(key string, expire int) {
    R.mu.Lock()
    defer R.mu.Unlock()
    key = R.md5Key(key)
    if v, err := R.Read(key); err != nil {
        R.Set(key, v, expire)
    }
}

func (R *Redis) Set(key string, value interface{}, expire int) error {
    R.mu.Lock()
    defer R.mu.Unlock()
    return R.save(R.md5Key(key), value, expire, false)
}

func (R *Redis) SetImmutable(key string, value interface{}, expire int) error {
    R.mu.Lock()
    defer R.mu.Unlock()
    return R.save(R.md5Key(key), value, expire, true)
}

func (R *Redis) Remove(key string) {
    R.mu.Lock()
    defer R.mu.Unlock()
    R.remove(R.md5Key(key))
}

func (R *Redis) LKeep(key string, value interface{}, maxLen, expire int) error {
    R.mu.Lock()
    defer R.mu.Unlock()
    key = R.md5Key(key)
    _, err := R.client.Pipelined(func(pip redis.Pipeliner) error {
        pip.LPush(key, value)
        pip.LTrim(key, 0, maxLen - 1)
        pip.Expire(key, time.Duration(expire) * time.Second)
        return nil;
    })
    return err;
}

func (R *Redis) LRange(key string, start, stop int) ([]string, error) {
    R.mu.Lock()
    defer R.mu.Unlock()
    key = R.md5Key(key)
    cmd := R.client.LRange(key, int64(start), int64(stop))
    return cmd.Val(), cmd.Err()
}

func (R *Redis) LExist(key string, value interface{}) bool {
    if strArr, err := R.LRange(key, 0, -1); err == nil {
        for _, v := range strArr {
            if v == value.(string) {
                return true
            }
        }
    }
    return false
}

func (R *Redis) remove(key string) error {
    status := R.client.Del(key)
    return status.Err()
}

func (R *Redis) save(key string, value interface{}, expire int, immutable bool) error {
    key = R.md5Key(key)
    cmd := R.client.HGet(key, "i")
    if find, _ := strconv.ParseBool(cmd.Val()); find {
        return fmt.Errorf("this key(%s) write protection", key)
    }
    R.client.Pipelined(func(pipe redis.Pipeliner) error {
        pipe.HSet(key, "v", value)
        pipe.HSet(key, "i", immutable)
        pipe.Expire(key, time.Duration(expire) * time.Second)
        return nil
    })
    return nil
}

func (R *Redis) md5Key(key string) string {
    hash := md5.New()
    hash.Write([]byte(key))
    return hex.EncodeToString(hash.Sum([]byte("jwt#")))
}

func init() {
    storage.Register("redis", &Redis{})
}<commit_msg>fix bug<commit_after>package redis

import (
    "strconv"
    "sync"
    "time"
    "github.com/BluePecker/JwtAuth/storage"
    "github.com/go-redis/redis"
    "crypto/md5"
    "encoding/hex"
    "fmt"
)

type Redis struct {
    create time.Time
    mu     sync.RWMutex
    client *redis.Client
}

func (R *Redis) Initializer(opt storage.Option) error {
    Database, _ := strconv.Atoi(opt.Database)
    R.client = redis.NewClient(&redis.Options{
        Network: "tcp",
        Addr: fmt.Sprintf("%s:%d", opt.Host, opt.Port),
        PoolSize: opt.PoolSize,
        DB: Database,
        MaxRetries: opt.MaxRetries,
    })
    err := R.client.Ping().Err()
    if err != nil {
        defer R.client.Close()
    }
    return err
}

func (R *Redis) TTL(key string) float64 {
    R.mu.RLock()
    defer R.mu.RUnlock()
    return R.client.TTL(R.md5Key(key)).Val().Seconds()
}

func (R *Redis) Read(key string) (interface{}, error) {
    R.mu.RLock()
    defer R.mu.RUnlock()
    status := R.client.Get(R.md5Key(key))
    return status.Val(), status.Err()
}

func (R *Redis) ReadInt(key string) (int, error) {
    R.mu.RLock()
    defer R.mu.RUnlock()
    status := R.client.Get(R.md5Key(key))
    if status.Err() != nil {
        return 0, status.Err()
    }
    return strconv.Atoi(status.Val())
}

func (R *Redis) ReadString(key string) (string, error) {
    R.mu.RLock()
    defer R.mu.RUnlock()
    status := R.client.Get(R.md5Key(key))
    if status.Err() != nil {
        return "", status.Err()
    }
    return status.Val(), nil
}

func (R *Redis) Upgrade(key string, expire int) {
    R.mu.Lock()
    defer R.mu.Unlock()
    key = R.md5Key(key)
    if v, err := R.Read(key); err != nil {
        R.Set(key, v, expire)
    }
}

func (R *Redis) Set(key string, value interface{}, expire int) error {
    R.mu.Lock()
    defer R.mu.Unlock()
    return R.save(R.md5Key(key), value, expire, false)
}

func (R *Redis) SetImmutable(key string, value interface{}, expire int) error {
    R.mu.Lock()
    defer R.mu.Unlock()
    return R.save(R.md5Key(key), value, expire, true)
}

func (R *Redis) Remove(key string) {
    R.mu.Lock()
    defer R.mu.Unlock()
    R.remove(R.md5Key(key))
}

func (R *Redis) LKeep(key string, value interface{}, maxLen, expire int) error {
    R.mu.Lock()
    defer R.mu.Unlock()
    key = R.md5Key(key)
    _, err := R.client.Pipelined(func(pip redis.Pipeliner) error {
        pip.LPush(key, value)
        pip.LTrim(key, 0, int64(maxLen - 1))
        pip.Expire(key, time.Duration(expire) * time.Second)
        return nil;
    })
    return err;
}

func (R *Redis) LRange(key string, start, stop int) ([]string, error) {
    R.mu.Lock()
    defer R.mu.Unlock()
    key = R.md5Key(key)
    cmd := R.client.LRange(key, int64(start), int64(stop))
    return cmd.Val(), cmd.Err()
}

func (R *Redis) LExist(key string, value interface{}) bool {
    if strArr, err := R.LRange(key, 0, -1); err == nil {
        for _, v := range strArr {
            if v == value.(string) {
                return true
            }
        }
    }
    return false
}

func (R *Redis) remove(key string) error {
    status := R.client.Del(key)
    return status.Err()
}

func (R *Redis) save(key string, value interface{}, expire int, immutable bool) error {
    key = R.md5Key(key)
    cmd := R.client.HGet(key, "i")
    if find, _ := strconv.ParseBool(cmd.Val()); find {
        return fmt.Errorf("this key(%s) write protection", key)
    }
    R.client.Pipelined(func(pipe redis.Pipeliner) error {
        pipe.HSet(key, "v", value)
        pipe.HSet(key, "i", immutable)
        pipe.Expire(key, time.Duration(expire) * time.Second)
        return nil
    })
    return nil
}

func (R *Redis) md5Key(key string) string {
    hash := md5.New()
    hash.Write([]byte(key))
    return hex.EncodeToString(hash.Sum([]byte("jwt#")))
}

func init() {
    storage.Register("redis", &Redis{})
}<|endoftext|>


<commit_before>package scp

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/ssh"
	"github.com/viant/toolbox/storage"
	"io"
	"io/ioutil"
	"net/url"
	"strings"

	"path"
	"github.com/viant/toolbox/cred"
)

const defaultSSHPort = 22

const (
	fileInfoPermission = iota
	_
	fileInfoOwner
	fileInfoGroup
	fileInfoSize
	fileInfoDateMonth
	fileInfoDateDay
	fileInfoDateHour
	fileInfoDateYear
	fileInfoName
)

type service struct {
	config *cred.Config
}

func (s *service) runCommand(URL string, command string) (string, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return "", err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = 22
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return "", err
	}
	defer client.Close()
	session, err := client.NewSession()
	if err != nil {
		return "", err
	}
	defer session.Close()
	output, err := session.CombinedOutput(command)
	if err != nil {
		return "", err
	}
	return toolbox.AsString(output), err

}

//List returns a list of object for supplied URL
func (s *service) List(URL string) ([]storage.Object, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}

	var result = make([]storage.Object, 0)
	output, err := s.runCommand(URL, "ls -lTtr "+parsedUrl.Path)
	if strings.Contains(string(output), "No such file or directory") {
		return result, nil
	}

	var fileNameFilter = ""
	if err == nil && output == "" {
		parent, fileName := path.Split(parsedUrl.Path)
		fileNameFilter = fileName
		output, err = s.runCommand(URL, "ls -lTtr "+parent+" | grep "+fileName)
	}
	if err != nil {
		return nil, err
	}

	for _, line := range strings.Split(string(output), "\n") {
		fileInfo := extractFileInfo(line)
		if fileInfo.name == "" {
			continue
		}
		fileInfo.url = URL
		if fileNameFilter == "" || fileNameFilter == fileInfo.name {
			result = append(result, fileInfo)
		}
	}
	return result, nil
}

func extractFileInfo(line string) *object {
	fragmentCount := 0
	fileInfo := &object{}
	for i := range line {

		aChar := string(line[i])
		if aChar == " " || aChar == "\t" {
			if i+1 < len(line) {
				nextChar := string(line[i+1])
				if !(nextChar == " " || nextChar == "\t") {
					fragmentCount++
				}
			}
			continue
		}
		switch fragmentCount {

		case fileInfoPermission:
			fileInfo.permission += aChar
		case fileInfoOwner:
			fileInfo.owner += aChar
		case fileInfoGroup:
			fileInfo.group += aChar
		case fileInfoSize:
			fileInfo.size += aChar
		case fileInfoDateMonth:
			fileInfo.month += aChar
		case fileInfoDateDay:
			fileInfo.day += aChar
		case fileInfoDateHour:
			fileInfo.hour += aChar
		case fileInfoDateYear:
			fileInfo.year += aChar
		case fileInfoName:
			fileInfo.name += aChar
		}

	}
	return fileInfo
}

func (s *service) Exists(URL string) (bool, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return false, err
	}
	output, err := s.runCommand(URL, "ls -lTtr "+parsedUrl.Path)
	if strings.Contains(string(output), "No such file or directory") {
		return false, nil
	}
	return true, nil

}

func (s *service) StorageObject(URL string) (storage.Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, fmt.Errorf("No found %v", URL)
	}
	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *service) Download(object storage.Object) (io.Reader, error) {
	if object == nil {
		return nil, fmt.Errorf("Object was nil")
	}
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return nil, err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return nil, err
	}
	defer client.Close()

	content, err := client.Download(parsedUrl.Path)
	if err != nil {
		return nil, err
	}
	return bytes.NewReader(content), nil
}

//Upload uploads provided reader content for supplied URL.
func (s *service) Upload(URL string, reader io.Reader) error {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()

	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return fmt.Errorf("Failed to upload - unable read: %v", err)
	}
	return client.Upload(parsedUrl.Path, content)
}

func (s *service) Register(schema string, service storage.Service) error {
	return errors.New("unsupported")
}

//Delete removes passed in storage object
func (s *service) Delete(object storage.Object) error {
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()

	if parsedUrl.Path == "/" {
		return fmt.Errorf("Invalid removal path: %v", parsedUrl.Path)
	}
	_, err = session.Output("rm -rf " + parsedUrl.Path)
	return err
}

//NewService create a new gc storage service
func NewService(config *cred.Config) *service {
	return &service{
		config: config,
	}
}
<commit_msg>added scp verification<commit_after>package scp

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/ssh"
	"github.com/viant/toolbox/storage"
	"io"
	"io/ioutil"
	"net/url"
	"strings"

	"path"
	"github.com/viant/toolbox/cred"
)

const defaultSSHPort = 22

const (
	fileInfoPermission = iota
	_
	fileInfoOwner
	fileInfoGroup
	fileInfoSize
	fileInfoDateMonth
	fileInfoDateDay
	fileInfoDateHour
	fileInfoDateYear
	fileInfoName
)

type service struct {
	config *cred.Config
}

func (s *service) runCommand(URL string, command string) (string, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return "", err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = 22
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return "", err
	}
	defer client.Close()
	session, err := client.NewSession()
	if err != nil {
		return "", err
	}
	defer session.Close()
	output, err := session.CombinedOutput(command)
	if err != nil {
		return "", err
	}
	return toolbox.AsString(output), err

}

//List returns a list of object for supplied URL
func (s *service) List(URL string) ([]storage.Object, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}

	var result = make([]storage.Object, 0)
	output, err := s.runCommand(URL, "ls -lTtr "+parsedUrl.Path)
	if strings.Contains(string(output), "No such file or directory") {
		return result, nil
	}

	var fileNameFilter = ""
	if err == nil && output == "" {
		parent, fileName := path.Split(parsedUrl.Path)
		fileNameFilter = fileName
		output, err = s.runCommand(URL, "ls -lTtr "+parent+" | grep "+fileName)
	}
	if err != nil {
		return nil, err
	}

	for _, line := range strings.Split(string(output), "\n") {
		fileInfo := extractFileInfo(line)
		if fileInfo.name == "" {
			continue
		}
		fileInfo.url = URL
		if fileNameFilter == "" || fileNameFilter == fileInfo.name {
			result = append(result, fileInfo)
		}
	}
	return result, nil
}

func extractFileInfo(line string) *object {
	fragmentCount := 0
	fileInfo := &object{}
	for i := range line {

		aChar := string(line[i])
		if aChar == " " || aChar == "\t" {
			if i+1 < len(line) {
				nextChar := string(line[i+1])
				if !(nextChar == " " || nextChar == "\t") {
					fragmentCount++
				}
			}
			continue
		}
		switch fragmentCount {

		case fileInfoPermission:
			fileInfo.permission += aChar
		case fileInfoOwner:
			fileInfo.owner += aChar
		case fileInfoGroup:
			fileInfo.group += aChar
		case fileInfoSize:
			fileInfo.size += aChar
		case fileInfoDateMonth:
			fileInfo.month += aChar
		case fileInfoDateDay:
			fileInfo.day += aChar
		case fileInfoDateHour:
			fileInfo.hour += aChar
		case fileInfoDateYear:
			fileInfo.year += aChar
		case fileInfoName:
			fileInfo.name += aChar
		}

	}
	return fileInfo
}

func (s *service) Exists(URL string) (bool, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return false, err
	}
	output, err := s.runCommand(URL, "ls -lTtr "+parsedUrl.Path)
	if strings.Contains(string(output), "No such file or directory") {
		return false, nil
	}
	return true, nil

}

func (s *service) StorageObject(URL string) (storage.Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, fmt.Errorf("No found %v", URL)
	}
	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *service) Download(object storage.Object) (io.Reader, error) {
	if object == nil {
		return nil, fmt.Errorf("Object was nil")
	}
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return nil, err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return nil, err
	}
	defer client.Close()

	content, err := client.Download(parsedUrl.Path)
	if err != nil {
		return nil, err
	}

	//download verification (as sometimes scp failed) with one retry
	if int(object.Size()) != len(content) {
		content, err = client.Download(parsedUrl.Path)
		if err != nil {
			return nil, err
		}
		if int(object.Size()) != len(content) {
			return nil, fmt.Errorf("Faled to download from %v,  object size was: %v, but scp download was %v", object.URL(), object.Size(), len(content))
		}
	}
	return bytes.NewReader(content), nil
}

//Upload uploads provided reader content for supplied URL.
func (s *service) Upload(URL string, reader io.Reader) error {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()

	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return fmt.Errorf("Failed to upload - unable read: %v", err)
	}

	err = client.Upload(parsedUrl.Path, content)
	object, err := s.StorageObject(URL)
	if err != nil {
		return  err
	}
	if int(object.Size()) != len(content) {
		err = client.Upload(parsedUrl.Path, content)
		object, err = s.StorageObject(URL)
		if err != nil {
			return  err
		}
		if int(object.Size()) != len(content) {
			return fmt.Errorf("Failed to upload to %v, actual size was:%v,  but uploaded size was  ", URL, len(content), int(object.Size()))
		}
	}
	return err
}

func (s *service) Register(schema string, service storage.Service) error {
	return errors.New("unsupported")
}

//Delete removes passed in storage object
func (s *service) Delete(object storage.Object) error {
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()

	if parsedUrl.Path == "/" {
		return fmt.Errorf("Invalid removal path: %v", parsedUrl.Path)
	}
	_, err = session.Output("rm -rf " + parsedUrl.Path)
	return err
}

//NewService create a new gc storage service
func NewService(config *cred.Config) *service {
	return &service{
		config: config,
	}
}
<|endoftext|>


<commit_before>package scp

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/ssh"
	"github.com/viant/toolbox/storage"
	"io"
	"io/ioutil"
	"strings"

	"path"
	"github.com/viant/toolbox/cred"
	"net/url"
)

const defaultSSHPort = 22

const (
	fileInfoPermission = iota
	_
	fileInfoOwner
	fileInfoGroup
	fileInfoSize
	fileInfoDateMonth
	fileInfoDateDay
	fileInfoDateHour
	fileInfoDateYear
	fileInfoName

)

const (
	fileIsoInfoPermission = iota
	_
	fileIsoInfoOwner
	fileIsoInfoGroup
	fileIsoInfoSize
	fileIsoDate
	fileIsoTime
	fileIsoTimezone
	fileIsoInfoName
)

type service struct {
	config *cred.Config

}

func (s *service) runCommand(session *ssh.MultiCommandSession, URL string, command string) (string, error) {
	output, _ := session.Run(command, 0, "$ ", "usage")
	return toolbox.AsString(output), nil
}

func (s *service) canListWithTimeStyle(session *ssh.MultiCommandSession, URL string) (bool, error) {
	output, err := s.runCommand(session, URL, "ls -ltr --time-style=full-iso")
	if err != nil {
		return false, err
	}
	return ! strings.Contains(string(output), "usage"), nil
}

func normalizeFileInfoOutput(lines string) string {
	var result = make([]string, 0)
	for _, line := range strings.Split(lines, "\n") {
		if strings.HasPrefix(strings.ToLower(line), "total") {
			continue
		}
		result = append(result, line)
	}
	return strings.Join(result, "\n")
}

func  (s *service) getClient(parsedURL *url.URL) (*ssh.Client, error) {
	port := toolbox.AsInt(parsedURL.Port())
	if port == 0 {
		port = 22
	}
	return ssh.NewClient(parsedURL.Hostname(), toolbox.AsInt(port), s.config)
}

//List returns a list of object for supplied URL
func (s *service) List(URL string) ([]storage.Object, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}
	client, err := s.getClient(parsedUrl)
	if err != nil {
		return nil, err
	}
	defer client.Close()
	session, err := client.OpenMultiCommandSession(&ssh.SessionConfig{})
	if err != nil {
		return nil, err
	}
	defer session.Close()

	var urlPath = strings.Replace(parsedUrl.Path, "//", "/", len(parsedUrl.Path))
	var result = make([]storage.Object, 0)

	canListWithTimeStyle, err := s.canListWithTimeStyle(session, URL)
	if err != nil {
		return nil, err
	}

	var lsCommand = "ls -ltr"
	if canListWithTimeStyle {
		lsCommand += " --time-style=full-iso"
	} else {
		lsCommand +="T"
	}

	output, err := s.runCommand(session, URL, lsCommand + " "+parsedUrl.Path)
	stdout := normalizeFileInfoOutput(string(output))
	if strings.Contains(stdout, "No such file or directory") {
		return result, nil
	}
	var fileNameFilter = ""

	if err == nil && stdout == "" {
		parent, fileName := path.Split(urlPath )
		fileNameFilter = fileName
		output, err = s.runCommand(session, URL, lsCommand + " "+parent+" | grep "+fileName)
	}
	if err != nil {
		return nil, err
	}

	stdout = normalizeFileInfoOutput(string(output))
	for _, line := range strings.Split(stdout, "\n") {
		fileInfo := ExtractFileInfo(line, canListWithTimeStyle)
		if fileInfo == nil {
			continue
		}
		if fileInfo.name == "" {
			continue
		}
		fileInfo.url = URL
		if fileNameFilter == "" || fileNameFilter == fileInfo.name {
			result = append(result, fileInfo)
		}
	}
	return result, nil
}

//file info with iso -rw-r--r-- 1 awitas awitas 2002 2017-11-04 22:29:33.363458941 +0000 aerospikeciads_aerospike.conf
//file info without iso // -rw-r--r--  1 awitas  1742120565   414 Jun  8 14:14:08 2017 id_rsa.pub

func ExtractFileInfo(line string, isoTimeStyle bool) *object {
	fragmentCount := 0
	fileInfo := &object{}
	if strings.TrimSpace(line) == "" {
		return nil
	}
	for i := range line {

		aChar := string(line[i])
		if aChar == " " || aChar == "\t" {
			if i+1 < len(line) {
				nextChar := string(line[i+1])
				if !(nextChar == " " || nextChar == "\t") {
					fragmentCount++
				}
			}
			continue
		}

		if isoTimeStyle {
			switch fragmentCount {
			case fileIsoInfoPermission:
				fileInfo.permission += aChar
			case fileIsoInfoOwner:
				fileInfo.owner += aChar
			case fileIsoInfoGroup:
				fileInfo.group += aChar
			case fileIsoInfoSize:
				fileInfo.size += aChar
			case fileIsoDate:
				fileInfo.date += aChar
			case fileIsoTime:
				fileInfo.time += aChar
			case fileIsoTimezone:
				fileInfo.timezone += aChar
			case fileIsoInfoName:
				fileInfo.name += aChar
			}
			continue
		}

		switch fragmentCount {

		case fileInfoPermission:
			fileInfo.permission += aChar
		case fileInfoOwner:
			fileInfo.owner += aChar
		case fileInfoGroup:
			fileInfo.group += aChar
		case fileInfoSize:
			fileInfo.size += aChar
		case fileInfoDateMonth:
			fileInfo.month += aChar
		case fileInfoDateDay:
			fileInfo.day += aChar
		case fileInfoDateHour:
			fileInfo.hour += aChar
		case fileInfoDateYear:
			fileInfo.year += aChar
		case fileInfoName:
			fileInfo.name += aChar
		}

	}
	return fileInfo
}

func (s *service) Exists(URL string) (bool, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return false, err
	}

	client, err := s.getClient(parsedUrl)
	if err != nil {
		return false, err
	}
	defer client.Close()
	session, err := client.OpenMultiCommandSession(&ssh.SessionConfig{})
	if err != nil {
		return false, err
	}
	defer session.Close()

	output, err := s.runCommand(session, URL, "ls -ltr "+parsedUrl.Path)
	if strings.Contains(string(output), "No such file or directory") {
		return false, nil
	}
	return true, nil

}

func (s *service) StorageObject(URL string) (storage.Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, fmt.Errorf("No found %v", URL)
	}
	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *service) Download(object storage.Object) (io.Reader, error) {
	if object == nil {
		return nil, fmt.Errorf("Object was nil")
	}
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return nil, err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return nil, err
	}
	defer client.Close()

	content, err := client.Download(parsedUrl.Path)
	if err != nil {
		return nil, err
	}

	//download verification (as sometimes scp failed) with one retry
	if int(object.Size()) != len(content) {
		content, err = client.Download(parsedUrl.Path)
		if err != nil {
			return nil, err
		}
		if int(object.Size()) != len(content) {
			return nil, fmt.Errorf("Faled to download from %v,  object size was: %v, but scp download was %v", object.URL(), object.Size(), len(content))
		}
	}
	return bytes.NewReader(content), nil
}

//Upload uploads provided reader content for supplied URL.
func (s *service) Upload(URL string, reader io.Reader) error {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()

	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return fmt.Errorf("Failed to upload - unable read: %v", err)
	}

	err = client.Upload(parsedUrl.Path, content)
	if err != nil {
		return  fmt.Errorf("Failed to upload: %v %v", URL,  err)
	}

	object, err := s.StorageObject(URL)
	if err != nil {
		return  fmt.Errorf("Failed to get upload object  %v for verification: %v", URL, err)
	}

	if int(object.Size()) != len(content) {
		err = client.Upload(parsedUrl.Path, content)
		object, err = s.StorageObject(URL)
		if err != nil {
			return  err
		}
		if int(object.Size()) != len(content) {
			return fmt.Errorf("Failed to upload to %v, actual size was:%v,  but uploaded size was  ", URL, len(content), int(object.Size()))
		}
	}
	return err
}

func (s *service) Register(schema string, service storage.Service) error {
	return errors.New("unsupported")
}

//Delete removes passed in storage object
func (s *service) Delete(object storage.Object) error {
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()

	if parsedUrl.Path == "/" {
		return fmt.Errorf("Invalid removal path: %v", parsedUrl.Path)
	}
	_, err = session.Output("rm -rf " + parsedUrl.Path)
	return err
}

//NewService create a new gc storage service
func NewService(config *cred.Config) *service {
	return &service{
		config: config,
	}
}
<commit_msg>patched verification performance<commit_after>package scp

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/ssh"
	"github.com/viant/toolbox/storage"
	"io"
	"io/ioutil"
	"strings"

	"path"
	"github.com/viant/toolbox/cred"
	"net/url"
)

const defaultSSHPort = 22

const (
	fileInfoPermission = iota
	_
	fileInfoOwner
	fileInfoGroup
	fileInfoSize
	fileInfoDateMonth
	fileInfoDateDay
	fileInfoDateHour
	fileInfoDateYear
	fileInfoName

)

const (
	fileIsoInfoPermission = iota
	_
	fileIsoInfoOwner
	fileIsoInfoGroup
	fileIsoInfoSize
	fileIsoDate
	fileIsoTime
	fileIsoTimezone
	fileIsoInfoName
)

type service struct {
	config *cred.Config

}

func (s *service) runCommand(session *ssh.MultiCommandSession, URL string, command string) (string, error) {
	output, _ := session.Run(command, 0, "$ ", "usage", "No such file or directory")
	return toolbox.AsString(output), nil
}

func (s *service) canListWithTimeStyle(session *ssh.MultiCommandSession, URL string) (bool, error) {
	output, err := s.runCommand(session, URL, "ls -ltr --time-style=full-iso")
	if err != nil {
		return false, err
	}
	return ! strings.Contains(string(output), "usage"), nil
}

func normalizeFileInfoOutput(lines string) string {
	var result = make([]string, 0)
	for _, line := range strings.Split(lines, "\n") {
		if strings.HasPrefix(strings.ToLower(line), "total") {
			continue
		}
		result = append(result, line)
	}
	return strings.Join(result, "\n")
}

func  (s *service) getClient(parsedURL *url.URL) (*ssh.Client, error) {
	port := toolbox.AsInt(parsedURL.Port())
	if port == 0 {
		port = 22
	}
	return ssh.NewClient(parsedURL.Hostname(), toolbox.AsInt(port), s.config)
}

//List returns a list of object for supplied URL
func (s *service) List(URL string) ([]storage.Object, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}
	client, err := s.getClient(parsedUrl)
	if err != nil {
		return nil, err
	}
	defer client.Close()
	session, err := client.OpenMultiCommandSession(&ssh.SessionConfig{})
	if err != nil {
		return nil, err
	}
	defer session.Close()

	var urlPath = strings.Replace(parsedUrl.Path, "//", "/", len(parsedUrl.Path))
	var result = make([]storage.Object, 0)

	canListWithTimeStyle, err := s.canListWithTimeStyle(session, URL)
	if err != nil {
		return nil, err
	}

	var lsCommand = "ls -ltr"
	if canListWithTimeStyle {
		lsCommand += " --time-style=full-iso"
	} else {
		lsCommand +="T"
	}

	output, err := s.runCommand(session, URL, lsCommand + " "+parsedUrl.Path)
	stdout := normalizeFileInfoOutput(string(output))
	if strings.Contains(stdout, "No such file or directory") {
		return result, nil
	}
	var fileNameFilter = ""

	if err == nil && stdout == "" {
		parent, fileName := path.Split(urlPath )
		fileNameFilter = fileName
		output, err = s.runCommand(session, URL, lsCommand + " "+parent+" | grep "+fileName)
	}
	if err != nil {
		return nil, err
	}

	stdout = normalizeFileInfoOutput(string(output))
	for _, line := range strings.Split(stdout, "\n") {
		fileInfo := ExtractFileInfo(line, canListWithTimeStyle)
		if fileInfo == nil {
			continue
		}
		if fileInfo.name == "" {
			continue
		}
		fileInfo.url = URL
		if fileNameFilter == "" || fileNameFilter == fileInfo.name {
			result = append(result, fileInfo)
		}
	}
	return result, nil
}

//file info with iso -rw-r--r-- 1 awitas awitas 2002 2017-11-04 22:29:33.363458941 +0000 aerospikeciads_aerospike.conf
//file info without iso // -rw-r--r--  1 awitas  1742120565   414 Jun  8 14:14:08 2017 id_rsa.pub

func ExtractFileInfo(line string, isoTimeStyle bool) *object {
	fragmentCount := 0
	fileInfo := &object{}
	if strings.TrimSpace(line) == "" {
		return nil
	}
	for i := range line {

		aChar := string(line[i])
		if aChar == " " || aChar == "\t" {
			if i+1 < len(line) {
				nextChar := string(line[i+1])
				if !(nextChar == " " || nextChar == "\t") {
					fragmentCount++
				}
			}
			continue
		}

		if isoTimeStyle {
			switch fragmentCount {
			case fileIsoInfoPermission:
				fileInfo.permission += aChar
			case fileIsoInfoOwner:
				fileInfo.owner += aChar
			case fileIsoInfoGroup:
				fileInfo.group += aChar
			case fileIsoInfoSize:
				fileInfo.size += aChar
			case fileIsoDate:
				fileInfo.date += aChar
			case fileIsoTime:
				fileInfo.time += aChar
			case fileIsoTimezone:
				fileInfo.timezone += aChar
			case fileIsoInfoName:
				fileInfo.name += aChar
			}
			continue
		}

		switch fragmentCount {

		case fileInfoPermission:
			fileInfo.permission += aChar
		case fileInfoOwner:
			fileInfo.owner += aChar
		case fileInfoGroup:
			fileInfo.group += aChar
		case fileInfoSize:
			fileInfo.size += aChar
		case fileInfoDateMonth:
			fileInfo.month += aChar
		case fileInfoDateDay:
			fileInfo.day += aChar
		case fileInfoDateHour:
			fileInfo.hour += aChar
		case fileInfoDateYear:
			fileInfo.year += aChar
		case fileInfoName:
			fileInfo.name += aChar
		}

	}
	return fileInfo
}

func (s *service) Exists(URL string) (bool, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return false, err
	}

	client, err := s.getClient(parsedUrl)
	if err != nil {
		return false, err
	}
	defer client.Close()
	session, err := client.OpenMultiCommandSession(&ssh.SessionConfig{})
	if err != nil {
		return false, err
	}
	defer session.Close()

	output, err := s.runCommand(session, URL, "ls -ltr "+parsedUrl.Path)
	if strings.Contains(string(output), "No such file or directory") {
		return false, nil
	}
	return true, nil

}

func (s *service) StorageObject(URL string) (storage.Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, fmt.Errorf("No found %v", URL)
	}
	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *service) Download(object storage.Object) (io.Reader, error) {
	if object == nil {
		return nil, fmt.Errorf("Object was nil")
	}
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return nil, err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return nil, err
	}
	defer client.Close()

	content, err := client.Download(parsedUrl.Path)
	if err != nil {
		return nil, err
	}

	//download verification (as sometimes scp failed) with one retry
	if int(object.Size()) != len(content) {
		content, err = client.Download(parsedUrl.Path)
		if err != nil {
			return nil, err
		}
		if int(object.Size()) != len(content) {
			return nil, fmt.Errorf("Faled to download from %v,  object size was: %v, but scp download was %v", object.URL(), object.Size(), len(content))
		}
	}
	return bytes.NewReader(content), nil
}

//Upload uploads provided reader content for supplied URL.
func (s *service) Upload(URL string, reader io.Reader) error {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()

	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return fmt.Errorf("Failed to upload - unable read: %v", err)
	}

	err = client.Upload(parsedUrl.Path, content)
	if err != nil {
		return  fmt.Errorf("Failed to upload: %v %v", URL,  err)
	}

	object, err := s.StorageObject(URL)
	if err != nil {
		return  fmt.Errorf("Failed to get upload object  %v for verification: %v", URL, err)
	}

	if int(object.Size()) != len(content) {
		err = client.Upload(parsedUrl.Path, content)
		object, err = s.StorageObject(URL)
		if err != nil {
			return  err
		}
		if int(object.Size()) != len(content) {
			return fmt.Errorf("Failed to upload to %v, actual size was:%v,  but uploaded size was  ", URL, len(content), int(object.Size()))
		}
	}
	return err
}

func (s *service) Register(schema string, service storage.Service) error {
	return errors.New("unsupported")
}

//Delete removes passed in storage object
func (s *service) Delete(object storage.Object) error {
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()

	if parsedUrl.Path == "/" {
		return fmt.Errorf("Invalid removal path: %v", parsedUrl.Path)
	}
	_, err = session.Output("rm -rf " + parsedUrl.Path)
	return err
}

//NewService create a new gc storage service
func NewService(config *cred.Config) *service {
	return &service{
		config: config,
	}
}
<|endoftext|>


<commit_before>// +build linux
// +build cgo

package shared

import (
	"errors"
	"fmt"
	"os"
	"syscall"
	"unsafe"
)

// #cgo LDFLAGS: -lutil -lpthread
/*
#define _GNU_SOURCE
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <stdlib.h>
#include <grp.h>
#include <pty.h>
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <string.h>
#include <stdio.h>

#ifndef AT_SYMLINK_FOLLOW
#define AT_SYMLINK_FOLLOW    0x400
#endif

#ifndef AT_EMPTY_PATH
#define AT_EMPTY_PATH       0x1000
#endif

// This is an adaption from https://codereview.appspot.com/4589049, to be
// included in the stdlib with the stdlib's license.

static int mygetgrgid_r(int gid, struct group *grp,
	char *buf, size_t buflen, struct group **result) {
	return getgrgid_r(gid, grp, buf, buflen, result);
}

void configure_pty(int fd) {
	struct termios term_settings;
	struct winsize win;

	if (tcgetattr(fd, &term_settings) < 0) {
		fprintf(stderr, "Failed to get settings: %s\n", strerror(errno));
		return;
	}

	term_settings.c_iflag |= IMAXBEL;
	term_settings.c_iflag |= IUTF8;
	term_settings.c_iflag |= BRKINT;
	term_settings.c_iflag |= IXANY;

	term_settings.c_cflag |= HUPCL;

	if (tcsetattr(fd, TCSANOW, &term_settings) < 0) {
		fprintf(stderr, "Failed to set settings: %s\n", strerror(errno));
		return;
	}

	if (ioctl(fd, TIOCGWINSZ, &win) < 0) {
		fprintf(stderr, "Failed to get the terminal size: %s\n", strerror(errno));
		return;
	}

	win.ws_col = 80;
	win.ws_row = 25;

	if (ioctl(fd, TIOCSWINSZ, &win) < 0) {
		fprintf(stderr, "Failed to set the terminal size: %s\n", strerror(errno));
		return;
	}

	if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) {
		fprintf(stderr, "Failed to set FD_CLOEXEC: %s\n", strerror(errno));
		return;
	}

	return;
}

void create_pty(int *master, int *slave, int uid, int gid) {
	if (openpty(master, slave, NULL, NULL, NULL) < 0) {
		fprintf(stderr, "Failed to openpty: %s\n", strerror(errno));
		return;
	}

	configure_pty(*master);
	configure_pty(*slave);

	if (fchown(*slave, uid, gid) < 0) {
		fprintf(stderr, "Warning: error chowning pty to container root\n");
		fprintf(stderr, "Continuing...\n");
	}
	if (fchown(*master, uid, gid) < 0) {
		fprintf(stderr, "Warning: error chowning pty to container root\n");
		fprintf(stderr, "Continuing...\n");
	}
}

void create_pipe(int *master, int *slave) {
	int pipefd[2];

	if (pipe2(pipefd, O_CLOEXEC) < 0) {
		fprintf(stderr, "Failed to create a pipe: %s\n", strerror(errno));
		return;
	}

	*master = pipefd[0];
	*slave = pipefd[1];
}

int shiftowner(char *basepath, char *path, int uid, int gid) {
	struct stat sb;
	int fd, r;
	char fdpath[PATH_MAX];
	char realpath[PATH_MAX];

	fd = open(path, O_PATH|O_NOFOLLOW);
	if (fd < 0 ) {
		perror("Failed open");
		return 1;
	}

	r = sprintf(fdpath, "/proc/self/fd/%d", fd);
	if (r < 0) {
		perror("Failed sprintf");
		close(fd);
		return 1;
	}

	r = readlink(fdpath, realpath, PATH_MAX);
	if (r < 0) {
		perror("Failed readlink");
		close(fd);
		return 1;
	}

	if (strlen(realpath) < strlen(basepath)) {
		printf("Invalid path, source (%s) is outside of basepath (%s).\n", realpath, basepath);
		close(fd);
		return 1;
	}

	if (strncmp(realpath, basepath, strlen(basepath))) {
		printf("Invalid path, source (%s) is outside of basepath (%s).\n", realpath, basepath);
		close(fd);
		return 1;
	}

	r = fstat(fd, &sb);
	if (r < 0) {
		perror("Failed fstat");
		close(fd);
		return 1;
	}

	r = fchownat(fd, "", uid, gid, AT_EMPTY_PATH|AT_SYMLINK_NOFOLLOW);
	if (r < 0) {
		perror("Failed chown");
		close(fd);
		return 1;
	}

	if (!S_ISLNK(sb.st_mode)) {
		r = chmod(fdpath, sb.st_mode);
		if (r < 0) {
			perror("Failed chmod");
			close(fd);
			return 1;
		}
	}

	close(fd);
	return 0;
}
*/
import "C"

func ShiftOwner(basepath string, path string, uid int, gid int) error {
	cbasepath := C.CString(basepath)
	defer C.free(unsafe.Pointer(cbasepath))

	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))

	r := C.shiftowner(cbasepath, cpath, C.int(uid), C.int(gid))
	if r != 0 {
		return fmt.Errorf("Failed to change ownership of: %s", path)
	}
	return nil
}

func OpenPty(uid, gid int) (master *os.File, slave *os.File, err error) {
	fd_master := C.int(-1)
	fd_slave := C.int(-1)
	rootUid := C.int(uid)
	rootGid := C.int(gid)

	C.create_pty(&fd_master, &fd_slave, rootUid, rootGid)

	if fd_master == -1 || fd_slave == -1 {
		return nil, nil, errors.New("Failed to create a new pts pair")
	}

	master = os.NewFile(uintptr(fd_master), "master")
	slave = os.NewFile(uintptr(fd_slave), "slave")

	return master, slave, nil
}

func Pipe() (master *os.File, slave *os.File, err error) {
	fd_master := C.int(-1)
	fd_slave := C.int(-1)

	C.create_pipe(&fd_master, &fd_slave)

	if fd_master == -1 || fd_slave == -1 {
		return nil, nil, errors.New("Failed to create a new pipe")
	}

	master = os.NewFile(uintptr(fd_master), "master")
	slave = os.NewFile(uintptr(fd_slave), "slave")

	return master, slave, nil
}

// GroupName is an adaption from https://codereview.appspot.com/4589049.
func GroupName(gid int) (string, error) {
	var grp C.struct_group
	var result *C.struct_group

	bufSize := C.size_t(C.sysconf(C._SC_GETGR_R_SIZE_MAX))
	buf := C.malloc(bufSize)
	if buf == nil {
		return "", fmt.Errorf("allocation failed")
	}
	defer C.free(buf)

	// mygetgrgid_r is a wrapper around getgrgid_r to
	// to avoid using gid_t because C.gid_t(gid) for
	// unknown reasons doesn't work on linux.
	rv := C.mygetgrgid_r(C.int(gid),
		&grp,
		(*C.char)(buf),
		bufSize,
		&result)

	if rv != 0 {
		return "", fmt.Errorf("failed group lookup: %s", syscall.Errno(rv))
	}

	if result == nil {
		return "", fmt.Errorf("unknown group %d", gid)
	}

	return C.GoString(result.gr_name), nil
}

// GroupId is an adaption from https://codereview.appspot.com/4589049.
func GroupId(name string) (int, error) {
	var grp C.struct_group
	var result *C.struct_group

	bufSize := C.size_t(C.sysconf(C._SC_GETGR_R_SIZE_MAX))
	buf := C.malloc(bufSize)
	if buf == nil {
		return -1, fmt.Errorf("allocation failed")
	}
	defer C.free(buf)

	// mygetgrgid_r is a wrapper around getgrgid_r to
	// to avoid using gid_t because C.gid_t(gid) for
	// unknown reasons doesn't work on linux.
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))

	rv := C.getgrnam_r(cname,
		&grp,
		(*C.char)(buf),
		bufSize,
		&result)

	if rv != 0 {
		return -1, fmt.Errorf("failed group lookup: %s", syscall.Errno(rv))
	}

	if result == nil {
		return -1, fmt.Errorf("unknown group %s", name)
	}

	return int(C.int(result.gr_gid)), nil
}

// --- pure Go functions ---

func GetFileStat(p string) (uid int, gid int, major int, minor int,
	inode uint64, nlink int, err error) {
	var stat syscall.Stat_t
	err = syscall.Lstat(p, &stat)
	if err != nil {
		return
	}
	uid = int(stat.Uid)
	gid = int(stat.Gid)
	inode = uint64(stat.Ino)
	nlink = int(stat.Nlink)
	major = -1
	minor = -1
	if stat.Mode&syscall.S_IFBLK != 0 || stat.Mode&syscall.S_IFCHR != 0 {
		major = int(stat.Rdev / 256)
		minor = int(stat.Rdev % 256)
	}

	return
}

func IsMountPoint(name string) bool {
	stat, err := os.Stat(name)
	if err != nil {
		return false
	}

	rootStat, err := os.Lstat(name + "/..")
	if err != nil {
		return false
	}
	// If the directory has the same device as parent, then it's not a mountpoint.
	return stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev
}

func ReadLastNLines(f *os.File, lines int) (string, error) {
	if lines <= 0 {
		return "", fmt.Errorf("invalid line count")
	}

	stat, err := f.Stat()
	if err != nil {
		return "", err
	}

	data, err := syscall.Mmap(int(f.Fd()), 0, int(stat.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		return "", err
	}
	defer syscall.Munmap(data)

	for i := len(data) - 1; i >= 0; i-- {
		if data[i] == '\n' {
			lines--
		}

		if lines < 0 {
			return string(data[i+1 : len(data)]), nil
		}
	}

	return string(data), nil
}

func SetSize(fd int, width int, height int) (err error) {
	var dimensions [4]uint16
	dimensions[0] = uint16(height)
	dimensions[1] = uint16(width)

	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TIOCSWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 {
		return err
	}
	return nil
}
<commit_msg>Use the mountpoint command when available<commit_after>// +build linux
// +build cgo

package shared

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"syscall"
	"unsafe"
)

// #cgo LDFLAGS: -lutil -lpthread
/*
#define _GNU_SOURCE
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <stdlib.h>
#include <grp.h>
#include <pty.h>
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <string.h>
#include <stdio.h>

#ifndef AT_SYMLINK_FOLLOW
#define AT_SYMLINK_FOLLOW    0x400
#endif

#ifndef AT_EMPTY_PATH
#define AT_EMPTY_PATH       0x1000
#endif

// This is an adaption from https://codereview.appspot.com/4589049, to be
// included in the stdlib with the stdlib's license.

static int mygetgrgid_r(int gid, struct group *grp,
	char *buf, size_t buflen, struct group **result) {
	return getgrgid_r(gid, grp, buf, buflen, result);
}

void configure_pty(int fd) {
	struct termios term_settings;
	struct winsize win;

	if (tcgetattr(fd, &term_settings) < 0) {
		fprintf(stderr, "Failed to get settings: %s\n", strerror(errno));
		return;
	}

	term_settings.c_iflag |= IMAXBEL;
	term_settings.c_iflag |= IUTF8;
	term_settings.c_iflag |= BRKINT;
	term_settings.c_iflag |= IXANY;

	term_settings.c_cflag |= HUPCL;

	if (tcsetattr(fd, TCSANOW, &term_settings) < 0) {
		fprintf(stderr, "Failed to set settings: %s\n", strerror(errno));
		return;
	}

	if (ioctl(fd, TIOCGWINSZ, &win) < 0) {
		fprintf(stderr, "Failed to get the terminal size: %s\n", strerror(errno));
		return;
	}

	win.ws_col = 80;
	win.ws_row = 25;

	if (ioctl(fd, TIOCSWINSZ, &win) < 0) {
		fprintf(stderr, "Failed to set the terminal size: %s\n", strerror(errno));
		return;
	}

	if (fcntl(fd, F_SETFD, FD_CLOEXEC) < 0) {
		fprintf(stderr, "Failed to set FD_CLOEXEC: %s\n", strerror(errno));
		return;
	}

	return;
}

void create_pty(int *master, int *slave, int uid, int gid) {
	if (openpty(master, slave, NULL, NULL, NULL) < 0) {
		fprintf(stderr, "Failed to openpty: %s\n", strerror(errno));
		return;
	}

	configure_pty(*master);
	configure_pty(*slave);

	if (fchown(*slave, uid, gid) < 0) {
		fprintf(stderr, "Warning: error chowning pty to container root\n");
		fprintf(stderr, "Continuing...\n");
	}
	if (fchown(*master, uid, gid) < 0) {
		fprintf(stderr, "Warning: error chowning pty to container root\n");
		fprintf(stderr, "Continuing...\n");
	}
}

void create_pipe(int *master, int *slave) {
	int pipefd[2];

	if (pipe2(pipefd, O_CLOEXEC) < 0) {
		fprintf(stderr, "Failed to create a pipe: %s\n", strerror(errno));
		return;
	}

	*master = pipefd[0];
	*slave = pipefd[1];
}

int shiftowner(char *basepath, char *path, int uid, int gid) {
	struct stat sb;
	int fd, r;
	char fdpath[PATH_MAX];
	char realpath[PATH_MAX];

	fd = open(path, O_PATH|O_NOFOLLOW);
	if (fd < 0 ) {
		perror("Failed open");
		return 1;
	}

	r = sprintf(fdpath, "/proc/self/fd/%d", fd);
	if (r < 0) {
		perror("Failed sprintf");
		close(fd);
		return 1;
	}

	r = readlink(fdpath, realpath, PATH_MAX);
	if (r < 0) {
		perror("Failed readlink");
		close(fd);
		return 1;
	}

	if (strlen(realpath) < strlen(basepath)) {
		printf("Invalid path, source (%s) is outside of basepath (%s).\n", realpath, basepath);
		close(fd);
		return 1;
	}

	if (strncmp(realpath, basepath, strlen(basepath))) {
		printf("Invalid path, source (%s) is outside of basepath (%s).\n", realpath, basepath);
		close(fd);
		return 1;
	}

	r = fstat(fd, &sb);
	if (r < 0) {
		perror("Failed fstat");
		close(fd);
		return 1;
	}

	r = fchownat(fd, "", uid, gid, AT_EMPTY_PATH|AT_SYMLINK_NOFOLLOW);
	if (r < 0) {
		perror("Failed chown");
		close(fd);
		return 1;
	}

	if (!S_ISLNK(sb.st_mode)) {
		r = chmod(fdpath, sb.st_mode);
		if (r < 0) {
			perror("Failed chmod");
			close(fd);
			return 1;
		}
	}

	close(fd);
	return 0;
}
*/
import "C"

func ShiftOwner(basepath string, path string, uid int, gid int) error {
	cbasepath := C.CString(basepath)
	defer C.free(unsafe.Pointer(cbasepath))

	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))

	r := C.shiftowner(cbasepath, cpath, C.int(uid), C.int(gid))
	if r != 0 {
		return fmt.Errorf("Failed to change ownership of: %s", path)
	}
	return nil
}

func OpenPty(uid, gid int) (master *os.File, slave *os.File, err error) {
	fd_master := C.int(-1)
	fd_slave := C.int(-1)
	rootUid := C.int(uid)
	rootGid := C.int(gid)

	C.create_pty(&fd_master, &fd_slave, rootUid, rootGid)

	if fd_master == -1 || fd_slave == -1 {
		return nil, nil, errors.New("Failed to create a new pts pair")
	}

	master = os.NewFile(uintptr(fd_master), "master")
	slave = os.NewFile(uintptr(fd_slave), "slave")

	return master, slave, nil
}

func Pipe() (master *os.File, slave *os.File, err error) {
	fd_master := C.int(-1)
	fd_slave := C.int(-1)

	C.create_pipe(&fd_master, &fd_slave)

	if fd_master == -1 || fd_slave == -1 {
		return nil, nil, errors.New("Failed to create a new pipe")
	}

	master = os.NewFile(uintptr(fd_master), "master")
	slave = os.NewFile(uintptr(fd_slave), "slave")

	return master, slave, nil
}

// GroupName is an adaption from https://codereview.appspot.com/4589049.
func GroupName(gid int) (string, error) {
	var grp C.struct_group
	var result *C.struct_group

	bufSize := C.size_t(C.sysconf(C._SC_GETGR_R_SIZE_MAX))
	buf := C.malloc(bufSize)
	if buf == nil {
		return "", fmt.Errorf("allocation failed")
	}
	defer C.free(buf)

	// mygetgrgid_r is a wrapper around getgrgid_r to
	// to avoid using gid_t because C.gid_t(gid) for
	// unknown reasons doesn't work on linux.
	rv := C.mygetgrgid_r(C.int(gid),
		&grp,
		(*C.char)(buf),
		bufSize,
		&result)

	if rv != 0 {
		return "", fmt.Errorf("failed group lookup: %s", syscall.Errno(rv))
	}

	if result == nil {
		return "", fmt.Errorf("unknown group %d", gid)
	}

	return C.GoString(result.gr_name), nil
}

// GroupId is an adaption from https://codereview.appspot.com/4589049.
func GroupId(name string) (int, error) {
	var grp C.struct_group
	var result *C.struct_group

	bufSize := C.size_t(C.sysconf(C._SC_GETGR_R_SIZE_MAX))
	buf := C.malloc(bufSize)
	if buf == nil {
		return -1, fmt.Errorf("allocation failed")
	}
	defer C.free(buf)

	// mygetgrgid_r is a wrapper around getgrgid_r to
	// to avoid using gid_t because C.gid_t(gid) for
	// unknown reasons doesn't work on linux.
	cname := C.CString(name)
	defer C.free(unsafe.Pointer(cname))

	rv := C.getgrnam_r(cname,
		&grp,
		(*C.char)(buf),
		bufSize,
		&result)

	if rv != 0 {
		return -1, fmt.Errorf("failed group lookup: %s", syscall.Errno(rv))
	}

	if result == nil {
		return -1, fmt.Errorf("unknown group %s", name)
	}

	return int(C.int(result.gr_gid)), nil
}

// --- pure Go functions ---

func GetFileStat(p string) (uid int, gid int, major int, minor int,
	inode uint64, nlink int, err error) {
	var stat syscall.Stat_t
	err = syscall.Lstat(p, &stat)
	if err != nil {
		return
	}
	uid = int(stat.Uid)
	gid = int(stat.Gid)
	inode = uint64(stat.Ino)
	nlink = int(stat.Nlink)
	major = -1
	minor = -1
	if stat.Mode&syscall.S_IFBLK != 0 || stat.Mode&syscall.S_IFCHR != 0 {
		major = int(stat.Rdev / 256)
		minor = int(stat.Rdev % 256)
	}

	return
}

func IsMountPoint(name string) bool {
	_, err := exec.LookPath("mountpoint")
	if err == nil {
		err = exec.Command("mountpoint", "-q", name).Run()
		if err != nil {
			return false
		}

		return true
	}

	stat, err := os.Stat(name)
	if err != nil {
		return false
	}

	rootStat, err := os.Lstat(name + "/..")
	if err != nil {
		return false
	}
	// If the directory has the same device as parent, then it's not a mountpoint.
	return stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev
}

func ReadLastNLines(f *os.File, lines int) (string, error) {
	if lines <= 0 {
		return "", fmt.Errorf("invalid line count")
	}

	stat, err := f.Stat()
	if err != nil {
		return "", err
	}

	data, err := syscall.Mmap(int(f.Fd()), 0, int(stat.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		return "", err
	}
	defer syscall.Munmap(data)

	for i := len(data) - 1; i >= 0; i-- {
		if data[i] == '\n' {
			lines--
		}

		if lines < 0 {
			return string(data[i+1 : len(data)]), nil
		}
	}

	return string(data), nil
}

func SetSize(fd int, width int, height int) (err error) {
	var dimensions [4]uint16
	dimensions[0] = uint16(height)
	dimensions[1] = uint16(width)

	if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(syscall.TIOCSWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package svnwatch

import (
	"encoding/xml"

	"github.com/jackwilsdon/svnwatch/svn"
	"github.com/pkg/errors"
)

type Repositories struct {
	XMLName      xml.Name     `xml:"repositories"`
	Repositories []Repository `xml:"repository"`
}

func (r *Repositories) ForURL(url string) *Repository {
	for i, _ := range r.Repositories {
		if url == r.Repositories[i].URL {
			return &r.Repositories[i]
		}
	}

	r.Repositories = append(r.Repositories, Repository{
		Revision: 0,
		URL:      url,
	})

	return &r.Repositories[len(r.Repositories)-1]
}

type Repository struct {
	XMLName  xml.Name `xml:"repository"`
	URL      string   `xml:"url,attr"`
	Revision int      `xml:",chardata"`
}

func (r *Repository) Update() ([]svn.Revision, error) {
	revisions, err := svn.GetLogRange(r.URL, r.Revision, nil)

	if err != nil {
		return nil, errors.Wrapf(err, "failed to get log range for %s (range %d:HEAD)", r.URL, r.Revision)
	}

	originalRevision := r.Revision

	for _, revision := range revisions {
		if revision.Revision > r.Revision {
			r.Revision = revision.Revision
		}
	}

	// If it's our first update /or/ the revision hasn't changed then return nothing
	if originalRevision == 0 || r.Revision == originalRevision {
		return nil, nil
	}

	// Return everything but the first revision, as that is the revision we passed to GetLogRange
	return revisions[1:], nil
}

func (r *Repository) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	repo := struct {
		URL      *string `xml:"url,attr"`
		Revision *int    `xml:",chardata"`
	}{}

	if err := d.DecodeElement(&repo, &start); err != nil {
		return err
	}

	if repo.URL == nil {
		return errors.New("missing URL from repository")
	}

	if repo.Revision == nil {
		return errors.New("missing revision from repository")
	}

	r.URL = *repo.URL
	r.Revision = *repo.Revision

	return nil
}
<commit_msg>Simplify for loop<commit_after>package svnwatch

import (
	"encoding/xml"

	"github.com/jackwilsdon/svnwatch/svn"
	"github.com/pkg/errors"
)

type Repositories struct {
	XMLName      xml.Name     `xml:"repositories"`
	Repositories []Repository `xml:"repository"`
}

func (r *Repositories) ForURL(url string) *Repository {
	for i := range r.Repositories {
		if url == r.Repositories[i].URL {
			return &r.Repositories[i]
		}
	}

	r.Repositories = append(r.Repositories, Repository{
		Revision: 0,
		URL:      url,
	})

	return &r.Repositories[len(r.Repositories)-1]
}

type Repository struct {
	XMLName  xml.Name `xml:"repository"`
	URL      string   `xml:"url,attr"`
	Revision int      `xml:",chardata"`
}

func (r *Repository) Update() ([]svn.Revision, error) {
	revisions, err := svn.GetLogRange(r.URL, r.Revision, nil)

	if err != nil {
		return nil, errors.Wrapf(err, "failed to get log range for %s (range %d:HEAD)", r.URL, r.Revision)
	}

	originalRevision := r.Revision

	for _, revision := range revisions {
		if revision.Revision > r.Revision {
			r.Revision = revision.Revision
		}
	}

	// If it's our first update /or/ the revision hasn't changed then return nothing
	if originalRevision == 0 || r.Revision == originalRevision {
		return nil, nil
	}

	// Return everything but the first revision, as that is the revision we passed to GetLogRange
	return revisions[1:], nil
}

func (r *Repository) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	repo := struct {
		URL      *string `xml:"url,attr"`
		Revision *int    `xml:",chardata"`
	}{}

	if err := d.DecodeElement(&repo, &start); err != nil {
		return err
	}

	if repo.URL == nil {
		return errors.New("missing URL from repository")
	}

	if repo.Revision == nil {
		return errors.New("missing revision from repository")
	}

	r.URL = *repo.URL
	r.Revision = *repo.Revision

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2022 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/gravitational/teleport/api/types"
	"github.com/gravitational/teleport/lib/asciitable"
	"github.com/gravitational/teleport/lib/auth"
	libclient "github.com/gravitational/teleport/lib/client"
	"github.com/gravitational/teleport/lib/service"

	"github.com/gravitational/kingpin"
	"github.com/gravitational/trace"

	"github.com/google/uuid"
)

// AlertCommand implements the `tctl alerts` family of commands.
type AlertCommand struct {
	config *service.Config

	message string

	labels string

	verbose bool

	alertList   *kingpin.CmdClause
	alertCreate *kingpin.CmdClause
}

// Initialize allows AlertCommand to plug itself into the CLI parser
func (c *AlertCommand) Initialize(app *kingpin.Application, config *service.Config) {
	c.config = config
	alert := app.Command("alerts", "Manage cluster alerts").Alias("alert")

	c.alertList = alert.Command("list", "List cluster alerts").Alias("ls")
	c.alertList.Flag("verbose", "Show detailed alert info").Short('v').BoolVar(&c.verbose)
	c.alertList.Flag("labels", labelHelp).StringVar(&c.labels)

	c.alertCreate = alert.Command("create", "Create cluster alerts").Hidden()
	c.alertCreate.Arg("message", "Alert body message").Required().StringVar(&c.message)
	c.alertCreate.Flag("labels", labelHelp).StringVar(&c.labels)
}

// TryRun takes the CLI command as an argument (like "alerts ls") and executes it.
func (c *AlertCommand) TryRun(ctx context.Context, cmd string, client auth.ClientI) (match bool, err error) {
	switch cmd {
	case c.alertList.FullCommand():
		err = c.List(ctx, client)
	case c.alertCreate.FullCommand():
		err = c.Create(ctx, client)
	default:
		return false, nil
	}
	return true, trace.Wrap(err)
}

func (c *AlertCommand) List(ctx context.Context, client auth.ClientI) error {
	labels, err := libclient.ParseLabelSpec(c.labels)
	if err != nil {
		return trace.Wrap(err)
	}

	alerts, err := client.GetClusterAlerts(ctx, types.GetClusterAlertsRequest{
		Labels: labels,
	})
	if err != nil {
		return trace.Wrap(err)
	}

	if len(alerts) == 0 {
		fmt.Println("no alerts")
		return nil
	}

	// sort so that newer/high-severity alerts show up higher.
	types.SortClusterAlerts(alerts)

	if c.verbose {
		table := asciitable.MakeTable([]string{"Severity", "Message", "Created", "Labels"})
		for _, alert := range alerts {
			var labelPairs []string
			for key, val := range alert.Metadata.Labels {
				// alert labels can be displayed unquoted because we enforce a
				// very limited charset.
				labelPairs = append(labelPairs, fmt.Sprintf("%s=%s", key, val))
			}
			table.AddRow([]string{
				alert.Spec.Severity.String(),
				fmt.Sprintf("%q", alert.Spec.Message),
				alert.Spec.Created.Format(time.RFC822),
				strings.Join(labelPairs, ", "),
			})
		}
		fmt.Println(table.AsBuffer().String())
	} else {
		table := asciitable.MakeTable([]string{"Severity", "Message"})
		for _, alert := range alerts {
			table.AddRow([]string{alert.Spec.Severity.String(), fmt.Sprintf("%q", alert.Spec.Message)})
		}
		fmt.Println(table.AsBuffer().String())
	}

	return nil
}

func (c *AlertCommand) Create(ctx context.Context, client auth.ClientI) error {
	labels, err := libclient.ParseLabelSpec(c.labels)
	if err != nil {
		return trace.Wrap(err)
	}

	alert, err := types.NewClusterAlert(uuid.New().String(), c.message)
	if err != nil {
		return trace.Wrap(err)
	}

	alert.Metadata.Labels = labels

	return trace.Wrap(client.UpsertClusterAlert(ctx, alert))
}
<commit_msg>Unhide tctl alert create (#16271)<commit_after>/*
Copyright 2022 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/gravitational/teleport/api/types"
	"github.com/gravitational/teleport/lib/asciitable"
	"github.com/gravitational/teleport/lib/auth"
	libclient "github.com/gravitational/teleport/lib/client"
	"github.com/gravitational/teleport/lib/service"

	"github.com/gravitational/kingpin"
	"github.com/gravitational/trace"

	"github.com/google/uuid"
)

// AlertCommand implements the `tctl alerts` family of commands.
type AlertCommand struct {
	config *service.Config

	message  string
	labels   string
	severity string
	ttl      time.Duration

	verbose bool

	alertList   *kingpin.CmdClause
	alertCreate *kingpin.CmdClause
}

// Initialize allows AlertCommand to plug itself into the CLI parser
func (c *AlertCommand) Initialize(app *kingpin.Application, config *service.Config) {
	c.config = config
	alert := app.Command("alerts", "Manage cluster alerts").Alias("alert")

	c.alertList = alert.Command("list", "List cluster alerts").Alias("ls")
	c.alertList.Flag("verbose", "Show detailed alert info").Short('v').BoolVar(&c.verbose)
	c.alertList.Flag("labels", labelHelp).StringVar(&c.labels)

	c.alertCreate = alert.Command("create", "Create cluster alerts")
	c.alertCreate.Arg("message", "Alert body message").Required().StringVar(&c.message)
	c.alertCreate.Flag("ttl", "Time duration after which the alert expires.").DurationVar(&c.ttl)
	c.alertCreate.Flag("severity", "Severity of the alert (low, medium, or high)").Default("low").EnumVar(&c.severity, "low", "medium", "high")
	c.alertCreate.Flag("labels", labelHelp).StringVar(&c.labels)
}

// TryRun takes the CLI command as an argument (like "alerts ls") and executes it.
func (c *AlertCommand) TryRun(ctx context.Context, cmd string, client auth.ClientI) (match bool, err error) {
	switch cmd {
	case c.alertList.FullCommand():
		err = c.List(ctx, client)
	case c.alertCreate.FullCommand():
		err = c.Create(ctx, client)
	default:
		return false, nil
	}
	return true, trace.Wrap(err)
}

func (c *AlertCommand) List(ctx context.Context, client auth.ClientI) error {
	labels, err := libclient.ParseLabelSpec(c.labels)
	if err != nil {
		return trace.Wrap(err)
	}

	alerts, err := client.GetClusterAlerts(ctx, types.GetClusterAlertsRequest{
		Labels: labels,
	})
	if err != nil {
		return trace.Wrap(err)
	}

	if len(alerts) == 0 {
		fmt.Println("no alerts")
		return nil
	}

	// sort so that newer/high-severity alerts show up higher.
	types.SortClusterAlerts(alerts)

	if c.verbose {
		table := asciitable.MakeTable([]string{"Severity", "Message", "Created", "Labels"})
		for _, alert := range alerts {
			var labelPairs []string
			for key, val := range alert.Metadata.Labels {
				// alert labels can be displayed unquoted because we enforce a
				// very limited charset.
				labelPairs = append(labelPairs, fmt.Sprintf("%s=%s", key, val))
			}
			table.AddRow([]string{
				alert.Spec.Severity.String(),
				fmt.Sprintf("%q", alert.Spec.Message),
				alert.Spec.Created.Format(time.RFC822),
				strings.Join(labelPairs, ", "),
			})
		}
		fmt.Println(table.AsBuffer().String())
	} else {
		table := asciitable.MakeTable([]string{"Severity", "Message"})
		for _, alert := range alerts {
			table.AddRow([]string{alert.Spec.Severity.String(), fmt.Sprintf("%q", alert.Spec.Message)})
		}
		fmt.Println(table.AsBuffer().String())
	}

	return nil
}

func (c *AlertCommand) Create(ctx context.Context, client auth.ClientI) error {
	labels, err := libclient.ParseLabelSpec(c.labels)
	if err != nil {
		return trace.Wrap(err)
	}

	var sev types.AlertSeverity
	switch c.severity {
	case "low":
		sev = types.AlertSeverity_LOW
	case "medium":
		sev = types.AlertSeverity_MEDIUM
	case "high":
		sev = types.AlertSeverity_HIGH
	}

	alert, err := types.NewClusterAlert(uuid.New().String(), c.message, types.WithAlertSeverity(sev))
	if err != nil {
		return trace.Wrap(err)
	}

	if len(labels) == 0 {
		labels[types.AlertOnLogin] = "yes"
		labels[types.AlertPermitAll] = "yes"
	}
	alert.Metadata.Labels = labels

	if c.ttl > 0 {
		alert.SetExpiry(time.Now().UTC().Add(c.ttl))
	}

	return trace.Wrap(client.UpsertClusterAlert(ctx, alert))
}
<|endoftext|>