<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	tid "github.com/Financial-Times/transactionid-utils-go"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/stretchr/testify/assert"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
)

var contentPreviewService *httptest.Server
var methodeApiMock *httptest.Server
var methodeArticleTransformerMock *httptest.Server

func startMethodeApiMock(status string) {
	r := mux.NewRouter()
	if status == "happy" {
		r.Path("/eom-file/{uuid}").Handler(handlers.MethodHandler{"GET": http.HandlerFunc(methodeApiHandlerMock)})
	} else {
		r.Path("/eom-file/{uuid}").Handler(handlers.MethodHandler{"GET": http.HandlerFunc(unhappyHandler)})
	}
	methodeApiMock = httptest.NewServer(r)
}

func methodeApiHandlerMock(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")

	vars := mux.Vars(r)
	uuid := vars["uuid"]

	if r.Header.Get(tid.TransactionIDHeader) != "" {
		w.Header().Set(tid.TransactionIDHeader, r.Header.Get(tid.TransactionIDHeader))
	} else {
		w.Header().Set(tid.TransactionIDHeader, "tid_w58gqvazux")
	}

	if r.Header.Get("Authorization") == "Basic default" && uuid == "d7db73ec-cf53-11e5-92a1-c5e23ef99c77" {
		file, err := os.Open("test-resources/methode-api-output.json")
		if err != nil {
			return
		}
		defer file.Close()
		io.Copy(w, file)
	} else {
		w.WriteHeader(http.StatusNotFound)
		json.NewEncoder(w).Encode("{\"message\":\"404 Not Found - null\"}")
	}

}

func unhappyHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusInternalServerError)
}

func startMethodeArticleTransformerMock(status string) {
	r := mux.NewRouter()
	if status == "happy" {
		r.Path("/content-transform/{uuid}").Queries("preview", "true").Handler(handlers.MethodHandler{"POST": http.HandlerFunc(methodeArticleTransformerHandlerMock)})
	} else {
		r.Path("/content-transform/{uuid}").Handler(handlers.MethodHandler{"POST": http.HandlerFunc(unhappyHandler)})
	}

	methodeArticleTransformerMock = httptest.NewServer(r)
}

func methodeArticleTransformerHandlerMock(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")

	vars := mux.Vars(r)
	uuid := vars["uuid"]

	if r.Header.Get(tid.TransactionIDHeader) != "" && isEqualToMethodeApiOutput(r.Body) {
		w.Header().Set(tid.TransactionIDHeader, r.Header.Get(tid.TransactionIDHeader))
	} else {
		w.Header().Set(tid.TransactionIDHeader, "tid_w58gqvazux")
	}

	if uuid == "d7db73ec-cf53-11e5-92a1-c5e23ef99c77" {
		file, err := os.Open("test-resources/methode-article-transformer-output.json")
		if err != nil {
			return
		}
		defer file.Close()
		io.Copy(w, file)
	} else {
		w.WriteHeader(http.StatusNotFound)
	}

}

func isEqualToMethodeApiOutput(r io.Reader) bool {
	file, err := os.Open("test-resources/methode-api-output.json")
	if err != nil {
		return false
	}
	defer file.Close()
	methodeApiOutput := getStringFromReader(file)
	readerOutput := getStringFromReader(r)
	return methodeApiOutput == readerOutput
}

func stopServices() {
	methodeApiMock.Close()
	methodeArticleTransformerMock.Close()
	contentPreviewService.Close()
}

func startContentPreviewService() {

	methodeApiUrl := methodeApiMock.URL + "/eom-file/"
	nativeContentAppHealthUri := methodeApiMock.URL + "/build-info"
	methodArticleTransformerUrl := methodeArticleTransformerMock.URL + "/content-transform/"
	transformAppHealthUrl := methodeArticleTransformerMock.URL + "/build-info"

	sc := ServiceConfig{
		"content-preview",
		"8084",
		"default",
		"methode-article-transformer",
		methodeApiUrl,
		methodArticleTransformerUrl,
		nativeContentAppHealthUri,
		transformAppHealthUrl,
		"Native Content Service",
		"Native Content Transformer Service",
	}

	appLogger := NewAppLogger()
	metricsHandler := NewMetrics()
	contentHandler := ContentHandler{&sc, appLogger, &metricsHandler}

	h := setupServiceHandler(sc, metricsHandler, contentHandler)

	contentPreviewService = httptest.NewServer(h)
}

func TestShouldReturn200AndTrasformerOutput(t *testing.T) {
	startMethodeApiMock("happy")
	startMethodeArticleTransformerMock("happy")
	startContentPreviewService()
	defer stopServices()
	resp, err := http.Get(contentPreviewService.URL + "/content-preview/d7db73ec-cf53-11e5-92a1-c5e23ef99c77")
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusOK, resp.StatusCode, "Response status should be 200")

	file, _ := os.Open("test-resources/methode-article-transformer-output.json")
	defer file.Close()

	expectedOutput := getStringFromReader(file)
	actualOutput := getStringFromReader(resp.Body)

	assert.Equal(t, expectedOutput, actualOutput, "Response body shoud be equal to transformer response body")
}

func getStringFromReader(r io.Reader) string {
	buf := new(bytes.Buffer)
	buf.ReadFrom(r)
	return buf.String()
}

func TestShouldReturn404(t *testing.T) {
	startMethodeApiMock("happy")
	startMethodeArticleTransformerMock("happy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/content-preview/158ab514-f989-4abc-b42b-b3edf0811899")
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusNotFound, resp.StatusCode, "Response status should be 404")

	contentPreviewService.Close()

}

func TestShouldReturn503whenMethodeApiIsNotHappy(t *testing.T) {
	startMethodeApiMock("unhappy")
	startMethodeArticleTransformerMock("happy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/content-preview/d7db73ec-cf53-11e5-92a1-c5e23ef99c77")
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode, "Response status should be 503")
}

func TestShouldReturn503whenMethodeApiIsNotAvailable(t *testing.T) {
	startMethodeArticleTransformerMock("happy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/content-preview/d7db73ec-cf53-11e5-92a1-c5e23ef99c77")
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode, "Response status should be 503")
}

func TestShouldReturn503whenTransformerIsNotHappy(t *testing.T) {
	startMethodeApiMock("happy")
	startMethodeArticleTransformerMock("unhappy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/content-preview/d7db73ec-cf53-11e5-92a1-c5e23ef99c77")
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode, "Response status should be 503")
}

func TestShouldReturn503whenTransformerNotAvailable(t *testing.T) {
	startMethodeApiMock("happy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/content-preview/d7db73ec-cf53-11e5-92a1-c5e23ef99c77")
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode, "Response status should be 503")
}
<commit_msg>Implemented tests for health-check<commit_after>package main

import (
	"bytes"
	"encoding/json"
	fthealth "github.com/Financial-Times/go-fthealth/v1a"
	tid "github.com/Financial-Times/transactionid-utils-go"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/stretchr/testify/assert"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
)

var contentPreviewService *httptest.Server
var methodeApiMock *httptest.Server
var methodeArticleTransformerMock *httptest.Server

func startMethodeApiMock(status string) {
	r := mux.NewRouter()
	if status == "happy" {
		r.Path("/eom-file/{uuid}").Handler(handlers.MethodHandler{"GET": http.HandlerFunc(methodeApiHandlerMock)})
		r.Path("/build-info").Handler(handlers.MethodHandler{"GET": http.HandlerFunc(happyHandler)})
	} else {
		r.Path("/eom-file/{uuid}").Handler(handlers.MethodHandler{"GET": http.HandlerFunc(unhappyHandler)})
		r.Path("/build-info").Handler(handlers.MethodHandler{"GET": http.HandlerFunc(unhappyHandler)})
	}
	methodeApiMock = httptest.NewServer(r)
}

func methodeApiHandlerMock(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")

	vars := mux.Vars(r)
	uuid := vars["uuid"]

	if r.Header.Get(tid.TransactionIDHeader) != "" {
		w.Header().Set(tid.TransactionIDHeader, r.Header.Get(tid.TransactionIDHeader))
	} else {
		w.Header().Set(tid.TransactionIDHeader, "tid_w58gqvazux")
	}

	if r.Header.Get("Authorization") == "Basic default" && uuid == "d7db73ec-cf53-11e5-92a1-c5e23ef99c77" {
		file, err := os.Open("test-resources/methode-api-output.json")
		if err != nil {
			return
		}
		defer file.Close()
		io.Copy(w, file)
	} else {
		w.WriteHeader(http.StatusNotFound)
		json.NewEncoder(w).Encode("{\"message\":\"404 Not Found - null\"}")
	}

}

func unhappyHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusInternalServerError)
}

func happyHandler(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
}

func startMethodeArticleTransformerMock(status string) {
	r := mux.NewRouter()
	if status == "happy" {
		r.Path("/content-transform/{uuid}").Queries("preview", "true").Handler(handlers.MethodHandler{"POST": http.HandlerFunc(methodeArticleTransformerHandlerMock)})
		r.Path("/build-info").Handler(handlers.MethodHandler{"GET": http.HandlerFunc(happyHandler)})
	} else {
		r.Path("/content-transform/{uuid}").Handler(handlers.MethodHandler{"POST": http.HandlerFunc(unhappyHandler)})
		r.Path("/build-info").Handler(handlers.MethodHandler{"GET": http.HandlerFunc(unhappyHandler)})
	}

	methodeArticleTransformerMock = httptest.NewServer(r)
}

func methodeArticleTransformerHandlerMock(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json; charset=utf-8")

	vars := mux.Vars(r)
	uuid := vars["uuid"]

	if r.Header.Get(tid.TransactionIDHeader) != "" && isEqualToMethodeApiOutput(r.Body) {
		w.Header().Set(tid.TransactionIDHeader, r.Header.Get(tid.TransactionIDHeader))
	} else {
		w.Header().Set(tid.TransactionIDHeader, "tid_w58gqvazux")
	}

	if uuid == "d7db73ec-cf53-11e5-92a1-c5e23ef99c77" {
		file, err := os.Open("test-resources/methode-article-transformer-output.json")
		if err != nil {
			return
		}
		defer file.Close()
		io.Copy(w, file)
	} else {
		w.WriteHeader(http.StatusNotFound)
	}

}

func isEqualToMethodeApiOutput(r io.Reader) bool {
	file, err := os.Open("test-resources/methode-api-output.json")
	if err != nil {
		return false
	}
	defer file.Close()
	methodeApiOutput := getStringFromReader(file)
	readerOutput := getStringFromReader(r)
	return methodeApiOutput == readerOutput
}

func stopServices() {
	methodeApiMock.Close()
	methodeArticleTransformerMock.Close()
	contentPreviewService.Close()
}

func startContentPreviewService() {

	methodeApiUrl := methodeApiMock.URL + "/eom-file/"
	nativeContentAppHealthUri := methodeApiMock.URL + "/build-info"
	methodArticleTransformerUrl := methodeArticleTransformerMock.URL + "/content-transform/"
	transformAppHealthUrl := methodeArticleTransformerMock.URL + "/build-info"

	sc := ServiceConfig{
		"content-preview",
		"8084",
		"default",
		"methode-article-transformer",
		methodeApiUrl,
		methodArticleTransformerUrl,
		nativeContentAppHealthUri,
		transformAppHealthUrl,
		"Native Content Service",
		"Native Content Transformer Service",
	}

	appLogger := NewAppLogger()
	metricsHandler := NewMetrics()
	contentHandler := ContentHandler{&sc, appLogger, &metricsHandler}

	h := setupServiceHandler(sc, metricsHandler, contentHandler)

	contentPreviewService = httptest.NewServer(h)
}

func TestShouldReturn200AndTrasformerOutput(t *testing.T) {
	startMethodeApiMock("happy")
	startMethodeArticleTransformerMock("happy")
	startContentPreviewService()
	defer stopServices()
	resp, err := http.Get(contentPreviewService.URL + "/content-preview/d7db73ec-cf53-11e5-92a1-c5e23ef99c77")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusOK, resp.StatusCode, "Response status should be 200")

	file, _ := os.Open("test-resources/methode-article-transformer-output.json")
	defer file.Close()

	expectedOutput := getStringFromReader(file)
	actualOutput := getStringFromReader(resp.Body)

	assert.Equal(t, expectedOutput, actualOutput, "Response body shoud be equal to transformer response body")
}

func getStringFromReader(r io.Reader) string {
	buf := new(bytes.Buffer)
	buf.ReadFrom(r)
	return buf.String()
}

func TestShouldReturn404(t *testing.T) {
	startMethodeApiMock("happy")
	startMethodeArticleTransformerMock("happy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/content-preview/158ab514-f989-4abc-b42b-b3edf0811899")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusNotFound, resp.StatusCode, "Response status should be 404")

	contentPreviewService.Close()

}

func TestShouldReturn503whenMethodeApiIsNotHappy(t *testing.T) {
	startMethodeApiMock("unhappy")
	startMethodeArticleTransformerMock("happy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/content-preview/d7db73ec-cf53-11e5-92a1-c5e23ef99c77")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode, "Response status should be 503")
}

func TestShouldReturn503whenMethodeApiIsNotAvailable(t *testing.T) {
	startMethodeArticleTransformerMock("happy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/content-preview/d7db73ec-cf53-11e5-92a1-c5e23ef99c77")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode, "Response status should be 503")
}

func TestShouldReturn503whenTransformerIsNotHappy(t *testing.T) {
	startMethodeApiMock("happy")
	startMethodeArticleTransformerMock("unhappy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/content-preview/d7db73ec-cf53-11e5-92a1-c5e23ef99c77")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode, "Response status should be 503")
}

func TestShouldReturn503whenTransformerNotAvailable(t *testing.T) {
	startMethodeApiMock("happy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/content-preview/d7db73ec-cf53-11e5-92a1-c5e23ef99c77")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusServiceUnavailable, resp.StatusCode, "Response status should be 503")
}

func TestShouldBeHealthy(t *testing.T) {
	startMethodeApiMock("happy")
	startMethodeArticleTransformerMock("happy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/__health")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusOK, resp.StatusCode, "Response status should be 200")

	var res fthealth.HealthResult

	json.NewDecoder(resp.Body).Decode(&res)

	assert.Equal(t, true, res.Ok, "The service should be healthy")
}

func TestShouldBeUnhealthyWhenMethodeApiIsNotHappy(t *testing.T) {
	startMethodeApiMock("unhappy")
	startMethodeArticleTransformerMock("happy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/__health")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusOK, resp.StatusCode, "Response status should be 200")

	var res fthealth.HealthResult

	json.NewDecoder(resp.Body).Decode(&res)

	assert.Equal(t, false, res.Ok, "The service should be unhealthy")

	for i := 0; i < len(res.Checks); i++ {
		switch res.Checks[i].Name {
		case "Native Content Service Availabililty Check":
			assert.Equal(t, false, res.Checks[i].Ok, "The native content should be unhealthy")
		case "Native Content Transformer Service Availabililty":
			assert.Equal(t, true, res.Checks[i].Ok, "The transformer should be healthy")
		}
	}
}

func TestShouldBeUnhealthyWhenTransformerIsNotHappy(t *testing.T) {
	startMethodeApiMock("happy")
	startMethodeArticleTransformerMock("unhappy")
	startContentPreviewService()
	defer stopServices()

	resp, err := http.Get(contentPreviewService.URL + "/__health")
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()

	assert.Equal(t, http.StatusOK, resp.StatusCode, "Response status should be 200")

	var res fthealth.HealthResult

	json.NewDecoder(resp.Body).Decode(&res)

	assert.Equal(t, false, res.Ok, "The service should be unhealthy")

	for i := 0; i < len(res.Checks); i++ {
		switch res.Checks[i].Name {
		case "Native Content Service Availabililty Check":
			assert.Equal(t, true, res.Checks[i].Ok, "The native content should be healthy")
		case "Native Content Transformer Service Availabililty":
			assert.Equal(t, false, res.Checks[i].Ok, "The transformer should be unhealthy")
		}
	}

}
<|endoftext|>


<commit_before>package scp

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/ssh"
	"github.com/viant/toolbox/storage"
	"io"
	"io/ioutil"
	"net/url"
	"strings"

	"path"
	"github.com/viant/toolbox/cred"
)

const defaultSSHPort = 22

const (
	fileInfoPermission = iota
	_
	fileInfoOwner
	fileInfoGroup
	fileInfoSize
	fileInfoDateMonth
	fileInfoDateDay
	fileInfoDateHour
	fileInfoDateYear
	fileInfoName

)

const (
	fileIsoInfoPermission = iota
	_
	fileIsoInfoOwner
	fileIsoInfoGroup
	fileIsoInfoSize
	fileIsoDate
	fileIsoTime
	fileIsoTimezone
	fileIsoInfoName
)

type service struct {
	config *cred.Config
}

func (s *service) runCommand(URL string, command string) (string, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return "", err
	}
	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = 22
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return "", err
	}
	defer client.Close()
	session, err := client.NewSession()
	if err != nil {
		return "", err
	}
	defer session.Close()
	output, _ := session.CombinedOutput(command)
	return toolbox.AsString(output), err

}

func (s *service) canListWithTimeStyle(URL string) (bool, error) {
	output, err := s.runCommand(URL, "ls -ltr --time-style=full-iso")
	if err != nil {
		return false, err
	}
	return ! strings.Contains(string(output), "usage"), nil
}

func normalizeFileInfoOutput(lines string) string {
	var result = make([]string, 0)
	for _, line := range strings.Split(lines, "\n") {
		if strings.HasPrefix(strings.ToLower(line), "total") {
			continue
		}
		result = append(result, line)
	}
	return strings.Join(result, "\n")
}

//List returns a list of object for supplied URL
func (s *service) List(URL string) ([]storage.Object, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}

	var urlPath = strings.Replace(parsedUrl.Path, "//", "/", len(parsedUrl.Path))
	var result = make([]storage.Object, 0)

	canListWithTimeStyle, err := s.canListWithTimeStyle(URL)
	if err != nil {
		return nil, err
	}

	var lsCommand = "ls -ltr"
	if canListWithTimeStyle {
		lsCommand += " --time-style=full-iso"
	} else {
		lsCommand +="T"
	}

	output, err := s.runCommand(URL, lsCommand + " "+parsedUrl.Path)
	stdout := normalizeFileInfoOutput(string(output))
	if strings.Contains(stdout, "No such file or directory") {
		return result, nil
	}
	var fileNameFilter = ""

	if err == nil && stdout == "" {
		parent, fileName := path.Split(urlPath )
		fileNameFilter = fileName
		output, err = s.runCommand(URL, lsCommand + " "+parent+" | grep "+fileName)
	}
	if err != nil {
		return nil, err
	}

	stdout = normalizeFileInfoOutput(string(output))
	for _, line := range strings.Split(stdout, "\n") {
		fileInfo := ExtractFileInfo(line, canListWithTimeStyle)
		if fileInfo == nil {
			continue
		}
		if fileInfo.name == "" {
			continue
		}
		fileInfo.url = URL
		if fileNameFilter == "" || fileNameFilter == fileInfo.name {
			result = append(result, fileInfo)
		}
	}
	return result, nil
}

//file info with iso -rw-r--r-- 1 awitas awitas 2002 2017-11-04 22:29:33.363458941 +0000 aerospikeciads_aerospike.conf
//file info without iso // -rw-r--r--  1 awitas  1742120565   414 Jun  8 14:14:08 2017 id_rsa.pub

func ExtractFileInfo(line string, isoTimeStyle bool) *object {
	fragmentCount := 0
	fileInfo := &object{}
	if strings.TrimSpace(line) == "" {
		return nil
	}
	for i := range line {

		aChar := string(line[i])
		if aChar == " " || aChar == "\t" {
			if i+1 < len(line) {
				nextChar := string(line[i+1])
				if !(nextChar == " " || nextChar == "\t") {
					fragmentCount++
				}
			}
			continue
		}

		if isoTimeStyle {
			switch fragmentCount {
			case fileIsoInfoPermission:
				fileInfo.permission += aChar
			case fileIsoInfoOwner:
				fileInfo.owner += aChar
			case fileIsoInfoGroup:
				fileInfo.group += aChar
			case fileIsoInfoSize:
				fileInfo.size += aChar
			case fileIsoDate:
				fileInfo.date += aChar
			case fileIsoTime:
				fileInfo.time += aChar
			case fileIsoTimezone:
				fileInfo.timezone += aChar
			case fileIsoInfoName:
				fileInfo.name += aChar
			}
			continue
		}

		switch fragmentCount {

		case fileInfoPermission:
			fileInfo.permission += aChar
		case fileInfoOwner:
			fileInfo.owner += aChar
		case fileInfoGroup:
			fileInfo.group += aChar
		case fileInfoSize:
			fileInfo.size += aChar
		case fileInfoDateMonth:
			fileInfo.month += aChar
		case fileInfoDateDay:
			fileInfo.day += aChar
		case fileInfoDateHour:
			fileInfo.hour += aChar
		case fileInfoDateYear:
			fileInfo.year += aChar
		case fileInfoName:
			fileInfo.name += aChar
		}

	}
	return fileInfo
}

func (s *service) Exists(URL string) (bool, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return false, err
	}
	output, err := s.runCommand(URL, "ls -lTtr "+parsedUrl.Path)
	if strings.Contains(string(output), "No such file or directory") {
		return false, nil
	}
	return true, nil

}

func (s *service) StorageObject(URL string) (storage.Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, fmt.Errorf("No found %v", URL)
	}
	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *service) Download(object storage.Object) (io.Reader, error) {
	if object == nil {
		return nil, fmt.Errorf("Object was nil")
	}
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return nil, err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return nil, err
	}
	defer client.Close()

	content, err := client.Download(parsedUrl.Path)
	if err != nil {
		return nil, err
	}

	//download verification (as sometimes scp failed) with one retry
	if int(object.Size()) != len(content) {
		content, err = client.Download(parsedUrl.Path)
		if err != nil {
			return nil, err
		}
		if int(object.Size()) != len(content) {
			return nil, fmt.Errorf("Faled to download from %v,  object size was: %v, but scp download was %v", object.URL(), object.Size(), len(content))
		}
	}
	return bytes.NewReader(content), nil
}

//Upload uploads provided reader content for supplied URL.
func (s *service) Upload(URL string, reader io.Reader) error {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()

	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return fmt.Errorf("Failed to upload - unable read: %v", err)
	}

	err = client.Upload(parsedUrl.Path, content)
	if err != nil {
		return  fmt.Errorf("Failed to upload: %v %v", URL,  err)
	}

	object, err := s.StorageObject(URL)
	if err != nil {
		return  fmt.Errorf("Failed to get upload object  %v for verification: %v", URL, err)
	}

	if int(object.Size()) != len(content) {
		err = client.Upload(parsedUrl.Path, content)
		object, err = s.StorageObject(URL)
		if err != nil {
			return  err
		}
		if int(object.Size()) != len(content) {
			return fmt.Errorf("Failed to upload to %v, actual size was:%v,  but uploaded size was  ", URL, len(content), int(object.Size()))
		}
	}
	return err
}

func (s *service) Register(schema string, service storage.Service) error {
	return errors.New("unsupported")
}

//Delete removes passed in storage object
func (s *service) Delete(object storage.Object) error {
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()

	if parsedUrl.Path == "/" {
		return fmt.Errorf("Invalid removal path: %v", parsedUrl.Path)
	}
	_, err = session.Output("rm -rf " + parsedUrl.Path)
	return err
}

//NewService create a new gc storage service
func NewService(config *cred.Config) *service {
	return &service{
		config: config,
	}
}
<commit_msg>patched verification performance<commit_after>package scp

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/ssh"
	"github.com/viant/toolbox/storage"
	"io"
	"io/ioutil"
	"strings"

	"path"
	"github.com/viant/toolbox/cred"
	"net/url"
)

const defaultSSHPort = 22

const (
	fileInfoPermission = iota
	_
	fileInfoOwner
	fileInfoGroup
	fileInfoSize
	fileInfoDateMonth
	fileInfoDateDay
	fileInfoDateHour
	fileInfoDateYear
	fileInfoName

)

const (
	fileIsoInfoPermission = iota
	_
	fileIsoInfoOwner
	fileIsoInfoGroup
	fileIsoInfoSize
	fileIsoDate
	fileIsoTime
	fileIsoTimezone
	fileIsoInfoName
)

type service struct {
	config *cred.Config

}

func (s *service) runCommand(session *ssh.MultiCommandSession, URL string, command string) (string, error) {
	output, _ := session.Run(command, 0, "$ ", "usage")
	fmt.Printf("Command: %v %v", command, output)
	return toolbox.AsString(output), nil
}

func (s *service) canListWithTimeStyle(session *ssh.MultiCommandSession, URL string) (bool, error) {
	output, err := s.runCommand(session, URL, "ls -ltr --time-style=full-iso")
	if err != nil {
		return false, err
	}
	return ! strings.Contains(string(output), "usage"), nil
}

func normalizeFileInfoOutput(lines string) string {
	var result = make([]string, 0)
	for _, line := range strings.Split(lines, "\n") {
		if strings.HasPrefix(strings.ToLower(line), "total") {
			continue
		}
		result = append(result, line)
	}
	return strings.Join(result, "\n")
}

func  (s *service) getClient(parsedURL *url.URL) (*ssh.Client, error) {
	port := toolbox.AsInt(parsedURL.Port())
	if port == 0 {
		port = 22
	}
	return ssh.NewClient(parsedURL.Hostname(), toolbox.AsInt(port), s.config)
}

//List returns a list of object for supplied URL
func (s *service) List(URL string) ([]storage.Object, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}
	client, err := s.getClient(parsedUrl)
	if err != nil {
		return nil, err
	}
	defer client.Close()
	session, err := client.OpenMultiCommandSession(&ssh.SessionConfig{})
	if err != nil {
		return nil, err
	}
	defer session.Close()

	var urlPath = strings.Replace(parsedUrl.Path, "//", "/", len(parsedUrl.Path))
	var result = make([]storage.Object, 0)

	canListWithTimeStyle, err := s.canListWithTimeStyle(session, URL)
	if err != nil {
		return nil, err
	}

	var lsCommand = "ls -ltr"
	if canListWithTimeStyle {
		lsCommand += " --time-style=full-iso"
	} else {
		lsCommand +="T"
	}

	output, err := s.runCommand(session, URL, lsCommand + " "+parsedUrl.Path)
	stdout := normalizeFileInfoOutput(string(output))
	if strings.Contains(stdout, "No such file or directory") {
		return result, nil
	}
	var fileNameFilter = ""

	if err == nil && stdout == "" {
		parent, fileName := path.Split(urlPath )
		fileNameFilter = fileName
		output, err = s.runCommand(session, URL, lsCommand + " "+parent+" | grep "+fileName)
	}
	if err != nil {
		return nil, err
	}

	stdout = normalizeFileInfoOutput(string(output))
	for _, line := range strings.Split(stdout, "\n") {
		fileInfo := ExtractFileInfo(line, canListWithTimeStyle)
		if fileInfo == nil {
			continue
		}
		if fileInfo.name == "" {
			continue
		}
		fileInfo.url = URL
		if fileNameFilter == "" || fileNameFilter == fileInfo.name {
			result = append(result, fileInfo)
		}
	}
	return result, nil
}

//file info with iso -rw-r--r-- 1 awitas awitas 2002 2017-11-04 22:29:33.363458941 +0000 aerospikeciads_aerospike.conf
//file info without iso // -rw-r--r--  1 awitas  1742120565   414 Jun  8 14:14:08 2017 id_rsa.pub

func ExtractFileInfo(line string, isoTimeStyle bool) *object {
	fragmentCount := 0
	fileInfo := &object{}
	if strings.TrimSpace(line) == "" {
		return nil
	}
	for i := range line {

		aChar := string(line[i])
		if aChar == " " || aChar == "\t" {
			if i+1 < len(line) {
				nextChar := string(line[i+1])
				if !(nextChar == " " || nextChar == "\t") {
					fragmentCount++
				}
			}
			continue
		}

		if isoTimeStyle {
			switch fragmentCount {
			case fileIsoInfoPermission:
				fileInfo.permission += aChar
			case fileIsoInfoOwner:
				fileInfo.owner += aChar
			case fileIsoInfoGroup:
				fileInfo.group += aChar
			case fileIsoInfoSize:
				fileInfo.size += aChar
			case fileIsoDate:
				fileInfo.date += aChar
			case fileIsoTime:
				fileInfo.time += aChar
			case fileIsoTimezone:
				fileInfo.timezone += aChar
			case fileIsoInfoName:
				fileInfo.name += aChar
			}
			continue
		}

		switch fragmentCount {

		case fileInfoPermission:
			fileInfo.permission += aChar
		case fileInfoOwner:
			fileInfo.owner += aChar
		case fileInfoGroup:
			fileInfo.group += aChar
		case fileInfoSize:
			fileInfo.size += aChar
		case fileInfoDateMonth:
			fileInfo.month += aChar
		case fileInfoDateDay:
			fileInfo.day += aChar
		case fileInfoDateHour:
			fileInfo.hour += aChar
		case fileInfoDateYear:
			fileInfo.year += aChar
		case fileInfoName:
			fileInfo.name += aChar
		}

	}
	return fileInfo
}

func (s *service) Exists(URL string) (bool, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return false, err
	}

	client, err := s.getClient(parsedUrl)
	if err != nil {
		return false, err
	}
	defer client.Close()
	session, err := client.OpenMultiCommandSession(&ssh.SessionConfig{})
	if err != nil {
		return false, err
	}
	defer session.Close()

	output, err := s.runCommand(session, URL, "ls -ltr "+parsedUrl.Path)
	if strings.Contains(string(output), "No such file or directory") {
		return false, nil
	}
	return true, nil

}

func (s *service) StorageObject(URL string) (storage.Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, fmt.Errorf("No found %v", URL)
	}
	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *service) Download(object storage.Object) (io.Reader, error) {
	if object == nil {
		return nil, fmt.Errorf("Object was nil")
	}
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return nil, err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return nil, err
	}
	defer client.Close()

	content, err := client.Download(parsedUrl.Path)
	if err != nil {
		return nil, err
	}

	//download verification (as sometimes scp failed) with one retry
	if int(object.Size()) != len(content) {
		content, err = client.Download(parsedUrl.Path)
		if err != nil {
			return nil, err
		}
		if int(object.Size()) != len(content) {
			return nil, fmt.Errorf("Faled to download from %v,  object size was: %v, but scp download was %v", object.URL(), object.Size(), len(content))
		}
	}
	return bytes.NewReader(content), nil
}

//Upload uploads provided reader content for supplied URL.
func (s *service) Upload(URL string, reader io.Reader) error {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()

	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return fmt.Errorf("Failed to upload - unable read: %v", err)
	}

	err = client.Upload(parsedUrl.Path, content)
	if err != nil {
		return  fmt.Errorf("Failed to upload: %v %v", URL,  err)
	}

	object, err := s.StorageObject(URL)
	if err != nil {
		return  fmt.Errorf("Failed to get upload object  %v for verification: %v", URL, err)
	}

	if int(object.Size()) != len(content) {
		err = client.Upload(parsedUrl.Path, content)
		object, err = s.StorageObject(URL)
		if err != nil {
			return  err
		}
		if int(object.Size()) != len(content) {
			return fmt.Errorf("Failed to upload to %v, actual size was:%v,  but uploaded size was  ", URL, len(content), int(object.Size()))
		}
	}
	return err
}

func (s *service) Register(schema string, service storage.Service) error {
	return errors.New("unsupported")
}

//Delete removes passed in storage object
func (s *service) Delete(object storage.Object) error {
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	client, err := ssh.NewClient(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}
	defer client.Close()
	session, err := client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()

	if parsedUrl.Path == "/" {
		return fmt.Errorf("Invalid removal path: %v", parsedUrl.Path)
	}
	_, err = session.Output("rm -rf " + parsedUrl.Path)
	return err
}

//NewService create a new gc storage service
func NewService(config *cred.Config) *service {
	return &service{
		config: config,
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package placement

import (
	"net/http"
	"path"
	"time"

	"github.com/m3db/m3/src/cluster/kv"
	"github.com/m3db/m3/src/cluster/placement"
	"github.com/m3db/m3/src/query/api/v1/handler"
	"github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions"
	"github.com/m3db/m3/src/query/generated/proto/admin"
	"github.com/m3db/m3/src/query/util/logging"
	xhttp "github.com/m3db/m3/src/x/net/http"

	"github.com/gogo/protobuf/jsonpb"
	"go.uber.org/zap"
)

const (
	// SetHTTPMethod is the HTTP method for the the upsert endpoint.
	SetHTTPMethod = http.MethodPost

	setPathName = "set"
)

var (
	// M3DBSetURL is the url for the m3db replace handler (method POST).
	M3DBSetURL = path.Join(handler.RoutePrefixV1,
		M3DBServicePlacementPathName, setPathName)

	// M3AggSetURL is the url for the m3aggregator replace handler (method
	// POST).
	M3AggSetURL = path.Join(handler.RoutePrefixV1,
		M3AggServicePlacementPathName, setPathName)

	// M3CoordinatorSetURL is the url for the m3coordinator replace handler
	// (method POST).
	M3CoordinatorSetURL = path.Join(handler.RoutePrefixV1,
		M3CoordinatorServicePlacementPathName, setPathName)
)

// SetHandler is the type for placement replaces.
type SetHandler Handler

// NewSetHandler returns a new SetHandler.
func NewSetHandler(opts HandlerOptions) *SetHandler {
	return &SetHandler{HandlerOptions: opts, nowFn: time.Now}
}

func (h *SetHandler) ServeHTTP(
	svc handleroptions.ServiceNameAndDefaults,
	w http.ResponseWriter,
	r *http.Request,
) {
	ctx := r.Context()
	logger := logging.WithContext(ctx, h.instrumentOptions)

	req, pErr := h.parseRequest(r)
	if pErr != nil {
		xhttp.Error(w, pErr.Inner(), pErr.Code())
		return
	}

	serviceOpts := handleroptions.NewServiceOptions(svc,
		r.Header, h.m3AggServiceOptions)
	service, _, err := ServiceWithAlgo(h.clusterClient,
		serviceOpts, h.nowFn(), nil)
	if err != nil {
		logger.Error("unable to create placement service", zap.Error(err))
		xhttp.Error(w, err, http.StatusInternalServerError)
		return
	}

	curPlacement, err := service.Placement()
	if err == kv.ErrNotFound {
		logger.Error("placement not found", zap.Any("req", req), zap.Error(err))
		xhttp.Error(w, err, http.StatusNotFound)
		return
	}
	if err != nil {
		logger.Error("unable to get current placement", zap.Error(err))
		xhttp.Error(w, err, http.StatusInternalServerError)
		return
	}

	var (
		placementProto   = req.Placement
		placementVersion int
	)
	newPlacement, err := placement.NewPlacementFromProto(req.Placement)
	if err != nil {
		logger.Error("unable to create new placement from proto", zap.Error(err))
		xhttp.Error(w, err, http.StatusBadRequest)
		return
	}

	dryRun := !req.Confirm
	if dryRun {
		logger.Info("performing dry run for set placement, not confirmed")
		placementVersion = curPlacement.Version() + 1
	} else {
		logger.Info("performing live run for set placement, confirmed")
		// Ensure the placement we're updating is still the one on which we validated
		// all shards are available.
		updatedPlacement, err := service.CheckAndSet(newPlacement,
			curPlacement.Version())
		if err != nil {
			logger.Error("unable to update placement", zap.Error(err))
			xhttp.Error(w, err, http.StatusInternalServerError)
			return
		}

		placementVersion = updatedPlacement.Version()
	}

	resp := &admin.PlacementSetResponse{
		Placement: placementProto,
		Version:   int32(placementVersion),
		DryRun:    dryRun,
	}

	xhttp.WriteProtoMsgJSONResponse(w, resp, logger)
}

func (h *SetHandler) parseRequest(r *http.Request) (*admin.PlacementSetRequest, *xhttp.ParseError) {
	defer r.Body.Close()

	req := &admin.PlacementSetRequest{}
	if err := jsonpb.Unmarshal(r.Body, req); err != nil {
		return nil, xhttp.NewParseError(err, http.StatusBadRequest)
	}

	return req, nil
}
<commit_msg>Add log message on placement set parse error (#2457)<commit_after>// Copyright (c) 2018 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package placement

import (
	"net/http"
	"path"
	"time"

	"github.com/m3db/m3/src/cluster/kv"
	"github.com/m3db/m3/src/cluster/placement"
	"github.com/m3db/m3/src/query/api/v1/handler"
	"github.com/m3db/m3/src/query/api/v1/handler/prometheus/handleroptions"
	"github.com/m3db/m3/src/query/generated/proto/admin"
	"github.com/m3db/m3/src/query/util/logging"
	xhttp "github.com/m3db/m3/src/x/net/http"

	"github.com/gogo/protobuf/jsonpb"
	"go.uber.org/zap"
)

const (
	// SetHTTPMethod is the HTTP method for the the upsert endpoint.
	SetHTTPMethod = http.MethodPost

	setPathName = "set"
)

var (
	// M3DBSetURL is the url for the m3db replace handler (method POST).
	M3DBSetURL = path.Join(handler.RoutePrefixV1,
		M3DBServicePlacementPathName, setPathName)

	// M3AggSetURL is the url for the m3aggregator replace handler (method
	// POST).
	M3AggSetURL = path.Join(handler.RoutePrefixV1,
		M3AggServicePlacementPathName, setPathName)

	// M3CoordinatorSetURL is the url for the m3coordinator replace handler
	// (method POST).
	M3CoordinatorSetURL = path.Join(handler.RoutePrefixV1,
		M3CoordinatorServicePlacementPathName, setPathName)
)

// SetHandler is the type for placement replaces.
type SetHandler Handler

// NewSetHandler returns a new SetHandler.
func NewSetHandler(opts HandlerOptions) *SetHandler {
	return &SetHandler{HandlerOptions: opts, nowFn: time.Now}
}

func (h *SetHandler) ServeHTTP(
	svc handleroptions.ServiceNameAndDefaults,
	w http.ResponseWriter,
	r *http.Request,
) {
	ctx := r.Context()
	logger := logging.WithContext(ctx, h.instrumentOptions)

	req, pErr := h.parseRequest(r)
	if pErr != nil {
		logger.Error("unable to parse request", zap.Error(pErr))
		xhttp.Error(w, pErr, http.StatusBadRequest)
		return
	}

	serviceOpts := handleroptions.NewServiceOptions(svc,
		r.Header, h.m3AggServiceOptions)
	service, _, err := ServiceWithAlgo(h.clusterClient,
		serviceOpts, h.nowFn(), nil)
	if err != nil {
		logger.Error("unable to create placement service", zap.Error(err))
		xhttp.Error(w, err, http.StatusInternalServerError)
		return
	}

	curPlacement, err := service.Placement()
	if err == kv.ErrNotFound {
		logger.Error("placement not found", zap.Any("req", req), zap.Error(err))
		xhttp.Error(w, err, http.StatusNotFound)
		return
	}
	if err != nil {
		logger.Error("unable to get current placement", zap.Error(err))
		xhttp.Error(w, err, http.StatusInternalServerError)
		return
	}

	var (
		placementProto   = req.Placement
		placementVersion int
	)
	newPlacement, err := placement.NewPlacementFromProto(req.Placement)
	if err != nil {
		logger.Error("unable to create new placement from proto", zap.Error(err))
		xhttp.Error(w, err, http.StatusBadRequest)
		return
	}

	dryRun := !req.Confirm
	if dryRun {
		logger.Info("performing dry run for set placement, not confirmed")
		placementVersion = curPlacement.Version() + 1
	} else {
		logger.Info("performing live run for set placement, confirmed")
		// Ensure the placement we're updating is still the one on which we validated
		// all shards are available.
		updatedPlacement, err := service.CheckAndSet(newPlacement,
			curPlacement.Version())
		if err != nil {
			logger.Error("unable to update placement", zap.Error(err))
			xhttp.Error(w, err, http.StatusInternalServerError)
			return
		}

		placementVersion = updatedPlacement.Version()
	}

	resp := &admin.PlacementSetResponse{
		Placement: placementProto,
		Version:   int32(placementVersion),
		DryRun:    dryRun,
	}

	xhttp.WriteProtoMsgJSONResponse(w, resp, logger)
}

func (h *SetHandler) parseRequest(r *http.Request) (*admin.PlacementSetRequest, error) {
	defer r.Body.Close()

	req := &admin.PlacementSetRequest{}
	if err := jsonpb.Unmarshal(r.Body, req); err != nil {
		return nil, err
	}

	return req, nil
}
<|endoftext|>


<commit_before>// package name: reef
package main

import (
	"fmt"

	"github.com/exis-io/core"
	"github.com/gopherjs/gopherjs/js"
)

// A good resource on working with gopherjs
// http://legacytotheedge.blogspot.de/2014/03/gopherjs-go-to-javascript-transpiler.html

/*
TODO
    Remove fmt from core to reduce size of library
    Inject writer for logging and saving
*/

var fabric string = core.FabricLocal

func main() {
	// Functions are autoexported on non-pointer types-- dont need "Subdomain" listed here
	js.Global.Set("MantleDomain", map[string]interface{}{
		"New": New,
	})

	js.Global.Set("Config", map[string]interface{}{
		"SetLogLevelOff":      SetLogLevelOff,
		"SetLogLevelApp":      SetLogLevelApp,
		"SetLogLevelErr":      SetLogLevelErr,
		"SetLogLevelWarn":     SetLogLevelWarn,
		"SetLogLevelInfo":     SetLogLevelInfo,
		"SetLogLevelDebug":    SetLogLevelDebug,
		"SetFabricDev":        SetFabricDev,
		"SetFabricSandbox":    SetFabricSandbox,
		"SetFabricProduction": SetFabricProduction,
		"SetFabricLocal":      SetFabricLocal,
		"SetFabric":           SetFabric,
		"Application":         Application,
		"Debug":               Debug,
		"Info":                Info,
		"Warn":                Warn,
		"Error":               Error,
	})

}

type Domain struct {
	coreDomain core.Domain
}

type Conn struct {
	wrapper *js.Object
	app     core.App
	domain  *Domain
}

func (c Conn) OnMessage(msg *js.Object) {
	c.app.ReceiveString(msg.String())
}

func (c Conn) OnOpen(msg *js.Object) {
	go c.domain.FinishJoin(&c)
}

func (c Conn) OnClose(msg *js.Object) {
	c.app.Close(msg.String())
}

func (c Conn) Send(data []byte) {
	c.wrapper.Get("conn").Call("send", string(data))
}

func (c Conn) Close(reason string) error {
	fmt.Println("Asked to close: ", reason)
	c.wrapper.Get("conn").Call("close", 1001, reason)
	return nil
}

func (c Conn) SetApp(app core.App) {
	c.app = app
}

func New(name string) *js.Object {
	d := Domain{
		coreDomain: core.NewDomain(name, nil),
	}

	return js.MakeWrapper(&d)
}

func (d *Domain) Subdomain(name string) *js.Object {
	n := Domain{
		coreDomain: d.coreDomain.Subdomain(name),
	}

	return js.MakeWrapper(&n)
}

// var conn Conn

// Blocks on callbacks from the core.
// TODO: trigger a close meta callback when connection is lost
func (d *Domain) Receive() string {
	return core.MantleMarshall(d.coreDomain.GetApp().CallbackListen())
}

func (d *Domain) Join() {
	w := js.Global.Get("WsWrapper")

	conn := Conn{
		wrapper: w,
		domain:  d,
		app:     d.coreDomain.GetApp(),
	}

	w.Set("onmessage", conn.OnMessage)
	w.Set("onopen", conn.OnOpen)
	w.Set("onclose", conn.OnClose)

	w.Call("open", fabric)
}

// The actual join method
func (d *Domain) FinishJoin(c *Conn) {
	if err := d.coreDomain.Join(c); err != nil {
		// d.coreDomain.GetApp().CallbackSend(eb, err.Error())
		fmt.Println("Cant join: ", err)
	} else {
		// d.coreDomain.GetApp().CallbackSend(cb)
		fmt.Println("Joined!")
	}

}

func (d *Domain) Subscribe(cb uint, endpoint string) {
	go func() {
		d.coreDomain.Subscribe(endpoint, cb, make([]interface{}, 0))
	}()
}

func (d *Domain) Register(cb uint, endpoint string) {
	go func() {
		d.coreDomain.Register(endpoint, cb, make([]interface{}, 0))
	}()
}

// Args are string encoded json
func (d *Domain) Publish(cb uint, endpoint string, args string) {
	go func() {
		d.coreDomain.Publish(endpoint, cb, core.MantleUnmarshal(args))
	}()
}

func (d *Domain) Call(cb uint, endpoint string, args string) {
	go func() {
		d.coreDomain.Call(endpoint, cb, core.MantleUnmarshal(args))
	}()
}

func (d *Domain) Yield(request uint, args string) {
	go func() {
		d.coreDomain.GetApp().Yield(request, core.MantleUnmarshal(args))
	}()
}

func (d *Domain) Unsubscribe(endpoint string) {
	go func() {
		d.coreDomain.Unsubscribe(endpoint)
	}()
}

func (d *Domain) Unregister(endpoint string) {
	go func() {
		d.coreDomain.Unregister(endpoint)
	}()
}

func (d *Domain) Leave() {
	go func() {
		d.coreDomain.Leave()
	}()
}

func SetLogLevelOff()   { core.LogLevel = core.LogLevelOff }
func SetLogLevelApp()   { core.LogLevel = core.LogLevelApp }
func SetLogLevelErr()   { core.LogLevel = core.LogLevelErr }
func SetLogLevelWarn()  { core.LogLevel = core.LogLevelWarn }
func SetLogLevelInfo()  { core.LogLevel = core.LogLevelInfo }
func SetLogLevelDebug() { core.LogLevel = core.LogLevelDebug }

func SetFabricDev()        { fabric = core.FabricDev }
func SetFabricSandbox()    { fabric = core.FabricSandbox }
func SetFabricProduction() { fabric = core.FabricProduction }
func SetFabricLocal()      { fabric = core.FabricLocal }
func SetFabric(url string) { fabric = url }

func Application(s string) { core.Application("%s", s) }
func Debug(s string)       { core.Debug("%s", s) }
func Info(s string)        { core.Info("%s", s) }
func Warn(s string)        { core.Warn("%s", s) }
func Error(s string)       { core.Error("%s", s) }

// Do we want a GetName? Most likely yes

/*
type wrapper struct {
	app    core.App
	conn   *js.Object
	opened chan bool
}

type domain struct {
	wrapper  *wrapper
	mirror   core.Domain
	handlers map[uint]*js.Object
	kill     chan bool
}

type Domain interface {
	Subscribe(string, interface{}) error
	Register(string, interface{}) error

	Publish(string, ...interface{}) error
	Call(string, ...interface{}) ([]interface{}, error)

	Unsubscribe(string) error
	Unregister(string) error

	Join() error
	Leave() error
}

var wrap *wrapper

// Required main method
func main() {
	js.Global.Set("Core", map[string]interface{}{
		"SetLoggingDebug": core.SetLoggingDebug,
		"SetLoggingInfo":  core.SetLoggingInfo,
		"SetLoggingWarn":  core.SetLoggingWarn,
	})

	// Change Wrapper to Pool
	js.Global.Set("Wrapper", map[string]interface{}{
		"New":              NewWrapper,
		"SetConnection":    SetConnection,
		"ConnectionOpened": ConnectionOpened,
		"NewMessage":       NewMessage,
	})

	js.Global.Set("Domain", map[string]interface{}{
		"New": NewDomain,
	})

	// core.SetLogWriter()
}

/////////////////////////////////////////////
// Connection Wrapper
/////////////////////////////////////////////

func NewWrapper() {
	if wrap == nil {
		h := core.NewApp()

		wrap = &wrapper{
			app:    h,
			opened: make(chan bool),
		}
	}
}

func (w *wrapper) Send(data []byte) {
	w.conn.Call("send", string(data))
}

func (w *wrapper) Close(reason string) error {
	w.conn.Call("close", 1000, reason)
	return nil
}

// Call SetConnection, then Join
func SetConnection(c *js.Object) {
	fmt.Println("Connection set: ", c)
	wrap.conn = c
	// c.Set("onmessage", wrap.app.ReceiveString)
}

func NewMessage(c *js.Object) {
	fmt.Println("Message Receive: ", c.String())
	wrap.app.ReceiveString(c.String())
}

func ConnectionOpened() {
	wrap.opened <- true
}

/////////////////////////////////////////////
// Domain Functions
/////////////////////////////////////////////

func NewDomain(name string) *js.Object {
	fmt.Println("Created a new domain")

	if wrap == nil {
		fmt.Println("WARN: wrapper hasn't been created yet!")
	}

	d := domain{
		wrapper:  wrap,
		handlers: make(map[uint]*js.Object),
		kill:     make(chan bool),
	}

	d.mirror = wrap.app.NewDomain(name, d)
	return js.MakeWrapper(&d)
}

func (d *domain) Subscribe(endpoint string, handler *js.Object) error {
	// Cute, but not the best idea long term. Deferreds are going to be easiest (?)
	go func() {
		if i, err := d.mirror.Subscribe(endpoint, []interface{}{}); err != nil {
			// return err
			fmt.Println("Unable to subscribe: ", err.Error())
		} else {
			// fmt.Println("Subscribedd with id, handler: ", i, handler)
			d.handlers[i] = handler
			// return nil
		}
	}()

	return nil
}

func (d *domain) Register(endpoint string, handler *js.Object) error {
	if i, err := d.mirror.Register(endpoint, []interface{}{}); err != nil {
		return err
	} else {
		d.handlers[i] = handler
		return nil
	}
}

func (d *domain) Publish(endpoint string, args ...interface{}) error {
	err := d.mirror.Publish(endpoint, args)
	return err
}

func (d *domain) Call(endpoint string, args ...interface{}) ([]interface{}, error) {
	args, err := d.mirror.Call(endpoint, args)
	return args, err
}

func (d *domain) Unsubscribe(endpoint string) error {
	err := d.mirror.Unsubscribe(endpoint)
	return err
}

func (d *domain) Unregister(endpoint string) error {
	err := d.mirror.Unregister(endpoint)
	return err
}

func (d *domain) Join() error {
	// If this domain doesnt have a pool, create one now and obtain a connection
	// If we can't call out because of the platform, the wrapper must push us a connection when a domain calls join

	go func() {
		// wait for onopen from the connection
		<-d.wrapper.opened
		d.mirror.Join(d.wrapper)
	}()

	return nil
}

func (d *domain) Leave() error {
	err := d.mirror.Leave()

	// for each subscription
	// for each registration

	return err
}

func (d domain) Invoke(endpoint string, id uint, args []interface{}) ([]interface{}, error) {
	// return core.Cumin(d.handlers[id], args)
	d.handlers[id].Invoke(args)
	return nil, nil
}

func (d domain) OnJoin(string) {
	fmt.Println("Delegate joined!")
}

func (d domain) OnLeave(string) {
	fmt.Println("Delegate left!!")
	d.kill <- true
}
*/
<commit_msg>cleaning and refactoring<commit_after>// package name: reef
package main

import (
	"fmt"

	_ "github.com/augustoroman/promise"
	"github.com/exis-io/core"
	"github.com/gopherjs/gopherjs/js"
)

// A good resource on working with gopherjs
// http://legacytotheedge.blogspot.de/2014/03/gopherjs-go-to-javascript-transpiler.html

/*
TODO
    Remove fmt from core to reduce size of library
    Inject writer for logging and saving
*/

var fabric string = core.FabricLocal

func main() {
	// Functions are autoexported on non-pointer types-- dont need "Subdomain" listed here
	js.Global.Set("Domain", map[string]interface{}{
		"New": New,
	})

	js.Global.Set("Config", map[string]interface{}{
		"SetLogLevelOff":      SetLogLevelOff,
		"SetLogLevelApp":      SetLogLevelApp,
		"SetLogLevelErr":      SetLogLevelErr,
		"SetLogLevelWarn":     SetLogLevelWarn,
		"SetLogLevelInfo":     SetLogLevelInfo,
		"SetLogLevelDebug":    SetLogLevelDebug,
		"SetFabricDev":        SetFabricDev,
		"SetFabricSandbox":    SetFabricSandbox,
		"SetFabricProduction": SetFabricProduction,
		"SetFabricLocal":      SetFabricLocal,
		"SetFabric":           SetFabric,
		"Application":         Application,
		"Debug":               Debug,
		"Info":                Info,
		"Warn":                Warn,
		"Error":               Error,
	})

}

type Domain struct {
	coreDomain core.Domain
}

type Conn struct {
	wrapper *js.Object
	app     core.App
	domain  *Domain
}

func (c Conn) OnMessage(msg *js.Object) {
	c.app.ReceiveString(msg.String())
}

func (c Conn) OnOpen(msg *js.Object) {
	go c.domain.FinishJoin(&c)
}

func (c Conn) OnClose(msg *js.Object) {
	c.app.Close(msg.String())
}

func (c Conn) Send(data []byte) {
	c.wrapper.Get("conn").Call("send", string(data))
}

func (c Conn) Close(reason string) error {
	fmt.Println("Asked to close: ", reason)
	c.wrapper.Get("conn").Call("close", 1001, reason)
	return nil
}

func (c Conn) SetApp(app core.App) {
	c.app = app
}

func New(name string) *js.Object {
	d := Domain{
		coreDomain: core.NewDomain(name, nil),
	}

	return js.MakeWrapper(&d)
}

func (d *Domain) Subdomain(name string) *js.Object {
	n := Domain{
		coreDomain: d.coreDomain.Subdomain(name),
	}

	return js.MakeWrapper(&n)
}

// var conn Conn

// Blocks on callbacks from the core.
// TODO: trigger a close meta callback when connection is lost
func (d *Domain) Receive() string {
	return core.MantleMarshall(d.coreDomain.GetApp().CallbackListen())
}

func (d *Domain) Join() {
	w := js.Global.Get("WsWrapper")

	conn := Conn{
		wrapper: w,
		domain:  d,
		app:     d.coreDomain.GetApp(),
	}

	w.Set("onmessage", conn.OnMessage)
	w.Set("onopen", conn.OnOpen)
	w.Set("onclose", conn.OnClose)

	w.Call("open", fabric)
}

// The actual join method
func (d *Domain) FinishJoin(c *Conn) {
	if err := d.coreDomain.Join(c); err != nil {
		// d.coreDomain.GetApp().CallbackSend(eb, err.Error())
		fmt.Println("Cant join: ", err)
	} else {
		// d.coreDomain.GetApp().CallbackSend(cb)
		fmt.Println("Joined!")
	}

}

func (d *Domain) Subscribe(cb uint, endpoint string) {
	go func() {
		d.coreDomain.Subscribe(endpoint, cb, make([]interface{}, 0))
	}()
}

func (d *Domain) Register(cb uint, endpoint string) {
	go func() {
		d.coreDomain.Register(endpoint, cb, make([]interface{}, 0))
	}()
}

// Args are string encoded json
func (d *Domain) Publish(cb uint, endpoint string, args string) {
	go func() {
		d.coreDomain.Publish(endpoint, cb, core.MantleUnmarshal(args))
	}()
}

func (d *Domain) Call(cb uint, endpoint string, args string) {
	go func() {
		d.coreDomain.Call(endpoint, cb, core.MantleUnmarshal(args))
	}()
}

func (d *Domain) Yield(request uint, args string) {
	go func() {
		d.coreDomain.GetApp().Yield(request, core.MantleUnmarshal(args))
	}()
}

func (d *Domain) Unsubscribe(endpoint string) {
	go func() {
		d.coreDomain.Unsubscribe(endpoint)
	}()
}

func (d *Domain) Unregister(endpoint string) {
	go func() {
		d.coreDomain.Unregister(endpoint)
	}()
}

func (d *Domain) Leave() {
	go func() {
		d.coreDomain.Leave()
	}()
}

func SetLogLevelOff()   { core.LogLevel = core.LogLevelOff }
func SetLogLevelApp()   { core.LogLevel = core.LogLevelApp }
func SetLogLevelErr()   { core.LogLevel = core.LogLevelErr }
func SetLogLevelWarn()  { core.LogLevel = core.LogLevelWarn }
func SetLogLevelInfo()  { core.LogLevel = core.LogLevelInfo }
func SetLogLevelDebug() { core.LogLevel = core.LogLevelDebug }

func SetFabricDev()        { fabric = core.FabricDev }
func SetFabricSandbox()    { fabric = core.FabricSandbox }
func SetFabricProduction() { fabric = core.FabricProduction }
func SetFabricLocal()      { fabric = core.FabricLocal }
func SetFabric(url string) { fabric = url }

func Application(s string) { core.Application("%s", s) }
func Debug(s string)       { core.Debug("%s", s) }
func Info(s string)        { core.Info("%s", s) }
func Warn(s string)        { core.Warn("%s", s) }
func Error(s string)       { core.Error("%s", s) }

// Do we want a GetName? Most likely yes

/*
type wrapper struct {
	app    core.App
	conn   *js.Object
	opened chan bool
}

type domain struct {
	wrapper  *wrapper
	mirror   core.Domain
	handlers map[uint]*js.Object
	kill     chan bool
}

type Domain interface {
	Subscribe(string, interface{}) error
	Register(string, interface{}) error

	Publish(string, ...interface{}) error
	Call(string, ...interface{}) ([]interface{}, error)

	Unsubscribe(string) error
	Unregister(string) error

	Join() error
	Leave() error
}

var wrap *wrapper

// Required main method
func main() {
	js.Global.Set("Core", map[string]interface{}{
		"SetLoggingDebug": core.SetLoggingDebug,
		"SetLoggingInfo":  core.SetLoggingInfo,
		"SetLoggingWarn":  core.SetLoggingWarn,
	})

	// Change Wrapper to Pool
	js.Global.Set("Wrapper", map[string]interface{}{
		"New":              NewWrapper,
		"SetConnection":    SetConnection,
		"ConnectionOpened": ConnectionOpened,
		"NewMessage":       NewMessage,
	})

	js.Global.Set("Domain", map[string]interface{}{
		"New": NewDomain,
	})

	// core.SetLogWriter()
}

/////////////////////////////////////////////
// Connection Wrapper
/////////////////////////////////////////////

func NewWrapper() {
	if wrap == nil {
		h := core.NewApp()

		wrap = &wrapper{
			app:    h,
			opened: make(chan bool),
		}
	}
}

func (w *wrapper) Send(data []byte) {
	w.conn.Call("send", string(data))
}

func (w *wrapper) Close(reason string) error {
	w.conn.Call("close", 1000, reason)
	return nil
}

// Call SetConnection, then Join
func SetConnection(c *js.Object) {
	fmt.Println("Connection set: ", c)
	wrap.conn = c
	// c.Set("onmessage", wrap.app.ReceiveString)
}

func NewMessage(c *js.Object) {
	fmt.Println("Message Receive: ", c.String())
	wrap.app.ReceiveString(c.String())
}

func ConnectionOpened() {
	wrap.opened <- true
}

/////////////////////////////////////////////
// Domain Functions
/////////////////////////////////////////////

func NewDomain(name string) *js.Object {
	fmt.Println("Created a new domain")

	if wrap == nil {
		fmt.Println("WARN: wrapper hasn't been created yet!")
	}

	d := domain{
		wrapper:  wrap,
		handlers: make(map[uint]*js.Object),
		kill:     make(chan bool),
	}

	d.mirror = wrap.app.NewDomain(name, d)
	return js.MakeWrapper(&d)
}

func (d *domain) Subscribe(endpoint string, handler *js.Object) error {
	// Cute, but not the best idea long term. Deferreds are going to be easiest (?)
	go func() {
		if i, err := d.mirror.Subscribe(endpoint, []interface{}{}); err != nil {
			// return err
			fmt.Println("Unable to subscribe: ", err.Error())
		} else {
			// fmt.Println("Subscribedd with id, handler: ", i, handler)
			d.handlers[i] = handler
			// return nil
		}
	}()

	return nil
}

func (d *domain) Register(endpoint string, handler *js.Object) error {
	if i, err := d.mirror.Register(endpoint, []interface{}{}); err != nil {
		return err
	} else {
		d.handlers[i] = handler
		return nil
	}
}

func (d *domain) Publish(endpoint string, args ...interface{}) error {
	err := d.mirror.Publish(endpoint, args)
	return err
}

func (d *domain) Call(endpoint string, args ...interface{}) ([]interface{}, error) {
	args, err := d.mirror.Call(endpoint, args)
	return args, err
}

func (d *domain) Unsubscribe(endpoint string) error {
	err := d.mirror.Unsubscribe(endpoint)
	return err
}

func (d *domain) Unregister(endpoint string) error {
	err := d.mirror.Unregister(endpoint)
	return err
}

func (d *domain) Join() error {
	// If this domain doesnt have a pool, create one now and obtain a connection
	// If we can't call out because of the platform, the wrapper must push us a connection when a domain calls join

	go func() {
		// wait for onopen from the connection
		<-d.wrapper.opened
		d.mirror.Join(d.wrapper)
	}()

	return nil
}

func (d *domain) Leave() error {
	err := d.mirror.Leave()

	// for each subscription
	// for each registration

	return err
}

func (d domain) Invoke(endpoint string, id uint, args []interface{}) ([]interface{}, error) {
	// return core.Cumin(d.handlers[id], args)
	d.handlers[id].Invoke(args)
	return nil, nil
}

func (d domain) OnJoin(string) {
	fmt.Println("Delegate joined!")
}

func (d domain) OnLeave(string) {
	fmt.Println("Delegate left!!")
	d.kill <- true
}
*/
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"fmt"
	pop3 "github.com/d3xter/GoPOP3"
	"github.com/parkghost/spamdefender/common"
	"io/ioutil"
	"log"
	"net/mail"
	"os"
	"strconv"
	"strings"
)

var (
	username      = "your email account"
	password      = "your password "
	serverAddress = "pop3.live.com:995"
	secure        = true
	destination   = "mailbox"
	mailFrom      = "webmaster@mail.javaworld.com.tw"
	start         = 0
)

// TODO: retry download when failure
// TODO: multi-client for speed up

func main() {

	log.Printf("Connect to %s\n", serverAddress)
	var client *pop3.Client
	var dialErr error
	if secure {
		client, dialErr = DialTLS(serverAddress)
	} else {
		client, dialErr = pop3.Dial(serverAddress)
	}
	checkErr(dialErr)

	log.Println("Authenticating")
	plainAuth := pop3.CreatePlainAuthentication(username, password)
	authErr := client.Authenticate(plainAuth)
	checkErr(authErr)

	log.Println("Fetching mailbox stat")
	cmdErr := PrintMailBoxStat(client)
	checkErr(cmdErr)

	log.Println("Downloading mails")
	err := ScanMailbox(client)
	checkErr(err)

	log.Println("Finished to download all of mails")

}

func DialTLS(addr string) (client *pop3.Client, err error) {
	conn, err := tls.Dial("tcp", addr, nil)
	if err != nil {
		return nil, err
	}
	host := addr[:strings.Index(addr, ":")]

	return pop3.NewClient(conn, host)
}

func checkErr(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func PrintMailBoxStat(client *pop3.Client) error {
	mailCount, mailBoxSize, err := client.GetStatus()
	if err != nil {
		return err
	}

	fmt.Printf("Account:%s, MailCount:%d, MailBoxSize:%s\n", username, mailCount, common.HumanReadableSize(uint64(mailBoxSize)))
	return nil
}

func ScanMailbox(client *pop3.Client) error {
	rawMailList, respErr := client.GetRawMailList()
	checkErr(respErr)

	scanner := bufio.NewScanner(strings.NewReader(rawMailList))
	scanner.Split(ScanFirstWordAtLine)

	lastNo := 0
	for scanner.Scan() {
		if scanner.Err() == nil {
			index, err := strconv.Atoi(scanner.Text())
			if err != nil {
				log.Fatalf("Stop scan due to Err:%v, Token:%s", err, scanner.Text())
			}

			if index >= start {
				err = CheckAndDownloadMailContent(client, index)
				if err != nil {
					log.Fatalf("Download mail failure, Err:%v, No:%d, ", err, index)
				}
				lastNo = index
			}
		} else {
			log.Fatalf("Stop scan due to Err:%v, lastNo:%d", scanner.Err(), lastNo)
			break
		}
	}
	return nil
}

func ScanFirstWordAtLine(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.IndexByte(data, '\n'); i >= 0 {
		_, t, e := bufio.ScanWords(data, atEOF)
		return i + 1, t, e
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		_, t, e := bufio.ScanWords(data, atEOF)
		return len(data), t, e
	}
	// Request more data.
	return 0, nil, nil
}

func CheckAndDownloadMailContent(client *pop3.Client, index int) (err error) {
	filePath := destination + string(os.PathSeparator) + strconv.Itoa(index)
	if _, errFileStat := os.Stat(filePath); errFileStat != nil { // mail file not found

		log.Printf("Downloading mail, No:%d", index)
		// TODO: store binary file or handle non-utf8 encoding
		// THINK: check content integrity
		var mailContent string
		mailContent, err = client.GetRawMail(index)
		if err != nil {
			return
		}

		//find newline at head of file
		var lastNewline = 0
		for i, b := range mailContent {
			if b != '\n' {
				lastNewline = i
				break
			}
		}
		rawMailContent := []byte(mailContent)[lastNewline:]

		// it's safe to ignore parse error
		fromAddress, _ := getFromAddress(rawMailContent)

		if fromAddress == mailFrom {
			log.Printf("store mail content to file:%s", filePath)
			err = ioutil.WriteFile(filePath, rawMailContent, 0644)
			if err != nil {
				return
			}
		}

	}
	return
}

func getFromAddress(rawMailContent []byte) (string, error) {
	var message *mail.Message
	message, err := mail.ReadMessage(bytes.NewReader(rawMailContent))
	if err != nil {
		return "", err
	}

	address, err := mail.ParseAddress(message.Header.Get("From"))
	if err != nil {
		log.Printf("parse from address failure, Err:%v, Raw:%s", err, message.Header.Get("From"))
		return "", err
	}

	return address.Address, nil
}
<commit_msg>change POP3 client library<commit_after>package main

import (
	"bytes"
	"fmt"
	pop3 "github.com/bytbox/go-pop3"
	"github.com/parkghost/pkg/net/mail"
	"github.com/parkghost/spamdefender/common"
	"io/ioutil"
	"log"
	"os"
	"strconv"
)

var (
	username      = "your email account"
	password      = "your password "
	serverAddress = "pop3.live.com:995"
	secure        = true
	destination   = "mailbox"
	mailFrom      = "webmaster@mail.javaworld.com.tw"
	offset        = 0
)

// TODO: retry download on failure

func main() {

	log.Printf("Connect to %s\n", serverAddress)

	var client *pop3.Client
	var dialErr error
	if secure {
		client, dialErr = pop3.DialTLS(serverAddress)
	} else {
		client, dialErr = pop3.Dial(serverAddress)
	}
	checkErr(dialErr)

	authErr := client.Auth(username, password)
	checkErr(authErr)

	log.Println("Fetching mailbox stat")
	cmdErr := PrintMailBoxStat(client)
	checkErr(cmdErr)

	log.Println("Downloading mails")
	err := ScanMailbox(client)
	checkErr(err)

	log.Println("Finished to download all of mails")

}

func checkErr(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func PrintMailBoxStat(client *pop3.Client) error {
	mailCount, mailBoxSize, err := client.Stat()
	if err != nil {
		return err
	}

	fmt.Printf("Account:%s, MailCount:%d, MailBoxSize:%s\n", username, mailCount, common.HumanReadableSize(uint64(mailBoxSize)))
	return nil
}

func ScanMailbox(client *pop3.Client) error {
	msgs, _, err := client.ListAll()
	checkErr(err)

	for i := offset; i < len(msgs); i++ {
		err = CheckAndDownloadMailContent(client, msgs[i])
		if err != nil {
			log.Fatalf("Download mail failure, Err:%v, Mail:%d, ", err, msgs[i])
		}
	}

	return nil
}

func CheckAndDownloadMailContent(client *pop3.Client, index int) (err error) {
	filePath := destination + string(os.PathSeparator) + strconv.Itoa(index)
	if _, errFileStat := os.Stat(filePath); errFileStat != nil { // mail file not found

		log.Printf("Downloading Mail:%d", index)
		// TODO: handle non-utf8 encoding
		// THINK: check content integrity
		var mailContent string
		mailContent, err = client.Retr(index)
		if err != nil {
			return
		}

		//find newline at head of file
		var lastNewline = 0
		for i, b := range mailContent {
			if b != '\n' {
				lastNewline = i
				break
			}
		}
		rawMailContent := []byte(mailContent)[lastNewline:]

		// it's safe to ignore parse error
		fromAddress, _ := getFromAddress(rawMailContent)

		if fromAddress == mailFrom {
			log.Printf("Store mail content to file:%s", filePath)
			err = ioutil.WriteFile(filePath, rawMailContent, 0644)
			if err != nil {
				return
			}
		}

	}
	return
}

func getFromAddress(rawMailContent []byte) (string, error) {
	var message *mail.Message
	message, err := mail.ReadMessage(bytes.NewReader(rawMailContent))
	if err != nil {
		return "", err
	}

	address, err := mail.ParseAddress(message.Header.Get("From"))
	if err != nil {
		log.Printf("parse from address failure, Err:%v, Raw:%s", err, message.Header.Get("From"))
		return "", err
	}

	return address.Address, nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"github.com/gophergala2016/goad/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws"
	"github.com/gophergala2016/goad/queue"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

func main() {
	address := os.Args[1]
	concurrencycount, err := strconv.Atoi(os.Args[2])
	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	maxRequestCount, err := strconv.Atoi(os.Args[3])
	sqsurl := os.Args[4]
	awsregion := os.Args[5]
	clientTimeout, _ := time.ParseDuration("1s")
	if len(os.Args) > 6 {
		newClientTimeout, err := time.ParseDuration(os.Args[6])
		if err == nil {
			clientTimeout = newClientTimeout
		} else {
			fmt.Printf("Error parsing timeout: %s\n", err)
			return
		}
	}
	fmt.Printf("Using a timeout of %d nanoseconds\n", clientTimeout.Nanoseconds())
	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	client := &http.Client{}
	client.Timeout = clientTimeout
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		return errors.New("redirect")
	}
	fmt.Printf("Will spawn %d workers making %d requests to %s\n", concurrencycount, maxRequestCount, address)
	runLoadTest(client, sqsurl, address, maxRequestCount, concurrencycount, awsregion)
}

type Job struct{}

type RequestResult struct {
	Time             int64  `json:"time"`
	Host             string `json:"host"`
	Type             string `json:"type"`
	Status           int    `json:"status"`
	ElapsedFirstByte int64  `json:"elapsed-first-byte"`
	ElapsedLastByte  int64  `json:"elapsed-last-byte"`
	Elapsed          int64  `json:"elapsed"`
	Bytes            int    `json:"bytes"`
	Timeout          bool   `json:"timeout"`
	ConnectionError  bool   `json:"connection-error"`
	State            string `json:"state"`
}

func runLoadTest(client *http.Client, sqsurl string, url string, totalRequests int, concurrencycount int, awsregion string) {
	awsConfig := aws.NewConfig().WithRegion(awsregion)
	sqsAdaptor := queue.NewSQSAdaptor(awsConfig, sqsurl)
	//sqsAdaptor := queue.NewDummyAdaptor(sqsurl)
	jobs := make(chan Job, totalRequests)
	ch := make(chan RequestResult, totalRequests)
	var wg sync.WaitGroup
	loadTestStartTime := time.Now()
	var requestsSoFar int
	for i := 0; i < totalRequests; i++ {
		jobs <- Job{}
	}
	close(jobs)
	for i := 0; i < concurrencycount; i++ {
		wg.Add(1)
		go fetch(loadTestStartTime, client, url, totalRequests, jobs, ch, &wg, awsregion)
	}
	fmt.Println("Waiting for results…")

	ticker := time.NewTicker(5 * time.Second)
	quit := make(chan struct{})
	quitting := false

	for requestsSoFar < totalRequests && !quitting {
		i := 0

		var timeToFirstTotal int64
		var requestTimeTotal int64
		totBytesRead := 0
		statuses := make(map[string]int)
		var firstRequestTime int64
		var lastRequestTime int64
		var slowest int64
		var fastest int64
		var totalTimedOut int
		var totalConnectionError int

		resetStats := false

		for requestsSoFar < totalRequests && !quitting && !resetStats {
			select {
			case r := <-ch:
				i++
				requestsSoFar++
				if requestsSoFar%10 == 0 {
					fmt.Printf("\r%.2f%% done (%d requests out of %d)", (float64(requestsSoFar)/float64(totalRequests))*100.0, requestsSoFar, totalRequests)
				}
				if firstRequestTime == 0 {
					firstRequestTime = r.Time
				}

				lastRequestTime = r.Time

				if r.Timeout {
					totalTimedOut++
					continue
				}
				if r.ConnectionError {
					totalConnectionError++
					continue
				}

				if r.ElapsedLastByte > slowest {
					slowest = r.ElapsedLastByte
				}
				if fastest == 0 {
					fastest = r.ElapsedLastByte
				} else {
					if r.ElapsedLastByte < fastest {
						fastest = r.ElapsedLastByte
					}
				}

				timeToFirstTotal += r.ElapsedFirstByte
				totBytesRead += r.Bytes
				statusStr := strconv.Itoa(r.Status)
				_, ok := statuses[statusStr]
				if !ok {
					statuses[statusStr] = 1
				} else {
					statuses[statusStr]++
				}
				requestTimeTotal += r.Elapsed
				if requestsSoFar == totalRequests {
					quitting = true
					continue
				}
			case <-ticker.C:
				if i == 0 {
					continue
				}
				durationNanoSeconds := lastRequestTime - firstRequestTime
				durationSeconds := float32(durationNanoSeconds) / float32(1000000000)
				fatalError := ""
				if (totalTimedOut + totalConnectionError) > i/2 {
					fatalError = "Over 50% of requests failed, aborting"
					quitting = true
				}
				aggData := queue.AggData{
					i,
					totalTimedOut,
					totalConnectionError,
					timeToFirstTotal / int64(i),
					totBytesRead,
					statuses,
					requestTimeTotal / int64(i),
					float32(i) / durationSeconds,
					(float32(totBytesRead) / durationSeconds) / 1024.0,
					slowest,
					fastest,
					awsregion,
					fatalError,
				}
				sqsAdaptor.SendResult(aggData)
				resetStats = true
				continue
			case <-quit:
				ticker.Stop()
				quitting = true
			}
		}
	}
	fmt.Printf("\nYay🎈  - %d requests completed\n", requestsSoFar)

}

func fetch(loadTestStartTime time.Time, client *http.Client, address string, requestcount int, jobs <-chan Job, ch chan RequestResult, wg *sync.WaitGroup, awsregion string) {
	defer wg.Done()
	fmt.Printf("Fetching %s\n", address)
	for _ = range jobs {
		start := time.Now()
		req, err := http.NewRequest("GET", address, nil)
		req.Header.Add("User-Agent", "GOAD/0.1")
		response, err := client.Do(req)
		var status string
		var elapsedFirstByte time.Duration
		var elapsedLastByte time.Duration
		var elapsed time.Duration
		var statusCode int
		var bytesRead int
		buf := []byte(" ")
		timedOut := false
		connectionError := false
		if err != nil && !strings.Contains(err.Error(), "redirect") {
			status = fmt.Sprintf("ERROR: %s\n", err)
			switch err := err.(type) {
			case *url.Error:
				if err, ok := err.Err.(net.Error); ok && err.Timeout() {
					timedOut = true
				}
			case net.Error:
				if err.Timeout() {
					timedOut = true
				}
			}

			if !timedOut {
				connectionError = true
			}
		} else {
			statusCode = response.StatusCode
			_, err = response.Body.Read(buf)
			if err != nil {
				status = fmt.Sprintf("reading first byte failed: %s\n", err)
			}
			elapsedFirstByte = time.Since(start)
			body, err := ioutil.ReadAll(response.Body)
			bytesRead = len(body) + 1
			elapsedLastByte = time.Since(start)
			if err != nil {
				status = fmt.Sprintf("reading response body failed: %s\n", err)
			} else {
				status = "Success"
			}
			elapsed = time.Since(start)
		}
		result := RequestResult{
			start.Sub(loadTestStartTime).Nanoseconds(),
			req.URL.Host,
			req.Method,
			statusCode,
			elapsed.Nanoseconds(),
			elapsedFirstByte.Nanoseconds(),
			elapsedLastByte.Nanoseconds(),
			bytesRead,
			timedOut,
			connectionError,
			status,
		}
		ch <- result
	}
}
<commit_msg>Less verbose output to stdout<commit_after>package main

import (
	"errors"
	"fmt"
	"github.com/gophergala2016/goad/Godeps/_workspace/src/github.com/aws/aws-sdk-go/aws"
	"github.com/gophergala2016/goad/queue"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

func main() {
	address := os.Args[1]
	concurrencycount, err := strconv.Atoi(os.Args[2])
	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	maxRequestCount, err := strconv.Atoi(os.Args[3])
	sqsurl := os.Args[4]
	awsregion := os.Args[5]
	clientTimeout, _ := time.ParseDuration("1s")
	if len(os.Args) > 6 {
		newClientTimeout, err := time.ParseDuration(os.Args[6])
		if err == nil {
			clientTimeout = newClientTimeout
		} else {
			fmt.Printf("Error parsing timeout: %s\n", err)
			return
		}
	}
	fmt.Printf("Using a timeout of %.2f seconds\n", float32(clientTimeout.Nanoseconds())/float32(1000000000))
	if err != nil {
		fmt.Printf("ERROR %s\n", err)
		return
	}
	client := &http.Client{}
	client.Timeout = clientTimeout
	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
		return errors.New("redirect")
	}
	fmt.Printf("Will spawn %d workers making %d requests to %s\n", concurrencycount, maxRequestCount, address)
	runLoadTest(client, sqsurl, address, maxRequestCount, concurrencycount, awsregion)
}

type Job struct{}

type RequestResult struct {
	Time             int64  `json:"time"`
	Host             string `json:"host"`
	Type             string `json:"type"`
	Status           int    `json:"status"`
	ElapsedFirstByte int64  `json:"elapsed-first-byte"`
	ElapsedLastByte  int64  `json:"elapsed-last-byte"`
	Elapsed          int64  `json:"elapsed"`
	Bytes            int    `json:"bytes"`
	Timeout          bool   `json:"timeout"`
	ConnectionError  bool   `json:"connection-error"`
	State            string `json:"state"`
}

func runLoadTest(client *http.Client, sqsurl string, url string, totalRequests int, concurrencycount int, awsregion string) {
	awsConfig := aws.NewConfig().WithRegion(awsregion)
	sqsAdaptor := queue.NewSQSAdaptor(awsConfig, sqsurl)
	//sqsAdaptor := queue.NewDummyAdaptor(sqsurl)
	jobs := make(chan Job, totalRequests)
	ch := make(chan RequestResult, totalRequests)
	var wg sync.WaitGroup
	loadTestStartTime := time.Now()
	var requestsSoFar int
	for i := 0; i < totalRequests; i++ {
		jobs <- Job{}
	}
	close(jobs)
	fmt.Print("Spawning workers…")
	for i := 0; i < concurrencycount; i++ {
		wg.Add(1)
		go fetch(loadTestStartTime, client, url, totalRequests, jobs, ch, &wg, awsregion)
		fmt.Print(".")
	}
	fmt.Println(" done.\nWaiting for results…")

	ticker := time.NewTicker(5 * time.Second)
	quit := make(chan struct{})
	quitting := false

	for requestsSoFar < totalRequests && !quitting {
		i := 0

		var timeToFirstTotal int64
		var requestTimeTotal int64
		totBytesRead := 0
		statuses := make(map[string]int)
		var firstRequestTime int64
		var lastRequestTime int64
		var slowest int64
		var fastest int64
		var totalTimedOut int
		var totalConnectionError int

		resetStats := false

		for requestsSoFar < totalRequests && !quitting && !resetStats {
			select {
			case r := <-ch:
				i++
				requestsSoFar++
				if requestsSoFar%10 == 0 {
					fmt.Printf("\r%.2f%% done (%d requests out of %d)", (float64(requestsSoFar)/float64(totalRequests))*100.0, requestsSoFar, totalRequests)
				}
				if firstRequestTime == 0 {
					firstRequestTime = r.Time
				}

				lastRequestTime = r.Time

				if r.Timeout {
					totalTimedOut++
					continue
				}
				if r.ConnectionError {
					totalConnectionError++
					continue
				}

				if r.ElapsedLastByte > slowest {
					slowest = r.ElapsedLastByte
				}
				if fastest == 0 {
					fastest = r.ElapsedLastByte
				} else {
					if r.ElapsedLastByte < fastest {
						fastest = r.ElapsedLastByte
					}
				}

				timeToFirstTotal += r.ElapsedFirstByte
				totBytesRead += r.Bytes
				statusStr := strconv.Itoa(r.Status)
				_, ok := statuses[statusStr]
				if !ok {
					statuses[statusStr] = 1
				} else {
					statuses[statusStr]++
				}
				requestTimeTotal += r.Elapsed
				if requestsSoFar == totalRequests {
					quitting = true
					continue
				}
			case <-ticker.C:
				if i == 0 {
					continue
				}
				durationNanoSeconds := lastRequestTime - firstRequestTime
				durationSeconds := float32(durationNanoSeconds) / float32(1000000000)
				fatalError := ""
				if (totalTimedOut + totalConnectionError) > i/2 {
					fatalError = "Over 50% of requests failed, aborting"
					quitting = true
				}
				aggData := queue.AggData{
					i,
					totalTimedOut,
					totalConnectionError,
					timeToFirstTotal / int64(i),
					totBytesRead,
					statuses,
					requestTimeTotal / int64(i),
					float32(i) / durationSeconds,
					(float32(totBytesRead) / durationSeconds) / 1024.0,
					slowest,
					fastest,
					awsregion,
					fatalError,
				}
				sqsAdaptor.SendResult(aggData)
				resetStats = true
				continue
			case <-quit:
				ticker.Stop()
				quitting = true
			}
		}
	}
	fmt.Printf("\nYay🎈  - %d requests completed\n", requestsSoFar)

}

func fetch(loadTestStartTime time.Time, client *http.Client, address string, requestcount int, jobs <-chan Job, ch chan RequestResult, wg *sync.WaitGroup, awsregion string) {
	defer wg.Done()
	for _ = range jobs {
		start := time.Now()
		req, err := http.NewRequest("GET", address, nil)
		req.Header.Add("User-Agent", "GOAD/0.1")
		response, err := client.Do(req)
		var status string
		var elapsedFirstByte time.Duration
		var elapsedLastByte time.Duration
		var elapsed time.Duration
		var statusCode int
		var bytesRead int
		buf := []byte(" ")
		timedOut := false
		connectionError := false
		if err != nil && !strings.Contains(err.Error(), "redirect") {
			status = fmt.Sprintf("ERROR: %s\n", err)
			switch err := err.(type) {
			case *url.Error:
				if err, ok := err.Err.(net.Error); ok && err.Timeout() {
					timedOut = true
				}
			case net.Error:
				if err.Timeout() {
					timedOut = true
				}
			}

			if !timedOut {
				connectionError = true
			}
		} else {
			statusCode = response.StatusCode
			_, err = response.Body.Read(buf)
			if err != nil {
				status = fmt.Sprintf("reading first byte failed: %s\n", err)
			}
			elapsedFirstByte = time.Since(start)
			body, err := ioutil.ReadAll(response.Body)
			bytesRead = len(body) + 1
			elapsedLastByte = time.Since(start)
			if err != nil {
				status = fmt.Sprintf("reading response body failed: %s\n", err)
			} else {
				status = "Success"
			}
			elapsed = time.Since(start)
		}
		result := RequestResult{
			start.Sub(loadTestStartTime).Nanoseconds(),
			req.URL.Host,
			req.Method,
			statusCode,
			elapsed.Nanoseconds(),
			elapsedFirstByte.Nanoseconds(),
			elapsedLastByte.Nanoseconds(),
			bytesRead,
			timedOut,
			connectionError,
			status,
		}
		ch <- result
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Ls reads the directories in the command line and prints out the names.

The options are:
	-l		Long form.
	-r		raw (%v) form
	-R		recurse
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
)

var (
	long      = flag.Bool("l", false, "Long form")
	raw       = flag.Bool("r", false, "raw struct")
	recursive = flag.Bool("R", false, "Recurse")
)

func show(fullpath string, fi os.FileInfo) {
	switch {
	case *raw == true:
		fmt.Printf("%v\n", fi)
	case *long == false:
		fmt.Printf("%v\n", fi.Name())
	// -rw-r--r-- 1 root root 174 Aug 18 17:18 /etc/hosts
	case *long == true:
		fmt.Printf("%v\t%v\t%v\t%v", fi.Mode(), fi.Size(), fi.Name(), fi.ModTime())
		if link, err := os.Readlink(fullpath); err == nil {
			fmt.Printf(" -> %v", link)
		}
		fmt.Printf("\n")
	}

}

func main() {
	flag.Parse()

	dirs := flag.Args()

	if len(dirs) == 0 {
		dirs = []string{"."}
	}
	for _, v := range dirs {
		if len(dirs) > 1 {
			fmt.Printf("%v:\n", v)
		}
		err := filepath.Walk(v, func(path string, fi os.FileInfo, err error) error {
			show(path, fi)
			if fi.IsDir() && !*recursive && path != v {
				return filepath.SkipDir
			}

			return err
		})
		if err != nil {
			fmt.Printf("%s: %v\n", v, err)
		}
	}
}
<commit_msg>Have ls print out errors correctly.<commit_after>// Copyright 2013 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Ls reads the directories in the command line and prints out the names.

The options are:
	-l		Long form.
	-r		raw (%v) form
	-R		recurse
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
)

var (
	long      = flag.Bool("l", false, "Long form")
	raw       = flag.Bool("r", false, "raw struct")
	recursive = flag.Bool("R", false, "Recurse")
)

func show(fullpath string, fi os.FileInfo) {
	switch {
	case *raw == true:
		fmt.Printf("%v\n", fi)
	case *long == false:
		fmt.Printf("%v\n", fi.Name())
	// -rw-r--r-- 1 root root 174 Aug 18 17:18 /etc/hosts
	case *long == true:
		fmt.Printf("%v\t%v\t%v\t%v", fi.Mode(), fi.Size(), fi.Name(), fi.ModTime())
		if link, err := os.Readlink(fullpath); err == nil {
			fmt.Printf(" -> %v", link)
		}
		fmt.Printf("\n")
	}

}

func main() {
	flag.Parse()

	dirs := flag.Args()

	if len(dirs) == 0 {
		dirs = []string{"."}
	}
	for _, v := range dirs {
		if len(dirs) > 1 {
			fmt.Printf("%v:\n", v)
		}
		err := filepath.Walk(v, func(path string, fi os.FileInfo, err error) error {
			if err != nil {
				fmt.Printf("%v: %v\n", path, err)
				return err
			}
			show(path, fi)
			if fi.IsDir() && !*recursive && path != v {
				return filepath.SkipDir
			}

			return err
		})
		if err != nil {
			fmt.Printf("%s: %v\n", v, err)
		}
	}
}
<|endoftext|>