<commit_before>package gandalf

import (
	"bytes"
	"encoding/json"
	. "launchpad.net/gocheck"
	"net/http/httptest"
)

// should close the created servers
func (s *S) TestDoRequest(c *C) {
	h := TestHandler{content: `some return message`}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	body := bytes.NewBufferString(`{"foo":"bar"}`)
	response, err := client.doRequest("POST", "/test", body)
	c.Assert(err, IsNil)
	c.Assert(response.StatusCode, Equals, 200)
	c.Assert(string(h.body), Equals, `{"foo":"bar"}`)
	c.Assert(h.url, Equals, "/test")
}

func (s *S) TestDoRequestShouldNotSetContentTypeToJsonWhenBodyIsNil(c *C) {
	h := TestHandler{content: `some return message`}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	response, err := client.doRequest("DELETE", "/test", nil)
	c.Assert(err, IsNil)
	c.Assert(response.StatusCode, Equals, 200)
	c.Assert(h.header.Get("Content-Type"), Not(Equals), "application/json")
}

func (s *S) TestPost(c *C) {
	h := TestHandler{content: `some return message`}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	r := repository{Name: "test", Users: []string{"samwan"}}
	err := client.post(r, "/repository")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/repository")
	c.Assert(h.method, Equals, "POST")
	c.Assert(string(h.body), Equals, `{"name":"test","users":["samwan"],"ispublic":false}`)
}

func (s *S) TestPostWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	r := repository{Name: "test", Users: []string{"samwan"}}
	err := client.post(r, "/repository")
	c.Assert(err, ErrorMatches, "^Error performing requested operation\n$")
}

func (s *S) TestDelete(c *C) {
	h := TestHandler{content: `some return message`}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.delete(nil, "/user/someuser")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/user/someuser")
	c.Assert(h.method, Equals, "DELETE")
	c.Assert(string(h.body), Equals, "null")
}

func (s *S) TestDeleteWithConnectionError(c *C) {
	client := Client{Endpoint: "http://127.0.0.1:747399"}
	err := client.delete(nil, "/users/something")
	c.Assert(err, NotNil)
}

func (s *S) TestDeleteWithResponseError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.delete(nil, "/user/someuser")
	c.Assert(err, ErrorMatches, "^Error performing requested operation\n$")
	c.Assert(string(h.body), Equals, "null")
}

func (s *S) TestDeleteWithBody(c *C) {
	h := TestHandler{content: `some return message`}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.delete(map[string]string{"test": "foo"}, "/user/someuser")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/user/someuser")
	c.Assert(h.method, Equals, "DELETE")
	c.Assert(string(h.body), Equals, `{"test":"foo"}`)
}

func (s *S) TestGet(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.get("/user/someuser")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/user/someuser")
	c.Assert(h.method, Equals, "GET")
}

func (s *S) TestGetWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.get("/user/someuser")
	c.Assert(err, ErrorMatches, "^Error performing requested operation\n$")
}

func (s *S) TestFormatBody(c *C) {
	b, err := (&Client{}).formatBody(map[string]string{"test": "foo"})
	c.Assert(err, IsNil)
	c.Assert(b.String(), Equals, `{"test":"foo"}`)
}

func (s *S) TestFormatBodyReturnJsonNullWithNilBody(c *C) {
	b, err := (&Client{}).formatBody(nil)
	c.Assert(err, IsNil)
	c.Assert(b.String(), Equals, "null")
}

func (s *S) TestNewRepository(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	_, err := client.NewRepository("proj1", []string{"someuser"}, false)
	c.Assert(err, IsNil)
	c.Assert(string(h.body), Equals, `{"name":"proj1","users":["someuser"],"ispublic":false}`)
	c.Assert(h.url, Equals, "/repository")
	c.Assert(h.method, Equals, "POST")
}

func (s *S) TestNewRepositoryWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	_, err := client.NewRepository("proj1", []string{"someuser"}, false)
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestNewUser(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	_, err := client.NewUser("someuser", map[string]string{"testkey": "ssh-rsa somekey"})
	c.Assert(err, IsNil)
	c.Assert(string(h.body), Equals, `{"name":"someuser","keys":{"testkey":"ssh-rsa somekey"}}`)
	c.Assert(h.url, Equals, "/user")
	c.Assert(h.method, Equals, "POST")
}

func (s *S) TestNewUserWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	_, err := client.NewUser("someuser", map[string]string{"testkey": "ssh-rsa somekey"})
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestRemoveUser(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.RemoveUser("someuser")
	c.Assert(err, IsNil)
	c.Assert(string(h.body), Equals, "null")
	c.Assert(h.url, Equals, "/user/someuser")
	c.Assert(h.method, Equals, "DELETE")
}

func (s *S) TestRemoveUserWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.RemoveUser("someuser")
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestRemoveRepository(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.RemoveRepository("project1")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/repository/project1")
	c.Assert(h.method, Equals, "DELETE")
	c.Assert(string(h.body), Equals, "null")
}

func (s *S) TestRemoveRepositoryWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.RemoveRepository("proj2")
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestAddKey(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	key := map[string]string{"pubkey": "ssh-rsa somekey me@myhost"}
	err := client.AddKey("username", key)
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/user/username/key")
	c.Assert(h.method, Equals, "POST")
	c.Assert(string(h.body), Equals, `{"pubkey":"ssh-rsa somekey me@myhost"}`)
	c.Assert(h.header.Get("Content-Type"), Equals, "application/json")
}

func (s *S) TestAddKeyWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.AddKey("proj2", map[string]string{"key": "ssh-rsa keycontent user@host"})
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestRemoveKey(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.RemoveKey("username", "keyname")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/user/username/key/keyname")
	c.Assert(h.method, Equals, "DELETE")
	c.Assert(string(h.body), Equals, "null")
}

func (s *S) TestRemoveKeyWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.RemoveKey("proj2", "keyname")
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestGrantAccess(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	repositories := []string{"projectx", "projecty"}
	users := []string{"userx"}
	err := client.GrantAccess(repositories, users)
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/repository/grant")
	c.Assert(h.method, Equals, "POST")
	expected, err := json.Marshal(map[string][]string{"repositories": repositories, "users": users})
	c.Assert(err, IsNil)
	c.Assert(h.body, DeepEquals, expected)
	c.Assert(h.header.Get("Content-Type"), Equals, "application/json")
}

func (s *S) TestGrantAccessWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.GrantAccess([]string{"projectx", "projecty"}, []string{"userx"})
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestRevokeAccess(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	repositories := []string{"projectx", "projecty"}
	users := []string{"userx"}
	err := client.RevokeAccess(repositories, users)
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/repository/revoke")
	c.Assert(h.method, Equals, "DELETE")
	expected, err := json.Marshal(map[string][]string{"repositories": repositories, "users": users})
	c.Assert(err, IsNil)
	c.Assert(h.body, DeepEquals, expected)
	c.Assert(h.header.Get("Content-Type"), Equals, "application/json")
}

func (s *S) TestRevokeAccessWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	client := Client{Endpoint: ts.URL}
	err := client.RevokeAccess([]string{"projectx", "projecty"}, []string{"usery"})
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}
<commit_msg>tests: close test servers<commit_after>package gandalf

import (
	"bytes"
	"encoding/json"
	. "launchpad.net/gocheck"
	"net/http/httptest"
)

func (s *S) TestDoRequest(c *C) {
	h := TestHandler{content: `some return message`}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	body := bytes.NewBufferString(`{"foo":"bar"}`)
	response, err := client.doRequest("POST", "/test", body)
	c.Assert(err, IsNil)
	c.Assert(response.StatusCode, Equals, 200)
	c.Assert(string(h.body), Equals, `{"foo":"bar"}`)
	c.Assert(h.url, Equals, "/test")
}

func (s *S) TestDoRequestShouldNotSetContentTypeToJsonWhenBodyIsNil(c *C) {
	h := TestHandler{content: `some return message`}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	response, err := client.doRequest("DELETE", "/test", nil)
	c.Assert(err, IsNil)
	c.Assert(response.StatusCode, Equals, 200)
	c.Assert(h.header.Get("Content-Type"), Not(Equals), "application/json")
}

func (s *S) TestPost(c *C) {
	h := TestHandler{content: `some return message`}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	r := repository{Name: "test", Users: []string{"samwan"}}
	err := client.post(r, "/repository")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/repository")
	c.Assert(h.method, Equals, "POST")
	c.Assert(string(h.body), Equals, `{"name":"test","users":["samwan"],"ispublic":false}`)
}

func (s *S) TestPostWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	r := repository{Name: "test", Users: []string{"samwan"}}
	err := client.post(r, "/repository")
	c.Assert(err, ErrorMatches, "^Error performing requested operation\n$")
}

func (s *S) TestDelete(c *C) {
	h := TestHandler{content: `some return message`}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.delete(nil, "/user/someuser")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/user/someuser")
	c.Assert(h.method, Equals, "DELETE")
	c.Assert(string(h.body), Equals, "null")
}

func (s *S) TestDeleteWithConnectionError(c *C) {
	client := Client{Endpoint: "http://127.0.0.1:747399"}
	err := client.delete(nil, "/users/something")
	c.Assert(err, NotNil)
}

func (s *S) TestDeleteWithResponseError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.delete(nil, "/user/someuser")
	c.Assert(err, ErrorMatches, "^Error performing requested operation\n$")
	c.Assert(string(h.body), Equals, "null")
}

func (s *S) TestDeleteWithBody(c *C) {
	h := TestHandler{content: `some return message`}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.delete(map[string]string{"test": "foo"}, "/user/someuser")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/user/someuser")
	c.Assert(h.method, Equals, "DELETE")
	c.Assert(string(h.body), Equals, `{"test":"foo"}`)
}

func (s *S) TestGet(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.get("/user/someuser")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/user/someuser")
	c.Assert(h.method, Equals, "GET")
}

func (s *S) TestGetWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.get("/user/someuser")
	c.Assert(err, ErrorMatches, "^Error performing requested operation\n$")
}

func (s *S) TestFormatBody(c *C) {
	b, err := (&Client{}).formatBody(map[string]string{"test": "foo"})
	c.Assert(err, IsNil)
	c.Assert(b.String(), Equals, `{"test":"foo"}`)
}

func (s *S) TestFormatBodyReturnJsonNullWithNilBody(c *C) {
	b, err := (&Client{}).formatBody(nil)
	c.Assert(err, IsNil)
	c.Assert(b.String(), Equals, "null")
}

func (s *S) TestNewRepository(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	_, err := client.NewRepository("proj1", []string{"someuser"}, false)
	c.Assert(err, IsNil)
	c.Assert(string(h.body), Equals, `{"name":"proj1","users":["someuser"],"ispublic":false}`)
	c.Assert(h.url, Equals, "/repository")
	c.Assert(h.method, Equals, "POST")
}

func (s *S) TestNewRepositoryWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	_, err := client.NewRepository("proj1", []string{"someuser"}, false)
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestNewUser(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	_, err := client.NewUser("someuser", map[string]string{"testkey": "ssh-rsa somekey"})
	c.Assert(err, IsNil)
	c.Assert(string(h.body), Equals, `{"name":"someuser","keys":{"testkey":"ssh-rsa somekey"}}`)
	c.Assert(h.url, Equals, "/user")
	c.Assert(h.method, Equals, "POST")
}

func (s *S) TestNewUserWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	_, err := client.NewUser("someuser", map[string]string{"testkey": "ssh-rsa somekey"})
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestRemoveUser(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.RemoveUser("someuser")
	c.Assert(err, IsNil)
	c.Assert(string(h.body), Equals, "null")
	c.Assert(h.url, Equals, "/user/someuser")
	c.Assert(h.method, Equals, "DELETE")
}

func (s *S) TestRemoveUserWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.RemoveUser("someuser")
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestRemoveRepository(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.RemoveRepository("project1")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/repository/project1")
	c.Assert(h.method, Equals, "DELETE")
	c.Assert(string(h.body), Equals, "null")
}

func (s *S) TestRemoveRepositoryWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.RemoveRepository("proj2")
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestAddKey(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	key := map[string]string{"pubkey": "ssh-rsa somekey me@myhost"}
	err := client.AddKey("username", key)
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/user/username/key")
	c.Assert(h.method, Equals, "POST")
	c.Assert(string(h.body), Equals, `{"pubkey":"ssh-rsa somekey me@myhost"}`)
	c.Assert(h.header.Get("Content-Type"), Equals, "application/json")
}

func (s *S) TestAddKeyWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.AddKey("proj2", map[string]string{"key": "ssh-rsa keycontent user@host"})
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestRemoveKey(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.RemoveKey("username", "keyname")
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/user/username/key/keyname")
	c.Assert(h.method, Equals, "DELETE")
	c.Assert(string(h.body), Equals, "null")
}

func (s *S) TestRemoveKeyWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.RemoveKey("proj2", "keyname")
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestGrantAccess(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	repositories := []string{"projectx", "projecty"}
	users := []string{"userx"}
	err := client.GrantAccess(repositories, users)
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/repository/grant")
	c.Assert(h.method, Equals, "POST")
	expected, err := json.Marshal(map[string][]string{"repositories": repositories, "users": users})
	c.Assert(err, IsNil)
	c.Assert(h.body, DeepEquals, expected)
	c.Assert(h.header.Get("Content-Type"), Equals, "application/json")
}

func (s *S) TestGrantAccessWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.GrantAccess([]string{"projectx", "projecty"}, []string{"userx"})
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}

func (s *S) TestRevokeAccess(c *C) {
	h := TestHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	repositories := []string{"projectx", "projecty"}
	users := []string{"userx"}
	err := client.RevokeAccess(repositories, users)
	c.Assert(err, IsNil)
	c.Assert(h.url, Equals, "/repository/revoke")
	c.Assert(h.method, Equals, "DELETE")
	expected, err := json.Marshal(map[string][]string{"repositories": repositories, "users": users})
	c.Assert(err, IsNil)
	c.Assert(h.body, DeepEquals, expected)
	c.Assert(h.header.Get("Content-Type"), Equals, "application/json")
}

func (s *S) TestRevokeAccessWithError(c *C) {
	h := ErrorHandler{}
	ts := httptest.NewServer(&h)
	defer ts.Close()
	client := Client{Endpoint: ts.URL}
	err := client.RevokeAccess([]string{"projectx", "projecty"}, []string{"usery"})
	expected := "^Error performing requested operation\n$"
	c.Assert(err, ErrorMatches, expected)
}
<|endoftext|>


<commit_before>package twitch

import (
	"reflect"
	"testing"
	"time"
)

func TestCanCreateClient(t *testing.T) {
	client := NewClient("justinfan123123", "oauth:1123123")

	if reflect.TypeOf(*client) != reflect.TypeOf(Client{}) {
		t.Error("client is not of type Client")
	}
}

func TestCanConnect(t *testing.T) {
	client := NewClient("justinfan123123", "oauth:123123132")

	client.SetIrcAddress("irc.chat.twitch.tv:6667")

	go client.Connect()
	time.Sleep(time.Millisecond * 100)
}

func TestCanJoinChannel(t *testing.T) {
	client := NewClient("justinfan123123", "oauth:123123132")

	client.OnNewMessage(func(channel string, user User, message Message) {

	})

	client.OnNewRoomstateMessage(func(channel string, user User, message Message) {

	})

	client.OnNewClearchatMessage(func(channel string, user User, message Message) {

	})

	client.Join("gempir")
	time.Sleep(time.Millisecond * 100)
}
<commit_msg>real connection test<commit_after>package twitch

import (
	"bufio"
	"net"
	"reflect"
	"strings"
	"testing"
	"time"
)

func TestCanCreateClient(t *testing.T) {
	client := NewClient("justinfan123123", "oauth:1123123")

	if reflect.TypeOf(*client) != reflect.TypeOf(Client{}) {
		t.Error("client is not of type Client")
	}
}

func TestCanConnectAndAuthenticate(t *testing.T) {
	var nicknameMsg string
	var oauthMsg string

	go func() {
		ln, _ := net.Listen("tcp", ":4321")
		conn, _ := ln.Accept()

		for {
			message, _ := bufio.NewReader(conn).ReadString('\n')
			message = strings.Replace(message, "\r\n", "", 1)
			if strings.HasPrefix(message, "NICK") {
				nicknameMsg = message
			}
			if strings.HasPrefix(message, "PASS") {
				oauthMsg = message
			}
			if nicknameMsg != "" && oauthMsg != "" {
				ln.Close()
			}
		}
	}()

	client := NewClient("justinfan123123", "oauth:123123132")
	client.SetIrcAddress("127.0.0.1:4321")
	go client.Connect()

	// wait for client to connect and server to read messages
	time.Sleep(time.Second)

	if nicknameMsg != "NICK justinfan123123" || oauthMsg != "PASS oauth:123123132" {
		t.Fatalf("invalid authentication data: username: %s, oauth: %s", nicknameMsg, oauthMsg)
	}
}
<|endoftext|>


<commit_before>/*
 * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"fmt"
	"sort"
	"sync"
	"time"

	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/bpool"
	"github.com/minio/minio/pkg/color"
	"github.com/minio/minio/pkg/dsync"
	"github.com/minio/minio/pkg/madmin"
	"github.com/minio/minio/pkg/sync/errgroup"
)

// OfflineDisk represents an unavailable disk.
var OfflineDisk StorageAPI // zero value is nil

// partialOperation is a successful upload/delete of an object
// but not written in all disks (having quorum)
type partialOperation struct {
	bucket    string
	object    string
	versionID string
	failedSet int
}

// erasureObjects - Implements ER object layer.
type erasureObjects struct {
	GatewayUnsupported

	// getDisks returns list of storageAPIs.
	getDisks func() []StorageAPI

	// getLockers returns list of remote and local lockers.
	getLockers func() []dsync.NetLocker

	// getEndpoints returns list of endpoint strings belonging this set.
	// some may be local and some remote.
	getEndpoints func() []string

	// Locker mutex map.
	nsMutex *nsLockMap

	// Byte pools used for temporary i/o buffers.
	bp *bpool.BytePoolCap

	mrfOpCh chan partialOperation
}

// NewNSLock - initialize a new namespace RWLocker instance.
func (er erasureObjects) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
	return er.nsMutex.NewNSLock(ctx, er.getLockers, bucket, objects...)
}

// Shutdown function for object storage interface.
func (er erasureObjects) Shutdown(ctx context.Context) error {
	// Add any object layer shutdown activities here.
	closeStorageDisks(er.getDisks())
	return nil
}

// byDiskTotal is a collection satisfying sort.Interface.
type byDiskTotal []madmin.Disk

func (d byDiskTotal) Len() int      { return len(d) }
func (d byDiskTotal) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d byDiskTotal) Less(i, j int) bool {
	return d[i].TotalSpace < d[j].TotalSpace
}

func diskErrToDriveState(err error) (state string) {
	state = madmin.DriveStateUnknown
	switch err {
	case errDiskNotFound:
		state = madmin.DriveStateOffline
	case errCorruptedFormat:
		state = madmin.DriveStateCorrupt
	case errUnformattedDisk:
		state = madmin.DriveStateUnformatted
	case errDiskAccessDenied:
		state = madmin.DriveStatePermission
	case errFaultyDisk:
		state = madmin.DriveStateFaulty
	case nil:
		state = madmin.DriveStateOk
	}
	return
}

// getDisksInfo - fetch disks info across all other storage API.
func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Disk, errs []error, onlineDisks, offlineDisks madmin.BackendDisks) {
	disksInfo = make([]madmin.Disk, len(disks))
	onlineDisks = make(madmin.BackendDisks)
	offlineDisks = make(madmin.BackendDisks)

	for _, ep := range endpoints {
		if _, ok := offlineDisks[ep]; !ok {
			offlineDisks[ep] = 0
		}
		if _, ok := onlineDisks[ep]; !ok {
			onlineDisks[ep] = 0
		}
	}

	g := errgroup.WithNErrs(len(disks))
	for index := range disks {
		index := index
		g.Go(func() error {
			if disks[index] == OfflineDisk {
				disksInfo[index] = madmin.Disk{
					State:    diskErrToDriveState(errDiskNotFound),
					Endpoint: endpoints[index],
				}
				// Storage disk is empty, perhaps ignored disk or not available.
				return errDiskNotFound
			}
			info, err := disks[index].DiskInfo()
			if err != nil {
				if !IsErr(err, baseErrs...) {
					reqInfo := (&logger.ReqInfo{}).AppendTags("disk", disks[index].String())
					ctx := logger.SetReqInfo(GlobalContext, reqInfo)
					logger.LogIf(ctx, err)
				}
			}
			di := madmin.Disk{
				Endpoint:   endpoints[index],
				DrivePath:  info.MountPath,
				TotalSpace: info.Total,
				UsedSpace:  info.Used,
				UUID:       info.ID,
				State:      diskErrToDriveState(err),
			}
			if info.Total > 0 {
				di.Utilization = float64(info.Used / info.Total * 100)
			}
			disksInfo[index] = di
			return err
		}, index)
	}

	errs = g.Wait()
	// Wait for the routines.
	for i, diskInfoErr := range errs {
		ep := endpoints[i]
		if diskInfoErr != nil {
			offlineDisks[ep]++
			continue
		}
		onlineDisks[ep]++
	}

	// Success.
	return disksInfo, errs, onlineDisks, offlineDisks
}

// Get an aggregated storage info across all disks.
func getStorageInfo(disks []StorageAPI, endpoints []string) (StorageInfo, []error) {
	disksInfo, errs, onlineDisks, offlineDisks := getDisksInfo(disks, endpoints)

	// Sort so that the first element is the smallest.
	sort.Sort(byDiskTotal(disksInfo))

	storageInfo := StorageInfo{
		Disks: disksInfo,
	}

	storageInfo.Backend.Type = BackendErasure
	storageInfo.Backend.OnlineDisks = onlineDisks
	storageInfo.Backend.OfflineDisks = offlineDisks

	return storageInfo, errs
}

// StorageInfo - returns underlying storage statistics.
func (er erasureObjects) StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) {
	disks := er.getDisks()
	endpoints := er.getEndpoints()
	if local {
		var localDisks []StorageAPI
		var localEndpoints []string
		for i, disk := range disks {
			if disk != nil {
				if disk.IsLocal() {
					// Append this local disk since local flag is true
					localDisks = append(localDisks, disk)
					localEndpoints = append(localEndpoints, endpoints[i])
				}
			}
		}
		disks = localDisks
		endpoints = localEndpoints
	}
	return getStorageInfo(disks, endpoints)
}

// GetMetrics - is not implemented and shouldn't be called.
func (er erasureObjects) GetMetrics(ctx context.Context) (*Metrics, error) {
	logger.LogIf(ctx, NotImplemented{})
	return &Metrics{}, NotImplemented{}
}

// CrawlAndGetDataUsage collects usage from all buckets.
// updates are sent as different parts of the underlying
// structure has been traversed.
func (er erasureObjects) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error {
	return NotImplemented{API: "CrawlAndGetDataUsage"}
}

// CrawlAndGetDataUsage will start crawling buckets and send updated totals as they are traversed.
// Updates are sent on a regular basis and the caller *must* consume them.
func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error {
	var disks []StorageAPI

	for _, d := range er.getLoadBalancedDisks() {
		if d == nil || !d.IsOnline() {
			continue
		}
		disks = append(disks, d)
	}
	if len(disks) == 0 || len(buckets) == 0 {
		return nil
	}

	// Load bucket totals
	oldCache := dataUsageCache{}
	err := oldCache.load(ctx, er, dataUsageCacheName)
	if err != nil {
		return err
	}

	// New cache..
	cache := dataUsageCache{
		Info: dataUsageCacheInfo{
			Name:      dataUsageRoot,
			NextCycle: oldCache.Info.NextCycle,
		},
		Cache: make(map[string]dataUsageEntry, len(oldCache.Cache)),
	}
	bloom := bf.bytes()

	// Put all buckets into channel.
	bucketCh := make(chan BucketInfo, len(buckets))
	// Add new buckets first
	for _, b := range buckets {
		if oldCache.find(b.Name) == nil {
			bucketCh <- b
		}
	}

	// Add existing buckets if changes or lifecycles.
	for _, b := range buckets {
		e := oldCache.find(b.Name)
		if e != nil {
			cache.replace(b.Name, dataUsageRoot, *e)
			lc, err := globalLifecycleSys.Get(b.Name)
			activeLC := err == nil && lc.HasActiveRules("", true)
			if activeLC || bf == nil || bf.containsDir(b.Name) {
				bucketCh <- b
			} else {
				if intDataUpdateTracker.debug {
					logger.Info(color.Green("crawlAndGetDataUsage:")+" Skipping bucket %v, not updated", b.Name)
				}
			}
		}
	}

	close(bucketCh)
	bucketResults := make(chan dataUsageEntryInfo, len(disks))

	// Start async collector/saver.
	// This goroutine owns the cache.
	var saverWg sync.WaitGroup
	saverWg.Add(1)
	go func() {
		const updateTime = 30 * time.Second
		t := time.NewTicker(updateTime)
		defer t.Stop()
		defer saverWg.Done()
		var lastSave time.Time

	saveLoop:
		for {
			select {
			case <-ctx.Done():
				// Return without saving.
				return
			case <-t.C:
				if cache.Info.LastUpdate.Equal(lastSave) {
					continue
				}
				logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
				updates <- cache.clone()
				lastSave = cache.Info.LastUpdate
			case v, ok := <-bucketResults:
				if !ok {
					break saveLoop
				}
				cache.replace(v.Name, v.Parent, v.Entry)
				cache.Info.LastUpdate = time.Now()
			}
		}
		// Save final state...
		cache.Info.NextCycle++
		cache.Info.LastUpdate = time.Now()
		logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
		updates <- cache
	}()

	// Start one crawler per disk
	var wg sync.WaitGroup
	wg.Add(len(disks))
	for i := range disks {
		go func(i int) {
			defer wg.Done()
			disk := disks[i]

			for bucket := range bucketCh {
				select {
				case <-ctx.Done():
					return
				default:
				}

				// Load cache for bucket
				cacheName := pathJoin(bucket.Name, dataUsageCacheName)
				cache := dataUsageCache{}
				logger.LogIf(ctx, cache.load(ctx, er, cacheName))
				if cache.Info.Name == "" {
					cache.Info.Name = bucket.Name
				}
				cache.Info.BloomFilter = bloom
				if cache.Info.Name != bucket.Name {
					logger.LogIf(ctx, fmt.Errorf("cache name mismatch: %s != %s", cache.Info.Name, bucket.Name))
					cache.Info = dataUsageCacheInfo{
						Name:       bucket.Name,
						LastUpdate: time.Time{},
						NextCycle:  0,
					}
				}

				// Calc usage
				before := cache.Info.LastUpdate
				cache, err = disk.CrawlAndGetDataUsage(ctx, cache)
				cache.Info.BloomFilter = nil
				if err != nil {
					logger.LogIf(ctx, err)
					if cache.Info.LastUpdate.After(before) {
						logger.LogIf(ctx, cache.save(ctx, er, cacheName))
					}
					continue
				}

				var root dataUsageEntry
				if r := cache.root(); r != nil {
					root = cache.flatten(*r)
				}
				bucketResults <- dataUsageEntryInfo{
					Name:   cache.Info.Name,
					Parent: dataUsageRoot,
					Entry:  root,
				}
				// Save cache
				logger.LogIf(ctx, cache.save(ctx, er, cacheName))
			}
		}(i)
	}
	wg.Wait()
	close(bucketResults)
	saverWg.Wait()

	return nil
}

// IsReady - shouldn't be called will panic.
func (er erasureObjects) IsReady(ctx context.Context) bool {
	logger.CriticalIf(ctx, NotImplemented{})
	return true
}
<commit_msg>add missing available space from metrics (#10065)<commit_after>/*
 * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"fmt"
	"sort"
	"sync"
	"time"

	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/bpool"
	"github.com/minio/minio/pkg/color"
	"github.com/minio/minio/pkg/dsync"
	"github.com/minio/minio/pkg/madmin"
	"github.com/minio/minio/pkg/sync/errgroup"
)

// OfflineDisk represents an unavailable disk.
var OfflineDisk StorageAPI // zero value is nil

// partialOperation is a successful upload/delete of an object
// but not written in all disks (having quorum)
type partialOperation struct {
	bucket    string
	object    string
	versionID string
	failedSet int
}

// erasureObjects - Implements ER object layer.
type erasureObjects struct {
	GatewayUnsupported

	// getDisks returns list of storageAPIs.
	getDisks func() []StorageAPI

	// getLockers returns list of remote and local lockers.
	getLockers func() []dsync.NetLocker

	// getEndpoints returns list of endpoint strings belonging this set.
	// some may be local and some remote.
	getEndpoints func() []string

	// Locker mutex map.
	nsMutex *nsLockMap

	// Byte pools used for temporary i/o buffers.
	bp *bpool.BytePoolCap

	mrfOpCh chan partialOperation
}

// NewNSLock - initialize a new namespace RWLocker instance.
func (er erasureObjects) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
	return er.nsMutex.NewNSLock(ctx, er.getLockers, bucket, objects...)
}

// Shutdown function for object storage interface.
func (er erasureObjects) Shutdown(ctx context.Context) error {
	// Add any object layer shutdown activities here.
	closeStorageDisks(er.getDisks())
	return nil
}

// byDiskTotal is a collection satisfying sort.Interface.
type byDiskTotal []madmin.Disk

func (d byDiskTotal) Len() int      { return len(d) }
func (d byDiskTotal) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d byDiskTotal) Less(i, j int) bool {
	return d[i].TotalSpace < d[j].TotalSpace
}

func diskErrToDriveState(err error) (state string) {
	state = madmin.DriveStateUnknown
	switch err {
	case errDiskNotFound:
		state = madmin.DriveStateOffline
	case errCorruptedFormat:
		state = madmin.DriveStateCorrupt
	case errUnformattedDisk:
		state = madmin.DriveStateUnformatted
	case errDiskAccessDenied:
		state = madmin.DriveStatePermission
	case errFaultyDisk:
		state = madmin.DriveStateFaulty
	case nil:
		state = madmin.DriveStateOk
	}
	return
}

// getDisksInfo - fetch disks info across all other storage API.
func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Disk, errs []error, onlineDisks, offlineDisks madmin.BackendDisks) {
	disksInfo = make([]madmin.Disk, len(disks))
	onlineDisks = make(madmin.BackendDisks)
	offlineDisks = make(madmin.BackendDisks)

	for _, ep := range endpoints {
		if _, ok := offlineDisks[ep]; !ok {
			offlineDisks[ep] = 0
		}
		if _, ok := onlineDisks[ep]; !ok {
			onlineDisks[ep] = 0
		}
	}

	g := errgroup.WithNErrs(len(disks))
	for index := range disks {
		index := index
		g.Go(func() error {
			if disks[index] == OfflineDisk {
				disksInfo[index] = madmin.Disk{
					State:    diskErrToDriveState(errDiskNotFound),
					Endpoint: endpoints[index],
				}
				// Storage disk is empty, perhaps ignored disk or not available.
				return errDiskNotFound
			}
			info, err := disks[index].DiskInfo()
			if err != nil {
				if !IsErr(err, baseErrs...) {
					reqInfo := (&logger.ReqInfo{}).AppendTags("disk", disks[index].String())
					ctx := logger.SetReqInfo(GlobalContext, reqInfo)
					logger.LogIf(ctx, err)
				}
			}
			di := madmin.Disk{
				Endpoint:       endpoints[index],
				DrivePath:      info.MountPath,
				TotalSpace:     info.Total,
				UsedSpace:      info.Used,
				AvailableSpace: info.Free,
				UUID:           info.ID,
				State:          diskErrToDriveState(err),
			}
			if info.Total > 0 {
				di.Utilization = float64(info.Used / info.Total * 100)
			}
			disksInfo[index] = di
			return err
		}, index)
	}

	errs = g.Wait()
	// Wait for the routines.
	for i, diskInfoErr := range errs {
		ep := endpoints[i]
		if diskInfoErr != nil {
			offlineDisks[ep]++
			continue
		}
		onlineDisks[ep]++
	}

	// Success.
	return disksInfo, errs, onlineDisks, offlineDisks
}

// Get an aggregated storage info across all disks.
func getStorageInfo(disks []StorageAPI, endpoints []string) (StorageInfo, []error) {
	disksInfo, errs, onlineDisks, offlineDisks := getDisksInfo(disks, endpoints)

	// Sort so that the first element is the smallest.
	sort.Sort(byDiskTotal(disksInfo))

	storageInfo := StorageInfo{
		Disks: disksInfo,
	}

	storageInfo.Backend.Type = BackendErasure
	storageInfo.Backend.OnlineDisks = onlineDisks
	storageInfo.Backend.OfflineDisks = offlineDisks

	return storageInfo, errs
}

// StorageInfo - returns underlying storage statistics.
func (er erasureObjects) StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) {
	disks := er.getDisks()
	endpoints := er.getEndpoints()
	if local {
		var localDisks []StorageAPI
		var localEndpoints []string
		for i, disk := range disks {
			if disk != nil {
				if disk.IsLocal() {
					// Append this local disk since local flag is true
					localDisks = append(localDisks, disk)
					localEndpoints = append(localEndpoints, endpoints[i])
				}
			}
		}
		disks = localDisks
		endpoints = localEndpoints
	}
	return getStorageInfo(disks, endpoints)
}

// GetMetrics - is not implemented and shouldn't be called.
func (er erasureObjects) GetMetrics(ctx context.Context) (*Metrics, error) {
	logger.LogIf(ctx, NotImplemented{})
	return &Metrics{}, NotImplemented{}
}

// CrawlAndGetDataUsage collects usage from all buckets.
// updates are sent as different parts of the underlying
// structure has been traversed.
func (er erasureObjects) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error {
	return NotImplemented{API: "CrawlAndGetDataUsage"}
}

// CrawlAndGetDataUsage will start crawling buckets and send updated totals as they are traversed.
// Updates are sent on a regular basis and the caller *must* consume them.
func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error {
	var disks []StorageAPI

	for _, d := range er.getLoadBalancedDisks() {
		if d == nil || !d.IsOnline() {
			continue
		}
		disks = append(disks, d)
	}
	if len(disks) == 0 || len(buckets) == 0 {
		return nil
	}

	// Load bucket totals
	oldCache := dataUsageCache{}
	err := oldCache.load(ctx, er, dataUsageCacheName)
	if err != nil {
		return err
	}

	// New cache..
	cache := dataUsageCache{
		Info: dataUsageCacheInfo{
			Name:      dataUsageRoot,
			NextCycle: oldCache.Info.NextCycle,
		},
		Cache: make(map[string]dataUsageEntry, len(oldCache.Cache)),
	}
	bloom := bf.bytes()

	// Put all buckets into channel.
	bucketCh := make(chan BucketInfo, len(buckets))
	// Add new buckets first
	for _, b := range buckets {
		if oldCache.find(b.Name) == nil {
			bucketCh <- b
		}
	}

	// Add existing buckets if changes or lifecycles.
	for _, b := range buckets {
		e := oldCache.find(b.Name)
		if e != nil {
			cache.replace(b.Name, dataUsageRoot, *e)
			lc, err := globalLifecycleSys.Get(b.Name)
			activeLC := err == nil && lc.HasActiveRules("", true)
			if activeLC || bf == nil || bf.containsDir(b.Name) {
				bucketCh <- b
			} else {
				if intDataUpdateTracker.debug {
					logger.Info(color.Green("crawlAndGetDataUsage:")+" Skipping bucket %v, not updated", b.Name)
				}
			}
		}
	}

	close(bucketCh)
	bucketResults := make(chan dataUsageEntryInfo, len(disks))

	// Start async collector/saver.
	// This goroutine owns the cache.
	var saverWg sync.WaitGroup
	saverWg.Add(1)
	go func() {
		const updateTime = 30 * time.Second
		t := time.NewTicker(updateTime)
		defer t.Stop()
		defer saverWg.Done()
		var lastSave time.Time

	saveLoop:
		for {
			select {
			case <-ctx.Done():
				// Return without saving.
				return
			case <-t.C:
				if cache.Info.LastUpdate.Equal(lastSave) {
					continue
				}
				logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
				updates <- cache.clone()
				lastSave = cache.Info.LastUpdate
			case v, ok := <-bucketResults:
				if !ok {
					break saveLoop
				}
				cache.replace(v.Name, v.Parent, v.Entry)
				cache.Info.LastUpdate = time.Now()
			}
		}
		// Save final state...
		cache.Info.NextCycle++
		cache.Info.LastUpdate = time.Now()
		logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
		updates <- cache
	}()

	// Start one crawler per disk
	var wg sync.WaitGroup
	wg.Add(len(disks))
	for i := range disks {
		go func(i int) {
			defer wg.Done()
			disk := disks[i]

			for bucket := range bucketCh {
				select {
				case <-ctx.Done():
					return
				default:
				}

				// Load cache for bucket
				cacheName := pathJoin(bucket.Name, dataUsageCacheName)
				cache := dataUsageCache{}
				logger.LogIf(ctx, cache.load(ctx, er, cacheName))
				if cache.Info.Name == "" {
					cache.Info.Name = bucket.Name
				}
				cache.Info.BloomFilter = bloom
				if cache.Info.Name != bucket.Name {
					logger.LogIf(ctx, fmt.Errorf("cache name mismatch: %s != %s", cache.Info.Name, bucket.Name))
					cache.Info = dataUsageCacheInfo{
						Name:       bucket.Name,
						LastUpdate: time.Time{},
						NextCycle:  0,
					}
				}

				// Calc usage
				before := cache.Info.LastUpdate
				cache, err = disk.CrawlAndGetDataUsage(ctx, cache)
				cache.Info.BloomFilter = nil
				if err != nil {
					logger.LogIf(ctx, err)
					if cache.Info.LastUpdate.After(before) {
						logger.LogIf(ctx, cache.save(ctx, er, cacheName))
					}
					continue
				}

				var root dataUsageEntry
				if r := cache.root(); r != nil {
					root = cache.flatten(*r)
				}
				bucketResults <- dataUsageEntryInfo{
					Name:   cache.Info.Name,
					Parent: dataUsageRoot,
					Entry:  root,
				}
				// Save cache
				logger.LogIf(ctx, cache.save(ctx, er, cacheName))
			}
		}(i)
	}
	wg.Wait()
	close(bucketResults)
	saverWg.Wait()

	return nil
}

// IsReady - shouldn't be called will panic.
func (er erasureObjects) IsReady(ctx context.Context) bool {
	logger.CriticalIf(ctx, NotImplemented{})
	return true
}
<|endoftext|>


<commit_before>package models

import (
	"archive/zip"
	"bytes"
	"compress/gzip"
	"encoding/xml"
	"fmt"
	"github.com/fatih/color"
	"github.com/pquerna/ffjson/ffjson"
	"github.com/ulrf/ulrf/modules/setting"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func GetSvul(ogrn string, docLoc string, id int) (s Svul, e error) {
	s, e = GetSvulFromLevelDb(ogrn)
	if e == nil {
		color.Green("found with ogrn")
		return s, nil
	}
	s, e = GetSvulFromLevelDb(docLoc + fmt.Sprint(id))
	if e == nil {
		go SetSvultoLevelDb(&s)
		color.Green("found with docloc")
		return s, nil
	}
	s, e = GetFromZipDb(docLoc, id)
	if e != nil {
		color.Red("%s", e)
		return s, nil
	}
	color.Green("found in xmlzip")

	e = RemoveFromLevelDb(docLoc + fmt.Sprint(id))
	if e != nil {
		color.Red("%s", e)
		return s, nil
	}
	e = SetSvultoLevelDb(&s)
	if e != nil {
		color.Red("%s", e)
		return s, nil
	}
	return s, e
}

func RemoveFromLevelDb(key string) error {
	return ldb.Delete([]byte(key), nil)
}

func GetSvulFromLevelDb(ogrn string) (s Svul, e error) {
	var (
		res Svul
		bts []byte
	)
	bts, e = ldb.Get([]byte(ogrn), nil)
	if e != nil {
		return
	}
	var (
		b = bytes.NewReader(bts)
		r *gzip.Reader
	)
	r, e = gzip.NewReader(b)
	if e != nil {
		return
	}
	dec := ffjson.NewDecoder()
	e = dec.DecodeReader(r, &res)
	if e != nil {
		return
	}
	s = res
	return
}

func GetFromZipDb(docLoc string, id int) (s Svul, e error) {
	color.Yellow("%s %d", docLoc, id)
	start := time.Now()
	dname := setting.XMLDBZIP.Path

	var rc *zip.ReadCloser
	dl := getXmlLoc(docLoc)
	if dl == "" {
		e = fmt.Errorf("%s empty doc with id %d", dl, id)
		return
	}
	rc, e = zip.OpenReader(dname + "/" + dl)
	if e != nil {
		return
	}
	color.Green("[reader opened] %s", time.Since(start))
	for _, v := range rc.File {
		if v.Name == docLoc {
			color.Green("[file founded] %s", time.Since(start))
			var (
				xm  io.ReadCloser
				bts []byte
			)
			xm, e = v.Open()
			if e != nil {
				return
			}
			bts, e = fixed(xm)
			if e != nil {
				return
			}
			color.Green("[file fixed] %s", time.Since(start))
			e = xm.Close()
			if e != nil {
				return
			}

			s, e = unmarshal(bts, id)
			if e != nil {
				return
			}
			color.Green("[xml unmarshaled] %s", time.Since(start))
			return
		}
	}
	rc.Close()

	return
}

func SetSvultoLevelDb(s *Svul) error {
	bts, e := ffjson.Marshal(s)
	if e != nil {
		return e
	}
	var b bytes.Buffer
	w := gzip.NewWriter(&b)
	_, e = w.Write(bts)
	if e != nil {
		return e
	}
	w.Close()
	return ldb.Put([]byte(s.OGRN), b.Bytes(), nil)
}

func fixed(in io.Reader) ([]byte, error) {
	//r := bytes.NewReader(in)
	tr := transform.NewReader(in, charmap.Windows1251.NewDecoder())
	buf, e := ioutil.ReadAll(tr)
	if e != e {
		return nil, e
	}
	buf = bytes.Replace(buf, []byte("windows-1251"), []byte("utf-8"), 1)
	return buf, e
}

func unmarshal(in []byte, id int) (s Svul, e error) {
	f := bytes.NewReader(in)
	dec := xml.NewDecoder(f)
	i := 0
	for {
		t, _ := dec.Token()
		if t == nil {
			break
		}
		switch se := t.(type) {
		case xml.StartElement:
			if se.Name.Local == "СвЮЛ" {
				if i != id {
					i++
					continue
				}
				e = dec.DecodeElement(&s, &se)
				return
			}
		}
	}
	return
}

var (
	xmlLocCacheInited = false
	xmlLocCache       = map[string]string{}
)

func fillCacheNames() {
	dname := setting.XMLDBZIP.Path
	var e error

	w := func(path string, info os.FileInfo, err error) error {
		if strings.HasSuffix(info.Name(), ".zip") {
			var rc *zip.ReadCloser
			rc, e = zip.OpenReader(path)
			if e != nil {
				return e
			}
			for _, v := range rc.File {
				color.Green("%s %s", v.Name, info.Name())
				xmlLocCache[v.Name] = strings.TrimPrefix(path, dname)
			}
		}
		return nil
	}
	e = filepath.Walk(dname, w)
	if e != nil {
		panic(e)
	}

	xmlLocCacheInited = true
}

func getXmlLoc(name string) string {
	if !xmlLocCacheInited {
		fillCacheNames()
	}
	if fname, ok := xmlLocCache[name]; ok {
		return fname
	}
	return ""
}
<commit_msg>fix panic<commit_after>package models

import (
	"archive/zip"
	"bytes"
	"compress/gzip"
	"encoding/xml"
	"fmt"
	"github.com/fatih/color"
	"github.com/pquerna/ffjson/ffjson"
	"github.com/ulrf/ulrf/modules/setting"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func GetSvul(ogrn string, docLoc string, id int) (s Svul, e error) {
	s, e = GetSvulFromLevelDb(ogrn)
	if e == nil {
		color.Green("found with ogrn")
		return s, nil
	}
	s, e = GetSvulFromLevelDb(docLoc + fmt.Sprint(id))
	if e == nil {
		go SetSvultoLevelDb(&s)
		color.Green("found with docloc")
		return s, nil
	}
	s, e = GetFromZipDb(docLoc, id)
	if e != nil {
		color.Red("%s", e)
		return s, nil
	}
	color.Green("found in xmlzip")

	e = RemoveFromLevelDb(docLoc + fmt.Sprint(id))
	if e != nil {
		color.Red("%s", e)
		return s, nil
	}
	e = SetSvultoLevelDb(&s)
	if e != nil {
		color.Red("%s", e)
		return s, nil
	}
	return s, e
}

func RemoveFromLevelDb(key string) error {
	return ldb.Delete([]byte(key), nil)
}

func GetSvulFromLevelDb(ogrn string) (s Svul, e error) {
	var (
		res Svul
		bts []byte
	)
	bts, e = ldb.Get([]byte(ogrn), nil)
	if e != nil {
		return
	}
	var (
		b = bytes.NewReader(bts)
		r *gzip.Reader
	)
	r, e = gzip.NewReader(b)
	if e != nil {
		return
	}
	dec := ffjson.NewDecoder()
	e = dec.DecodeReader(r, &res)
	if e != nil {
		return
	}
	s = res
	return
}

func GetFromZipDb(docLoc string, id int) (s Svul, e error) {
	color.Yellow("%s %d", docLoc, id)
	start := time.Now()
	dname := setting.XMLDBZIP.Path

	var rc *zip.ReadCloser
	dl := getXmlLoc(docLoc)
	if dl == "" {
		e = fmt.Errorf("%s empty doc with id %d", dl, id)
		return
	}
	rc, e = zip.OpenReader(dname + "/" + dl)
	if e != nil {
		return
	}
	color.Green("[reader opened] %s", time.Since(start))
	for _, v := range rc.File {
		if v.Name == docLoc {
			color.Green("[file founded] %s", time.Since(start))
			var (
				xm  io.ReadCloser
				bts []byte
			)
			xm, e = v.Open()
			if e != nil {
				return
			}
			bts, e = fixed(xm)
			if e != nil {
				return
			}
			color.Green("[file fixed] %s", time.Since(start))
			e = xm.Close()
			if e != nil {
				return
			}

			s, e = unmarshal(bts, id)
			if e != nil {
				return
			}
			color.Green("[xml unmarshaled] %s", time.Since(start))
			return
		}
	}
	rc.Close()

	return
}

func SetSvultoLevelDb(s *Svul) error {
	bts, e := ffjson.Marshal(s)
	if e != nil {
		return e
	}
	var b bytes.Buffer
	w := gzip.NewWriter(&b)
	_, e = w.Write(bts)
	if e != nil {
		return e
	}
	w.Close()
	return ldb.Put([]byte(s.OGRN), b.Bytes(), nil)
}

func fixed(in io.Reader) ([]byte, error) {
	//r := bytes.NewReader(in)
	tr := transform.NewReader(in, charmap.Windows1251.NewDecoder())
	buf, e := ioutil.ReadAll(tr)
	if e != e {
		return nil, e
	}
	buf = bytes.Replace(buf, []byte("windows-1251"), []byte("utf-8"), 1)
	return buf, e
}

func unmarshal(in []byte, id int) (s Svul, e error) {
	f := bytes.NewReader(in)
	dec := xml.NewDecoder(f)
	i := 0
	for {
		t, _ := dec.Token()
		if t == nil {
			break
		}
		switch se := t.(type) {
		case xml.StartElement:
			if se.Name.Local == "СвЮЛ" {
				if i != id {
					i++
					continue
				}
				e = dec.DecodeElement(&s, &se)
				return
			}
		}
	}
	return
}

var (
	xmlLocCacheInited = false
	xmlLocCache       = map[string]string{}
)

func fillCacheNames() {
	dname := setting.XMLDBZIP.Path
	var e error

	w := func(path string, info os.FileInfo, err error) error {
		if strings.HasSuffix(info.Name(), ".zip") {
			var rc *zip.ReadCloser
			rc, e = zip.OpenReader(path)
			if e != nil {
				return e
			}
			for _, v := range rc.File {
				color.Green("%s %s", v.Name, info.Name())
				xmlLocCache[v.Name] = strings.TrimPrefix(path, dname)
			}
		}
		return nil
	}
	e = filepath.Walk(dname, w)
	if e != nil {
		color.Red("%s", e)
	}

	xmlLocCacheInited = true
}

func getXmlLoc(name string) string {
	if !xmlLocCacheInited {
		fillCacheNames()
	}
	if fname, ok := xmlLocCache[name]; ok {
		return fname
	}
	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/build"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"

	"goa.design/goa/v3/codegen"
	"golang.org/x/tools/go/packages"
)

// Generator is the code generation management data structure.
type Generator struct {
	// Command is the name of the command to run.
	Command string

	// DesignPath is the Go import path to the design package.
	DesignPath string

	// Output is the absolute path to the output directory.
	Output string

	// DesignVersion is the major component of the Goa version used by the design DSL.
	// DesignVersion is either 2 or 3.
	DesignVersion int

	// bin is the filename of the generated generator.
	bin string

	// tmpDir is the temporary directory used to compile the generator.
	tmpDir string
}

// NewGenerator creates a Generator.
func NewGenerator(cmd string, path, output string) *Generator {
	bin := "goa"
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	var version int
	{
		version = 2
		matched := false
		pkgs, _ := packages.Load(&packages.Config{Mode: packages.NeedFiles}, path)
		fset := token.NewFileSet()
		p := regexp.MustCompile(`goa.design/goa/v(\d+)/dsl`)
		for _, pkg := range pkgs {
			for _, gof := range pkg.GoFiles {
				if bs, err := ioutil.ReadFile(gof); err == nil {
					if f, err := parser.ParseFile(fset, "", string(bs), parser.ImportsOnly); err == nil {
						for _, s := range f.Imports {
							matches := p.FindStringSubmatch(s.Path.Value)
							if len(matches) == 2 {
								matched = true
								version, _ = strconv.Atoi(matches[1]) // We know it's an integer
							}
						}
					}
				}
				if matched {
					break
				}
			}
			if matched {
				break
			}
		}
	}

	return &Generator{
		Command:       cmd,
		DesignPath:    path,
		Output:        output,
		DesignVersion: version,
		bin:           bin,
	}
}

// Write writes the main file.
func (g *Generator) Write(debug bool) error {
	var tmpDir string
	{
		wd := "."
		if cwd, err := os.Getwd(); err != nil {
			wd = cwd
		}
		tmp, err := ioutil.TempDir(wd, "goa")
		if err != nil {
			return err
		}
		tmpDir = tmp
	}
	g.tmpDir = tmpDir

	var sections []*codegen.SectionTemplate
	{
		data := map[string]interface{}{
			"Command":       g.Command,
			"CleanupDirs":   cleanupDirs(g.Command, g.Output),
			"DesignVersion": g.DesignVersion,
		}
		ver := ""
		if g.DesignVersion > 2 {
			ver = "v" + strconv.Itoa(g.DesignVersion) + "/"
		}
		imports := []*codegen.ImportSpec{
			codegen.SimpleImport("flag"),
			codegen.SimpleImport("fmt"),
			codegen.SimpleImport("os"),
			codegen.SimpleImport("path/filepath"),
			codegen.SimpleImport("sort"),
			codegen.SimpleImport("strconv"),
			codegen.SimpleImport("strings"),
			codegen.SimpleImport("goa.design/goa/" + ver + "codegen"),
			codegen.SimpleImport("goa.design/goa/" + ver + "codegen/generator"),
			codegen.SimpleImport("goa.design/goa/" + ver + "eval"),
			codegen.NewImport("goa", "goa.design/goa/"+ver+"pkg"),
			codegen.NewImport("_", g.DesignPath),
		}
		sections = []*codegen.SectionTemplate{
			codegen.Header("Code Generator", "main", imports),
			{
				Name:   "main",
				Source: mainT,
				Data:   data,
			},
		}
	}

	f := &codegen.File{Path: "main.go", SectionTemplates: sections}
	_, err := f.Render(tmpDir)
	return err
}

// Compile compiles the generator.
func (g *Generator) Compile() error {
	// We first need to go get the generated package to make sure that all
	// dependencies are added to go.sum prior to compiling.
	pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName}, g.tmpDir)
	if err != nil {
		return err
	}
	if len(pkgs) != 1 {
		return fmt.Errorf("expected to find one package in %s", g.tmpDir)
	}
	if err := g.runGoCmd("get", pkgs[0].PkgPath); err != nil {
		return err
	}
	return g.runGoCmd("build", "-o", g.bin)
}

// Run runs the compiled binary and return the output lines.
func (g *Generator) Run() ([]string, error) {
	var cmdl string
	{
		args := make([]string, len(os.Args)-1)
		gopaths := filepath.SplitList(os.Getenv("GOPATH"))
		if len(gopaths) == 0 {
			gopaths = []string{build.Default.GOPATH}
		}
		for i, a := range os.Args[1:] {
			for _, p := range gopaths {
				if strings.HasPrefix(a, p) {
					args[i] = strings.Replace(a, p, "$(GOPATH)", 1)
					break
				}
			}
			if args[i] == "" {
				args[i] = a
			}
		}
		cmdl = " " + strings.Join(args, " ")
		rawcmd := filepath.Base(os.Args[0])
		// Remove .exe suffix to avoid different output on Windows.
		rawcmd = strings.TrimSuffix(rawcmd, ".exe")

		cmdl = fmt.Sprintf("$ %s%s", rawcmd, cmdl)
	}

	args := []string{"--version=" + strconv.Itoa(g.DesignVersion), "--output=" + g.Output, "--cmd=" + cmdl}
	cmd := exec.Command(filepath.Join(g.tmpDir, g.bin), args...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s\n%s", err, string(out))
	}
	res := strings.Split(string(out), "\n")
	for (len(res) > 0) && (res[len(res)-1] == "") {
		res = res[:len(res)-1]
	}
	return res, nil
}

// Remove deletes the package files.
func (g *Generator) Remove() {
	if g.tmpDir != "" {
		os.RemoveAll(g.tmpDir)
		g.tmpDir = ""
	}
}

func (g *Generator) runGoCmd(args ...string) error {
	gobin, err := exec.LookPath("go")
	if err != nil {
		return fmt.Errorf(`failed to find a go compiler, looked in "%s"`, os.Getenv("PATH"))
	}
	if g.DesignVersion > 2 {
		os.Setenv("GO111MODULE", "on")
	}
	c := exec.Cmd{
		Path: gobin,
		Args: append([]string{gobin}, args...),
		Dir:  g.tmpDir,
	}
	out, err := c.CombinedOutput()
	if err != nil {
		if len(out) > 0 {
			return fmt.Errorf(string(out))
		}
		return fmt.Errorf("failed to compile generator: %s", err)
	}
	return nil
}

// cleanupDirs returns the paths of the subdirectories under gendir to delete
// before generating code.
func cleanupDirs(cmd, output string) []string {
	if cmd == "gen" {
		gendirPath := filepath.Join(output, codegen.Gendir)
		gendir, err := os.Open(gendirPath)
		if err != nil {
			return nil
		}
		defer gendir.Close()
		finfos, err := gendir.Readdir(-1)
		if err != nil {
			return []string{gendirPath}
		}
		dirs := []string{}
		for _, fi := range finfos {
			if fi.IsDir() {
				dirs = append(dirs, filepath.Join(gendirPath, fi.Name()))
			}
		}
		return dirs
	}
	return nil
}

// mainT is the template for the generator main.
const mainT = `func main() {
	var (
		out     = flag.String("output", "", "")
		version = flag.String("version", "", "")
		cmdl    = flag.String("cmd", "", "")
		ver int
	)
	{
		flag.Parse()
		if *out == "" {
			fail("missing output flag")
		}
		if *version == "" {
			fail("missing version flag")
		}
		if *cmdl == "" {
			fail("missing cmd flag")
		}
		v, err := strconv.Atoi(*version)
		if err != nil {
			fail("invalid version %s", *version)
		}
		ver = v
	}

	if ver > goa.Major {
		fail("cannot run goa %s on design using goa v%s\n", goa.Version(), *version)
	}
	if err := eval.Context.Errors; err != nil {
		fail(err.Error())
	}
	if err := eval.RunDSL(); err != nil {
		fail(err.Error())
	}
{{- range .CleanupDirs }}
	if err := os.RemoveAll({{ printf "%q" . }}); err != nil {
		fail(err.Error())
	}
{{- end }}
{{- if gt .DesignVersion 2 }}
	codegen.DesignVersion = ver
{{- end }}
	outputs, err := generator.Generate(*out, {{ printf "%q" .Command }})
	if err != nil {
		fail(err.Error())
	}

	fmt.Println(strings.Join(outputs, "\n"))
}

func fail(msg string, vals ...interface{}) {
	fmt.Fprintf(os.Stderr, msg, vals...)
	os.Exit(1)
}
`
<commit_msg>Add . to tmpdir for loading correct module path (#2921)<commit_after>package main

import (
	"fmt"
	"go/build"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"

	"goa.design/goa/v3/codegen"
	"golang.org/x/tools/go/packages"
)

// Generator is the code generation management data structure.
type Generator struct {
	// Command is the name of the command to run.
	Command string

	// DesignPath is the Go import path to the design package.
	DesignPath string

	// Output is the absolute path to the output directory.
	Output string

	// DesignVersion is the major component of the Goa version used by the design DSL.
	// DesignVersion is either 2 or 3.
	DesignVersion int

	// bin is the filename of the generated generator.
	bin string

	// tmpDir is the temporary directory used to compile the generator.
	tmpDir string
}

// NewGenerator creates a Generator.
func NewGenerator(cmd string, path, output string) *Generator {
	bin := "goa"
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	var version int
	{
		version = 2
		matched := false
		pkgs, _ := packages.Load(&packages.Config{Mode: packages.NeedFiles}, path)
		fset := token.NewFileSet()
		p := regexp.MustCompile(`goa.design/goa/v(\d+)/dsl`)
		for _, pkg := range pkgs {
			for _, gof := range pkg.GoFiles {
				if bs, err := ioutil.ReadFile(gof); err == nil {
					if f, err := parser.ParseFile(fset, "", string(bs), parser.ImportsOnly); err == nil {
						for _, s := range f.Imports {
							matches := p.FindStringSubmatch(s.Path.Value)
							if len(matches) == 2 {
								matched = true
								version, _ = strconv.Atoi(matches[1]) // We know it's an integer
							}
						}
					}
				}
				if matched {
					break
				}
			}
			if matched {
				break
			}
		}
	}

	return &Generator{
		Command:       cmd,
		DesignPath:    path,
		Output:        output,
		DesignVersion: version,
		bin:           bin,
	}
}

// Write writes the main file.
func (g *Generator) Write(debug bool) error {
	var tmpDir string
	{
		wd := "."
		if cwd, err := os.Getwd(); err != nil {
			wd = cwd
		}
		tmp, err := ioutil.TempDir(wd, "goa")
		if err != nil {
			return err
		}
		tmpDir = tmp
	}
	g.tmpDir = tmpDir

	var sections []*codegen.SectionTemplate
	{
		data := map[string]interface{}{
			"Command":       g.Command,
			"CleanupDirs":   cleanupDirs(g.Command, g.Output),
			"DesignVersion": g.DesignVersion,
		}
		ver := ""
		if g.DesignVersion > 2 {
			ver = "v" + strconv.Itoa(g.DesignVersion) + "/"
		}
		imports := []*codegen.ImportSpec{
			codegen.SimpleImport("flag"),
			codegen.SimpleImport("fmt"),
			codegen.SimpleImport("os"),
			codegen.SimpleImport("path/filepath"),
			codegen.SimpleImport("sort"),
			codegen.SimpleImport("strconv"),
			codegen.SimpleImport("strings"),
			codegen.SimpleImport("goa.design/goa/" + ver + "codegen"),
			codegen.SimpleImport("goa.design/goa/" + ver + "codegen/generator"),
			codegen.SimpleImport("goa.design/goa/" + ver + "eval"),
			codegen.NewImport("goa", "goa.design/goa/"+ver+"pkg"),
			codegen.NewImport("_", g.DesignPath),
		}
		sections = []*codegen.SectionTemplate{
			codegen.Header("Code Generator", "main", imports),
			{
				Name:   "main",
				Source: mainT,
				Data:   data,
			},
		}
	}

	f := &codegen.File{Path: "main.go", SectionTemplates: sections}
	_, err := f.Render(tmpDir)
	return err
}

// Compile compiles the generator.
func (g *Generator) Compile() error {
	// We first need to go get the generated package to make sure that all
	// dependencies are added to go.sum prior to compiling.
	pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName}, fmt.Sprintf(".%c%s", filepath.Separator, g.tmpDir))
	if err != nil {
		return err
	}
	if len(pkgs) != 1 {
		return fmt.Errorf("expected to find one package in %s", g.tmpDir)
	}
	if err := g.runGoCmd("get", pkgs[0].PkgPath); err != nil {
		return err
	}
	return g.runGoCmd("build", "-o", g.bin)
}

// Run runs the compiled binary and return the output lines.
func (g *Generator) Run() ([]string, error) {
	var cmdl string
	{
		args := make([]string, len(os.Args)-1)
		gopaths := filepath.SplitList(os.Getenv("GOPATH"))
		if len(gopaths) == 0 {
			gopaths = []string{build.Default.GOPATH}
		}
		for i, a := range os.Args[1:] {
			for _, p := range gopaths {
				if strings.HasPrefix(a, p) {
					args[i] = strings.Replace(a, p, "$(GOPATH)", 1)
					break
				}
			}
			if args[i] == "" {
				args[i] = a
			}
		}
		cmdl = " " + strings.Join(args, " ")
		rawcmd := filepath.Base(os.Args[0])
		// Remove .exe suffix to avoid different output on Windows.
		rawcmd = strings.TrimSuffix(rawcmd, ".exe")

		cmdl = fmt.Sprintf("$ %s%s", rawcmd, cmdl)
	}

	args := []string{"--version=" + strconv.Itoa(g.DesignVersion), "--output=" + g.Output, "--cmd=" + cmdl}
	cmd := exec.Command(filepath.Join(g.tmpDir, g.bin), args...)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("%s\n%s", err, string(out))
	}
	res := strings.Split(string(out), "\n")
	for (len(res) > 0) && (res[len(res)-1] == "") {
		res = res[:len(res)-1]
	}
	return res, nil
}

// Remove deletes the package files.
func (g *Generator) Remove() {
	if g.tmpDir != "" {
		os.RemoveAll(g.tmpDir)
		g.tmpDir = ""
	}
}

func (g *Generator) runGoCmd(args ...string) error {
	gobin, err := exec.LookPath("go")
	if err != nil {
		return fmt.Errorf(`failed to find a go compiler, looked in "%s"`, os.Getenv("PATH"))
	}
	if g.DesignVersion > 2 {
		os.Setenv("GO111MODULE", "on")
	}
	c := exec.Cmd{
		Path: gobin,
		Args: append([]string{gobin}, args...),
		Dir:  g.tmpDir,
	}
	out, err := c.CombinedOutput()
	if err != nil {
		if len(out) > 0 {
			return fmt.Errorf(string(out))
		}
		return fmt.Errorf("failed to compile generator: %s", err)
	}
	return nil
}

// cleanupDirs returns the paths of the subdirectories under gendir to delete
// before generating code.
func cleanupDirs(cmd, output string) []string {
	if cmd == "gen" {
		gendirPath := filepath.Join(output, codegen.Gendir)
		gendir, err := os.Open(gendirPath)
		if err != nil {
			return nil
		}
		defer gendir.Close()
		finfos, err := gendir.Readdir(-1)
		if err != nil {
			return []string{gendirPath}
		}
		dirs := []string{}
		for _, fi := range finfos {
			if fi.IsDir() {
				dirs = append(dirs, filepath.Join(gendirPath, fi.Name()))
			}
		}
		return dirs
	}
	return nil
}

// mainT is the template for the generator main.
const mainT = `func main() {
	var (
		out     = flag.String("output", "", "")
		version = flag.String("version", "", "")
		cmdl    = flag.String("cmd", "", "")
		ver int
	)
	{
		flag.Parse()
		if *out == "" {
			fail("missing output flag")
		}
		if *version == "" {
			fail("missing version flag")
		}
		if *cmdl == "" {
			fail("missing cmd flag")
		}
		v, err := strconv.Atoi(*version)
		if err != nil {
			fail("invalid version %s", *version)
		}
		ver = v
	}

	if ver > goa.Major {
		fail("cannot run goa %s on design using goa v%s\n", goa.Version(), *version)
	}
	if err := eval.Context.Errors; err != nil {
		fail(err.Error())
	}
	if err := eval.RunDSL(); err != nil {
		fail(err.Error())
	}
{{- range .CleanupDirs }}
	if err := os.RemoveAll({{ printf "%q" . }}); err != nil {
		fail(err.Error())
	}
{{- end }}
{{- if gt .DesignVersion 2 }}
	codegen.DesignVersion = ver
{{- end }}
	outputs, err := generator.Generate(*out, {{ printf "%q" .Command }})
	if err != nil {
		fail(err.Error())
	}

	fmt.Println(strings.Join(outputs, "\n"))
}

func fail(msg string, vals ...interface{}) {
	fmt.Fprintf(os.Stderr, msg, vals...)
	os.Exit(1)
}
`
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/colinmarc/hdfs"
	"os"
	"strings"
)

func ls(paths []string, long, all bool) {
	paths, nn, err := normalizePaths(paths)
	if err != nil {
		fatal(err)
	}

	client, err := getClient(nn)
	if err != nil {
		fatal(err)
	}

	expanded, err := expandPaths(client, paths)
	if err != nil {
		fatal(err)
	}

	files := make([]os.FileInfo, 0, len(expanded))
	dirs := make([]string, 0, len(expanded))
	for _, p := range expanded {
		fi, err := stat(client, p)
		if err != nil {
			fatal(err)
		}

		if fi.IsDir() {
			dirs = append(dirs, p)
		} else {
			files = append(files, fi)
		}
	}

	if len(files) == 0 && len(dirs) == 1 {
		printDir(client, dirs[0], long, all)
	} else {
		printFiles(files, long, all)

		for _, dir := range dirs {
			fmt.Printf("\n%s/:\n", dir)
			printDir(client, dir, long, all)
		}
	}
}

func printDir(client *hdfs.Client, dir string, long, all bool) {
	files, err := readDir(client, dir, "")
	if err != nil {
		fatal(err)
	}

	if all {
		fmt.Println(".")
		fmt.Println("..")
	}

	printFiles(files, long, all)
}

func printFiles(files []os.FileInfo, long, all bool) {
	for _, file := range files {
		if all || !strings.HasPrefix(file.Name(), ".") {
			fmt.Println(file.Name())
		}
	}
}
<commit_msg>implement ls -l<commit_after>package main

import (
	"fmt"
	"github.com/colinmarc/hdfs"
	"os"
	"path"
	"strings"
	"text/tabwriter"
	"time"
)

func ls(paths []string, long, all bool) {
	paths, nn, err := normalizePaths(paths)
	if err != nil {
		fatal(err)
	}

	client, err := getClient(nn)
	if err != nil {
		fatal(err)
	}

	expanded, err := expandPaths(client, paths)
	if err != nil {
		fatal(err)
	}

	files := make([]os.FileInfo, 0, len(expanded))
	dirs := make([]string, 0, len(expanded))
	for _, p := range expanded {
		fi, err := stat(client, p)
		if err != nil {
			fatal(err)
		}

		if fi.IsDir() {
			dirs = append(dirs, p)
		} else {
			files = append(files, fi)
		}
	}

	if len(files) == 0 && len(dirs) == 1 {
		printDir(client, dirs[0], long, all)
	} else {
		var tw *tabwriter.Writer
		if long {
			tw = defaultTabWriter()
			defer tw.Flush()
		}

		printFiles(tw, files, long, all)

		for _, dir := range dirs {
			fmt.Printf("\n%s/:\n", dir)
			printDir(client, dir, long, all)
		}
	}
}

func printDir(client *hdfs.Client, dir string, long, all bool) {
	files, err := readDir(client, dir, "")
	if err != nil {
		fatal(err)
	}

	var tw *tabwriter.Writer
	if long {
		tw = defaultTabWriter()
		defer tw.Flush()
	}

	if all {
		if long {
			dot, err := stat(client, dir)
			if err != nil {
				fatal(err)
			}

			dotdot, err := stat(client, path.Join(dir, ".."))
			if err != nil {
				fatal(err)
			}

			printLong(tw, ".", dot)
			printLong(tw, "..", dotdot)
		} else {
			fmt.Println(".")
			fmt.Println("..")
		}
	}

	printFiles(tw, files, long, all)
}

func printFiles(tw *tabwriter.Writer, files []os.FileInfo, long, all bool) {
	for _, file := range files {
		if all || !strings.HasPrefix(file.Name(), ".") {
			if long {
				printLong(tw, file.Name(), file)
			} else {
				fmt.Println(file.Name())
			}
		}
	}
}

func printLong(tw *tabwriter.Writer, name string, info os.FileInfo) {
	fi := info.(*hdfs.FileInfo)
	// mode owner group size date(\w tab) time/year name
	mode := fi.Mode().String()
	owner := fi.Owner()
	group := fi.OwnerGroup()
	size := fi.Size()

	modtime := fi.ModTime()
	date := modtime.Format("Jan\t2")
	var timeOrYear string
	if modtime.Year() == time.Now().Year() {
		timeOrYear = modtime.Format("15:04")
	} else {
		timeOrYear = string(modtime.Year())
	}

	fmt.Fprintf(tw, "%s \t%s \t %s \t %d \t%s \t%s \t%s\n",
		mode, owner, group, size, date, timeOrYear, name)
}

func defaultTabWriter() *tabwriter.Writer {
	return tabwriter.NewWriter(os.Stdout, 3, 0, 0, ' ', tabwriter.AlignRight|tabwriter.TabIndent)
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"net/http"
	"os"

	"io/ioutil"
	"os/exec"

	"errors"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"strings"
)

func getExternalIP() (string, error) {
	resp, err := http.Get("https://myexternalip.com/raw")
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode == 200 {
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", nil
		}
		return fmt.Sprintf("%s/%s", strings.TrimSpace(string(b)), "32"), nil
	}

	return "", errors.New("Could not get external ip address.")
}

func getEc2Client(region string) (*ec2.EC2, error) {
	session, err := session.NewSession()
	if err != nil {
		return nil, err
	}

	config := aws.Config{
		Region: aws.String(region),
	}

	return ec2.New(session, &config), nil
}

func getVpcID(svc *ec2.EC2) (string, error) {
	vpc, err := svc.DescribeVpcs(&ec2.DescribeVpcsInput{})

	if err != nil {
		return "", err
	}

	if len(vpc.Vpcs) < 1 {
		fmt.Println(`You have at some point manually deleted the default VPC created by AWS for this region.
parsec-ec2 will not function for this region until a default VPC is recreated for it.
You can still try to launch Parsec EC2 instances in other regions that have retained
their default VPC.`)
		os.Exit(0)
	}

	return *vpc.Vpcs[0].VpcId, nil
}

func getSubnetID(svc *ec2.EC2, availabilityZone string) (string, error) {
	values := []*string{&availabilityZone}

	filter := ec2.Filter{
		Name:   aws.String("availability-zone"),
		Values: values,
	}

	filters := []*ec2.Filter{&filter}

	describeSubnetsInput := ec2.DescribeSubnetsInput{
		Filters: filters,
	}

	result, err := svc.DescribeSubnets(&describeSubnetsInput)

	if err != nil {
		return "", err
	}

	if len(result.Subnets) == 0 {
		fmt.Printf("Could not get the subnet id for availability zone %s.\n", availabilityZone)
		os.Exit(1)
	}

	return *result.Subnets[0].SubnetId, nil
}

func copy(source, destination string) error {
	b, err := ioutil.ReadFile(source)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(destination, b, 0644)
}

func hasServerKey(serverKey string) bool {
	return len(serverKey) > 0
}

func tfCmd(args []string) *exec.Cmd {
	command := exec.Command(Terraform, args...)
	command.Dir = installPath
	command.Env = os.Environ()

	return command
}

func tfCmdVars(p TfVars, args []string) *exec.Cmd {
	command := exec.Command(Terraform, args...)

	command.Dir = installPath

	command.Env = os.Environ()
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_instance_type=%s", p.InstanceType))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_region=%s", p.Region))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_server_key=%s", p.ServerKey))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_spot_price=%s", p.SpotPrice))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_subnet_id=%s", p.SubnetID))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_vpc_id=%s", p.VpcID))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_ami=%s", p.AMI))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_ip=%s", p.IP))

	return command
}

func executeSilent(command *exec.Cmd) error {
	commandErr, err := command.StderrPipe()
	if err != nil {
		return err
	}

	err = command.Start()
	if err != nil {
		return err
	}

	errOutput, err := ioutil.ReadAll(commandErr)
	if err != nil {
		return err
	}

	if len(errOutput) > 0 {
		return fmt.Errorf("Error executing Terraform command: %s\nError Output: %s", command.Args[1], errOutput)
	}

	return nil
}

func executeReturn(command *exec.Cmd) ([]byte, error) {
	initOut, err := command.StdoutPipe()
	if err != nil {
		return []byte{}, err
	}

	initErr, err := command.StderrPipe()
	if err != nil {
		return []byte{}, err
	}

	err = command.Start()
	if err != nil {
		return []byte{}, err
	}

	stdOutput, err := ioutil.ReadAll(initOut)
	if err != nil {
		return []byte{}, err
	}

	errOutput, err := ioutil.ReadAll(initErr)
	if err != nil {
		return []byte{}, err
	}

	if len(errOutput) > 0 {
		return []byte{}, fmt.Errorf("Error executing Terraform command: %s\nError Output: %s", command.Args[1], errOutput)
	}

	return stdOutput, nil
}

func executePrint(command *exec.Cmd) error {
	commandOut, err := command.StdoutPipe()
	if err != nil {
		return err
	}

	commandErr, err := command.StderrPipe()
	if err != nil {
		return err
	}

	err = command.Start()
	if err != nil {
		return err
	}

	stdOutput, err := ioutil.ReadAll(commandOut)
	if err != nil {
		return err
	}

	errOutput, err := ioutil.ReadAll(commandErr)
	if err != nil {
		return err
	}

	if len(errOutput) > 0 {
		return fmt.Errorf("Error executing Terraform command: %s\nError output: %s", command.Args[1], errOutput)
	}

	fmt.Printf("%s\n", stdOutput)
	fmt.Printf("%s\n", errOutput)

	return nil
}
<commit_msg>Switch to icanhazip because myexternalip was pretty flaky the last few days<commit_after>package cmd

import (
	"fmt"
	"net/http"
	"os"

	"io/ioutil"
	"os/exec"

	"errors"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"strings"
)

func getExternalIP() (string, error) {
	resp, err := http.Get("https://icanhazip.com")
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode == 200 {
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", nil
		}
		return fmt.Sprintf("%s/%s", strings.TrimSpace(string(b)), "32"), nil
	}

	return "", errors.New("Could not get external ip address.")
}

func getEc2Client(region string) (*ec2.EC2, error) {
	session, err := session.NewSession()
	if err != nil {
		return nil, err
	}

	config := aws.Config{
		Region: aws.String(region),
	}

	return ec2.New(session, &config), nil
}

func getVpcID(svc *ec2.EC2) (string, error) {
	vpc, err := svc.DescribeVpcs(&ec2.DescribeVpcsInput{})

	if err != nil {
		return "", err
	}

	if len(vpc.Vpcs) < 1 {
		fmt.Println(`You have at some point manually deleted the default VPC created by AWS for this region.
parsec-ec2 will not function for this region until a default VPC is recreated for it.
You can still try to launch Parsec EC2 instances in other regions that have retained
their default VPC.`)
		os.Exit(0)
	}

	return *vpc.Vpcs[0].VpcId, nil
}

func getSubnetID(svc *ec2.EC2, availabilityZone string) (string, error) {
	values := []*string{&availabilityZone}

	filter := ec2.Filter{
		Name:   aws.String("availability-zone"),
		Values: values,
	}

	filters := []*ec2.Filter{&filter}

	describeSubnetsInput := ec2.DescribeSubnetsInput{
		Filters: filters,
	}

	result, err := svc.DescribeSubnets(&describeSubnetsInput)

	if err != nil {
		return "", err
	}

	if len(result.Subnets) == 0 {
		fmt.Printf("Could not get the subnet id for availability zone %s.\n", availabilityZone)
		os.Exit(1)
	}

	return *result.Subnets[0].SubnetId, nil
}

func copy(source, destination string) error {
	b, err := ioutil.ReadFile(source)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(destination, b, 0644)
}

func hasServerKey(serverKey string) bool {
	return len(serverKey) > 0
}

func tfCmd(args []string) *exec.Cmd {
	command := exec.Command(Terraform, args...)
	command.Dir = installPath
	command.Env = os.Environ()

	return command
}

func tfCmdVars(p TfVars, args []string) *exec.Cmd {
	command := exec.Command(Terraform, args...)

	command.Dir = installPath

	command.Env = os.Environ()
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_instance_type=%s", p.InstanceType))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_region=%s", p.Region))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_server_key=%s", p.ServerKey))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_spot_price=%s", p.SpotPrice))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_subnet_id=%s", p.SubnetID))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_vpc_id=%s", p.VpcID))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_ami=%s", p.AMI))
	command.Env = append(command.Env, fmt.Sprintf("TF_VAR_ip=%s", p.IP))

	return command
}

func executeSilent(command *exec.Cmd) error {
	commandErr, err := command.StderrPipe()
	if err != nil {
		return err
	}

	err = command.Start()
	if err != nil {
		return err
	}

	errOutput, err := ioutil.ReadAll(commandErr)
	if err != nil {
		return err
	}

	if len(errOutput) > 0 {
		return fmt.Errorf("Error executing Terraform command: %s\nError Output: %s", command.Args[1], errOutput)
	}

	return nil
}

func executeReturn(command *exec.Cmd) ([]byte, error) {
	initOut, err := command.StdoutPipe()
	if err != nil {
		return []byte{}, err
	}

	initErr, err := command.StderrPipe()
	if err != nil {
		return []byte{}, err
	}

	err = command.Start()
	if err != nil {
		return []byte{}, err
	}

	stdOutput, err := ioutil.ReadAll(initOut)
	if err != nil {
		return []byte{}, err
	}

	errOutput, err := ioutil.ReadAll(initErr)
	if err != nil {
		return []byte{}, err
	}

	if len(errOutput) > 0 {
		return []byte{}, fmt.Errorf("Error executing Terraform command: %s\nError Output: %s", command.Args[1], errOutput)
	}

	return stdOutput, nil
}

func executePrint(command *exec.Cmd) error {
	commandOut, err := command.StdoutPipe()
	if err != nil {
		return err
	}

	commandErr, err := command.StderrPipe()
	if err != nil {
		return err
	}

	err = command.Start()
	if err != nil {
		return err
	}

	stdOutput, err := ioutil.ReadAll(commandOut)
	if err != nil {
		return err
	}

	errOutput, err := ioutil.ReadAll(commandErr)
	if err != nil {
		return err
	}

	if len(errOutput) > 0 {
		return fmt.Errorf("Error executing Terraform command: %s\nError output: %s", command.Args[1], errOutput)
	}

	fmt.Printf("%s\n", stdOutput)
	fmt.Printf("%s\n", errOutput)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// ht generates HTTP requests and checks the received responses.
//
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/vdobler/ht/ht"
)

// A Command is one of the subcommands of ht.
type Command struct {
	// Run the command.
	// The args are the arguments after the command name.
	Run func(cmd *Command, suites []*ht.Suite)

	Usage       string       // must start with command name
	Description string       // short description for ' go help'
	Help        string       // the output of 'ht help <cmd>'
	Flag        flag.FlagSet // the flags for this command
}

// Name returns the command's name: the first word in the usage line.
func (c *Command) Name() string {
	name := c.Usage
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (c *Command) usage() {
	fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.Usage)
	fmt.Fprintf(os.Stderr, "%s\n", c.Help)
	os.Exit(2)
}

// Commands lists the available commands and help topics.
// The order here is the order in which they are printed by 'go help'.
var commands = []*Command{
	cmdList,
	cmdRun,
	cmdExec,
	cmdBench,
	cmdPerf,
}

func usage() {
	formatedCmdList := ""

	for _, cmd := range commands {
		formatedCmdList += fmt.Sprintf("    %-8s %s\n",
			cmd.Name(), cmd.Description)
	}

	fmt.Printf(`Ht is a tool to generate http request and test the response.

Usage:

    ht <command> [flags...] <suite>...

The commands are:
%s
Run  ht help <command> to display the usage of <command>.

Tests IDs have the following format <suite>.<type><test> with <suite> and
<test> the sequential numbers of the suite and the test inside the suite.
Type is either empty, "u" for setUp test or "d" for tearDown tests. <test>
maybe a single number like "3" or a range like "3-7".
`, formatedCmdList)
	os.Exit(2)
}

// Variables which can be set via the command line. Statisfied flag.Value interface.
type cmdlVar map[string]string

func (v cmdlVar) String() string { return "" }
func (v cmdlVar) Set(s string) error {
	part := strings.SplitN(s, "=", 2)
	if len(part) != 2 {
		return fmt.Errorf("Bad argument '%s' to -D commandline parameter", s)
	}
	v[part[0]] = part[1]
	return nil
}

// Includepath which can be set via the command line. Statisfied flag.Value interface.
type cmdlIncl []string

func (i *cmdlIncl) String() string { return "" }
func (i *cmdlIncl) Set(s string) error {
	s = strings.TrimRight(s, "/")
	*i = append(*i, s)
	return nil
}

// The common flags.
var (
	variablesFlag cmdlVar = make(cmdlVar) // flag -D
	onlyFlag      string
	skipFlag      string
	verbosity     int
)

func addVariablesFlag(fs *flag.FlagSet) {
	fs.Var(variablesFlag, "D", "set `parameter=value`")
}

func addOnlyFlag(fs *flag.FlagSet) {
	fs.StringVar(&onlyFlag, "only", "", "run only tests given by `testID`")
}

func addSkipFlag(fs *flag.FlagSet) {
	fs.StringVar(&skipFlag, "skip", "", "skip tests identified by `testID`")
}

func addVerbosityFlag(fs *flag.FlagSet) {
	fs.IntVar(&verbosity, "verbosity", -99, "verbosity to `level`")
}

func main() {
	flag.Usage = usage
	flag.Parse()

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}

	if args[0] == "help" {
		help(args[1:])
		return
	}
	var suites []*ht.Suite
	for _, cmd := range commands {
		if cmd.Name() == args[0] {
			cmd.Flag.Usage = func() { cmd.usage() }
			cmd.Flag.Parse(args[1:])
			args = cmd.Flag.Args()
			if cmd.Name() == "run" {
				suites = loadTests(args)
			} else {
				suites = loadSuites(args)
			}
			cmd.Run(cmd, suites)
			return
		}
	}

	fmt.Fprintf(os.Stderr, "go: unknown subcommand %q\nRun 'go help' for usage.\n",
		args[0])
	os.Exit(2)
}

// The help command.
func help(args []string) {
	if len(args) == 0 {
		usage() // TODO: this is not a failure
	}
	if len(args) != 1 {
		fmt.Fprintf(os.Stderr, "usage: ht help <command>\n\nToo many arguments given.\n")
		os.Exit(2)
	}

	arg := args[0]

	for _, cmd := range commands {
		if cmd.Name() == arg {
			fmt.Printf(`Usage:

    ht %s
%s
Flags:
`, cmd.Usage, cmd.Help)
			cmd.Flag.PrintDefaults()
			return
		}
	}

	fmt.Fprintf(os.Stderr, "Unknown help topic %#q.  Run 'ht help'.\n", arg)
	os.Exit(2) // failed at 'go help cmd'
}

func loadSuites(args []string) []*ht.Suite {
	var suites []*ht.Suite

	logger := log.New(os.Stdout, "", log.LstdFlags)

	// Handle -only and -skip flags.
	only, skip := splitTestIDs(onlyFlag), splitTestIDs(skipFlag)

	// Input and setup suites from command line arguments.
	for _, s := range args {
		suite, err := ht.LoadSuite(s)
		if err != nil {
			log.Fatalf("Cannot read suite %q: %s", s, err)
		}
		for varName, varVal := range variablesFlag {
			suite.Variables[varName] = varVal
		}
		suite.Log = logger
		err = suite.Prepare()
		if err != nil {
			log.Fatal(err.Error())
		}
		if verbosity != -99 {
			for i := range suite.Setup {
				suite.Setup[i].Verbosity = verbosity
			}
			for i := range suite.Tests {
				suite.Tests[i].Verbosity = verbosity
			}
			for i := range suite.Teardown {
				suite.Teardown[i].Verbosity = verbosity
			}
		}
		suites = append(suites, suite)
	}

	// Disable tests based on the -only and -skip flags.
	for sNo, suite := range suites {
		for tNo, test := range suite.Setup {
			shouldRun(test, fmt.Sprintf("%d.U%d", sNo+1, tNo+1), only, skip)
		}
		for tNo, test := range suite.Tests {
			shouldRun(test, fmt.Sprintf("%d.%d", sNo+1, tNo+1), only, skip)
		}
		for tNo, test := range suite.Teardown {
			shouldRun(test, fmt.Sprintf("%d.D%d", sNo+1, tNo+1), only, skip)
		}
	}

	return suites
}

// loadTests loads single Test and combines them into an artificial
// Suite, ready for execution. Unrolling happens, but only the first
// unrolled test gets included into the suite.
func loadTests(args []string) []*ht.Suite {
	logger := log.New(os.Stdout, "", log.LstdFlags)
	suite := &ht.Suite{
		Name: "Autogenerated suite for run",
		Log:  logger,
	}

	// Input and setup tests from command line arguments.
	for _, t := range args {
		tests, err := ht.LoadTest(t)
		if err != nil {
			log.Fatalf("Cannot read test %q: %s", t, err)
		}
		suite.Tests = append(suite.Tests, tests[0])
	}

	err := suite.Prepare()
	if err != nil {
		log.Fatal(err.Error())
	}
	if verbosity != -99 {
		for i := range suite.Tests {
			suite.Tests[i].Verbosity = verbosity
		}
	}

	for varName, varVal := range variablesFlag {
		suite.Variables[varName] = varVal
	}

	suites := []*ht.Suite{suite}
	return suites
}

// shouldRun disables t if needed.
func shouldRun(t *ht.Test, id string, only, skip map[string]struct{}) {
	if _, ok := skip[id]; ok {
		t.Poll.Max = -1
		log.Printf("Skipping test %s %q", id, t.Name)
		return
	}
	if _, ok := only[id]; !ok && len(only) > 0 {
		t.Poll.Max = -1
		log.Printf("Not running test %s %q", id, t.Name)
		return
	}
}

func splitTestIDs(f string) (ids map[string]struct{}) {
	ids = make(map[string]struct{})
	if len(f) == 0 {
		return
	}
	fp := strings.Split(f, ",")
	for _, x := range fp {
		xp := strings.SplitN(x, ".", 2)
		s, t := "1", xp[0]
		if len(xp) == 2 {
			s, t = xp[0], xp[1]
		}
		typ := ""
		switch t[0] {
		case 'U', 'u', 'S', 's':
			typ = "U"
			t = t[1:]
		case 'D', 'd', 'T', 't':
			typ = "D"
			t = t[1:]
		default:
			typ = ""
		}
		// TODO: support ranges like "3.1-5"
		sNo := mustAtoi(s)
		beg, end := 1, 99
		if i := strings.Index(t, "-"); i > -1 {
			if i > 0 {
				beg = mustAtoi(t[:i])
			}
			if i < len(t)-1 {
				end = mustAtoi(t[i+1:])
			}
		} else {
			beg = mustAtoi(t)
			end = beg
		}
		for tNo := beg; tNo <= end; tNo++ {
			id := fmt.Sprintf("%d.%s%d", sNo, typ, tNo)
			ids[id] = struct{}{}
		}
	}
	return ids
}

func mustAtoi(s string) int {
	n, err := strconv.Atoi(s)
	if err != nil {
		log.Fatalf("%s", err.Error())
	}
	return n
}

// add current working direcory to end of include path slice if not already
// there.
func addCWD(i *cmdlIncl) {
	for _, p := range *i {
		if p == "." {
			return
		}
	}
	*i = append(*i, ".")
}
<commit_msg>cmd/ht: remove obsolete TODO<commit_after>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// ht generates HTTP requests and checks the received responses.
//
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/vdobler/ht/ht"
)

// A Command is one of the subcommands of ht.
type Command struct {
	// Run the command.
	// The args are the arguments after the command name.
	Run func(cmd *Command, suites []*ht.Suite)

	Usage       string       // must start with command name
	Description string       // short description for ' go help'
	Help        string       // the output of 'ht help <cmd>'
	Flag        flag.FlagSet // the flags for this command
}

// Name returns the command's name: the first word in the usage line.
func (c *Command) Name() string {
	name := c.Usage
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

func (c *Command) usage() {
	fmt.Fprintf(os.Stderr, "usage: %s\n\n", c.Usage)
	fmt.Fprintf(os.Stderr, "%s\n", c.Help)
	os.Exit(2)
}

// Commands lists the available commands and help topics.
// The order here is the order in which they are printed by 'go help'.
var commands = []*Command{
	cmdList,
	cmdRun,
	cmdExec,
	cmdBench,
	cmdPerf,
}

func usage() {
	formatedCmdList := ""

	for _, cmd := range commands {
		formatedCmdList += fmt.Sprintf("    %-8s %s\n",
			cmd.Name(), cmd.Description)
	}

	fmt.Printf(`Ht is a tool to generate http request and test the response.

Usage:

    ht <command> [flags...] <suite>...

The commands are:
%s
Run  ht help <command> to display the usage of <command>.

Tests IDs have the following format <suite>.<type><test> with <suite> and
<test> the sequential numbers of the suite and the test inside the suite.
Type is either empty, "u" for setUp test or "d" for tearDown tests. <test>
maybe a single number like "3" or a range like "3-7".
`, formatedCmdList)
	os.Exit(2)
}

// Variables which can be set via the command line. Statisfied flag.Value interface.
type cmdlVar map[string]string

func (v cmdlVar) String() string { return "" }
func (v cmdlVar) Set(s string) error {
	part := strings.SplitN(s, "=", 2)
	if len(part) != 2 {
		return fmt.Errorf("Bad argument '%s' to -D commandline parameter", s)
	}
	v[part[0]] = part[1]
	return nil
}

// Includepath which can be set via the command line. Statisfied flag.Value interface.
type cmdlIncl []string

func (i *cmdlIncl) String() string { return "" }
func (i *cmdlIncl) Set(s string) error {
	s = strings.TrimRight(s, "/")
	*i = append(*i, s)
	return nil
}

// The common flags.
var (
	variablesFlag cmdlVar = make(cmdlVar) // flag -D
	onlyFlag      string
	skipFlag      string
	verbosity     int
)

func addVariablesFlag(fs *flag.FlagSet) {
	fs.Var(variablesFlag, "D", "set `parameter=value`")
}

func addOnlyFlag(fs *flag.FlagSet) {
	fs.StringVar(&onlyFlag, "only", "", "run only tests given by `testID`")
}

func addSkipFlag(fs *flag.FlagSet) {
	fs.StringVar(&skipFlag, "skip", "", "skip tests identified by `testID`")
}

func addVerbosityFlag(fs *flag.FlagSet) {
	fs.IntVar(&verbosity, "verbosity", -99, "verbosity to `level`")
}

func main() {
	flag.Usage = usage
	flag.Parse()

	args := flag.Args()
	if len(args) < 1 {
		usage()
	}

	if args[0] == "help" {
		help(args[1:])
		return
	}
	var suites []*ht.Suite
	for _, cmd := range commands {
		if cmd.Name() == args[0] {
			cmd.Flag.Usage = func() { cmd.usage() }
			cmd.Flag.Parse(args[1:])
			args = cmd.Flag.Args()
			if cmd.Name() == "run" {
				suites = loadTests(args)
			} else {
				suites = loadSuites(args)
			}
			cmd.Run(cmd, suites)
			return
		}
	}

	fmt.Fprintf(os.Stderr, "go: unknown subcommand %q\nRun 'go help' for usage.\n",
		args[0])
	os.Exit(2)
}

// The help command.
func help(args []string) {
	if len(args) == 0 {
		usage() // TODO: this is not a failure
	}
	if len(args) != 1 {
		fmt.Fprintf(os.Stderr, "usage: ht help <command>\n\nToo many arguments given.\n")
		os.Exit(2)
	}

	arg := args[0]

	for _, cmd := range commands {
		if cmd.Name() == arg {
			fmt.Printf(`Usage:

    ht %s
%s
Flags:
`, cmd.Usage, cmd.Help)
			cmd.Flag.PrintDefaults()
			return
		}
	}

	fmt.Fprintf(os.Stderr, "Unknown help topic %#q.  Run 'ht help'.\n", arg)
	os.Exit(2) // failed at 'go help cmd'
}

func loadSuites(args []string) []*ht.Suite {
	var suites []*ht.Suite

	logger := log.New(os.Stdout, "", log.LstdFlags)

	// Handle -only and -skip flags.
	only, skip := splitTestIDs(onlyFlag), splitTestIDs(skipFlag)

	// Input and setup suites from command line arguments.
	for _, s := range args {
		suite, err := ht.LoadSuite(s)
		if err != nil {
			log.Fatalf("Cannot read suite %q: %s", s, err)
		}
		for varName, varVal := range variablesFlag {
			suite.Variables[varName] = varVal
		}
		suite.Log = logger
		err = suite.Prepare()
		if err != nil {
			log.Fatal(err.Error())
		}
		if verbosity != -99 {
			for i := range suite.Setup {
				suite.Setup[i].Verbosity = verbosity
			}
			for i := range suite.Tests {
				suite.Tests[i].Verbosity = verbosity
			}
			for i := range suite.Teardown {
				suite.Teardown[i].Verbosity = verbosity
			}
		}
		suites = append(suites, suite)
	}

	// Disable tests based on the -only and -skip flags.
	for sNo, suite := range suites {
		for tNo, test := range suite.Setup {
			shouldRun(test, fmt.Sprintf("%d.U%d", sNo+1, tNo+1), only, skip)
		}
		for tNo, test := range suite.Tests {
			shouldRun(test, fmt.Sprintf("%d.%d", sNo+1, tNo+1), only, skip)
		}
		for tNo, test := range suite.Teardown {
			shouldRun(test, fmt.Sprintf("%d.D%d", sNo+1, tNo+1), only, skip)
		}
	}

	return suites
}

// loadTests loads single Test and combines them into an artificial
// Suite, ready for execution. Unrolling happens, but only the first
// unrolled test gets included into the suite.
func loadTests(args []string) []*ht.Suite {
	logger := log.New(os.Stdout, "", log.LstdFlags)
	suite := &ht.Suite{
		Name: "Autogenerated suite for run",
		Log:  logger,
	}

	// Input and setup tests from command line arguments.
	for _, t := range args {
		tests, err := ht.LoadTest(t)
		if err != nil {
			log.Fatalf("Cannot read test %q: %s", t, err)
		}
		suite.Tests = append(suite.Tests, tests[0])
	}

	err := suite.Prepare()
	if err != nil {
		log.Fatal(err.Error())
	}
	if verbosity != -99 {
		for i := range suite.Tests {
			suite.Tests[i].Verbosity = verbosity
		}
	}

	for varName, varVal := range variablesFlag {
		suite.Variables[varName] = varVal
	}

	suites := []*ht.Suite{suite}
	return suites
}

// shouldRun disables t if needed.
func shouldRun(t *ht.Test, id string, only, skip map[string]struct{}) {
	if _, ok := skip[id]; ok {
		t.Poll.Max = -1
		log.Printf("Skipping test %s %q", id, t.Name)
		return
	}
	if _, ok := only[id]; !ok && len(only) > 0 {
		t.Poll.Max = -1
		log.Printf("Not running test %s %q", id, t.Name)
		return
	}
}

func splitTestIDs(f string) (ids map[string]struct{}) {
	ids = make(map[string]struct{})
	if len(f) == 0 {
		return
	}
	fp := strings.Split(f, ",")
	for _, x := range fp {
		xp := strings.SplitN(x, ".", 2)
		s, t := "1", xp[0]
		if len(xp) == 2 {
			s, t = xp[0], xp[1]
		}
		typ := ""
		switch t[0] {
		case 'U', 'u', 'S', 's':
			typ = "U"
			t = t[1:]
		case 'D', 'd', 'T', 't':
			typ = "D"
			t = t[1:]
		default:
			typ = ""
		}

		sNo := mustAtoi(s)
		beg, end := 1, 99
		if i := strings.Index(t, "-"); i > -1 {
			if i > 0 {
				beg = mustAtoi(t[:i])
			}
			if i < len(t)-1 {
				end = mustAtoi(t[i+1:])
			}
		} else {
			beg = mustAtoi(t)
			end = beg
		}
		for tNo := beg; tNo <= end; tNo++ {
			id := fmt.Sprintf("%d.%s%d", sNo, typ, tNo)
			ids[id] = struct{}{}
		}
	}
	return ids
}

func mustAtoi(s string) int {
	n, err := strconv.Atoi(s)
	if err != nil {
		log.Fatalf("%s", err.Error())
	}
	return n
}

// add current working direcory to end of include path slice if not already
// there.
func addCWD(i *cmdlIncl) {
	for _, p := range *i {
		if p == "." {
			return
		}
	}
	*i = append(*i, ".")
}
<|endoftext|>