<commit_before>//the icws_golib package wraps CIC ICWS fuctionality in a GO library.
//To get started, call icws_golib.NewIcws() to instansiate a new Icws
//struct.  Then call icws.Login(...) and continue on from there.
package icws_golib

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"crypto/tls"
)

type Icws struct {
	CurrentToken, CurrentCookie, CurrentSession, CurrentServer, UserId,HttpScheme string
	Port int

}

//Version information for the server
type ServerVersion struct{
	//The product's two-digit release year. For the release "CIC 2015 R1" this value will be "15"
	MajorVersion string
	//The product's release number. For the release "CIC 2015 R1" this value will be "1".
	MinorVersion string
	//The patch number of the release. The value "0" indicates the release without any patches. For the release "CIC 2015 R1" this value will be "0", and for the release "CIC 2015 R1 Patch2" this value will be "2".
	Su string
	//The product line identifier.
	ProductId string
	//The codebase identifier.
	CodeBaseId string
	//The build number.
	Build string
	//The display string for the release. This does not include patch information.
	ProductReleaseDisplayString string
	//The display string for the release including patch information. This string is recommended for use on application "About" screens.
	ProductPatchDisplayString string
}

//Definition of a server feature
type ServerFeature struct{
	//Id of the features
	FeatureId string
	//version of the feature
	Version int
}

//Creates a new ICWS struct
func NewIcws() (icws *Icws) {
	icws = &Icws{}
	icws.HttpScheme = "https"
	icws.Port = 8019
	return
}

func (i *Icws) loginWithData(applicationName, server, username, password string, loginData map[string]string) (err error) {

	server = fmt.Sprintf("%s://%s:%d", i.HttpScheme, server, i.Port)

	log.Printf("Logging into %s with user %s", server, username)

	i.CurrentSession = ""
	i.CurrentServer = server
	body, statusCode, err, cookie := i.httpPostAdvanced("connection", loginData)

	nextServerIndex := 0
	for statusCode == 503 {

		var returnData map[string]interface{}
		json.Unmarshal(body, &returnData)

		nextServer := returnData["alternateHostList"].([]interface{})[0]

		server = fmt.Sprintf("http://%s:8018", nextServer)

		log.Printf("Redirected to server %s", server)

		i.CurrentServer = server
		body, statusCode, err, cookie = i.httpPostAdvanced("connection", loginData)

		nextServerIndex++
	}

	if err == nil {

		var returnData map[string]string
		json.Unmarshal(body, &returnData)
		i.CurrentToken = returnData["csrfToken"]
		i.CurrentSession = returnData["sessionId"]
		i.CurrentServer = server
		i.CurrentCookie = cookie
		i.UserId = username

	} else {
		log.Printf("ERROR: %s\n", err.Error())
	}
	return
}

//Logs into a CIC server.  Server should be a server name e.g. MyServer.domain.com
func (i *Icws) Login(applicationName, server, username, password string) (err error) {

	var loginData = map[string]string{
		"__type":          "urn:inin.com:connection:icAuthConnectionRequestSettings",
		"applicationName": applicationName,
		"userID":          username,
		"password":        password,
	}

	return i.loginWithData(applicationName, server, username, password, loginData)
}

//Logs into a CIC server for a MarketPlace application using the app's custom license.  Server should be a url e.g. https://MyServer:8019
func (i *Icws) LoginMarketPlaceApp(applicationName, server, username, password, marketplaceLicense, markeplaceAppKey string) (err error) {

	var loginData = map[string]string{
		"__type":          "urn:inin.com:connection:icAuthConnectionRequestSettings",
		"applicationName": applicationName,
		"userID":          username,
		"password":        password,
	}

	return i.loginWithData(applicationName, server, username, password, loginData)
}

//gets server version details.  Does not need to be connected to the server
func (i *Icws) GetVersion() (version ServerVersion, err error) {

	body, err := i.httpNoConnectionGet("/connection/version")

	err = json.Unmarshal(body, &version)

	return
}

//Gets a list of features and their version from the server
func (i *Icws) GetFeatures() (features []ServerFeature, err error) {

	body, err := i.httpNoConnectionGet("/connection/features")

	if err != nil {
		return nil, err
	}

	var featureMap map[string][]ServerFeature
	err = json.Unmarshal(body, &featureMap)

	features = featureMap["featureInfoList"]
	return

}

func (i *Icws) httpDelete(url string) (err error) {

	req, err := i.httpRequest("DELETE", url, nil)

	if err != nil {
		return
	}

	response, err := i.httpClient().Do(req)
	if err != nil {
		return
	}
	defer response.Body.Close()
	if response.StatusCode == 401 {
		err = errors.New("authorization expired")
		return
	}
	body, _ := ioutil.ReadAll(response.Body)
	if response.StatusCode/100 != 2 {
		err = errors.New(createErrorMessage(response.StatusCode, body))

		return
	}

	return
}

func (i *Icws) httpGet(url string) (body []byte, err error) {

	req, err := i.httpRequest("GET", url, nil)

	if err != nil {
		return
	}

	response, err := i.httpClient().Do(req)
	if err != nil {
		return
	}
	defer response.Body.Close()
	if response.StatusCode == 401 {
		err = errors.New("authorization expired")
		return
	}
	body, err = ioutil.ReadAll(response.Body)
	if response.StatusCode/100 != 2 {
		err = errors.New(createErrorMessage(response.StatusCode, body))

		return
	}

	return
}

func (i *Icws) httpNoConnectionGet(url string) (body []byte, err error) {

	req, err := i.httpNoConnectionRequest("GET", url, nil)

	if err != nil {
		return
	}

	response, err := i.httpClient().Do(req)
	if err != nil {
		return
	}
	defer response.Body.Close()
	if response.StatusCode == 401 {
		err = errors.New("authorization expired")
		return
	}
	body, err = ioutil.ReadAll(response.Body)
	if response.StatusCode/100 != 2 {
		err = errors.New(createErrorMessage(response.StatusCode, body))

		return
	}

	return
}

func (i *Icws) httpPost(url string, attrs map[string]string) (body []byte, err error) {
	body, _, err, _ = i.httpPostAdvanced(url, attrs)
	return
}

func (i *Icws) httpPostAdvanced(url string, attrs map[string]string) (body []byte, statusCode int, err error, cookie string) {

	rbody, _ := json.Marshal(attrs)
	req, err := i.httpRequest("POST", url, bytes.NewReader(rbody))
	if err != nil {
		return
	}

	response, err := i.httpClient().Do(req)
	if err != nil {
		return

	}
	defer response.Body.Close()

	statusCode = response.StatusCode

	if response.StatusCode == 401 {
		err = errors.New("authorization expired")
		return
	}
	body, err = ioutil.ReadAll(response.Body)

	if response.StatusCode/100 != 2 {
		err = errors.New(createErrorMessage(response.StatusCode, body))

		return
	}

	if response.Header["Set-Cookie"] != nil {
		cookie = response.Header["Set-Cookie"][0]
	}
	return
}

func (i *Icws) httpPut(url string, attrs map[string]string) (body []byte, err error) {

	rbody, _ := json.Marshal(attrs)
	req, err := i.httpRequest("PUT", url, bytes.NewReader(rbody))
	if err != nil {
		return
	}

	response, err := i.httpClient().Do(req)
	if err != nil {
		return

	}
	defer response.Body.Close()
	if response.StatusCode == 401 {
		err = errors.New("authorization expired")
		return
	}
	body, err = ioutil.ReadAll(response.Body)

	if response.StatusCode/100 != 2 {
		err = errors.New(createErrorMessage(response.StatusCode, body))

		return
	}

	return
}

func (i *Icws) httpClient() (client *http.Client) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify : true},
	}
	client = &http.Client{Transport: tr}
	return
}

func (i *Icws) httpNoConnectionRequest(method, url string, body io.Reader) (request *http.Request, err error) {
	request, err = http.NewRequest(method, i.CurrentServer+"/icws"+url, body)
	if err != nil {
		return
	}

	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Accept-Language", "en-us")

	return
}

func (i *Icws) httpRequest(method, url string, body io.Reader) (request *http.Request, err error) {
	request, err = http.NewRequest(method, i.CurrentServer+"/icws/"+i.CurrentSession+url, body)
	if err != nil {
		return
	}

	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Accept-Language", "en-us")

	if len(i.CurrentCookie) > 0 {
		request.Header.Add("Cookie", i.CurrentCookie)
	} else {
		return
	}

	if len(i.CurrentToken) > 0 {
		request.Header.Add("ININ-ICWS-CSRF-Token", i.CurrentToken)
	} else {
		return
	}

	// request.Header.Add("User-Agent", fmt.Sprintf("cic cli (%s-%s)", runtime.GOOS, runtime.GOARCH))
	return
}

func createErrorMessage(statusCode int, body []byte) string {

	var errorDescription string

	switch statusCode {
	case 400:
		errorDescription = "Bad Request (400)"
	case 401:
		errorDescription = "Unauthorized (401)"
	case 403:
		errorDescription = "Forbidden (403)"
	case 404:
		errorDescription = "Not Found (404)"
	case 410:
		errorDescription = "Gone (410)"
	case 500:
		errorDescription = "Internal Server Error (500)"
	}

	var message map[string]interface{}
	json.Unmarshal(body, &message)

	if message["errorId"] != nil {
		errorDescription += ": " + message["errorId"].(string)
	}

	if message["message"] != nil {
		errorDescription += " " + message["message"].(string)
	}
	return errorDescription
}
<commit_msg>adding support for proxy login<commit_after>//the icws_golib package wraps CIC ICWS fuctionality in a GO library.
//To get started, call icws_golib.NewIcws() to instansiate a new Icws
//struct.  Then call icws.Login(...) and continue on from there.
package icws_golib

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"crypto/tls"
)

type Icws struct {
	CurrentToken, CurrentCookie, CurrentSession, CurrentServer, UserId,HttpScheme string
	Port int

}

//Version information for the server
type ServerVersion struct{
	//The product's two-digit release year. For the release "CIC 2015 R1" this value will be "15"
	MajorVersion string
	//The product's release number. For the release "CIC 2015 R1" this value will be "1".
	MinorVersion string
	//The patch number of the release. The value "0" indicates the release without any patches. For the release "CIC 2015 R1" this value will be "0", and for the release "CIC 2015 R1 Patch2" this value will be "2".
	Su string
	//The product line identifier.
	ProductId string
	//The codebase identifier.
	CodeBaseId string
	//The build number.
	Build string
	//The display string for the release. This does not include patch information.
	ProductReleaseDisplayString string
	//The display string for the release including patch information. This string is recommended for use on application "About" screens.
	ProductPatchDisplayString string
}

//Definition of a server feature
type ServerFeature struct{
	//Id of the features
	FeatureId string
	//version of the feature
	Version int
}

//Creates a new ICWS struct
func NewIcws() (icws *Icws) {
	icws = &Icws{}
	icws.HttpScheme = "https"
	icws.Port = 8019
	return
}

func (i *Icws) loginWithData(applicationName, server, username, password string, loginData map[string]string) (err error) {

	urlFormat := "%s://%s:%d"
	server = fmt.Sprintf(urlFormat, i.HttpScheme, server, i.Port)

	log.Printf("Logging into %s with user %s", server, username)

	i.CurrentSession = ""
	i.CurrentServer = server
	body, statusCode, err, cookie := i.httpPostAdvanced("connection", loginData)

	nextServerIndex := 0
	for statusCode == 503 {

		var returnData map[string]interface{}
		json.Unmarshal(body, &returnData)

		nextServer := returnData["alternateHostList"].([]interface{})[0]

		server = fmt.Sprintf(urlFormat, nextServer)

		log.Printf("Redirected to server %s", server)

		i.CurrentServer = server
		body, statusCode, err, cookie = i.httpPostAdvanced("connection", loginData)

		nextServerIndex++
	}

	if err == nil {

		var returnData map[string]string
		json.Unmarshal(body, &returnData)
		i.CurrentToken = returnData["csrfToken"]
		i.CurrentSession = returnData["sessionId"]
		i.CurrentServer = server
		i.CurrentCookie = cookie
		i.UserId = username

	} else {
		log.Printf("ERROR: %s\n", err.Error())
	}
	return
}

//logs into CIC through a reverse proxy
func (i *Icws) ProxyLoginWithData(applicationName, proxyUrl, serverName, username, password string, loginData map[string]string) (err error) {

	urlFormat := "%s://%s/%s"
	server := fmt.Sprintf(urlFormat, i.HttpScheme, proxyUrl, serverName)

	log.Printf("Logging into %s with user %s", server, username)

	i.CurrentSession = ""
	i.CurrentServer = server
	body, statusCode, err, cookie := i.httpPostAdvanced("connection", loginData)

	nextServerIndex := 0
	for statusCode == 503 {

		var returnData map[string]interface{}
		json.Unmarshal(body, &returnData)

		nextServer := returnData["alternateHostList"].([]interface{})[0]

		server = fmt.Sprintf(urlFormat, i.HttpScheme, proxyUrl, nextServer)

		log.Printf("Redirected to server %s", server)

		i.CurrentServer = server
		body, statusCode, err, cookie = i.httpPostAdvanced("connection", loginData)

		nextServerIndex++
	}

	if err == nil {

		var returnData map[string]string
		json.Unmarshal(body, &returnData)
		i.CurrentToken = returnData["csrfToken"]
		i.CurrentSession = returnData["sessionId"]
		i.CurrentServer = server
		i.CurrentCookie = cookie
		i.UserId = username

	} else {
		log.Printf("ERROR: %s\n", err.Error())
	}
	return
}

//logs into a CIC server through a reverse proxy where the url to reach the server would be proxyUrl/serverName
func (i *Icws) ProxyLogin(applicationName, proxyUrl, serverName, username, password string) (err error) {

	var loginData = map[string]string{
		"__type":          "urn:inin.com:connection:icAuthConnectionRequestSettings",
		"applicationName": applicationName,
		"userID":          username,
		"password":        password,
	}

	return i.ProxyLoginWithData(applicationName, proxyUrl, serverName, username, password, loginData)
}

//Logs into a CIC server for a MarketPlace application using the app's custom license.  Server should be a url e.g. https://MyServer:8019
func (i *Icws) ProxyLoginMarketPlaceApp(applicationName, proxyUrl, serverName, username, password, marketplaceLicense, marketplaceAppKey string) (err error) {

	var loginData = map[string]string{
		"__type":          "urn:inin.com:connection:icAuthConnectionRequestSettings",
		"applicationName": applicationName,
		"userID":          username,
		"password":        password,
		"marketPlaceApplicationLicenseName": marketplaceLicense,
		"marketPlaceApplicationCode" : marketplaceAppKey,
	}

	return i.ProxyLoginWithData(applicationName, proxyUrl, serverName, username, password, loginData)
}

//Logs into a CIC server.  Server should be a server name e.g. MyServer.domain.com
func (i *Icws) Login(applicationName, server, username, password string) (err error) {

	var loginData = map[string]string{
		"__type":          "urn:inin.com:connection:icAuthConnectionRequestSettings",
		"applicationName": applicationName,
		"userID":          username,
		"password":        password,
	}

	return i.loginWithData(applicationName, server, username, password, loginData)
}

//Logs into a CIC server for a MarketPlace application using the app's custom license.  Server should be a url e.g. https://MyServer:8019
func (i *Icws) LoginMarketPlaceApp(applicationName, server, username, password, marketplaceLicense, marketplaceAppKey string) (err error) {

	var loginData = map[string]string{
		"__type":          "urn:inin.com:connection:icAuthConnectionRequestSettings",
		"applicationName": applicationName,
		"userID":          username,
		"password":        password,
		"marketPlaceApplicationLicenseName": marketplaceLicense,
		"marketPlaceApplicationCode" : marketplaceAppKey,
	}

	return i.loginWithData(applicationName, server, username, password, loginData)
}

//gets server version details.  Does not need to be connected to the server
func (i *Icws) GetVersion() (version ServerVersion, err error) {

	body, err := i.httpNoConnectionGet("/connection/version")

	err = json.Unmarshal(body, &version)

	return
}

//Gets a list of features and their version from the server
func (i *Icws) GetFeatures() (features []ServerFeature, err error) {

	body, err := i.httpNoConnectionGet("/connection/features")

	if err != nil {
		return nil, err
	}

	var featureMap map[string][]ServerFeature
	err = json.Unmarshal(body, &featureMap)

	features = featureMap["featureInfoList"]
	return

}

func (i *Icws) httpDelete(url string) (err error) {

	req, err := i.httpRequest("DELETE", url, nil)

	if err != nil {
		return
	}

	response, err := i.httpClient().Do(req)
	if err != nil {
		return
	}
	defer response.Body.Close()
	if response.StatusCode == 401 {
		err = errors.New("authorization expired")
		return
	}
	body, _ := ioutil.ReadAll(response.Body)
	if response.StatusCode/100 != 2 {
		err = errors.New(createErrorMessage(response.StatusCode, body))

		return
	}

	return
}

func (i *Icws) httpGet(url string) (body []byte, err error) {

	req, err := i.httpRequest("GET", url, nil)

	if err != nil {
		return
	}

	response, err := i.httpClient().Do(req)
	if err != nil {
		return
	}
	defer response.Body.Close()
	if response.StatusCode == 401 {
		err = errors.New("authorization expired")
		return
	}
	body, err = ioutil.ReadAll(response.Body)
	if response.StatusCode/100 != 2 {
		err = errors.New(createErrorMessage(response.StatusCode, body))

		return
	}

	return
}

func (i *Icws) httpNoConnectionGet(url string) (body []byte, err error) {

	req, err := i.httpNoConnectionRequest("GET", url, nil)

	if err != nil {
		return
	}

	response, err := i.httpClient().Do(req)
	if err != nil {
		return
	}
	defer response.Body.Close()
	if response.StatusCode == 401 {
		err = errors.New("authorization expired")
		return
	}
	body, err = ioutil.ReadAll(response.Body)
	if response.StatusCode/100 != 2 {
		err = errors.New(createErrorMessage(response.StatusCode, body))

		return
	}

	return
}

func (i *Icws) httpPost(url string, attrs map[string]string) (body []byte, err error) {
	body, _, err, _ = i.httpPostAdvanced(url, attrs)
	return
}

func (i *Icws) httpPostAdvanced(url string, attrs map[string]string) (body []byte, statusCode int, err error, cookie string) {

	rbody, _ := json.Marshal(attrs)
	req, err := i.httpRequest("POST", url, bytes.NewReader(rbody))
	if err != nil {
		return
	}

	response, err := i.httpClient().Do(req)
	if err != nil {
		return

	}
	defer response.Body.Close()

	statusCode = response.StatusCode

	if response.StatusCode == 401 {
		err = errors.New("authorization expired")
		return
	}
	body, err = ioutil.ReadAll(response.Body)

	if response.StatusCode/100 != 2 {
		err = errors.New(createErrorMessage(response.StatusCode, body))

		return
	}

	if response.Header["Set-Cookie"] != nil {
		cookie = response.Header["Set-Cookie"][0]
	}
	return
}

func (i *Icws) httpPut(url string, attrs map[string]string) (body []byte, err error) {

	rbody, _ := json.Marshal(attrs)
	req, err := i.httpRequest("PUT", url, bytes.NewReader(rbody))
	if err != nil {
		return
	}

	response, err := i.httpClient().Do(req)
	if err != nil {
		return

	}
	defer response.Body.Close()
	if response.StatusCode == 401 {
		err = errors.New("authorization expired")
		return
	}
	body, err = ioutil.ReadAll(response.Body)

	if response.StatusCode/100 != 2 {
		err = errors.New(createErrorMessage(response.StatusCode, body))

		return
	}

	return
}

func (i *Icws) httpClient() (client *http.Client) {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify : true},
	}
	client = &http.Client{Transport: tr}
	return
}

func (i *Icws) httpNoConnectionRequest(method, url string, body io.Reader) (request *http.Request, err error) {
	request, err = http.NewRequest(method, i.CurrentServer+"/icws"+url, body)
	if err != nil {
		return
	}

	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Accept-Language", "en-us")

	return
}

func (i *Icws) httpRequest(method, url string, body io.Reader) (request *http.Request, err error) {
	request, err = http.NewRequest(method, i.CurrentServer+"/icws/"+i.CurrentSession+url, body)
	if err != nil {
		return
	}

	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Accept-Language", "en-us")

	if len(i.CurrentCookie) > 0 {
		request.Header.Add("Cookie", i.CurrentCookie)
	} else {
		return
	}

	if len(i.CurrentToken) > 0 {
		request.Header.Add("ININ-ICWS-CSRF-Token", i.CurrentToken)
	} else {
		return
	}

	// request.Header.Add("User-Agent", fmt.Sprintf("cic cli (%s-%s)", runtime.GOOS, runtime.GOARCH))
	return
}

func createErrorMessage(statusCode int, body []byte) string {

	var errorDescription string

	switch statusCode {
	case 400:
		errorDescription = "Bad Request (400)"
	case 401:
		errorDescription = "Unauthorized (401)"
	case 403:
		errorDescription = "Forbidden (403)"
	case 404:
		errorDescription = "Not Found (404)"
	case 410:
		errorDescription = "Gone (410)"
	case 500:
		errorDescription = "Internal Server Error (500)"
	}

	var message map[string]interface{}
	json.Unmarshal(body, &message)

	if message["errorId"] != nil {
		errorDescription += ": " + message["errorId"].(string)
	}

	if message["message"] != nil {
		errorDescription += " " + message["message"].(string)
	}
	return errorDescription
}
<|endoftext|>


<commit_before>// Copyright 2015 Bobby Powers. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package tmux

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"
	"sync"
)

const (
	ImsgfHasFD = 1

	IbufReadLen = 65535
	MaxImsgLen = 16384
)

var (
	imsgHeaderLen = (&ImsgHeader{}).WireLen()
)

type WireSerializer interface {
	InitFromWireBytes([]byte) error
	WireBytes([]byte) error
	WireLen() int
}

type ImsgBuffer struct {
	conn   io.ReadWriteCloser
	mu     sync.Mutex
	wQueue [][]byte
	// Linux kernel defines pid_t as a 32-bit signed int in
	// include/uapi/asm-generic/posix_types.h
	pid int32
}

// ImsgHeader describes the current message.
type ImsgHeader struct {
	Type   uint32
	Len    uint16
	Flags  uint16
	PeerID uint32
	Pid    uint32
}

// header length is used often, calc it once
func (ihdr *ImsgHeader) WireLen() int {
	return 16
}

func (ihdr *ImsgHeader) WireBytes(buf []byte) error {
	var err error
	if len(buf) < 16 {
		return fmt.Errorf("ImsgHeader 1 bad len %d/16", len(buf))
	}
	bb := bytes.NewBuffer(buf)

	if err = binary.Write(bb, binary.LittleEndian, ihdr.Type); err != nil {
		return fmt.Errorf("Write(Type): %s", err)
	}
	if err = binary.Write(bb, binary.LittleEndian, ihdr.Len); err != nil {
		return fmt.Errorf("Write(Len): %s", err)
	}
	if err = binary.Write(bb, binary.LittleEndian, ihdr.Flags); err != nil {
		return fmt.Errorf("Write(Flags): %s", err)
	}
	if err = binary.Write(bb, binary.LittleEndian, ihdr.PeerID); err != nil {
		return fmt.Errorf("Write(PeerID): %s", err)
	}
	if err = binary.Write(bb, binary.LittleEndian, ihdr.Pid); err != nil {
		return fmt.Errorf("Write(Pid): %s", err)
	}
	return nil
}

func (ihdr *ImsgHeader) InitFromWireBytes(buf []byte) error {
	if len(buf) != 16 {
		return fmt.Errorf("ImsgHeader 2 bad len %d/16", len(buf))
	}
	ihdr.Type = binary.LittleEndian.Uint32(buf[0:4])
	ihdr.Len = binary.LittleEndian.Uint16(buf[4:6])
	ihdr.Flags = binary.LittleEndian.Uint16(buf[6:8])
	ihdr.PeerID = binary.LittleEndian.Uint32(buf[8:12])
	ihdr.Pid = binary.LittleEndian.Uint32(buf[12:16])
	return nil
}

func NewImsgBuffer(conn io.ReadWriteCloser) *ImsgBuffer {
	return &ImsgBuffer{
		conn: conn,
		pid:  int32(os.Getpid()),
	}
}

func (ibuf *ImsgBuffer) Compose(kind, peerID, pid uint32, data WireSerializer) error {
	var err error
	size := imsgHeaderLen + data.WireLen()
	header := ImsgHeader{
		Type:   kind,
		Len:    uint16(size),
		PeerID: peerID,
		Pid:    pid,
	}
	if header.Pid == 0 {
		header.Pid = uint32(ibuf.pid)
	}

	buf := make([]byte, size)

	// TODO: rights/send FD

	if err = header.WireBytes(buf[0:imsgHeaderLen]); err != nil {
		return fmt.Errorf("header.WireBytes: %s", err)
	}
	if err = data.WireBytes(buf[imsgHeaderLen:]); err != nil {
		return fmt.Errorf("header.WireBytes: %s", err)
	}

	ibuf.mu.Lock()
	ibuf.wQueue = append(ibuf.wQueue, buf)
	ibuf.mu.Unlock()

	return nil
}

// Flush writes all pending buffers to the socket
func (ibuf *ImsgBuffer) Flush() {
	ibuf.mu.Lock()
	defer ibuf.mu.Unlock()

	for _, buf := range ibuf.wQueue {
		n, err := ibuf.conn.Write(buf)
		if err != nil {
			log.Printf("ibuf.conn.Write: %s", err)
		} else if n != len(buf) {
			log.Printf("ibuf.conn.Write short: %d/%d", n, len(buf))
		}
	}
	ibuf.wQueue = nil
}
<commit_msg>work on imsg<commit_after>// Copyright 2015 Bobby Powers. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package tmux

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"log"
	"net"
	"os"
	"sync"
)

const (
	ImsgfHasFD = 1

	IbufReadLen = 65535
	MaxImsgLen = 16384
)

var (
	imsgHeaderLen = (&ImsgHeader{}).WireLen()
)

type WireSerializer interface {
	InitFromWireBytes([]byte) error
	WireBytes([]byte) error
	WireLen() int
}

type Imsg struct {
	Header ImsgHeader
	Data []byte
}

type ImsgBuffer struct {
	conn   *net.UnixConn
	mu     sync.Mutex
	wQueue [][]byte
	// Linux kernel defines pid_t as a 32-bit signed int in
	// include/uapi/asm-generic/posix_types.h
	pid int32
}

// ImsgHeader describes the current message.
type ImsgHeader struct {
	Type   uint32
	Len    uint16
	Flags  uint16
	PeerID uint32
	Pid    uint32
}

// header length is used often, calc it once
func (ihdr *ImsgHeader) WireLen() int {
	return 16
}

func (ihdr *ImsgHeader) WireBytes(buf []byte) error {
	var err error
	if len(buf) < 16 {
		return fmt.Errorf("ImsgHeader 1 bad len %d/16", len(buf))
	}
	bb := bytes.NewBuffer(buf)

	if err = binary.Write(bb, binary.LittleEndian, ihdr.Type); err != nil {
		return fmt.Errorf("Write(Type): %s", err)
	}
	if err = binary.Write(bb, binary.LittleEndian, ihdr.Len); err != nil {
		return fmt.Errorf("Write(Len): %s", err)
	}
	if err = binary.Write(bb, binary.LittleEndian, ihdr.Flags); err != nil {
		return fmt.Errorf("Write(Flags): %s", err)
	}
	if err = binary.Write(bb, binary.LittleEndian, ihdr.PeerID); err != nil {
		return fmt.Errorf("Write(PeerID): %s", err)
	}
	if err = binary.Write(bb, binary.LittleEndian, ihdr.Pid); err != nil {
		return fmt.Errorf("Write(Pid): %s", err)
	}
	return nil
}

func (ihdr *ImsgHeader) InitFromWireBytes(buf []byte) error {
	if len(buf) != 16 {
		return fmt.Errorf("ImsgHeader 2 bad len %d/16", len(buf))
	}
	ihdr.Type = binary.LittleEndian.Uint32(buf[0:4])
	ihdr.Len = binary.LittleEndian.Uint16(buf[4:6])
	ihdr.Flags = binary.LittleEndian.Uint16(buf[6:8])
	ihdr.PeerID = binary.LittleEndian.Uint32(buf[8:12])
	ihdr.Pid = binary.LittleEndian.Uint32(buf[12:16])
	return nil
}

func NewImsgBuffer(path string) (*ImsgBuffer, error) {
	addr, err := net.ResolveUnixAddr("unix", path)
	if err != nil {
		return nil, fmt.Errorf("ResolveAddrUnix(%s): %s", path, err)
	}
	conn, err := net.DialUnix("unix", nil, addr)
	if err != nil {
		return nil, fmt.Errorf("DialUnix(%s): %s", path, err)
	}
	ibuf := &ImsgBuffer{
		conn: conn,
		pid:  int32(os.Getpid()),
	}
	go ibuf.read()

	return ibuf, nil
}

func (ibuf *ImsgBuffer) Compose(kind, peerID, pid uint32, data WireSerializer) error {
	var err error
	size := imsgHeaderLen + data.WireLen()
	header := ImsgHeader{
		Type:   kind,
		Len:    uint16(size),
		PeerID: peerID,
		Pid:    pid,
	}
	if header.Pid == 0 {
		header.Pid = uint32(ibuf.pid)
	}

	buf := make([]byte, size)

	// TODO: rights/send FD

	if err = header.WireBytes(buf[0:imsgHeaderLen]); err != nil {
		return fmt.Errorf("header.WireBytes: %s", err)
	}
	if err = data.WireBytes(buf[imsgHeaderLen:]); err != nil {
		return fmt.Errorf("header.WireBytes: %s", err)
	}

	ibuf.mu.Lock()
	ibuf.wQueue = append(ibuf.wQueue, buf)
	ibuf.mu.Unlock()

	return nil
}

// Flush writes all pending buffers to the socket
func (ibuf *ImsgBuffer) Flush() {
	ibuf.mu.Lock()
	defer ibuf.mu.Unlock()

	for _, buf := range ibuf.wQueue {
		n, _, err := ibuf.conn.WriteMsgUnix(buf, nil, nil)
		if err != nil {
			log.Printf("ibuf.conn.Write: %s", err)
		} else if n != len(buf) {
			log.Printf("ibuf.conn.Write short: %d/%d", n, len(buf))
		}
	}
	ibuf.wQueue = nil
}

func (ibuf *ImsgBuffer) Get() (*Imsg, error) {

	// read header

	// check len

	// TODO: FD passing

	// get data

	return nil, nil
}

func (ibuf *ImsgBuffer) read() int {
	return -1
}
<|endoftext|>


<commit_before>package vcfgo

import (
	"bytes"
	"fmt"
	"strconv"
	"strings"
)

type InfoByte struct {
	Info []byte

	header *Header
}

func NewInfoByte(info []byte, h *Header) *InfoByte {
	return &InfoByte{Info: info, header: h}
}

// return the start and end positions of the value.
// for flag the value is the flag.
func getpositions(info []byte, key string) (start, end int) {
	bkey := []byte(key)
	pos := 0
	for {
		if pos >= len(info) {
			return -1, -1
		}
		ipos := bytes.Index(info[pos:], bkey)
		if ipos == -1 {
			return -1, -1
		}
		pos += ipos
		if pos != 0 && info[pos-1] != ';' {
			pos += 1
			continue
		}
		eq := pos + bytes.IndexByte(info[pos:], '=')
		// at end of field and we found an Flag
		var semi int
		if eq == -1 {
			return pos, len(info)
		} else if eq-pos != len(bkey) {
			// found a longer key with same prefix.
			semi = bytes.IndexByte(info[pos:], ';')
			// flag field
			if semi == -1 {
				semi = len(info)
			} else {
				semi += pos
			}
			if semi-pos == len(bkey) {
				return pos, semi - 1
			}
			pos = semi + 1
			continue
		} else {
			semi = bytes.IndexByte(info[pos+1:], byte(';'))
		}
		if semi > -1 && eq > pos+semi {
			// should be a flag.
			return pos, pos + semi
		}

		// not at end of info field
		if semi != -1 {
			semi += pos
		}
		return eq + 1, semi
	}
}

func (i InfoByte) Contains(key string) bool {
	// short-circuit common case.
	if !bytes.Contains(i.Info, []byte(key+"=")) {
		return false
	}
	s, _ := getpositions(i.Info, key)
	return s != -1
}

func (i InfoByte) Keys() []string {
	sp := bytes.Split(i.Info, []byte{';'})
	keys := make([]string, 0, len(sp))
	for _, pair := range sp {
		key := bytes.SplitN(pair, []byte{'='}, 2)[0]
		keys = append(keys, string(key))
	}
	return keys

}

func (i *InfoByte) Delete(key string) {
	s, e := getpositions(i.Info, key)
	if s == -1 {
		return
	}
	// check if it's a flag
	if s != 0 && i.Info[s-1] != ';' {
		s -= (len(key) + 1)
	}
	if s < 0 {
		s = 0
	}
	if e == -1 {
		e = len(i.Info)
	} else {
		e += 2
	}
	if s == 0 && e == len(i.Info) {
		i.Info = i.Info[:0]
	} else if e < len(i.Info) {
		i.Info = append(i.Info[:s], i.Info[e:]...)
	} else {
		i.Info = i.Info[:s-1]
	}
}

func ItoS(k string, v interface{}) string {
	if _, ok := v.(bool); ok {
		return k
	} else {
		switch v.(type) {
		case float32:
			return fmtFloat32(v.(float32))
		case float64:
			return fmtFloat64(v.(float64))
		case int:
			return fmt.Sprintf("%d", v.(int))
		case uint32:
			return fmt.Sprintf("%d", v.(uint32))
		case []float64:
			vals := v.([]float64)
			svals := make([]string, len(vals))
			for i, val := range vals {
				svals[i] = fmtFloat64(float64(val))
			}
			return strings.Join(svals, ",")
		case []float32:
			vals := v.([]float32)
			svals := make([]string, len(vals))
			for i, val := range vals {
				svals[i] = fmtFloat32(float32(val))
			}
			return strings.Join(svals, ",")
		case []int:
			vals := v.([]int)
			svals := make([]string, len(vals))
			for i, val := range vals {
				svals[i] = strconv.Itoa(int(val))
			}
			return strings.Join(svals, ",")
		case []int32:
			vals := v.([]int32)
			svals := make([]string, len(vals))
			for i, val := range vals {
				svals[i] = strconv.Itoa(int(int32(val)))
			}
			return strings.Join(svals, ",")
		case []string:
			return strings.Join(v.([]string), ",")

		default:
			return v.(string)
		}
	}
}

func (i InfoByte) SGet(key string) []byte {
	var sub []byte
	if key == "" || len(i.Info) == 1 {
		return sub
	}
	start, end := getpositions(i.Info, key)
	if start == -1 {
		return sub
	}
	if end == -1 {
		end = len(i.Info) - 1
	}
	val := i.Info[start : end+1]
	return val
}

// Get a value from the bytes typed according to the header.
func (i InfoByte) Get(key string) (interface{}, error) {
	v := string(i.SGet(key))
	skey := string(key)
	var ok bool
	var hi *Info
	if i.header == nil || i.header.Infos == nil {
		ok = false
	} else {
		i.header.RLock()
		hi, ok = i.header.Infos[key]
		i.header.RUnlock()
	}
	if !ok {
		err := fmt.Errorf("Info Error: %s not found in header", skey)
		// flag
		if skey == v {
			return true, err
		}
		return v, err
	}

	if len(v) == 0 {
		var err error
		var val interface{} = nil
		if hi.Type != "Flag" {
			err = fmt.Errorf("Info Error: %s not found in row", skey)
		} else {
			val = false
		}
		return val, err
	}
	if v == key {
		var err error
		if hi.Type != "Flag" {
			err = fmt.Errorf("Info Error: flag field (%s) should be specified as such in the header", skey)
		}
		return true, err
	}

	var err error
	var iv interface{}
	if hi.Number == "1" || (hi.Number == "." && strings.IndexByte(v, ',') == -1) {
		return parseOne(skey, v, hi.Type)
	}

	switch hi.Number {

	case "0":
		if hi.Type != "Flag" {
			err = fmt.Errorf("Info Error: flag field (%s) should have Number=0", skey)
		}
		return true, err

	case "R", "A", "G", "2", "3", ".":
		vals := strings.Split(v, ",")
		var vi interface{}
		switch hi.Type {
		case "Integer":
			vi = make([]int, len(vals))
			for j, val := range vals {
				iv, err = parseOne(skey, val, hi.Type)
				vi.([]int)[j] = iv.(int)
			}
		case "Float":
			vi = make([]float32, len(vals))
			for j, val := range vals {
				iv, err = parseOne(skey, val, hi.Type)
				if o, ok := iv.(float32); ok {
					vi.([]float32)[j] = o
				} else {
					vi.([]float32)[j] = float32(iv.(float64))
				}

			}
		case "String":
			vi = make([]string, len(vals))
			for j, val := range vals {
				iv, err = parseOne(skey, val, hi.Type)
				vi.([]string)[j] = iv.(string)
			}
		default:
			vi = make([]interface{}, len(vals))
			for j, val := range vals {
				iv, err = parseOne(skey, val, hi.Type)
				vi.([]interface{})[j] = iv
			}
		}
		return vi, nil

	default:
		vals := strings.Split(v, ",")
		var vi interface{} = make([]interface{}, len(vals))
		if _, err := strconv.Atoi(hi.Number); err == nil {
			for j, val := range vals {
				iv, err = parseOne(skey, val, hi.Type)
				vi.([]interface{})[j] = iv
			}
		}
		return vi, err

	}
}

func (i InfoByte) String() string {
	return string(i.Info)
}

func (i InfoByte) Bytes() []byte {
	return i.Info
}

func (i *InfoByte) UpdateHeader(key string, value interface{}) {
	i.header.Lock()
	defer i.header.Unlock()
	if i.header != nil {
		switch value.(type) {
		case bool:
			i.header.Infos[key] = &Info{Id: key, Description: key, Number: "0", Type: "Flag"}
		case string:
			i.header.Infos[key] = &Info{Id: key, Description: key, Number: "1", Type: "Character"}
		case int, int32, int64, uint32, uint64:
			i.header.Infos[key] = &Info{Id: key, Description: key, Number: "1", Type: "Integer"}
		case float32, float64:
			i.header.Infos[key] = &Info{Id: key, Description: key, Number: "1", Type: "Float"}
		case []interface{}:
			v := value.([]interface{})[0]
			i.UpdateHeader(key, v)
		}
	}
}

func (i *InfoByte) Set(key string, value interface{}) error {
	if len(i.Info) == 0 {
		if v, ok := value.(bool); ok {
			if v {
				i.Info = []byte(key)
			}
		} else {
			i.Info = []byte(fmt.Sprintf("%s=%s", key, ItoS(key, value)))
		}
		return nil
	}
	s, e := getpositions(i.Info, key)
	if s == -1 || s == len(i.Info) {
		if b, ok := value.(bool); ok {
			if b {
				i.Info = append(i.Info, ';')
				i.Info = append(i.Info, key...)
			}
			return nil
		}
		slug := []byte(fmt.Sprintf(";%s=%s", key, ItoS(key, value)))
		i.Info = append(i.Info, slug...)
		//i.UpdateHeader(key, value)
		return nil
	}
	if b, ok := value.(bool); ok {
		if !b {
			i.Delete(key)
		}
		return nil
	}
	slug := []byte(ItoS(key, value))
	if e == -1 {
		i.Info = append(i.Info[:s], slug...)
	} else {
		i.Info = append(i.Info[:s], append(slug, i.Info[e+1:]...)...)
	}
	return nil
}

func (i *InfoByte) Add(key string, value interface{}) {
	i.Set(key, value)
}
<commit_msg>special case the most common case for finding a key in an info string<commit_after>package vcfgo

import (
	"bytes"
	"fmt"
	"strconv"
	"strings"
)

type InfoByte struct {
	Info []byte

	header *Header
}

func NewInfoByte(info []byte, h *Header) *InfoByte {
	return &InfoByte{Info: info, header: h}
}

/*
func bytesToString(b []byte) string {
	bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
	sh := reflect.StringHeader{bh.Data, bh.Len}
	return *(*string)(unsafe.Pointer(&sh))
}
*/

// return the start and end positions of the value.
// for flag the value is the flag.
func getpositions(info []byte, key string) (start, end int) {
	//if ipos := strings.LastIndex(bytesToString(info), ";"+key+"="); ipos != -1 {
	if ipos := bytes.Index(info, []byte(";"+key+"=")); ipos != -1 {
		start = ipos + 2 + len(key)
		for end := start + 1; end < len(info)-1; end++ {
			if info[end] == ';' {
				return start, end - 1
			}
		}
		return start, len(info) - 1
	}

	bkey := []byte(key)
	ipos, pos := 0, 0

	for {
		if pos >= len(info) {
			return -1, -1
		}
		if pos == 0 {
			ipos = bytes.Index(info, bkey)
		} else {
			ipos = bytes.Index(info[pos:], bkey)
		}

		if ipos == -1 {
			return -1, -1
		}
		pos += ipos
		if pos != 0 && info[pos-1] != ';' {
			pos += 1
			continue
		}
		eq := pos + bytes.IndexByte(info[pos:], '=')
		// at end of field and we found an Flag
		var semi int
		if eq == -1 {
			return pos, len(info)
		} else if eq-pos != len(bkey) {
			// found a longer key with same prefix.
			semi = bytes.IndexByte(info[pos:], ';')
			// flag field
			if semi == -1 {
				semi = len(info)
			} else {
				semi += pos
			}
			if semi-pos == len(bkey) {
				return pos, semi - 1
			}
			pos = semi + 1
			continue
		} else {
			semi = bytes.IndexByte(info[pos+1:], byte(';'))
		}
		if semi > -1 && eq > pos+semi {
			// should be a flag.
			return pos, pos + semi
		}

		// not at end of info field
		if semi != -1 {
			semi += pos
		}
		return eq + 1, semi
	}
}

func (i InfoByte) Contains(key string) bool {
	// short-circuit common case.
	if !bytes.Contains(i.Info, []byte(key+"=")) {
		return false
	}
	s, _ := getpositions(i.Info, key)
	return s != -1
}

func (i InfoByte) Keys() []string {
	sp := bytes.Split(i.Info, []byte{';'})
	keys := make([]string, 0, len(sp))
	for _, pair := range sp {
		key := bytes.SplitN(pair, []byte{'='}, 2)[0]
		keys = append(keys, string(key))
	}
	return keys

}

func (i *InfoByte) Delete(key string) {
	s, e := getpositions(i.Info, key)
	if s == -1 {
		return
	}
	// check if it's a flag
	if s != 0 && i.Info[s-1] != ';' {
		s -= (len(key) + 1)
	}
	if s < 0 {
		s = 0
	}
	if e == -1 {
		e = len(i.Info)
	} else {
		e += 2
	}
	if s == 0 && e == len(i.Info) {
		i.Info = i.Info[:0]
	} else if e < len(i.Info) {
		i.Info = append(i.Info[:s], i.Info[e:]...)
	} else {
		i.Info = i.Info[:s-1]
	}
}

func ItoS(k string, v interface{}) string {
	if _, ok := v.(bool); ok {
		return k
	} else {
		switch v.(type) {
		case float32:
			return fmtFloat32(v.(float32))
		case float64:
			return fmtFloat64(v.(float64))
		case int:
			return fmt.Sprintf("%d", v.(int))
		case uint32:
			return fmt.Sprintf("%d", v.(uint32))
		case []float64:
			vals := v.([]float64)
			svals := make([]string, len(vals))
			for i, val := range vals {
				svals[i] = fmtFloat64(float64(val))
			}
			return strings.Join(svals, ",")
		case []float32:
			vals := v.([]float32)
			svals := make([]string, len(vals))
			for i, val := range vals {
				svals[i] = fmtFloat32(float32(val))
			}
			return strings.Join(svals, ",")
		case []int:
			vals := v.([]int)
			svals := make([]string, len(vals))
			for i, val := range vals {
				svals[i] = strconv.Itoa(int(val))
			}
			return strings.Join(svals, ",")
		case []int32:
			vals := v.([]int32)
			svals := make([]string, len(vals))
			for i, val := range vals {
				svals[i] = strconv.Itoa(int(int32(val)))
			}
			return strings.Join(svals, ",")
		case []string:
			return strings.Join(v.([]string), ",")

		default:
			return v.(string)
		}
	}
}

func (i InfoByte) SGet(key string) []byte {
	var sub []byte
	if key == "" || len(i.Info) == 1 {
		return sub
	}
	start, end := getpositions(i.Info, key)
	if start == -1 {
		return sub
	}
	if end == -1 {
		end = len(i.Info) - 1
	}
	val := i.Info[start : end+1]
	return val
}

// Get a value from the bytes typed according to the header.
func (i InfoByte) Get(key string) (interface{}, error) {
	v := string(i.SGet(key))
	skey := string(key)
	var ok bool
	var hi *Info
	if i.header == nil || i.header.Infos == nil {
		ok = false
	} else {
		i.header.RLock()
		hi, ok = i.header.Infos[key]
		i.header.RUnlock()
	}
	if !ok {
		err := fmt.Errorf("Info Error: %s not found in header", skey)
		// flag
		if skey == v {
			return true, err
		}
		return v, err
	}

	if len(v) == 0 {
		var err error
		var val interface{} = nil
		if hi.Type != "Flag" {
			err = fmt.Errorf("Info Error: %s not found in row", skey)
		} else {
			val = false
		}
		return val, err
	}
	if v == key {
		var err error
		if hi.Type != "Flag" {
			err = fmt.Errorf("Info Error: flag field (%s) should be specified as such in the header", skey)
		}
		return true, err
	}

	var err error
	var iv interface{}
	if hi.Number == "1" || (hi.Number == "." && strings.IndexByte(v, ',') == -1) {
		return parseOne(skey, v, hi.Type)
	}

	switch hi.Number {

	case "0":
		if hi.Type != "Flag" {
			err = fmt.Errorf("Info Error: flag field (%s) should have Number=0", skey)
		}
		return true, err

	case "R", "A", "G", "2", "3", ".":
		vals := strings.Split(v, ",")
		var vi interface{}
		switch hi.Type {
		case "Integer":
			vi = make([]int, len(vals))
			for j, val := range vals {
				iv, err = parseOne(skey, val, hi.Type)
				vi.([]int)[j] = iv.(int)
			}
		case "Float":
			vi = make([]float32, len(vals))
			for j, val := range vals {
				iv, err = parseOne(skey, val, hi.Type)
				if o, ok := iv.(float32); ok {
					vi.([]float32)[j] = o
				} else {
					vi.([]float32)[j] = float32(iv.(float64))
				}

			}
		case "String":
			vi = make([]string, len(vals))
			for j, val := range vals {
				iv, err = parseOne(skey, val, hi.Type)
				vi.([]string)[j] = iv.(string)
			}
		default:
			vi = make([]interface{}, len(vals))
			for j, val := range vals {
				iv, err = parseOne(skey, val, hi.Type)
				vi.([]interface{})[j] = iv
			}
		}
		return vi, nil

	default:
		vals := strings.Split(v, ",")
		var vi interface{} = make([]interface{}, len(vals))
		if _, err := strconv.Atoi(hi.Number); err == nil {
			for j, val := range vals {
				iv, err = parseOne(skey, val, hi.Type)
				vi.([]interface{})[j] = iv
			}
		}
		return vi, err

	}
}

func (i InfoByte) String() string {
	return string(i.Info)
}

func (i InfoByte) Bytes() []byte {
	return i.Info
}

func (i *InfoByte) UpdateHeader(key string, value interface{}) {
	i.header.Lock()
	defer i.header.Unlock()
	if i.header != nil {
		switch value.(type) {
		case bool:
			i.header.Infos[key] = &Info{Id: key, Description: key, Number: "0", Type: "Flag"}
		case string:
			i.header.Infos[key] = &Info{Id: key, Description: key, Number: "1", Type: "Character"}
		case int, int32, int64, uint32, uint64:
			i.header.Infos[key] = &Info{Id: key, Description: key, Number: "1", Type: "Integer"}
		case float32, float64:
			i.header.Infos[key] = &Info{Id: key, Description: key, Number: "1", Type: "Float"}
		case []interface{}:
			v := value.([]interface{})[0]
			i.UpdateHeader(key, v)
		}
	}
}

func (i *InfoByte) Set(key string, value interface{}) error {
	if len(i.Info) == 0 {
		if v, ok := value.(bool); ok {
			if v {
				i.Info = []byte(key)
			}
		} else {
			i.Info = []byte(fmt.Sprintf("%s=%s", key, ItoS(key, value)))
		}
		return nil
	}
	s, e := getpositions(i.Info, key)
	if s == -1 || s == len(i.Info) {
		if b, ok := value.(bool); ok {
			if b {
				i.Info = append(i.Info, ';')
				i.Info = append(i.Info, key...)
			}
			return nil
		}
		slug := fmt.Sprintf(";%s=%s", key, ItoS(key, value))
		i.Info = append(i.Info, slug...)
		//i.UpdateHeader(key, value)
		return nil
	}
	if b, ok := value.(bool); ok {
		if !b {
			i.Delete(key)
		}
		return nil
	}
	slug := []byte(ItoS(key, value))
	if e == -1 {
		i.Info = append(i.Info[:s], slug...)
	} else {
		i.Info = append(i.Info[:s], append(slug, i.Info[e+1:]...)...)
	}
	return nil
}

func (i *InfoByte) Add(key string, value interface{}) {
	i.Set(key, value)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

type info struct {
	os.FileInfo
	path  string
	depth int
	err   error
	infos []*info
}

//Think/Write pusodue code
//what is the right way to write this function...
func (inf *info) visit(opts *options) (dirs, files int) {
	fi, err := os.Stat(inf.path)
	if err != nil {
		inf.err = err
		return
	}
	inf.FileInfo = fi
	if !fi.IsDir() {
		return 0, 1
	}
	dir, err := os.Open(inf.path)
	if err != nil {
		inf.err = err
		return
	}
	names, err := dir.Readdirnames(-1)
	dir.Close()
	if err != nil {
		inf.err = err
		return
	}
	inf.infos = make([]*info, 0)
	for _, name := range names {
		if !opts.all && strings.HasPrefix(name, ".") {
			continue
		}
		_inf := &info{
			path:  filepath.Join(inf.path, name),
			depth: inf.depth + 1,
		}
		inf.infos = append(inf.infos, _inf)
		d, f := _inf.visit(opts)
		dirs, files = dirs+d, files+f
	}
	return dirs + 1, files
}

func (inf *info) print(indent string) {
	if inf.err != nil {
		err := strings.Split(inf.err.Error(), ": ")[1]
		fmt.Printf("%s [%s]\n", inf.path, err)
		return
	}
	if inf.depth == 0 {
		fmt.Println(inf.path)
	} else {
		fmt.Println(inf.Name())
	}
	add := "│   "
	for i, _inf := range inf.infos {
		if i == len(inf.infos)-1 {
			fmt.Printf(indent + "└── ")
			add = "    "
		} else {
			fmt.Printf(indent + "├── ")
		}
		_inf.print(indent + add)
	}
}
<commit_msg>feat(info): dirsOnly options<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

type info struct {
	os.FileInfo
	path  string
	depth int
	err   error
	infos []*info
}

//Think/Write pusodue code
//what is the right way to write this function...
func (inf *info) visit(opts *options) (dirs, files int) {
	fi, err := os.Stat(inf.path)
	if err != nil {
		inf.err = err
		return
	}
	inf.FileInfo = fi
	if !fi.IsDir() {
		return 0, 1
	}
	dir, err := os.Open(inf.path)
	if err != nil {
		inf.err = err
		return
	}
	names, err := dir.Readdirnames(-1)
	dir.Close()
	if err != nil {
		inf.err = err
		return
	}
	inf.infos = make([]*info, 0)
	for _, name := range names {
		// "all" option
		if !opts.all && strings.HasPrefix(name, ".") {
			continue
		}
		_inf := &info{
			path:  filepath.Join(inf.path, name),
			depth: inf.depth + 1,
		}
		d, f := _inf.visit(opts)
		// "dirs only" option
		if opts.dirsOnly && !_inf.IsDir() {
			continue
		}
		inf.infos = append(inf.infos, _inf)
		dirs, files = dirs+d, files+f
	}
	return dirs + 1, files
}

func (inf *info) print(indent string) {
	if inf.err != nil {
		err := strings.Split(inf.err.Error(), ": ")[1]
		fmt.Printf("%s [%s]\n", inf.path, err)
		return
	}
	if inf.depth == 0 {
		fmt.Println(inf.path)
	} else {
		fmt.Println(inf.Name())
	}
	add := "│   "
	for i, _inf := range inf.infos {
		if i == len(inf.infos)-1 {
			fmt.Printf(indent + "└── ")
			add = "    "
		} else {
			fmt.Printf(indent + "├── ")
		}
		_inf.print(indent + add)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/build"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

// list runs the list subcommand, listing all the dependencies of the package
// at the specified path, relative paths are resolved from the current working
// directory.
func list(ctx *build.Context, cwd, path string) error {
	pkg, err := getPackage(ctx, path)
	if err != nil {
		return err
	}
	// List imports
	imp := getImports(pkg, opt.tests)
	fimp := filterImports(ctx, cwd, pkg.ImportPath, imp, opt.standard, opt.child)
	for _, v := range fimp {
		info(ctx, cwd, v)
	}
	return nil
}

// info runs the info subcommand, printing information about a given package.
// Also used by the list command to output details about imports, the quite and
// verbose flags determine the output.
func info(ctx *build.Context, cwd, path string) error {
	if opt.quite {
		fmt.Println(path)
		return nil
	}
	pkg, err := getPackage(ctx, path)
	if err != nil {
		return err
	}
	// Default output
	printBold(pkg.ImportPath)
	// Print package doc with line breaks
	if len(pkg.Doc) > 0 {
		printWrap(72, pkg.Doc)
	} else {
		fmt.Println("No package documentation.")
	}
	// Verbose output
	if opt.verbose {
		fmt.Println()
		fmt.Println("  Standard : ", pkg.Goroot)
		fmt.Println("  Directory : ", pkg.Dir)
		if len(pkg.AllTags) > 0 {
			fmt.Println("  Tags : ",
				strings.Join(pkg.AllTags, " "))
		}
	}
	fmt.Println()
	return nil
}

// filterImports filters the imports by either ommitting or including standard
// and child packages, returns filtered slice of import paths.
// TODO rewrite filtering this isn't a good way to do this.
func filterImports(ctx *build.Context, cwd, parent string, imp []string, std, child bool) []string {
	r := make([]string, 0, len(imp))
	for _, v := range imp {
		// Filter child packages
		if !child && isChildPackage(parent, v) {
			continue
		}
		// Filter standard packages
		if !std && isStandardPackage(ctx, cwd, v) {
			continue
		}
		r = append(r, v)
	}
	return r
}

// isChildPackage checks if the child package is stationed in a subdirectory of
// the parent package. Pass in the import paths for both the parent and the
// child.
func isChildPackage(parent, child string) bool {
	return strings.HasPrefix(child, parent)
}

// isStandardPackage checks if the package is located in the standard library.
// If an error is thrown during import assumes it is not in the standard library.
func isStandardPackage(ctx *build.Context, cwd, path string) bool {
	if pkg, err := ctx.Import(path, cwd, build.FindOnly); err != nil {
		return false
	} else {
		return pkg.Goroot
	}
}

// getImports compiles a list of all the imports by appending TestImports and
// XTestImports to Imports as necessary. Returns a sorted slice with unique
// elements.
func getImports(pkg *build.Package, includeTests bool) []string {
	imp := pkg.Imports
	if includeTests {
		imp = appendUnique(imp, pkg.TestImports)
		imp = appendUnique(imp, pkg.XTestImports)
	}
	sort.Strings(imp)
	return imp
}

// getPackage compiles information about the package at the given path. First,
// it tries to resolve the path as a relative path to the current working
// directory and then as an import path inside the GOPATH.
// This function addresses issues with specifying a relative directory without
// a preceding `./` and it attempts to gather the information in a manner that
// allows it to set the ImportPath attribute.
// Returns an error if the resolved directory does not contain a buildable Go
// package.
// Compiles all the files ignoring build flags and any other build contstraints.
func getPackage(ctx *build.Context, path string) (pkg *build.Package, err error) {
	var stat os.FileInfo
	if abs, err := filepath.Abs(path); err == nil {
		// Withouth the absolute path, does not set the ImportPath
		// properly.
		if stat, err = os.Stat(path); err == nil && stat.IsDir() {
			return ctx.ImportDir(abs, 0)
		}
	}
	return ctx.Import(path, "", 0)
}
<commit_msg>Output package name.<commit_after>package main

import (
	"fmt"
	"go/build"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

// list runs the list subcommand, listing all the dependencies of the package
// at the specified path, relative paths are resolved from the current working
// directory.
func list(ctx *build.Context, cwd, path string) error {
	pkg, err := getPackage(ctx, path)
	if err != nil {
		return err
	}
	// List imports
	imp := getImports(pkg, opt.tests)
	fimp := filterImports(ctx, cwd, pkg.ImportPath, imp, opt.standard, opt.child)
	for _, v := range fimp {
		info(ctx, cwd, v)
	}
	return nil
}

// info runs the info subcommand, printing information about a given package.
// Also used by the list command to output details about imports, the quite and
// verbose flags determine the output.
func info(ctx *build.Context, cwd, path string) error {
	if opt.quite {
		fmt.Println(path)
		return nil
	}
	pkg, err := getPackage(ctx, path)
	if err != nil {
		return err
	}
	// Default output
	printBold(fmt.Sprintf("%s (%s)", pkg.Name, pkg.ImportPath))
	// Print package doc with line breaks
	if len(pkg.Doc) > 0 {
		printWrap(72, pkg.Doc)
	} else {
		fmt.Println("No package documentation.")
	}
	// Verbose output
	if opt.verbose {
		fmt.Println("  Standard :\t", pkg.Goroot)
		fmt.Println("  Directory :\t", pkg.Dir)
		if len(pkg.AllTags) > 0 {
			fmt.Println("  Tags :\t",
				strings.Join(pkg.AllTags, " "))
		}
	}
	return nil
}

// filterImports filters the imports by either ommitting or including standard
// and child packages, returns filtered slice of import paths.
// TODO rewrite filtering this isn't a good way to do this.
func filterImports(ctx *build.Context, cwd, parent string, imp []string, std, child bool) []string {
	r := make([]string, 0, len(imp))
	for _, v := range imp {
		// Filter child packages
		if !child && isChildPackage(parent, v) {
			continue
		}
		// Filter standard packages
		if !std && isStandardPackage(ctx, cwd, v) {
			continue
		}
		r = append(r, v)
	}
	return r
}

// isChildPackage checks if the child package is stationed in a subdirectory of
// the parent package. Pass in the import paths for both the parent and the
// child.
func isChildPackage(parent, child string) bool {
	return strings.HasPrefix(child, parent)
}

// isStandardPackage checks if the package is located in the standard library.
// If an error is thrown during import assumes it is not in the standard library.
func isStandardPackage(ctx *build.Context, cwd, path string) bool {
	if pkg, err := ctx.Import(path, cwd, build.FindOnly); err != nil {
		return false
	} else {
		return pkg.Goroot
	}
}

// getImports compiles a list of all the imports by appending TestImports and
// XTestImports to Imports as necessary. Returns a sorted slice with unique
// elements.
func getImports(pkg *build.Package, includeTests bool) []string {
	imp := pkg.Imports
	if includeTests {
		imp = appendUnique(imp, pkg.TestImports)
		imp = appendUnique(imp, pkg.XTestImports)
	}
	sort.Strings(imp)
	return imp
}

// getPackage compiles information about the package at the given path. First,
// it tries to resolve the path as a relative path to the current working
// directory and then as an import path inside the GOPATH.
// This function addresses issues with specifying a relative directory without
// a preceding `./` and it attempts to gather the information in a manner that
// allows it to set the ImportPath attribute.
// Returns an error if the resolved directory does not contain a buildable Go
// package.
// Compiles all the files ignoring build flags and any other build contstraints.
func getPackage(ctx *build.Context, path string) (pkg *build.Package, err error) {
	var stat os.FileInfo
	if abs, err := filepath.Abs(path); err == nil {
		// Withouth the absolute path, does not set the ImportPath
		// properly.
		if stat, err = os.Stat(path); err == nil && stat.IsDir() {
			return ctx.ImportDir(abs, 0)
		}
	}
	return ctx.Import(path, "", 0)
}
<|endoftext|>


<commit_before>package cli

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"qiniu/api.v6/auth/digest"
	"qiniu/api.v6/rs"
	"qshell"
	"strings"
)

const (
	BATCH_CDN_REFRESH_ALLOW_MAX  = 100
	BATCH_CDN_PREFETCH_ALLOW_MAX = 100
)

func CdnRefresh(cmd string, params ...string) {
	var isDirs bool
	flagSet := flag.NewFlagSet("cdnRefresh", flag.ExitOnError)
	flagSet.BoolVar(&isDirs, "dirs", false, "refresh dirs")
	flagSet.Parse(params)

	cmdParams := flagSet.Args()

	if len(cmdParams) == 1 {
		urlListFile := cmdParams[0]

		account, gErr := qshell.GetAccount()
		if gErr != nil {
			fmt.Println(gErr)
			os.Exit(qshell.STATUS_ERROR)
		}

		mac := digest.Mac{
			account.AccessKey,
			[]byte(account.SecretKey),
		}

		client := rs.NewMac(&mac)
		fp, err := os.Open(urlListFile)
		if err != nil {
			fmt.Println("Open refresh item list file error,", err)
			os.Exit(qshell.STATUS_HALT)
		}
		defer fp.Close()
		scanner := bufio.NewScanner(fp)
		scanner.Split(bufio.ScanLines)

		itemsToRefresh := make([]string, 0, 10)
		for scanner.Scan() {
			url := strings.TrimSpace(scanner.Text())
			if url == "" {
				continue
			}
			itemsToRefresh = append(itemsToRefresh, url)

			if len(itemsToRefresh) == BATCH_CDN_REFRESH_ALLOW_MAX {
				if isDirs {
					cdnRefresh(&client, nil, itemsToRefresh)
				} else {
					cdnRefresh(&client, itemsToRefresh, nil)
				}
				itemsToRefresh = make([]string, 0, 10)
			}
		}

		if len(itemsToRefresh) > 0 {
			if isDirs {
				cdnRefresh(&client, nil, itemsToRefresh)
			} else {
				cdnRefresh(&client, itemsToRefresh, nil)
			}
		}
	} else {
		CmdHelp(cmd)
	}
}

func cdnRefresh(client *rs.Client, urls []string, dirs []string) {
	resp, err := qshell.BatchRefresh(client, urls, dirs)
	if err != nil {
		fmt.Println("CDN refresh error,", err)
	} else {
		if resp.Error != "" {
			fmt.Println(fmt.Sprintf("Code: %d, Info: %s", resp.Code, resp.Error))
		}
	}
}

func CdnPrefetch(cmd string, params ...string) {
	if len(params) == 1 {
		urlListFile := params[0]

		account, gErr := qshell.GetAccount()
		if gErr != nil {
			fmt.Println(gErr)
			os.Exit(qshell.STATUS_ERROR)
		}

		mac := digest.Mac{
			account.AccessKey,
			[]byte(account.SecretKey),
		}

		client := rs.NewMac(&mac)
		fp, err := os.Open(urlListFile)
		if err != nil {
			fmt.Println("Open url list file error,", err)
			os.Exit(qshell.STATUS_HALT)
		}
		defer fp.Close()
		scanner := bufio.NewScanner(fp)
		scanner.Split(bufio.ScanLines)

		urlsToPrefetch := make([]string, 0, 10)
		for scanner.Scan() {
			url := strings.TrimSpace(scanner.Text())
			if url == "" {
				continue
			}
			urlsToPrefetch = append(urlsToPrefetch, url)

			if len(urlsToPrefetch) == BATCH_CDN_PREFETCH_ALLOW_MAX {
				cdnPrefetch(&client, urlsToPrefetch)
				urlsToPrefetch = make([]string, 0, 10)
			}
		}

		if len(urlsToPrefetch) > 0 {
			cdnPrefetch(&client, urlsToPrefetch)
		}
	} else {
		CmdHelp(cmd)
	}
}

func cdnPrefetch(client *rs.Client, urls []string) {
	resp, err := qshell.BatchPrefetch(client, urls)
	if err != nil {
		fmt.Println("CDN prefetch error,", err)
	} else {
		if resp.Error != "" {
			fmt.Println(fmt.Sprintf("Code: %d, Info: %s", resp.Code, resp.Error))
		}
	}
}
<commit_msg>[ISSUE-70] fix the error of dirs refresh submit limit<commit_after>package cli

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"qiniu/api.v6/auth/digest"
	"qiniu/api.v6/rs"
	"qshell"
	"strings"
)

const (
	BATCH_CDN_REFRESH_URLS_ALLOW_MAX = 100
	BATCH_CDN_REFRESH_DIRS_ALLOW_MAX = 10
	BATCH_CDN_PREFETCH_ALLOW_MAX     = 100
)

func CdnRefresh(cmd string, params ...string) {
	var isDirs bool
	flagSet := flag.NewFlagSet("cdnRefresh", flag.ExitOnError)
	flagSet.BoolVar(&isDirs, "dirs", false, "refresh dirs")
	flagSet.Parse(params)

	cmdParams := flagSet.Args()

	if len(cmdParams) == 1 {
		urlListFile := cmdParams[0]

		account, gErr := qshell.GetAccount()
		if gErr != nil {
			fmt.Println(gErr)
			os.Exit(qshell.STATUS_ERROR)
		}

		mac := digest.Mac{
			account.AccessKey,
			[]byte(account.SecretKey),
		}

		client := rs.NewMac(&mac)
		fp, err := os.Open(urlListFile)
		if err != nil {
			fmt.Println("Open refresh item list file error,", err)
			os.Exit(qshell.STATUS_HALT)
		}
		defer fp.Close()
		scanner := bufio.NewScanner(fp)
		scanner.Split(bufio.ScanLines)

		itemsToRefresh := make([]string, 0, 100)

		if isDirs {
			for scanner.Scan() {
				item := strings.TrimSpace(scanner.Text())
				if item == "" {
					continue
				}
				itemsToRefresh = append(itemsToRefresh, item)

				if len(itemsToRefresh) == BATCH_CDN_REFRESH_DIRS_ALLOW_MAX {
					cdnRefresh(&client, nil, itemsToRefresh)
					itemsToRefresh = make([]string, 0, 10)
				}
			}
		} else {
			for scanner.Scan() {
				item := strings.TrimSpace(scanner.Text())
				if item == "" {
					continue
				}
				itemsToRefresh = append(itemsToRefresh, item)

				if len(itemsToRefresh) == BATCH_CDN_REFRESH_URLS_ALLOW_MAX {
					cdnRefresh(&client, itemsToRefresh, nil)
					itemsToRefresh = make([]string, 0, 100)
				}
			}
		}

		//check final items
		if len(itemsToRefresh) > 0 {
			if isDirs {
				cdnRefresh(&client, nil, itemsToRefresh)
			} else {
				cdnRefresh(&client, itemsToRefresh, nil)
			}
		}
	} else {
		CmdHelp(cmd)
	}
}

func cdnRefresh(client *rs.Client, urls []string, dirs []string) {
	resp, err := qshell.BatchRefresh(client, urls, dirs)
	if err != nil {
		fmt.Println("CDN refresh error,", err)
	} else {
		if resp.Error != "" {
			fmt.Println(fmt.Sprintf("Code: %d, Info: %s", resp.Code, resp.Error))
		}
	}
}

func CdnPrefetch(cmd string, params ...string) {
	if len(params) == 1 {
		urlListFile := params[0]

		account, gErr := qshell.GetAccount()
		if gErr != nil {
			fmt.Println(gErr)
			os.Exit(qshell.STATUS_ERROR)
		}

		mac := digest.Mac{
			account.AccessKey,
			[]byte(account.SecretKey),
		}

		client := rs.NewMac(&mac)
		fp, err := os.Open(urlListFile)
		if err != nil {
			fmt.Println("Open url list file error,", err)
			os.Exit(qshell.STATUS_HALT)
		}
		defer fp.Close()
		scanner := bufio.NewScanner(fp)
		scanner.Split(bufio.ScanLines)

		urlsToPrefetch := make([]string, 0, 10)
		for scanner.Scan() {
			url := strings.TrimSpace(scanner.Text())
			if url == "" {
				continue
			}
			urlsToPrefetch = append(urlsToPrefetch, url)

			if len(urlsToPrefetch) == BATCH_CDN_PREFETCH_ALLOW_MAX {
				cdnPrefetch(&client, urlsToPrefetch)
				urlsToPrefetch = make([]string, 0, 10)
			}
		}

		if len(urlsToPrefetch) > 0 {
			cdnPrefetch(&client, urlsToPrefetch)
		}
	} else {
		CmdHelp(cmd)
	}
}

func cdnPrefetch(client *rs.Client, urls []string) {
	resp, err := qshell.BatchPrefetch(client, urls)
	if err != nil {
		fmt.Println("CDN prefetch error,", err)
	} else {
		if resp.Error != "" {
			fmt.Println(fmt.Sprintf("Code: %d, Info: %s", resp.Code, resp.Error))
		}
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/schollz/croc/src/croc"
	"github.com/schollz/croc/src/utils"
	"github.com/skratchdot/open-golang/open"
	"github.com/urfave/cli"
)

var Version string
var cr *croc.Croc

func Run() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	app := cli.NewApp()
	app.Name = "croc"
	if Version == "" {
		Version = "dev"
	}

	app.Version = Version
	app.Compiled = time.Now()
	app.Usage = "easily and securely transfer stuff from one computer to another"
	app.UsageText = "croc allows any two computers to directly and securely transfer files"
	// app.ArgsUsage = "[args and such]"
	app.Commands = []cli.Command{
		{
			Name:        "send",
			Usage:       "send a file",
			Description: "send a file over the relay",
			ArgsUsage:   "[filename]",
			Flags: []cli.Flag{
				cli.BoolFlag{Name: "no-compress, o", Usage: "disable compression"},
				cli.BoolFlag{Name: "no-encrypt, e", Usage: "disable encryption"},
				cli.StringFlag{Name: "code, c", Usage: "codephrase used to connect to relay"},
			},
			HelpName: "croc send",
			Action: func(c *cli.Context) error {
				return send(c)
			},
		},
		{
			Name:        "relay",
			Usage:       "start a croc relay",
			Description: "the croc relay will handle websocket and TCP connections",
			Flags:       []cli.Flag{},
			HelpName:    "croc relay",
			Action: func(c *cli.Context) error {
				return relay(c)
			},
		},
		{
			Name:        "config",
			Usage:       "generates a config file",
			Description: "the croc config can be used to set static parameters",
			Flags:       []cli.Flag{},
			HelpName:    "croc config",
			Action: func(c *cli.Context) error {
				return saveDefaultConfig(c)
			},
		},
	}
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "addr", Value: "croc4.schollz.com", Usage: "address of the public relay"},
		cli.StringFlag{Name: "addr-ws", Value: "8153", Usage: "port of the public relay websocket server to connect"},
		cli.StringFlag{Name: "addr-tcp", Value: "8154,8155,8156,8157,8158,8159,8160,8161", Usage: "tcp ports of the public relay server to connect"},
		cli.BoolFlag{Name: "no-local", Usage: "disable local mode"},
		cli.BoolFlag{Name: "local", Usage: "use only local mode"},
		cli.BoolFlag{Name: "debug", Usage: "increase verbosity (a lot)"},
		cli.BoolFlag{Name: "yes", Usage: "automatically agree to all prompts"},
		cli.BoolFlag{Name: "stdout", Usage: "redirect file to stdout"},
		cli.BoolFlag{Name: "force-tcp", Usage: "force TCP"},
		cli.BoolFlag{Name: "force-web", Usage: "force websockets"},
		cli.StringFlag{Name: "port", Value: "8153", Usage: "port that the websocket listens on"},
		cli.StringFlag{Name: "tcp-port", Value: "8154,8155,8156,8157,8158,8159,8160,8161", Usage: "ports that the tcp server listens on"},
		cli.StringFlag{Name: "curve", Value: "siec", Usage: "specify elliptic curve to use for PAKE (p256, p384, p521, siec)"},
		cli.StringFlag{Name: "out", Value: ".", Usage: "specify an output folder to receive the file"},
	}
	app.EnableBashCompletion = true
	app.HideHelp = false
	app.HideVersion = false
	app.BashComplete = func(c *cli.Context) {
		fmt.Fprintf(c.App.Writer, "send\nreceive\relay")
	}
	app.Action = func(c *cli.Context) error {
		return receive(c)
	}
	app.Before = func(c *cli.Context) error {
		cr = croc.Init(c.GlobalBool("debug"))
		cr.Version = Version
		cr.AllowLocalDiscovery = true
		cr.Address = c.GlobalString("addr")
		cr.AddressTCPPorts = strings.Split(c.GlobalString("addr-tcp"), ",")
		cr.AddressWebsocketPort = c.GlobalString("addr-ws")
		cr.NoRecipientPrompt = c.GlobalBool("yes")
		cr.Stdout = c.GlobalBool("stdout")
		cr.LocalOnly = c.GlobalBool("local")
		cr.NoLocal = c.GlobalBool("no-local")
		cr.ShowText = true
		cr.RelayWebsocketPort = c.String("port")
		cr.RelayTCPPorts = strings.Split(c.String("tcp-port"), ",")
		cr.CurveType = c.String("curve")
		if c.GlobalBool("force-tcp") {
			cr.ForceSend = 2
		}
		if c.GlobalBool("force-web") {
			cr.ForceSend = 1
		}
		return nil
	}

	err := app.Run(os.Args)
	if err != nil {
		fmt.Printf("\nerror: %s", err.Error())
	}
}

func saveDefaultConfig(c *cli.Context) error {
	return croc.SaveDefaultConfig()
}

func send(c *cli.Context) error {
	stat, _ := os.Stdin.Stat()
	var fname string
	if (stat.Mode() & os.ModeCharDevice) == 0 {
		f, err := ioutil.TempFile(".", "croc-stdin-")
		if err != nil {
			return err
		}
		_, err = io.Copy(f, os.Stdin)
		if err != nil {
			return err
		}
		err = f.Close()
		if err != nil {
			return err
		}
		fname = f.Name()
		defer func() {
			err = os.Remove(fname)
			if err != nil {
				log.Println(err)
			}
		}()
	} else {
		fname = c.Args().First()
	}
	if fname == "" {
		return errors.New("must specify file: croc send [filename]")
	}
	cr.UseCompression = !c.Bool("no-compress")
	cr.UseEncryption = !c.Bool("no-encrypt")
	if c.String("code") != "" {
		cr.Codephrase = c.String("code")
	}
	cr.LoadConfig()
	if len(cr.Codephrase) == 0 {
		// generate code phrase
		cr.Codephrase = utils.GetRandomName()
	}

	// print the text
	finfo, err := os.Stat(fname)
	if err != nil {
		return err
	}
	fname, _ = filepath.Abs(fname)
	fname = filepath.Clean(fname)
	_, filename := filepath.Split(fname)
	fileOrFolder := "file"
	fsize := finfo.Size()
	if finfo.IsDir() {
		fileOrFolder = "folder"
		fsize, err = dirSize(fname)
		if err != nil {
			return err
		}
	}
	fmt.Fprintf(os.Stderr,
		"Sending %s %s named '%s'\nCode is: %s\nOn the other computer, please run:\n\ncroc %s\n\n",
		humanize.Bytes(uint64(fsize)),
		fileOrFolder,
		filename,
		cr.Codephrase,
		cr.Codephrase,
	)
	return cr.Send(fname, cr.Codephrase)
}

func receive(c *cli.Context) error {
	if c.GlobalString("code") != "" {
		cr.Codephrase = c.GlobalString("code")
	}
	if c.Args().First() != "" {
		cr.Codephrase = c.Args().First()
	}
	if c.GlobalString("out") != "" {
		os.Chdir(c.GlobalString("out"))
	}
	cr.LoadConfig()
	openFolder := false
	if len(os.Args) == 1 {
		// open folder since they didn't give any arguments
		openFolder = true
	}
	if cr.Codephrase == "" {
		cr.Codephrase = utils.GetInput("Enter receive code: ")
	}
	err := cr.Receive(cr.Codephrase)
	if err == nil && openFolder {
		cwd, _ := os.Getwd()
		open.Run(cwd)
	}
	return err
}

func relay(c *cli.Context) error {
	return cr.Relay()
}

func dirSize(path string) (int64, error) {
	var size int64
	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			size += info.Size()
		}
		return err
	})
	return size, err
}
<commit_msg>prompt user if accidently receiving<commit_after>package cli

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/schollz/croc/src/croc"
	"github.com/schollz/croc/src/utils"
	"github.com/skratchdot/open-golang/open"
	"github.com/urfave/cli"
)

var Version string
var cr *croc.Croc

func Run() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	app := cli.NewApp()
	app.Name = "croc"
	if Version == "" {
		Version = "dev"
	}

	app.Version = Version
	app.Compiled = time.Now()
	app.Usage = "easily and securely transfer stuff from one computer to another"
	app.UsageText = "croc allows any two computers to directly and securely transfer files"
	// app.ArgsUsage = "[args and such]"
	app.Commands = []cli.Command{
		{
			Name:        "send",
			Usage:       "send a file",
			Description: "send a file over the relay",
			ArgsUsage:   "[filename]",
			Flags: []cli.Flag{
				cli.BoolFlag{Name: "no-compress, o", Usage: "disable compression"},
				cli.BoolFlag{Name: "no-encrypt, e", Usage: "disable encryption"},
				cli.StringFlag{Name: "code, c", Usage: "codephrase used to connect to relay"},
			},
			HelpName: "croc send",
			Action: func(c *cli.Context) error {
				return send(c)
			},
		},
		{
			Name:        "relay",
			Usage:       "start a croc relay",
			Description: "the croc relay will handle websocket and TCP connections",
			Flags:       []cli.Flag{},
			HelpName:    "croc relay",
			Action: func(c *cli.Context) error {
				return relay(c)
			},
		},
		{
			Name:        "config",
			Usage:       "generates a config file",
			Description: "the croc config can be used to set static parameters",
			Flags:       []cli.Flag{},
			HelpName:    "croc config",
			Action: func(c *cli.Context) error {
				return saveDefaultConfig(c)
			},
		},
	}
	app.Flags = []cli.Flag{
		cli.StringFlag{Name: "addr", Value: "croc4.schollz.com", Usage: "address of the public relay"},
		cli.StringFlag{Name: "addr-ws", Value: "8153", Usage: "port of the public relay websocket server to connect"},
		cli.StringFlag{Name: "addr-tcp", Value: "8154,8155,8156,8157,8158,8159,8160,8161", Usage: "tcp ports of the public relay server to connect"},
		cli.BoolFlag{Name: "no-local", Usage: "disable local mode"},
		cli.BoolFlag{Name: "local", Usage: "use only local mode"},
		cli.BoolFlag{Name: "debug", Usage: "increase verbosity (a lot)"},
		cli.BoolFlag{Name: "yes", Usage: "automatically agree to all prompts"},
		cli.BoolFlag{Name: "stdout", Usage: "redirect file to stdout"},
		cli.BoolFlag{Name: "force-tcp", Usage: "force TCP"},
		cli.BoolFlag{Name: "force-web", Usage: "force websockets"},
		cli.StringFlag{Name: "port", Value: "8153", Usage: "port that the websocket listens on"},
		cli.StringFlag{Name: "tcp-port", Value: "8154,8155,8156,8157,8158,8159,8160,8161", Usage: "ports that the tcp server listens on"},
		cli.StringFlag{Name: "curve", Value: "siec", Usage: "specify elliptic curve to use for PAKE (p256, p384, p521, siec)"},
		cli.StringFlag{Name: "out", Value: ".", Usage: "specify an output folder to receive the file"},
	}
	app.EnableBashCompletion = true
	app.HideHelp = false
	app.HideVersion = false
	app.BashComplete = func(c *cli.Context) {
		fmt.Fprintf(c.App.Writer, "send\nreceive\relay")
	}
	app.Action = func(c *cli.Context) error {
		// if trying to send but forgot send, let the user know
		if c.Args().First() != "" && utils.Exists(c.Args().First()) {
			_, fname := filepath.Split(c.Args().First())
			yn := utils.GetInput(fmt.Sprintf("Did you mean to send '%s'? (y/n)", fname))
			if strings.ToLower(yn) == "y" {
				return send(c)
			}
		}
		return receive(c)
	}
	app.Before = func(c *cli.Context) error {
		cr = croc.Init(c.GlobalBool("debug"))
		cr.Version = Version
		cr.AllowLocalDiscovery = true
		cr.Address = c.GlobalString("addr")
		cr.AddressTCPPorts = strings.Split(c.GlobalString("addr-tcp"), ",")
		cr.AddressWebsocketPort = c.GlobalString("addr-ws")
		cr.NoRecipientPrompt = c.GlobalBool("yes")
		cr.Stdout = c.GlobalBool("stdout")
		cr.LocalOnly = c.GlobalBool("local")
		cr.NoLocal = c.GlobalBool("no-local")
		cr.ShowText = true
		cr.RelayWebsocketPort = c.String("port")
		cr.RelayTCPPorts = strings.Split(c.String("tcp-port"), ",")
		cr.CurveType = c.String("curve")
		if c.GlobalBool("force-tcp") {
			cr.ForceSend = 2
		}
		if c.GlobalBool("force-web") {
			cr.ForceSend = 1
		}
		return nil
	}

	err := app.Run(os.Args)
	if err != nil {
		fmt.Printf("\nerror: %s", err.Error())
	}
}

func saveDefaultConfig(c *cli.Context) error {
	return croc.SaveDefaultConfig()
}

func send(c *cli.Context) error {
	stat, _ := os.Stdin.Stat()
	var fname string
	if (stat.Mode() & os.ModeCharDevice) == 0 {
		f, err := ioutil.TempFile(".", "croc-stdin-")
		if err != nil {
			return err
		}
		_, err = io.Copy(f, os.Stdin)
		if err != nil {
			return err
		}
		err = f.Close()
		if err != nil {
			return err
		}
		fname = f.Name()
		defer func() {
			err = os.Remove(fname)
			if err != nil {
				log.Println(err)
			}
		}()
	} else {
		fname = c.Args().First()
	}
	if fname == "" {
		return errors.New("must specify file: croc send [filename]")
	}
	cr.UseCompression = !c.Bool("no-compress")
	cr.UseEncryption = !c.Bool("no-encrypt")
	if c.String("code") != "" {
		cr.Codephrase = c.String("code")
	}
	cr.LoadConfig()
	if len(cr.Codephrase) == 0 {
		// generate code phrase
		cr.Codephrase = utils.GetRandomName()
	}

	// print the text
	finfo, err := os.Stat(fname)
	if err != nil {
		return err
	}
	fname, _ = filepath.Abs(fname)
	fname = filepath.Clean(fname)
	_, filename := filepath.Split(fname)
	fileOrFolder := "file"
	fsize := finfo.Size()
	if finfo.IsDir() {
		fileOrFolder = "folder"
		fsize, err = dirSize(fname)
		if err != nil {
			return err
		}
	}
	fmt.Fprintf(os.Stderr,
		"Sending %s %s named '%s'\nCode is: %s\nOn the other computer, please run:\n\ncroc %s\n\n",
		humanize.Bytes(uint64(fsize)),
		fileOrFolder,
		filename,
		cr.Codephrase,
		cr.Codephrase,
	)
	return cr.Send(fname, cr.Codephrase)
}

func receive(c *cli.Context) error {
	if c.GlobalString("code") != "" {
		cr.Codephrase = c.GlobalString("code")
	}
	if c.Args().First() != "" {
		cr.Codephrase = c.Args().First()
	}
	if c.GlobalString("out") != "" {
		os.Chdir(c.GlobalString("out"))
	}
	cr.LoadConfig()
	openFolder := false
	if len(os.Args) == 1 {
		// open folder since they didn't give any arguments
		openFolder = true
	}
	if cr.Codephrase == "" {
		cr.Codephrase = utils.GetInput("Enter receive code: ")
	}
	err := cr.Receive(cr.Codephrase)
	if err == nil && openFolder {
		cwd, _ := os.Getwd()
		open.Run(cwd)
	}
	return err
}

func relay(c *cli.Context) error {
	return cr.Relay()
}

func dirSize(path string) (int64, error) {
	var size int64
	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			size += info.Size()
		}
		return err
	})
	return size, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"

	"golang.org/x/net/html"
)

func getAttr(t html.Token, name string) (ok bool, href string) {
	for _, a := range t.Attr {
		if a.Key == name {
			href = a.Val
			ok = true
		}
	}

	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

var skipUrls = map[string]int{
	"https://build.opensuse.org/project/show/Virtualization:containers": 1,
	"http://10.0.0.2:5000":                                              1,
	"https://linux.oracle.com":                                          1,
	"http://supervisord.org/":                                           1,
	"http://goo.gl/HSz8UT":                                              1,
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	if _, ok := skipUrls[req.url]; ok {
		fmt.Printf("Skipping: %s\n", req.url)
		reply.code = 299
		chFinished <- reply
		return
	}
	fmt.Printf("Crawling: %s\n", req.url)
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	loc, err := resp.Location()
	if err == nil && req.url != loc.String() {
		fmt.Printf("\t crawled \"%s\"", req.url)
		fmt.Printf("\t\t to \"%s\"", loc)
	}

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch tt {
		case html.ErrorToken:
			// End of the document, we're done
			return
		case html.StartTagToken, html.SelfClosingTagToken:
			t := z.Token()
			var ok bool
			var newUrl string

			switch t.Data {
			case "a", "link":
				ok, newUrl = getAttr(t, "href")
				if !ok {
					continue
				}
				break
			case "img", "script":
				ok, newUrl = getAttr(t, "src")
				if !ok {
					continue
				}
				break
			default:
				continue
			}

			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl string

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	if len(os.Args[1:]) == 0 {
		seedUrl = os.Getenv("CHECKURL")
		if seedUrl == "" {
			fmt.Println("Please specify a URL to check")
			os.Exit(-1)
		}
	} else {
		seedUrl = os.Args[1]
	}

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 3000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 20; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	new := NewUrl{
		from: "",
		url:  seedUrl,
	}
	chUrls <- new

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || count > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{
					from: foundUrl.from,
					url:  resourceUrl,
				}
			}
			f.usageCount++
			f.from[foundUrl.from]++
			foundUrls[resourceUrl] = f

		case ret := <-chFinished:
			count--
			info := foundUrls[ret.url]
			//info.from[ret.from]++
			info.response = ret.code
			info.err = ret.err
			foundUrls[ret.url] = info
		}
		// fmt.Printf("(w%d, u%d, c%d)", len(chWork), len(chUrls), count)
	}

	// We're done! Print the results...
	fmt.Println("\nDone.")
	summary := make(map[int]int)
	for url, info := range foundUrls {
		summary[info.response]++
		if info.response != 200 && info.response != 900 {
			fmt.Printf(" - %d (%d): %s\n", info.response, info.usageCount, url)
			if info.err != nil {
				fmt.Printf("\t%s\n", info.err)
			}
			for from, count := range info.from {
				fmt.Printf("\t\t%d times from %s\n", count, from)
			}
		}
	}
	fmt.Println("\nFound", len(foundUrls), "unique urls\n")
	errorCount := 0
	explain := map[int]string{
		900: "mailto or irc",
		299: "url skipped",
		200: "ok",
		404: "forbidden",
		403: "forbidden",
	}
	for code, count := range summary {
		reason, ok := explain[code]
		if !ok {
			// TODO: I presume go has a text error mapping
			reason = "HTTP code"
		}
		fmt.Printf("\t\tStatus %d : %d - %s\n", code, count, reason)
		if code != 200 && code != 299 && code != 900 {
			errorCount += count
		}
	}
	fmt.Println("\nError Count:", errorCount)

	close(chUrls)

	// return the number of 404's to show that there are things to be fixed
	os.Exit(errorCount)
}
<commit_msg>Avoid parsing js and css, and resolve relative urls using html URL<commit_after>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"strings"

	"golang.org/x/net/html"
)

func getAttr(t html.Token, name string) (ok bool, href string) {
	for _, a := range t.Attr {
		if a.Key == name {
			href = a.Val
			ok = true
		}
	}

	return
}

type UrlResponse struct {
	from string
	url  string
	code int
	err  error
}

type NewUrl struct {
	from string
	url  string
}

var skipUrls = map[string]int{
	"https://build.opensuse.org/project/show/Virtualization:containers": 1,
	"http://10.0.0.2:5000":                                              1,
	"https://linux.oracle.com":                                          1,
	"http://supervisord.org/":                                           1,
	"http://goo.gl/HSz8UT":                                              1,
}

func crawl(chWork chan NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	for true {
		new := <-chWork
		crawlOne(new, ch, chFinished)
	}
}

// Extract all http** links from a given webpage
func crawlOne(req NewUrl, ch chan NewUrl, chFinished chan UrlResponse) {
	base, err := url.Parse(req.url)
	reply := UrlResponse{
		url:  req.url,
		from: req.from,
		code: 999,
	}
	if _, ok := skipUrls[req.url]; ok {
		fmt.Printf("Skipping: %s\n", req.url)
		reply.code = 299
		chFinished <- reply
		return
	}
	fmt.Printf("Crawling: %s\n", req.url)
	if err != nil {
		fmt.Println("ERROR: failed to Parse \"" + req.url + "\"")
		reply.err = err
		chFinished <- reply
		return
	}
	switch base.Scheme {
	case "mailto", "irc":
		reply.err = fmt.Errorf("%s on page %s", base.Scheme, req.from)
		reply.code = 900
		chFinished <- reply
		return
	}
	resp, err := http.Get(req.url)
	if err != nil {
		fmt.Println("ERROR: Failed to crawl \"" + req.url + "\"  " + err.Error())
		reply.err = err
		chFinished <- reply
		return
	}
	defer func() {
		// Notify that we're done after this function
		reply.code = resp.StatusCode
		chFinished <- reply
	}()

	loc, err := resp.Location()
	if err == nil && req.url != loc.String() {
		fmt.Printf("\t crawled \"%s\"", req.url)
		fmt.Printf("\t\t to \"%s\"", loc)
	}

	b := resp.Body
	defer b.Close() // close Body when the function returns

	// only parse if this page is on the original site
	// if we moved this check back to the main loop, we'd parse more sites
	if !strings.HasPrefix(req.url, seedUrl) {
		return
	}

	// don't parse js files..
	if strings.HasSuffix(req.url, ".js") || strings.HasSuffix(req.url, ".js") {
		return
	}

	// TODO: it seems to sucessfully parse non-html (like js/css)
	z := html.NewTokenizer(b)

	for {
		tt := z.Next()

		switch tt {
		case html.ErrorToken:
			// End of the document, we're done
			return
		case html.StartTagToken, html.SelfClosingTagToken:
			t := z.Token()
			var ok bool
			var newUrl string

			switch t.Data {
			case "base":
				// use the actual baseUrl set in the html file
				ok, baseUrl := getAttr(t, "href")
				if !ok {
					continue
				}

				newBase, err := url.Parse(baseUrl)
				if err != nil {
					continue
				}
				base = base.ResolveReference(newBase)
				continue
			case "a", "link":
				ok, newUrl = getAttr(t, "href")
				if !ok {
					continue
				}
			case "img", "script":
				ok, newUrl = getAttr(t, "src")
				if !ok {
					continue
				}
			default:
				continue
			}

			u, e := url.Parse(newUrl)
			if e != nil {
				fmt.Println("ERROR: failed to Parse \"" + newUrl + "\"")
				continue
			}
			new := NewUrl{
				from: req.url,
				url:  base.ResolveReference(u).String(),
			}
			ch <- new
		}
	}
}

var seedUrl string

type FoundUrls struct {
	response   int
	usageCount int
	err        error
	from       map[string]int
}

func main() {
	if len(os.Args[1:]) == 0 {
		seedUrl = os.Getenv("CHECKURL")
		if seedUrl == "" {
			fmt.Println("Please specify a URL to check")
			os.Exit(-1)
		}
	} else {
		seedUrl = os.Args[1]
	}

	// Channels
	chUrls := make(chan NewUrl, 1000)
	chWork := make(chan NewUrl, 3000)
	chFinished := make(chan UrlResponse)

	var foundUrls = make(map[string]FoundUrls)

	for w := 1; w <= 20; w++ {
		go crawl(chWork, chUrls, chFinished)
	}

	new := NewUrl{
		from: "",
		url:  seedUrl,
	}
	chUrls <- new

	// Subscribe to both channels
	count := 0
	for len(chUrls) > 0 || count > 0 {
		select {
		case foundUrl := <-chUrls:
			// don't need to check err - its already been checked before its put in the chUrls que
			u, _ := url.Parse(foundUrl.url)
			// TODO: need a different pipeline for ensuring anchor fragments exist
			// TODO: consider only removing the query/fragment for docs urls
			u.RawQuery = ""
			u.Fragment = ""
			resourceUrl := u.String()

			f, ok := foundUrls[resourceUrl]
			if !ok {
				count++
				f.usageCount = 0
				f.response = 0
				f.from = make(map[string]int)
				f.from[foundUrl.from] = 1
				chWork <- NewUrl{
					from: foundUrl.from,
					url:  resourceUrl,
				}
			}
			f.usageCount++
			f.from[foundUrl.from]++
			foundUrls[resourceUrl] = f

		case ret := <-chFinished:
			count--
			info := foundUrls[ret.url]
			//info.from[ret.from]++
			info.response = ret.code
			info.err = ret.err
			foundUrls[ret.url] = info
		}
		// fmt.Printf("(w%d, u%d, c%d)", len(chWork), len(chUrls), count)
	}

	// We're done! Print the results...
	fmt.Println("\nDone.")
	summary := make(map[int]int)
	for url, info := range foundUrls {
		summary[info.response]++
		if info.response != 200 && info.response != 900 {
			fmt.Printf(" - %d (%d): %s\n", info.response, info.usageCount, url)
			if info.err != nil {
				fmt.Printf("\t%s\n", info.err)
			}
			for from, count := range info.from {
				fmt.Printf("\t\t%d times from %s\n", count, from)
			}
		}
	}
	fmt.Println("\nFound", len(foundUrls), "unique urls\n")
	errorCount := 0
	explain := map[int]string{
		900: "mailto or irc",
		299: "url skipped",
		200: "ok",
		404: "forbidden",
		403: "forbidden",
	}
	for code, count := range summary {
		reason, ok := explain[code]
		if !ok {
			// TODO: I presume go has a text error mapping
			reason = "HTTP code"
		}
		fmt.Printf("\t\tStatus %d : %d - %s\n", code, count, reason)
		if code != 200 && code != 299 && code != 900 {
			errorCount += count
		}
	}
	fmt.Println("\nError Count:", errorCount)

	close(chUrls)

	// return the number of 404's to show that there are things to be fixed
	os.Exit(errorCount)
}
<|endoftext|>


<commit_before>package jsh

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"strconv"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestList(t *testing.T) {

	Convey("List Tests", t, func() {

		testObject := &Object{
			ID:         "ID123",
			Type:       "testConversion",
			Attributes: json.RawMessage(`{"foo":"bar"}`),
		}

		testList := List{testObject}
		req := &http.Request{Method: "GET"}

		Convey("->Validate()", func() {
			err := testList.Validate(req, true)
			So(err, ShouldBeNil)
		})

		Convey("->Send(List)", func() {

			Convey("should send a properly formatted List response", func() {
				writer := httptest.NewRecorder()
				err := Send(writer, req, testList)
				So(err, ShouldBeNil)
				So(writer.Code, ShouldEqual, http.StatusOK)

				contentLength, convErr := strconv.Atoi(writer.HeaderMap.Get("Content-Length"))
				So(convErr, ShouldBeNil)
				So(contentLength, ShouldBeGreaterThan, 0)
				So(writer.HeaderMap.Get("Content-Type"), ShouldEqual, ContentType)

				req, reqErr := testRequest(writer.Body.Bytes())
				So(reqErr, ShouldBeNil)

				responseList, parseErr := ParseList(req)
				So(parseErr, ShouldBeNil)
				So(len(responseList), ShouldEqual, 1)
			})

			Convey("should send a properly formatted empty List response", func() {
				writer := httptest.NewRecorder()
				err := Send(writer, req, List{})
				So(err, ShouldBeNil)
				So(writer.Code, ShouldEqual, http.StatusOK)

				contentLength, convErr := strconv.Atoi(writer.HeaderMap.Get("Content-Length"))
				So(convErr, ShouldBeNil)
				So(contentLength, ShouldBeGreaterThan, 0)
				So(writer.HeaderMap.Get("Content-Type"), ShouldEqual, ContentType)

				req, reqErr := testRequest(writer.Body.Bytes())
				So(reqErr, ShouldBeNil)

				responseList, parseErr := ParseList(req)
				So(parseErr, ShouldBeNil)
				So(len(responseList), ShouldEqual, 0)
			})

			Convey("should reject nil List", func() {
				writer := httptest.NewRecorder()
				var list List
				err := Send(writer, req, list)
				So(err, ShouldNotBeNil)
				So(writer.Code, ShouldEqual, http.StatusInternalServerError)
			})

			Convey("should accept empty List", func() {
				writer := httptest.NewRecorder()
				err := Send(writer, req, List{})
				So(err, ShouldBeNil)
			})
		})

		Convey("->UnmarshalJSON()", func() {

			Convey("should handle a data object", func() {
				jObj := `{"data": {"type": "user", "id": "sweetID123", "attributes": {"ID":"123"}}}`

				l := List{}
				err := l.UnmarshalJSON([]byte(jObj))
				So(err, ShouldBeNil)
				So(l, ShouldNotBeEmpty)
			})

			Convey("should handle a data list", func() {
				jList := `{"data": [{"type": "user", "id": "sweetID123", "attributes": {"ID":"123"}}]}`

				l := List{}
				err := l.UnmarshalJSON([]byte(jList))
				So(err, ShouldBeNil)
				So(l, ShouldNotBeEmpty)
			})

			Convey("should handle an empty array", func() {
				jObj := `{"data": []}`

				l := List{}
				err := l.UnmarshalJSON([]byte(jObj))
				So(err, ShouldBeNil)
				So(l, ShouldNotBeNil)
			})
		})

		Convey("->MarshalJSON()", func() {

			Convey("should preserve an empty list", func() {
				list := List{}

				jData, err := json.Marshal(list)
				So(err, ShouldBeNil)

				So(string(jData), ShouldEqual, "[]")
			})
		})
	})
}
<commit_msg>Add sortable test<commit_after>package jsh

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"sort"
	"strconv"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestList(t *testing.T) {

	Convey("List Tests", t, func() {

		testObject := &Object{
			ID:         "ID123",
			Type:       "testConversion",
			Attributes: json.RawMessage(`{"foo":"bar"}`),
		}

		testList := List{testObject}
		req := &http.Request{Method: "GET"}

		Convey("->Validate()", func() {
			err := testList.Validate(req, true)
			So(err, ShouldBeNil)
		})

		Convey("->Sort(List)", func() {
			firstObject := &Object{
				ID:         "ID1",
				Type:       testObject.Type,
				Attributes: testObject.Attributes,
			}
			thirdObject := &Object{
				ID:         "ID456",
				Type:       testObject.Type,
				Attributes: testObject.Attributes,
			}

			testList = append(testList, thirdObject)
			testList = append(testList, firstObject)
			sort.Sort(testList)
			So(testList[0].ID, ShouldEqual, firstObject.ID)
			So(testList[1].ID, ShouldEqual, testObject.ID)
			So(testList[2].ID, ShouldEqual, thirdObject.ID)
		})

		Convey("->Send(List)", func() {

			Convey("should send a properly formatted List response", func() {
				writer := httptest.NewRecorder()
				err := Send(writer, req, testList)
				So(err, ShouldBeNil)
				So(writer.Code, ShouldEqual, http.StatusOK)

				contentLength, convErr := strconv.Atoi(writer.HeaderMap.Get("Content-Length"))
				So(convErr, ShouldBeNil)
				So(contentLength, ShouldBeGreaterThan, 0)
				So(writer.HeaderMap.Get("Content-Type"), ShouldEqual, ContentType)

				req, reqErr := testRequest(writer.Body.Bytes())
				So(reqErr, ShouldBeNil)

				responseList, parseErr := ParseList(req)
				So(parseErr, ShouldBeNil)
				So(len(responseList), ShouldEqual, 1)
			})

			Convey("should send a properly formatted empty List response", func() {
				writer := httptest.NewRecorder()
				err := Send(writer, req, List{})
				So(err, ShouldBeNil)
				So(writer.Code, ShouldEqual, http.StatusOK)

				contentLength, convErr := strconv.Atoi(writer.HeaderMap.Get("Content-Length"))
				So(convErr, ShouldBeNil)
				So(contentLength, ShouldBeGreaterThan, 0)
				So(writer.HeaderMap.Get("Content-Type"), ShouldEqual, ContentType)

				req, reqErr := testRequest(writer.Body.Bytes())
				So(reqErr, ShouldBeNil)

				responseList, parseErr := ParseList(req)
				So(parseErr, ShouldBeNil)
				So(len(responseList), ShouldEqual, 0)
			})

			Convey("should reject nil List", func() {
				writer := httptest.NewRecorder()
				var list List
				err := Send(writer, req, list)
				So(err, ShouldNotBeNil)
				So(writer.Code, ShouldEqual, http.StatusInternalServerError)
			})

			Convey("should accept empty List", func() {
				writer := httptest.NewRecorder()
				err := Send(writer, req, List{})
				So(err, ShouldBeNil)
			})
		})

		Convey("->UnmarshalJSON()", func() {

			Convey("should handle a data object", func() {
				jObj := `{"data": {"type": "user", "id": "sweetID123", "attributes": {"ID":"123"}}}`

				l := List{}
				err := l.UnmarshalJSON([]byte(jObj))
				So(err, ShouldBeNil)
				So(l, ShouldNotBeEmpty)
			})

			Convey("should handle a data list", func() {
				jList := `{"data": [{"type": "user", "id": "sweetID123", "attributes": {"ID":"123"}}]}`

				l := List{}
				err := l.UnmarshalJSON([]byte(jList))
				So(err, ShouldBeNil)
				So(l, ShouldNotBeEmpty)
			})

			Convey("should handle an empty array", func() {
				jObj := `{"data": []}`

				l := List{}
				err := l.UnmarshalJSON([]byte(jObj))
				So(err, ShouldBeNil)
				So(l, ShouldNotBeNil)
			})
		})

		Convey("->MarshalJSON()", func() {

			Convey("should preserve an empty list", func() {
				list := List{}

				jData, err := json.Marshal(list)
				So(err, ShouldBeNil)

				So(string(jData), ShouldEqual, "[]")
			})
		})
	})
}
<|endoftext|>