<commit_before>package main

import (
	"encoding/json"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

var qa_urls = []string{
	"http://is.qa.ec2.srcclr.com:3000/services",
	"http://10.0.3.103:3000/services",
	"http://10.0.3.126:3000/services",
}

var prod_urls = []string{
	"http://is.ec2.srcclr.com:3000/services",
}

// A map for Puppet Versions JSON
type PuppetVersions interface{}

// The final map to be passed to template
type Compared map[string]map[string]map[string]string

// Our bare page
type Page struct {
	Title   string
	Body    []byte
	Compare Compared
}

// What do *you* think this does?
func colorize(versions []string) (color string, err error) {
	match_fail, _ := regexp.Compile(`Failed`)
	if len(versions) > 1 {
		for i := 0; i < len(versions); i++ {
			if match_fail.MatchString(versions[i]) || match_fail.MatchString(versions[i+1]) {
				return "orange", nil
			}
			if versions[i] == versions[i+1] {
				color = "green"
				return color, nil
			} else {
				color = "red"
				return color, nil
			}
		}
	} else {
		return "green", nil
	}
	return "versions not an array?", err
}

func compare(puppet_v map[string]interface{}, qa_v map[string]map[string]string, prod_v map[string]map[string]string) (Compared, error) {
	c := make(map[string]map[string]map[string]string)
	// Setup regex for QA match
	match_qa, err := regexp.Compile(`_qa`)
	if err != nil {
		log.Println("Couldn't compile regex")
		return nil, err
	}
	match_prod, err := regexp.Compile(`_production`)
	if err != nil {
		log.Println("Couldn't compile regex")
		return nil, err
	}
	// Get environments from PuppetVersions, populate top level map
	c["qa"] = make(map[string]map[string]string)
	c["production"] = make(map[string]map[string]string)

	for p_name, pv := range puppet_v {
		// Create regex match for service name
		p_name_arry := strings.Split(p_name, "_")
		match_name := p_name_arry[0]
		match_svc, _ := regexp.Compile(match_name)
		pv_string := pv.(string)
		// Match QA map
		if match_qa.MatchString(p_name) {
			// Add the name and puppet version to QA map
			c["qa"][match_name] = make(map[string]string)
			c["qa"][match_name]["pv"] = pv_string

			// Init new array, add versions for this service
			colorize_arry := []string{}
			colorize_arry = append(colorize_arry, pv_string)
			for svc_name, endpoints := range qa_v {
				if match_svc.MatchString(svc_name) {
					for ep, version := range endpoints {
						c["qa"][match_name][ep] = version
						colorize_arry = append(colorize_arry, version)
						color, _ := colorize(colorize_arry)
						c["qa"][match_name]["color"] = color
					}
				}
			}
		}
		// Do it again for production env
		if match_prod.MatchString(p_name) {
			log.Println("Production MATCH: ", p_name, " ", pv)
			c["production"][match_name] = make(map[string]string)
			c["production"][match_name]["pv"] = pv_string

			// Init new array, add versions for this service
			colorize_arry := []string{}
			colorize_arry = append(colorize_arry, pv_string)
			for svc_name, endpoints := range prod_v {
				if match_svc.MatchString(svc_name) {
					for ep, version := range endpoints {
						c["production"][match_name][ep] = version
						colorize_arry = append(colorize_arry, version)
						color, _ := colorize(colorize_arry)
						c["production"][match_name]["color"] = color
					}
				}
			}
		}
	}

	return c, nil
}

func puppetversions(url string) (PuppetVersions, error) {
	// Get response and handle any errors on return
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		log.Println(err)
		return nil, err
	}

	// Read JSON from request
	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Unmarshel the JSON to our struct
	var v PuppetVersions
	err = json.Unmarshal(jsonDataFromHttp, &v)
	if err != nil {
		return nil, err
	}

	return v, nil
}

func getServices(urls []string) (map[string][]string, error) {
	available_services := make(map[string][]string)

	for _, url := range urls {
		resp, err := http.Get(url)
		if err != nil {
			log.Println(err)
			return nil, err
		}
		defer resp.Body.Close()
		jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}

		var services map[string]map[string][]string //interface{}

		err = json.Unmarshal(jsonDataFromHttp, &services)
		if err != nil {
			return nil, err
		}

		for _, v := range services {
			log.Println("Values: ", v)
			for name, endpoints := range v {
				log.Println("Name: ", name, " ", endpoints)
				available_services[name] = []string{}
				for _, ep := range endpoints {
					available_services[name] = append(available_services[name], ep)
				}
			}
		}
	}

	return available_services, nil
}

func queryServiceVersion(endpoint string) (version string, err error) {
	log.Println("Querying SERVICE address: ", endpoint)
	query_arry := []string{"http://", endpoint, "/info"}
	query := strings.Join(query_arry, "")
	// Query the URI
	timeout := time.Duration(1 * time.Second)
	client := http.Client{
		Timeout: timeout,
	}
	resp, err := client.Get(query)
	if err != nil {
		log.Println("ERROR querying ", query, " ", err)
		return "Failed GET Request", err
	}
	defer resp.Body.Close()

	// Get data and unmarshel the JSON to our map
	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("ERROR unmarsheling data for ", jsonDataFromHttp)
		return "Failed to read JSON", err
	}
	var info_response interface{}
	err = json.Unmarshal(jsonDataFromHttp, &info_response)
	if err != nil {
		return "Failed unmarshel JSON", err
	}
	// Parse out the version from the response
	info_map := info_response.(map[string]interface{})
	log.Println("Response: ", info_response)
	for _, values := range info_map {
		sub_info_map := values.(map[string]interface{})
		for key, info := range sub_info_map {
			string_info := info.(string)
			if key == "version" {
				log.Println("Version: ", string_info)
				return string_info, nil
			}
		}
	}
	return version, nil
}

func getVersions(services map[string][]string) (runningversions map[string]map[string]string, err error) {

	rv := make(map[string]map[string]string)

	for name, endpoints := range services {
		rv[name] = make(map[string]string)
		for _, endpoint := range endpoints {
			query_arry := strings.Fields(endpoint)
			if len(query_arry) == 2 {
				info_ep := query_arry[1]
				version, _ := queryServiceVersion(info_ep)
				rv[name][endpoint] = version
			} else {
				info_ep := query_arry[0]
				version, _ := queryServiceVersion(info_ep)
				rv[name][endpoint] = version
			}
		}
	}
	runningversions = rv
	return runningversions, nil
}

func loadPage(title string) (*Page, error) {
	// Get the versions from the puppet master
	log.Println("Getting Puppet Versions - Make sure VPN is on!")
	pv, err := puppetversions("http://puppet.ec2.srcclr.com:1015/versions")
	if err != nil {
		log.Println("Failed to get Puppet Versions from http://puppet.ec2.srcclr.com:1015/versions\n")
		log.Println(err)
	}
	log.Println("Puppet Versions: ", pv)

	// QA
	log.Println("Getting available services...")
	qa_rs, err := getServices(qa_urls)
	if err != nil {
		log.Println("Failed getting qa versions")
	}

	log.Println("RUNNING SERVICES QA: ", qa_rs)

	qa_v, err := getVersions(qa_rs)
	if err != nil {
		log.Println("Failed getting versions for ", qa_rs)
	}

	log.Println("Running Versions QA: ", qa_v)

	// PRODUCTION
	log.Println("Getting available services...")
	prod_rs, err := getServices(prod_urls)
	if err != nil {
		log.Println("Failed getting production versions")
	}

	log.Println("RUNNING SERVICES QA: ", prod_rs)

	prod_v, err := getVersions(prod_rs)
	if err != nil {
		log.Println("Failed getting versions for ", prod_rs)
	}

	log.Println("Running Versions PRODUCTION: ", prod_v)

	// Build the compared map of maps of strings of other types ... blah blah blah
	pv_map := pv.(map[string]interface{})
	compared, _ := compare(pv_map, qa_v, prod_v)

	for k, v := range compared {
		log.Println(k, " ", v, "\n")
	}

	filename := title + ".html"
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	return &Page{
			Title:   title,
			Body:    body,
			Compare: compared,
		},
		nil
}

func viewHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Loading page view...")
	title := "versionctl"
	p, err := loadPage(title)
	if err != nil {
		p = &Page{Title: title}
	}

	// Parse the template, execute and write it to stdout for good measure
	log.Println("Parsing go template...")
	t, _ := template.ParseFiles("versionctl.html")
	t.Execute(w, p)
	//log.Println("Serving:\n", string(p.Title), string(p.Body))
}

func main() {
	log.Println("Starting vctl...")
	http.HandleFunc("/", viewHandler)
	http.ListenAndServe(":9000", nil)
}
<commit_msg>vctl.go<commit_after>package main

import (
	"encoding/json"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"
)

var qa_urls = []string{
	"http://is.qa.ec2.srcclr.com:3000/services",
	"http://10.0.3.103:3000/services",
	"http://10.0.3.126:3000/services",
}

var prod_urls = []string{
	"http://is.ec2.srcclr.com:3000/services",
}

// A map for Puppet Versions JSON
type PuppetVersions interface{}

// The final map to be passed to template
type Compared map[string]map[string]map[string]string

// Our bare page
type Page struct {
	Title   string
	Body    []byte
	Compare Compared
}

// What do *you* think this does?
func colorize(versions []string) (color string, err error) {
	match_fail, _ := regexp.Compile(`Failed`)
	if len(versions) > 1 {
		for i := 0; i < len(versions); i++ {
			if match_fail.MatchString(versions[i]) {
				return "orange", nil
			}
		}
		for i := 0; i < len(versions); i++ {
			if versions[i] == versions[i+1] {
				color = "green"
				return color, nil
			} else {
				color = "red"
				return color, nil
			}
		}
	} else {
		return "green", nil
	}
	return "versions not an array?", err
}

func compare(puppet_v map[string]interface{}, qa_v map[string]map[string]string, prod_v map[string]map[string]string) (Compared, error) {
	c := make(map[string]map[string]map[string]string)
	// Setup regex for QA match
	match_qa, err := regexp.Compile(`_qa`)
	if err != nil {
		log.Println("Couldn't compile regex")
		return nil, err
	}
	match_prod, err := regexp.Compile(`_production`)
	if err != nil {
		log.Println("Couldn't compile regex")
		return nil, err
	}
	// Get environments from PuppetVersions, populate top level map
	c["qa"] = make(map[string]map[string]string)
	c["production"] = make(map[string]map[string]string)

	for p_name, pv := range puppet_v {
		// Create regex match for service name
		p_name_arry := strings.Split(p_name, "_")
		match_name := p_name_arry[0]
		match_svc, _ := regexp.Compile(match_name)
		pv_string := pv.(string)
		// Match QA map
		if match_qa.MatchString(p_name) {
			// Add the name and puppet version to QA map
			c["qa"][match_name] = make(map[string]string)
			c["qa"][match_name]["pv"] = pv_string

			// Init new array, add versions for this service
			colorize_arry := []string{}
			colorize_arry = append(colorize_arry, pv_string)
			for svc_name, endpoints := range qa_v {
				if match_svc.MatchString(svc_name) {
					for ep, version := range endpoints {
						c["qa"][match_name][ep] = version
						colorize_arry = append(colorize_arry, version)
						color, _ := colorize(colorize_arry)
						c["qa"][match_name]["color"] = color
					}
				}
			}
		}
		// Do it again for production env
		if match_prod.MatchString(p_name) {
			log.Println("Production MATCH: ", p_name, " ", pv)
			c["production"][match_name] = make(map[string]string)
			c["production"][match_name]["pv"] = pv_string

			// Init new array, add versions for this service
			colorize_arry := []string{}
			colorize_arry = append(colorize_arry, pv_string)
			for svc_name, endpoints := range prod_v {
				if match_svc.MatchString(svc_name) {
					for ep, version := range endpoints {
						c["production"][match_name][ep] = version
						colorize_arry = append(colorize_arry, version)
						color, _ := colorize(colorize_arry)
						c["production"][match_name]["color"] = color
					}
				}
			}
		}
	}

	return c, nil
}

func puppetversions(url string) (PuppetVersions, error) {
	// Get response and handle any errors on return
	resp, err := http.Get(url)
	defer resp.Body.Close()
	if err != nil {
		log.Println(err)
		return nil, err
	}

	// Read JSON from request
	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Unmarshel the JSON to our struct
	var v PuppetVersions
	err = json.Unmarshal(jsonDataFromHttp, &v)
	if err != nil {
		return nil, err
	}

	return v, nil
}

func getServices(urls []string) (map[string][]string, error) {
	available_services := make(map[string][]string)

	for _, url := range urls {
		resp, err := http.Get(url)
		if err != nil {
			log.Println(err)
			return nil, err
		}
		defer resp.Body.Close()
		jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}

		var services map[string]map[string][]string //interface{}

		err = json.Unmarshal(jsonDataFromHttp, &services)
		if err != nil {
			return nil, err
		}

		for _, v := range services {
			log.Println("Values: ", v)
			for name, endpoints := range v {
				log.Println("Name: ", name, " ", endpoints)
				available_services[name] = []string{}
				for _, ep := range endpoints {
					available_services[name] = append(available_services[name], ep)
				}
			}
		}
	}

	return available_services, nil
}

func queryServiceVersion(endpoint string) (version string, err error) {
	log.Println("Querying SERVICE address: ", endpoint)
	query_arry := []string{"http://", endpoint, "/info"}
	query := strings.Join(query_arry, "")
	// Query the URI
	timeout := time.Duration(1 * time.Second)
	client := http.Client{
		Timeout: timeout,
	}
	resp, err := client.Get(query)
	if err != nil {
		log.Println("ERROR querying ", query, " ", err)
		return "Failed GET Request", err
	}
	defer resp.Body.Close()

	// Get data and unmarshel the JSON to our map
	jsonDataFromHttp, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("ERROR unmarsheling data for ", jsonDataFromHttp)
		return "Failed to read JSON", err
	}
	var info_response interface{}
	err = json.Unmarshal(jsonDataFromHttp, &info_response)
	if err != nil {
		return "Failed unmarshel JSON", err
	}
	// Parse out the version from the response
	info_map := info_response.(map[string]interface{})
	log.Println("Response: ", info_response)
	for _, values := range info_map {
		sub_info_map := values.(map[string]interface{})
		for key, info := range sub_info_map {
			string_info := info.(string)
			if key == "version" {
				log.Println("Version: ", string_info)
				return string_info, nil
			}
		}
	}
	return version, nil
}

func getVersions(services map[string][]string) (runningversions map[string]map[string]string, err error) {

	rv := make(map[string]map[string]string)

	for name, endpoints := range services {
		rv[name] = make(map[string]string)
		for _, endpoint := range endpoints {
			query_arry := strings.Fields(endpoint)
			if len(query_arry) == 2 {
				info_ep := query_arry[1]
				version, _ := queryServiceVersion(info_ep)
				rv[name][endpoint] = version
			} else {
				info_ep := query_arry[0]
				version, _ := queryServiceVersion(info_ep)
				rv[name][endpoint] = version
			}
		}
	}
	runningversions = rv
	return runningversions, nil
}

func loadPage(title string) (*Page, error) {
	// Get the versions from the puppet master
	log.Println("Getting Puppet Versions - Make sure VPN is on!")
	pv, err := puppetversions("http://puppet.ec2.srcclr.com:1015/versions")
	if err != nil {
		log.Println("Failed to get Puppet Versions from http://puppet.ec2.srcclr.com:1015/versions\n")
		log.Println(err)
	}
	log.Println("Puppet Versions: ", pv)

	// QA
	log.Println("Getting available services...")
	qa_rs, err := getServices(qa_urls)
	if err != nil {
		log.Println("Failed getting qa versions")
	}

	log.Println("RUNNING SERVICES QA: ", qa_rs)

	qa_v, err := getVersions(qa_rs)
	if err != nil {
		log.Println("Failed getting versions for ", qa_rs)
	}

	log.Println("Running Versions QA: ", qa_v)

	// PRODUCTION
	log.Println("Getting available services...")
	prod_rs, err := getServices(prod_urls)
	if err != nil {
		log.Println("Failed getting production versions")
	}

	log.Println("RUNNING SERVICES QA: ", prod_rs)

	prod_v, err := getVersions(prod_rs)
	if err != nil {
		log.Println("Failed getting versions for ", prod_rs)
	}

	log.Println("Running Versions PRODUCTION: ", prod_v)

	// Build the compared map of maps of strings of other types ... blah blah blah
	pv_map := pv.(map[string]interface{})
	compared, _ := compare(pv_map, qa_v, prod_v)

	for k, v := range compared {
		log.Println(k, " ", v, "\n")
	}

	filename := title + ".html"
	body, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	return &Page{
			Title:   title,
			Body:    body,
			Compare: compared,
		},
		nil
}

func viewHandler(w http.ResponseWriter, r *http.Request) {
	log.Println("Loading page view...")
	title := "versionctl"
	p, err := loadPage(title)
	if err != nil {
		p = &Page{Title: title}
	}

	// Parse the template, execute and write it to stdout for good measure
	log.Println("Parsing go template...")
	t, _ := template.ParseFiles("versionctl.html")
	t.Execute(w, p)
	//log.Println("Serving:\n", string(p.Title), string(p.Body))
}

func main() {
	log.Println("Starting vctl...")
	http.HandleFunc("/", viewHandler)
	http.ListenAndServe(":9000", nil)
}
<|endoftext|>


<commit_before>// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package deltas

import (
	"context"
	"database/sql"
	"fmt"
)

func UpPulishedAppservice(ctx context.Context, tx *sql.Tx) error {
	_, err := tx.ExecContext(ctx, `	ALTER TABLE roomserver_published RENAME TO roomserver_published_tmp;
CREATE TABLE IF NOT EXISTS roomserver_published (
    room_id TEXT NOT NULL,
    appservice_id TEXT NOT NULL,
    network_id TEXT NOT NULL,
    published BOOLEAN NOT NULL DEFAULT false,
    CONSTRAINT unique_published_idx PRIMARY KEY (room_id, appservice_id, network_id)
);
INSERT
    INTO roomserver_published (
      room_id, published
    ) SELECT
        room_id, published
    FROM roomserver_published_tmp
;
DROP TABLE roomserver_published_tmp;`)
	if err != nil {
		return fmt.Errorf("failed to execute upgrade: %w", err)
	}
	return nil
}

func DownPublishedAppservice(ctx context.Context, tx *sql.Tx) error {
	_, err := tx.ExecContext(ctx, `	ALTER TABLE roomserver_published RENAME TO roomserver_published_tmp;
CREATE TABLE IF NOT EXISTS roomserver_published (
    room_id TEXT NOT NULL PRIMARY KEY,
    published BOOLEAN NOT NULL DEFAULT false
);
INSERT
    INTO roomserver_published (
      room_id, published
    ) SELECT
        room_id, published
    FROM roomserver_published_tmp
;
DROP TABLE roomserver_published_tmp;`)
	if err != nil {
		return fmt.Errorf("failed to execute upgrade: %w", err)
	}
	return nil
}
<commit_msg>Fix SQLite `roomserver_published` migration<commit_after>// Copyright 2022 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package deltas

import (
	"context"
	"database/sql"
	"fmt"
)

func UpPulishedAppservice(ctx context.Context, tx *sql.Tx) error {
	_, err := tx.ExecContext(ctx, `	ALTER TABLE roomserver_published RENAME TO roomserver_published_tmp;
CREATE TABLE IF NOT EXISTS roomserver_published (
    room_id TEXT NOT NULL,
    appservice_id TEXT NOT NULL DEFAULT '',
    network_id TEXT NOT NULL DEFAULT '',
    published BOOLEAN NOT NULL DEFAULT false,
    CONSTRAINT unique_published_idx PRIMARY KEY (room_id, appservice_id, network_id)
);
INSERT
    INTO roomserver_published (
      room_id, published
    ) SELECT
        room_id, published
    FROM roomserver_published_tmp
;
DROP TABLE roomserver_published_tmp;`)
	if err != nil {
		return fmt.Errorf("failed to execute upgrade: %w", err)
	}
	return nil
}

func DownPublishedAppservice(ctx context.Context, tx *sql.Tx) error {
	_, err := tx.ExecContext(ctx, `	ALTER TABLE roomserver_published RENAME TO roomserver_published_tmp;
CREATE TABLE IF NOT EXISTS roomserver_published (
    room_id TEXT NOT NULL PRIMARY KEY,
    published BOOLEAN NOT NULL DEFAULT false
);
INSERT
    INTO roomserver_published (
      room_id, published
    ) SELECT
        room_id, published
    FROM roomserver_published_tmp
;
DROP TABLE roomserver_published_tmp;`)
	if err != nil {
		return fmt.Errorf("failed to execute upgrade: %w", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package transport

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

func newHTTP(addr string) (tsp *httpTsp, err error) {
	u, err := url.Parse(addr)

	if err != nil {
		return nil, err
	}

	return &httpTsp{
		client: &http.Client{},
		url:    u,
	}, nil
}

type httpTsp struct {
	client *http.Client
	url    *url.URL
}

func (t *httpTsp) Request(resource string, data devicehiveData, timeout time.Duration) (rawRes []byte, err *Error) {
	t.setTimeout(timeout)
	method := t.getRequestMethod(data)

	reqDataReader, err := t.createRequestDataReader(data)
	if err != nil {
		return nil, err
	}

	addr, err := t.createRequestAddr(data)
	if err != nil {
		return
	}

	req, reqErr := http.NewRequest(method, addr, reqDataReader)
	if reqErr != nil {
		return nil, &Error{name: InvalidRequestErr, reason: reqErr.Error()}
	}

	return t.doRequest(req)
}

func (t *httpTsp) setTimeout(timeout time.Duration) {
	if timeout == 0 {
		timeout = DefaultTimeout
	}

	t.client.Timeout = timeout
}

func (t *httpTsp) getRequestMethod(data devicehiveData) string {
	defaultMethod := "GET"

	if data == nil {
		return defaultMethod
	}

	if _, ok := data["method"]; !ok {
		return defaultMethod
	}

	if m, ok := data["method"].(string); ok {
		return m
	}

	return defaultMethod
}

func (t *httpTsp) createRequestDataReader(data devicehiveData) (dataReader *bytes.Reader, err *Error) {
	var reqData []byte

	if reqData, ok := data["request"]; ok {
		var err error
		reqData, err = json.Marshal(reqData)

		if err != nil {
			return nil, &Error{name: InvalidRequestErr, reason: err.Error()}
		}
	} else {
		reqData = []byte("{}")
	}

	return bytes.NewReader(reqData), nil
}

func (t *httpTsp) createRequestAddr(data devicehiveData) (addr string, err *Error) {
	u := t.url

	if resource, ok := data["resource"].(string); ok {
		var urlErr error
		u, urlErr = t.url.Parse(resource)

		if urlErr != nil {
			return "", &Error{name: InvalidRequestErr, reason: urlErr.Error()}
		}
	}

	return u.String(), nil
}

func (t *httpTsp) doRequest(req *http.Request) (rawRes []byte, err *Error) {
	res, resErr := t.client.Do(req)

	if resErr != nil {
		if isTimeoutErr(resErr) {
			return nil, &Error{name: TimeoutErr, reason: resErr.Error()}
		}

		return nil, &Error{name: InvalidRequestErr, reason: resErr.Error()}
	}
	defer res.Body.Close()

	rawRes, rErr := ioutil.ReadAll(res.Body)

	if rErr != nil {
		return nil, &Error{name: InvalidResponseErr, reason: rErr.Error()}
	}

	return rawRes, nil
}

func (t *httpTsp) Subscribe(subscriptionId string) (eventChan chan []byte) {
	return nil
}

func (t *httpTsp) Unsubscribe(subscriptionId string) {}
<commit_msg>Fix shadowed variable in httpTsp.createRequestDataReader()<commit_after>package transport

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

func newHTTP(addr string) (tsp *httpTsp, err error) {
	u, err := url.Parse(addr)

	if err != nil {
		return nil, err
	}

	return &httpTsp{
		client: &http.Client{},
		url:    u,
	}, nil
}

type httpTsp struct {
	client *http.Client
	url    *url.URL
}

func (t *httpTsp) Request(resource string, data devicehiveData, timeout time.Duration) (rawRes []byte, err *Error) {
	t.setTimeout(timeout)
	method := t.getRequestMethod(data)

	reqDataReader, err := t.createRequestDataReader(data)
	if err != nil {
		return nil, err
	}

	addr, err := t.createRequestAddr(data)
	if err != nil {
		return
	}

	req, reqErr := http.NewRequest(method, addr, reqDataReader)
	if reqErr != nil {
		return nil, &Error{name: InvalidRequestErr, reason: reqErr.Error()}
	}

	return t.doRequest(req)
}

func (t *httpTsp) setTimeout(timeout time.Duration) {
	if timeout == 0 {
		timeout = DefaultTimeout
	}

	t.client.Timeout = timeout
}

func (t *httpTsp) getRequestMethod(data devicehiveData) string {
	defaultMethod := "GET"

	if data == nil {
		return defaultMethod
	}

	if _, ok := data["method"]; !ok {
		return defaultMethod
	}

	if m, ok := data["method"].(string); ok {
		return m
	}

	return defaultMethod
}

func (t *httpTsp) createRequestDataReader(data devicehiveData) (dataReader *bytes.Reader, err *Error) {
	var rawReqData []byte

	if reqData, ok := data["request"]; ok {
		var err error
		rawReqData, err = json.Marshal(reqData)

		if err != nil {
			return nil, &Error{name: InvalidRequestErr, reason: err.Error()}
		}
	} else {
		rawReqData = []byte("{}")
	}

	return bytes.NewReader(rawReqData), nil
}

func (t *httpTsp) createRequestAddr(data devicehiveData) (addr string, err *Error) {
	u := t.url

	if resource, ok := data["resource"].(string); ok {
		var urlErr error
		u, urlErr = t.url.Parse(resource)

		if urlErr != nil {
			return "", &Error{name: InvalidRequestErr, reason: urlErr.Error()}
		}
	}

	return u.String(), nil
}

func (t *httpTsp) doRequest(req *http.Request) (rawRes []byte, err *Error) {
	res, resErr := t.client.Do(req)

	if resErr != nil {
		if isTimeoutErr(resErr) {
			return nil, &Error{name: TimeoutErr, reason: resErr.Error()}
		}

		return nil, &Error{name: InvalidRequestErr, reason: resErr.Error()}
	}
	defer res.Body.Close()

	rawRes, rErr := ioutil.ReadAll(res.Body)

	if rErr != nil {
		return nil, &Error{name: InvalidResponseErr, reason: rErr.Error()}
	}

	return rawRes, nil
}

func (t *httpTsp) Subscribe(subscriptionId string) (eventChan chan []byte) {
	return nil
}

func (t *httpTsp) Unsubscribe(subscriptionId string) {}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
)

var (
	reApplication = regexp.MustCompile("[ \n]*application:[ \t]*([a-zA-Z0-9-]+)")
	reVersion     = regexp.MustCompile("\nversion:[ \t]*([a-zA-Z0-9-]+)")
)

func pipeCmd(command string, args ...string) error {
	cmd := exec.Command(command, args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	err = cmd.Start()
	if err != nil {
		return err
	}
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	cmd.Wait()
	return nil
}

func run(action string) (err error) {
	switch action {
	case "init":
		if _, err = os.Stat("tailbone"); err == nil {
			return errors.New("Tailbone is already initialized.")
		}
		if _, err = os.Stat(".git"); os.IsNotExist(err) {
			return errors.New("Your current directory must be a git repo.")
		}
		args := strings.Split("git submodule add -b reorg https://github.com/doug/tailbone.git", " ")
		err = pipeCmd(args[0], args[1:]...)
		if err != nil {
			return errors.New("Problem checking out the tailbone git submodule.")
		}
		if apptemplate, err := os.Open("tailbone/app.template.yaml"); err == nil {
			if appyaml, err := os.Create("app.yaml"); err == nil {
				_, err = io.Copy(appyaml, apptemplate)
			}
		}
		if _, err = os.Stat("app/index.html"); os.IsNotExist(err) {
			os.Mkdir("app", 0644)
			if index, err := os.Create("app/index.html"); err == nil {
				index.WriteString(INDEX_TEMPLATE)
			}
		}
	case "serve":
		if _, err := os.Stat("tailbone"); os.IsNotExist(err) {
			return errors.New("Must run 'tailbone init' first.")
		}
		if _, err := exec.LookPath("dev_appserver.py"); err != nil {
			return errors.New("You must have the cloud sdk for python installed on your system. Download it at https://developers.google.com/cloud/sdk/")
		}
		args := flag.Args()[1:]
		args = append(args, "tailbone")
		err = pipeCmd("dev_appserver.py", args...)
	case "deploy":
		if _, err := os.Stat("tailbone"); os.IsNotExist(err) {
			return errors.New("Must run 'tailbone init' first.")
		}
		if _, err := os.Stat("app.yaml"); os.IsNotExist(err) {
			return errors.New("Must have app.yaml, this can be copied from tailbone/app.template.yaml.")
		}
		if _, err := exec.LookPath("appcfg.py"); err != nil {
			return errors.New("You must have the cloud sdk for python installed on your system. Download it at https://developers.google.com/cloud/sdk/")
		}
		version := flag.Arg(1)
		if version == "" {
			return errors.New("Must provide a version name. Example: tailbone deploy master")
		}
		if appbytes, err := ioutil.ReadFile("app.yaml"); err == nil {
			appyaml := string(appbytes)
			matches := reApplication.FindStringSubmatch(appyaml)
			if len(matches) != 2 {
				return errors.New("Incorrectly formated app.yaml could not find application.")
			}
			application := matches[1]
			if application == DEFAULT_APPLICATION {
				fmt.Printf("You must enter an application id. If you haven't already created a project, do so now at http://cloud.google.com\n")
				fmt.Scanf("%s", &application)
				appyaml = reApplication.ReplaceAllStringFunc(appyaml, func(s string) string { return "application: " + application })
			}
			appyaml = reVersion.ReplaceAllStringFunc(appyaml, func(s string) string { return "\nversion: " + version })
			ioutil.WriteFile("app.yaml", []byte(appyaml), 0644)
			err = pipeCmd("appcfg.py", "update", "--oauth2", "tailbone")
		}
	case "update":
		if _, err := os.Stat("tailbone"); os.IsNotExist(err) {
			return errors.New("Must run 'tailbone init' first.")
		}
		if _, err := exec.LookPath("git"); err != nil {
			return errors.New("You must have git installed.")
		}
		log.Printf("Update doesn't work yet do this manually by updating the git submodule tailbone.")
	default:
		flag.Usage()
	}
	return
}

func main() {
	flag.Parse()

	flag.Usage = func() {
		fmt.Printf(USAGE_TEMPLATE)
	}

	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(1)
	}

	cmdname := flag.Arg(0)

	if err := run(cmdname); err != nil {
		log.Println(err)
		os.Exit(1)
	}
}

const (
	DEFAULT_APPLICATION = "your-application-id"
	INDEX_TEMPLATE      = `<!doctype html>

hello tailbone

`
	USAGE_TEMPLATE = `
tailbone init:
  Initialize tailbone
tailbone serve:
  Serve tailbone locally
tailbone deploy {version_name}:
  Deploy tailbone to AppEngine
tailbone update:
  Update the version of tailbone to latest
tailbone version:
  Get the version of tailbone

`
)
<commit_msg>tailbone check out master<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strings"
)

var (
	reApplication = regexp.MustCompile("[ \n]*application:[ \t]*([a-zA-Z0-9-]+)")
	reVersion     = regexp.MustCompile("\nversion:[ \t]*([a-zA-Z0-9-]+)")
)

func pipeCmd(command string, args ...string) error {
	cmd := exec.Command(command, args...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	err = cmd.Start()
	if err != nil {
		return err
	}
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	cmd.Wait()
	return nil
}

func run(action string) (err error) {
	switch action {
	case "init":
		if _, err = os.Stat("tailbone"); err == nil {
			return errors.New("Tailbone is already initialized.")
		}
		if _, err = os.Stat(".git"); os.IsNotExist(err) {
			return errors.New("Your current directory must be a git repo.")
		}
		args := strings.Split("git submodule add https://github.com/dataarts/tailbone.git", " ")
		err = pipeCmd(args[0], args[1:]...)
		if err != nil {
			return errors.New("Problem checking out the tailbone git submodule.")
		}
		if apptemplate, err := os.Open("tailbone/app.template.yaml"); err == nil {
			if appyaml, err := os.Create("app.yaml"); err == nil {
				_, err = io.Copy(appyaml, apptemplate)
			}
		}
		if _, err = os.Stat("app/index.html"); os.IsNotExist(err) {
			os.Mkdir("app", 0644)
			if index, err := os.Create("app/index.html"); err == nil {
				index.WriteString(INDEX_TEMPLATE)
			}
		}
	case "serve":
		if _, err := os.Stat("tailbone"); os.IsNotExist(err) {
			return errors.New("Must run 'tailbone init' first.")
		}
		if _, err := exec.LookPath("dev_appserver.py"); err != nil {
			return errors.New("You must have the cloud sdk for python installed on your system. Download it at https://developers.google.com/cloud/sdk/")
		}
		args := flag.Args()[1:]
		args = append(args, "tailbone")
		err = pipeCmd("dev_appserver.py", args...)
	case "deploy":
		if _, err := os.Stat("tailbone"); os.IsNotExist(err) {
			return errors.New("Must run 'tailbone init' first.")
		}
		if _, err := os.Stat("app.yaml"); os.IsNotExist(err) {
			return errors.New("Must have app.yaml, this can be copied from tailbone/app.template.yaml.")
		}
		if _, err := exec.LookPath("appcfg.py"); err != nil {
			return errors.New("You must have the cloud sdk for python installed on your system. Download it at https://developers.google.com/cloud/sdk/")
		}
		version := flag.Arg(1)
		if version == "" {
			return errors.New("Must provide a version name. Example: tailbone deploy master")
		}
		if appbytes, err := ioutil.ReadFile("app.yaml"); err == nil {
			appyaml := string(appbytes)
			matches := reApplication.FindStringSubmatch(appyaml)
			if len(matches) != 2 {
				return errors.New("Incorrectly formated app.yaml could not find application.")
			}
			application := matches[1]
			if application == DEFAULT_APPLICATION {
				fmt.Printf("You must enter an application id. If you haven't already created a project, do so now at http://cloud.google.com\n")
				fmt.Scanf("%s", &application)
				appyaml = reApplication.ReplaceAllStringFunc(appyaml, func(s string) string { return "application: " + application })
			}
			appyaml = reVersion.ReplaceAllStringFunc(appyaml, func(s string) string { return "\nversion: " + version })
			ioutil.WriteFile("app.yaml", []byte(appyaml), 0644)
			err = pipeCmd("appcfg.py", "update", "--oauth2", "tailbone")
		}
	case "update":
		if _, err := os.Stat("tailbone"); os.IsNotExist(err) {
			return errors.New("Must run 'tailbone init' first.")
		}
		if _, err := exec.LookPath("git"); err != nil {
			return errors.New("You must have git installed.")
		}
		log.Printf("Update doesn't work yet do this manually by updating the git submodule tailbone.")
	default:
		flag.Usage()
	}
	return
}

func main() {
	flag.Parse()

	flag.Usage = func() {
		fmt.Printf(USAGE_TEMPLATE)
	}

	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(1)
	}

	cmdname := flag.Arg(0)

	if err := run(cmdname); err != nil {
		log.Println(err)
		os.Exit(1)
	}
}

const (
	DEFAULT_APPLICATION = "your-application-id"
	INDEX_TEMPLATE      = `<!doctype html>

hello tailbone

`
	USAGE_TEMPLATE = `
tailbone init:
  Initialize tailbone
tailbone serve:
  Serve tailbone locally
tailbone deploy {version_name}:
  Deploy tailbone to AppEngine
tailbone update:
  Update the version of tailbone to latest
tailbone version:
  Get the version of tailbone

`
)
<|endoftext|>


<commit_before>package talker

import (
	"fmt"

	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
)

// Notifaction new the notifaction object
type Notifaction struct {
	channelName  string
	projectName  string
	testStatus   string
	buildStatus  string
	depolyStatus string
	commitNotes  string
	duration     int64
}

const (
	testStage   = "test"
	buildStage  = "build"
	deployStage = "deploy"
)

const (
	// ChannelChatID defines the bot name
	// Hacked this on the web client
	// Regex format: *c(.*)_*
	ChannelChatID = 1378084890
	// ChannelName defines build/deploy channel name
	ChannelName = "@AstralServerNotifaction"
)

// NewNotifaction init the Notifaction instance
func NewNotifaction(repo string, stage map[string]string, commit string, duration int64) *Notifaction {
	if _, ok := stage[testStage]; !ok {
		return &Notifaction{}
	}
	if _, ok := stage[buildStage]; !ok {
		return &Notifaction{}
	}
	testStatus := stage[testStage]
	buildStatus := stage[buildStage]
	deployStatus := stage[deployStage]

	return &Notifaction{
		channelName:  "",
		projectName:  repo,
		testStatus:   testStatus,
		buildStatus:  buildStatus,
		depolyStatus: deployStatus,
		commitNotes:  commit,
		duration:     duration,
	}
}

// Notify sends the msg to the tg channel
func (n *Notifaction) Notify() tgbotapi.MessageConfig {
	text := fmt.Sprintf("**Commit Note**: `%s` ", n.commitNotes)
	text = fmt.Sprintf("%s\n **Test Status**: `%v`", text, n.testStatus)
	text = fmt.Sprintf("%s\n **Build Status**: `%v`", text, n.buildStatus)
	text = fmt.Sprintf("%s\n **Deploy Status**: `%v`", text, n.depolyStatus)
	text = fmt.Sprintf("%s\n **Duration**: `%v`", text, n.duration)
	return tgbotapi.MessageConfig{
		BaseChat: tgbotapi.BaseChat{
			ChannelUsername: ChannelName,
		},
		Text:      text,
		ParseMode: tgbotapi.ModeMarkdown,
	}
}
<commit_msg>talker:Remove ChannelID<commit_after>package talker

import (
	"fmt"

	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
)

// Notifaction new the notifaction object
type Notifaction struct {
	channelName  string
	projectName  string
	testStatus   string
	buildStatus  string
	depolyStatus string
	commitNotes  string
	duration     int64
}

const (
	testStage   = "test"
	buildStage  = "build"
	deployStage = "deploy"
)

const (
	// ChannelName defines build/deploy channel name
	ChannelName = "@AstralServerNotifaction"
)

// NewNotifaction init the Notifaction instance
func NewNotifaction(repo string, stage map[string]string, commit string, duration int64) *Notifaction {
	if _, ok := stage[testStage]; !ok {
		return &Notifaction{}
	}
	if _, ok := stage[buildStage]; !ok {
		return &Notifaction{}
	}
	testStatus := stage[testStage]
	buildStatus := stage[buildStage]
	deployStatus := stage[deployStage]

	return &Notifaction{
		channelName:  "",
		projectName:  repo,
		testStatus:   testStatus,
		buildStatus:  buildStatus,
		depolyStatus: deployStatus,
		commitNotes:  commit,
		duration:     duration,
	}
}

// Notify sends the msg to the tg channel
func (n *Notifaction) Notify() tgbotapi.MessageConfig {
	text := fmt.Sprintf("**Commit Note**: `%s` ", n.commitNotes)
	text = fmt.Sprintf("%s\n **Test Status**: `%v`", text, n.testStatus)
	text = fmt.Sprintf("%s\n **Build Status**: `%v`", text, n.buildStatus)
	text = fmt.Sprintf("%s\n **Deploy Status**: `%v`", text, n.depolyStatus)
	text = fmt.Sprintf("%s\n **Duration**: `%v s`", text, n.duration)
	return tgbotapi.MessageConfig{
		BaseChat: tgbotapi.BaseChat{
			ChannelUsername: ChannelName,
		},
		Text:      text,
		ParseMode: tgbotapi.ModeMarkdown,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"
	"time"

	"code.google.com/p/goauth2/oauth"
	"github.com/codegangsta/cli"
	"github.com/google/go-github/github"
)

const (
	dateFormat = "2006-01-02"
)

func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[options] {{end}}
"
VERSION:
   {{.Version}}{{if or .Author .Email}}

AUTHOR:{{if .Author}}
  {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
  {{.Email}}{{end}}{{end}}{{if .Flags}}

OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`
}

func main() {
	app := cli.NewApp()
	app.Name = "thisweek"
	app.Usage = "create a report of your team's activity this week (or for whenever you'd like)"
	app.Version = "0.0.1"
	app.Author = "Marcelo Silveira"
	app.Email = "marcelo@mhfs.com.br"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "repo, r",
			Value: "",
			Usage: "GitHub repository to analyze e.g. mhfs/thisweek",
		},
		cli.StringFlag{
			Name:  "since, s",
			Value: beginningOfWeek().In(time.Local).Format(dateFormat),
			Usage: "list issues since given date, inclusive",
		},
		cli.StringSliceFlag{
			Name:  "label, l",
			Value: &cli.StringSlice{},
			Usage: "label to process, defaults to all",
		},
	}

	app.Action = func(ctx *cli.Context) {
		repo := ctx.String("repo")

		if repo == "" {
			fmt.Println("\n***** Missing required flag --repo *****\n")
			cli.ShowAppHelp(ctx)
			return
		}

		since, err := time.ParseInLocation(dateFormat, ctx.String("since"), time.Local)
		if err != nil {
			panic("invalid date provided")
		}

		fmt.Printf("Starting work for repo '%s' since '%s'\n\n", repo, since.Format(dateFormat))

		parts := strings.Split(repo, "/")
		owner, repo := parts[0], parts[1]

		var finished bool
		for page := 1; !finished; page++ {
			events, err := fetchEvents(owner, repo, page)

			if err != nil {
				panic(err) // FIXME be smarter. handle 404s, 403s, ...
			}

			for _, event := range events {
				// if event is closed or issue didn't remain closed
				if *event.Event != "closed" || *event.Issue.State != "closed" {
					continue
				}

				// events are ordered by created at desc. stop if got all we wanted.
				if event.CreatedAt.Before(since) {
					finished = true
					break
				}

				printEvent(&event)
			}
		}
	}

	app.Run(os.Args)
}

func fetchEvents(owner string, repo string, page int) ([]github.IssueEvent, error) {
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: os.Getenv("GH_TOKEN")},
	}

	client := github.NewClient(t.Client())

	options := github.ListOptions{Page: page, PerPage: 100}
	events, _, err := client.Issues.ListRepositoryEvents(owner, repo, &options)

	return events, err
}

func printEvent(event *github.IssueEvent) {
	number := event.Issue.Number
	closedAt := event.Issue.ClosedAt.In(time.Local).Format(dateFormat)
	title := event.Issue.Title

	var assignee string
	if event.Issue.Assignee != nil {
		assignee = "@" + *event.Issue.Assignee.Login
	} else {
		assignee = "unknown"
	}

	fmt.Printf("#%d - %s - %s (%s)\n", *number, closedAt, *title, assignee)
}

func beginningOfWeek() time.Time {
	now := time.Now()
	// truncate internal HH:MM:SS to zero and compensate for local zone offset
	// 2015-01-31 10:45:54.720292964 -0800 PST > 2015-01-30 16:00:00 -0800 PST > 2015-01-31 00:00:00 -0800 PST
	_, offset := now.Zone()
	beginningOfDay := now.Truncate(24 * time.Hour).Add(-1 * time.Duration(offset) * time.Second)

	// subtract days to get to sunday
	beginningOfWeek := beginningOfDay.Add(-time.Duration(now.Weekday()) * 24 * time.Hour)

	return beginningOfWeek
}
<commit_msg>implement label matching and more tweaks to output<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
	"time"

	"code.google.com/p/goauth2/oauth"
	"github.com/codegangsta/cli"
	"github.com/google/go-github/github"
)

const (
	dateFormat = "2006-01-02"
)

func init() {
	cli.AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[options] {{end}}
"
VERSION:
   {{.Version}}{{if or .Author .Email}}

AUTHOR:{{if .Author}}
  {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
  {{.Email}}{{end}}{{end}}{{if .Flags}}

OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`
}

func main() {
	app := cli.NewApp()
	app.Name = "thisweek"
	app.Usage = "create a report of your team's activity this week (or for whenever you'd like)"
	app.Version = "0.0.1"
	app.Author = "Marcelo Silveira"
	app.Email = "marcelo@mhfs.com.br"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "repo, r",
			Value: "",
			Usage: "GitHub repository to analyze e.g. mhfs/thisweek",
		},
		cli.StringFlag{
			Name:  "since, s",
			Value: beginningOfWeek().In(time.Local).Format(dateFormat),
			Usage: "list issues since given date, inclusive",
		},
		cli.StringSliceFlag{
			Name:  "label, l",
			Value: &cli.StringSlice{},
			Usage: "label to process, defaults to all",
		},
	}

	app.Action = func(ctx *cli.Context) {
		repo := ctx.String("repo")

		if repo == "" {
			fmt.Println("\n***** Missing required flag --repo *****\n")
			cli.ShowAppHelp(ctx)
			return
		}

		since, err := time.ParseInLocation(dateFormat, ctx.String("since"), time.Local)
		if err != nil {
			panic("invalid date provided")
		}

		fmt.Printf("Starting work for repo '%s' since '%s'\n\n", repo, since.Format(dateFormat))

		parts := strings.Split(repo, "/")
		owner, repo := parts[0], parts[1]

		var finished bool
		for page := 1; !finished; page++ {
			events, err := fetchEvents(owner, repo, page)

			if err != nil {
				panic(err) // FIXME be smarter. handle 404s, 403s, ...
			}

			for _, event := range events {
				// events are ordered by created at desc. stop if got all we wanted.
				if event.CreatedAt.Before(since) {
					finished = true
					break
				}

				// if event is closed or issue didn't remain closed
				if *event.Event != "closed" || *event.Issue.State != "closed" {
					continue
				}

				// if label filtering set, skip labels we're not interested at
				if labels := ctx.StringSlice("label"); len(labels) > 0 {
					if !matchingLabels(event.Issue.Labels, labels) {
						continue
					}
				}

				printEvent(&event)
			}
		}
	}

	app.Run(os.Args)
}

func fetchEvents(owner string, repo string, page int) ([]github.IssueEvent, error) {
	t := &oauth.Transport{
		Token: &oauth.Token{AccessToken: os.Getenv("GH_TOKEN")},
	}

	client := github.NewClient(t.Client())

	options := github.ListOptions{Page: page, PerPage: 100}
	events, _, err := client.Issues.ListRepositoryEvents(owner, repo, &options)

	return events, err
}

func printEvent(event *github.IssueEvent) {
	number := event.Issue.Number
	closedAt := event.Issue.ClosedAt.In(time.Local).Format(dateFormat)
	title := event.Issue.Title

	var assignee string
	if event.Issue.Assignee != nil {
		assignee = " by @" + *event.Issue.Assignee.Login
	}

	var labelsNames []string
	if labels := event.Issue.Labels; len(labels) > 0 {
		for _, l := range labels {
			labelsNames = append(labelsNames, *l.Name)
		}
	}
	labelsString := strings.Join(labelsNames, ", ")

	if len(labelsString) > 0 {
		labelsString = " (" + labelsString + ")"
	}

	fmt.Printf("#%d - %s - %s%s%s\n", *number, closedAt, *title, assignee, labelsString)
}

func beginningOfWeek() time.Time {
	now := time.Now()
	// truncate internal HH:MM:SS to zero and compensate for local zone offset
	// 2015-01-31 10:45:54.720292964 -0800 PST > 2015-01-30 16:00:00 -0800 PST > 2015-01-31 00:00:00 -0800 PST
	_, offset := now.Zone()
	beginningOfDay := now.Truncate(24 * time.Hour).Add(-1 * time.Duration(offset) * time.Second)

	// subtract days to get to sunday
	beginningOfWeek := beginningOfDay.Add(-time.Duration(now.Weekday()) * 24 * time.Hour)

	return beginningOfWeek
}

func matchingLabels(issueLabels []github.Label, targetLabels []string) bool {
	for _, issueLabel := range issueLabels {
		for _, targetLabel := range targetLabels {
			if *issueLabel.Name == targetLabel {
				return true
			}
		}
	}
	return false
}
<|endoftext|>


<commit_before>package throttled

import (
	"sync"
	"time"
)

type Throttle struct {
	delay  time.Duration
	bursts int

	wg      sync.WaitGroup
	bucket  chan chan bool
	stop    chan struct{}
	mu      sync.Mutex
	started bool
	muproc  sync.Mutex
}

func New(freq Delayer, bursts int) *Throttle {
	return &Throttle{delay: freq.Delay(), bursts: bursts}
}

func (t *Throttle) Func(fn func()) func() {
	t.start()
	return func() {
		t.wg.Add(1)
		defer t.wg.Done()
		ch := t.try()
		ok := <-ch
		if ok {
			fn()
		}
	}
}

func (t *Throttle) Close() {
	// Make sure no new calls get through
	close(t.stop)
	// Wait for pending calls (drain)
	t.wg.Wait()
	// Safely close the bucket, so that t.process can exit
	close(t.bucket)
	// Wait for t.process to exit
	t.muproc.Lock()
	t.muproc.Unlock()
}

func (t *Throttle) start() {
	t.mu.Lock()
	defer t.mu.Unlock()
	if !t.started {
		if t.bursts <= 1 {
			t.bursts = 1
		}
		t.bucket = make(chan chan bool, t.bursts)
		t.stop = make(chan struct{})
		t.started = true
		go t.process()
	}
}

func (t *Throttle) try() <-chan bool {
	ch := make(chan bool, 1)
	select {
	case <-t.stop:
		// Draining, does not accept any new calls
		ch <- false
		return ch
	default:
	}

	select {
	case t.bucket <- ch:
		return ch
	default:
		// No more space in the bucket
		ch <- false
		return ch
	}
}

func (t *Throttle) process() {
	var after <-chan time.Time

	t.muproc.Lock()
	defer t.muproc.Unlock()
	for v := range t.bucket {
		if after != nil {
			<-after
		}
		v <- true
		after = time.After(t.delay)
	}
}
<commit_msg>add http throttle<commit_after>package throttled

import (
	"net/http"
	"sync"
	"time"
)

type Throttle struct {
	delay  time.Duration
	bursts int

	wg      sync.WaitGroup
	bucket  chan chan bool
	stop    chan struct{}
	mu      sync.Mutex
	started bool
	muproc  sync.Mutex
}

// New creates a new throttle using the specified frequency, and allowing
// bursts number of exceeding calls.
func New(freq Delayer, bursts int) *Throttle {
	return &Throttle{delay: freq.Delay(), bursts: bursts}
}

// Func wraps a function and returns a throttled function.
func (t *Throttle) Func(fn func()) func() {
	return t.FuncDropped(fn, func() {})
}

func (t *Throttle) FuncDropped(fn func(), droppedfn func()) func() {
	t.start()
	return func() {
		t.wg.Add(1)
		defer t.wg.Done()
		ch := t.try()
		ok := <-ch
		if ok {
			fn()
		} else {
			droppedfn()
		}
	}
}

func (t *Throttle) Handler(h http.Handler) http.Handler {
	return t.HandlerDropped(h, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "Throttled", http.StatusServiceUnavailable)
	}))
}

func (t *Throttle) HandlerDropped(h http.Handler, droppedh http.Handler) http.Handler {
	t.start()
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		t.wg.Add(1)
		defer t.wg.Done()
		ch := t.try()
		ok := <-ch
		if ok {
			h.ServeHTTP(w, r)
		} else {
			droppedh.ServeHTTP(w, r)
		}
	})
}

// Close the throttle, draining the pending calls. The function
// returns once all pending calls have been processed.
func (t *Throttle) Close() {
	// Make sure no new calls get through
	close(t.stop)
	// Wait for pending calls (drain)
	t.wg.Wait()
	// Safely close the bucket, so that t.process can exit
	close(t.bucket)
	// Wait for t.process to exit
	t.muproc.Lock()
	t.muproc.Unlock()
}

// Prepare the throttle and start processing calls. This function
// is idempotent.
func (t *Throttle) start() {
	t.mu.Lock()
	defer t.mu.Unlock()
	if !t.started {
		if t.bursts <= 1 {
			t.bursts = 1
		}
		t.bucket = make(chan chan bool, t.bursts)
		t.stop = make(chan struct{})
		t.started = true
		go t.process()
	}
}

// Attempt to add the call to the bucket, and return the channel
// that indicates if the call can go through or not.
func (t *Throttle) try() <-chan bool {
	ch := make(chan bool, 1)
	// Check if the bucket is closed.
	select {
	case <-t.stop:
		// Draining, does not accept any new calls
		ch <- false
		return ch
	default:
	}
	// Try to enqueue in the bucket.
	select {
	case t.bucket <- ch:
		return ch
	default:
		// No more space in the bucket
		ch <- false
		return ch
	}
}

// Process the pending calls from the bucket.
func (t *Throttle) process() {
	var after <-chan time.Time
	t.muproc.Lock()
	defer t.muproc.Unlock()
	for v := range t.bucket {
		if after != nil {
			<-after
		}
		v <- true
		after = time.After(t.delay)
	}
}
<|endoftext|>


<commit_before>// Packet vlog provides package level verbose logging.
// The actual logging is done with the standard log package.
//
// To define a package level logging variable,
//   var v = vlog.New()
//
// To log a message at INFO level,
//   v.I("a")
//
// The logging level can be set with either the flag -vlog or
// the environment variable GO_VLOG.
//
// The -vlog or GO_VLOG format is,
//  k=v(,k=v)*
//  k can be exact match like "foo/bar" or prefix match like "foo/*".
//  v can be w|i|v1|v2
// Default level can be set with prefix match "*".
package vlog

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync/atomic"
)

//go:generate stringer -type=Level
type Level int32

const (
	v2 Level = -2 + iota
	v1
	info
	err
)

// E logs error message.
// If args[0] is a format string, args is formatted with Printf,
// otherwise args is formatted with Println.
func (v *Level) E(args ...interface{}) {
	if *v <= err {
		lg.Log("E " + Format(args...))
	}
}

func E(args ...interface{}) {
	levelVars[0].Level.E(args...)
}

// I logs info message.
func (v *Level) I(args ...interface{}) {
	if *v <= info {
		lg.Log(Format(args...))
	}
}

func I(args ...interface{}) {
	levelVars[0].Level.I(args...)
}

// V1 logs verbose level 1 message.
func (v *Level) V1(args ...interface{}) {
	if *v <= v1 {
		lg.Log(Format(args...))
	}
}

func V1(args ...interface{}) {
	levelVars[0].Level.V1(args...)
}

// V2 logs verbose level 2 message.
func (v *Level) V2(args ...interface{}) {
	if *v <= v2 {
		lg.Log(Format(args...))
	}
}

func V2(args ...interface{}) {
	levelVars[0].Level.V2(args...)
}

// Vstack logs the message and the stacktrace of this goroutine.
// It is noop when verbose logging is not enabled.
func (v *Level) Vstack(args ...interface{}) {
	if *v >= info {
		return
	}
	s := Format(args...)
	lg.Log(stackTrace(s))
}

func Vstack(args ...interface{}) {
	levelVars[0].Level.Vstack(args...)
}

// On returns true if the specific verbose level 1-3 is enabled.
func (v *Level) On(l int) bool {
	lv := Level(-l)
	return *v <= lv
}

func On(l int) bool {
	return levelVars[0].Level.On(l)
}

// Vset sets the verbose logging level.
func (v *Level) Vset(l int) Level {
	lv := Level(-l)
	if lv < v2 || lv >= info {
		lg.Log(Format("invalid verbose level=%d", l))
		return *v
	}
	old := *v
	atomic.StoreInt32((*int32)(v), int32(lv))
	return old
}

func Vset(l int) Level {
	return levelVars[0].Level.Vset(l)
}

// Error returns an error. The message of the error is formatted
// from args. If verbose level 1 is enabled, the error messag includes
// the caller, and if verbose level 2 is enabled, the error message
// includes the call stack.
func (v *Level) Error(args ...interface{}) error {
	return v.newError(Format(args...))
}

func Error(args ...interface{}) error {
	return levelVars[0].Level.Error(args...)
}

// newError is necessary to get the correct call stack
func (v *Level) newError(s string) error {
	switch *v {
	default:
		return errors.New(s)

	case v1:
		_, fn, ln, ok := runtime.Caller(2)
		if !ok {
			return errors.New("???: " + s)
		}
		return errors.New(fn + ":" + strconv.Itoa(ln) + " " + s)

	case v2:
		return errors.New(stackTrace(s))
	}
}

func parseLevel(lvs string) Level {
	switch strings.ToLower(lvs) {
	case "2", "v2":
		return v2
	case "1", "v1":
		return v1
	case "i", "info":
		return info
	case "e", "err":
		return err
	default:
		lg.Log(Format("ignore invalid logging level=%s", lvs))
		return info
	}
}

// New returns a vlog Level variable.
// The name of the Level variable is inferred with these rules,
//  - file name of the caller must be under "/src/", to follow go path convention
//  - if the file is ".../src/<foo_pkg>/{main,cmd}/bar.go", name is "foo_pkg/bar".
//  - if the file is ".../src/<foo_pkg>/bar.go", name is "foo_pkg".
//
// New must be called before Parse() is callled.
func New() *Level {
	// Note: 1 to skip New
	_, fn, _, ok := runtime.Caller(1)
	if !ok {
		lg.Log("fail to get file from runtime.caller")
		return &levelVars[0].Level // [0] is default
	}
	name := inferName(fn)
	return newVar(name, fn)
}

func newVar(name, fn string) *Level {
	if name == "" {
		lg.Log(Format("fail to infer name from file=%s", fn))
		return &levelVars[0].Level // [0] is default
	}
	for _, lv := range levelVars {
		if lv.Name == name {
			lg.Log(Format("dup level name=%s inferred from file=%s", name, fn))
			return &lv.Level
		}
	}
	lv := &levelVar{
		Name: name,
		File: fn,
	}
	levelVars = append(levelVars, lv)
	return &lv.Level
}

func inferName(fn string) string {
	fn = strings.ToLower(fn)
	i := strings.LastIndex(fn, "/src/")
	if i < 0 {
		return ""
	}
	dn, fn := path.Split(fn[i+len("/src/"):])
	if !strings.HasSuffix(fn, ".go") {
		return ""
	}
	for _, p := range []string{"/main/", "/cmd/"} {
		if strings.HasSuffix(dn, p) {
			dn = dn[:len(dn)-len(p)]
			fn = fn[:len(fn)-len(".go")]
			return path.Join(dn, fn)
		}
	}
	return strings.TrimRight(dn, "/")
}

type levelVar struct {
	Name  string
	File  string
	Level Level
}

func (lv *levelVar) String() string {
	return fmt.Sprintf("%s@%s", lv.Name, lv.File)
}

var levelVars = []*levelVar{&levelVar{}} // default level

func Parse() {
	flag.Parse()
	setLevels(*vlogFlag)
	if *vlogHelp {
		lg.Log("vlog setting:" + printLevelVars())
		flag.Usage()
		os.Exit(2)
	}
	if *vlogFile != "" {
		lg = newRotateLogger(*vlogFile)
	}
}

func ParseEnv() {
	if val := os.Getenv("GO_VLOG"); val != "" { // for testing
		setLevels(val)
		lg.Log(printLevelVars())
	}
}

// TODO: set level from a string at runtime
func setLevels(value string) {
	exact, prefix := parseFlag(value)
	if v, ok := prefix["/"]; ok {
		levelVars[0].Level = v // default level
	}
	def := levelVars[0].Level
	for _, lv := range levelVars[1:] {
		lv.Level = def
	}
	if len(prefix) > 0 {
		prefixes := make([]string, 0, len(prefix))
		for k := range prefix {
			prefixes = append(prefixes, k)
		}
		sort.Strings(prefixes)

		for _, lv := range levelVars[1:] {
			for i := len(prefixes) - 1; i >= 0; i-- {
				k := prefixes[i]
				// Match "foo" with "foo/" and "foo/bar" with "foo/"
				if lv.Name == k[:len(k)-1] || strings.HasPrefix(lv.Name, k) {
					lv.Level = prefix[k]
					break
				}
			}
		}
	}
	for _, lv := range levelVars[1:] {
		if i, ok := exact[lv.Name]; ok {
			lv.Level = i
		}
	}
}

func parseFlag(value string) (exact, prefix map[string]Level) {
	exact = make(map[string]Level)
	prefix = make(map[string]Level)
	s := value
	for s != "" {
		k := s
		if i := strings.Index(s, ","); i >= 0 {
			k, s = s[:i], s[i+1:]
		} else {
			s = ""
		}
		j := strings.Index(k, "=")
		if j < 0 {
			panic(Format("malformed: no level", value))
		}
		k, v := k[:j], k[j+1:]
		lv := parseLevel(v)
		k = strings.ToLower(k)
		pre := false
		if k == "*" || strings.HasSuffix(k, "/*") {
			k = k[:len(k)-1]
			if strings.Contains(k, "*") {
				panic(Format("malformed: multiple star", s))
			}
			pre = true
		} else if strings.Contains(k, "*") {
			panic(Format("malformed: star in middle", s))
		}
		k = strings.TrimRight(k, "/")
		if pre {
			prefix[k+"/"] = lv
		} else {
			exact[k] = lv
		}
	}
	return exact, prefix
}

func printLevelVars() string {
	var b bytes.Buffer
	fmt.Fprintf(&b, "*=%v", levelVars[0].Level)
	for _, lv := range levelVars[1:] {
		fmt.Fprintf(&b, ",%s=%v", lv.Name, lv.Level)
	}
	return b.String()
}

var (
	vlogFlag = flag.String("vlog", "", "vlog settings, k=v(,k=v)*")
	vlogFile = flag.String("vlogfile", "", "vlog file prefix")
	vlogHelp = flag.Bool("vloghelp", false, "show vlog setting and flag help")
)

type Logger interface {
	Log(s string)
	Flush()
}

type stderrLogger struct {
	lg *log.Logger
}

func (l *stderrLogger) Log(s string) {
	l.lg.Output(3, s)
}

func (l *stderrLogger) Flush() {}

const logPrefix = log.Ldate | log.Lmicroseconds | log.Lshortfile

// lg should always be available
var lg Logger = &stderrLogger{lg: log.New(os.Stderr, "", logPrefix)}

var stackTraceBegin = []byte("/vlog.go:")

func stackTrace(s string) string {
	var buf [4 << 10]byte
	m := copy(buf[:], s)
	n := runtime.Stack(buf[m:], false)
	n += m

	// Trim the frames in vlog.go, any line that contains "/vlog.go:"
	b := buf[m:n]
	j := bytes.LastIndex(b, stackTraceBegin)
	if j < 0 {
		return string(buf[:n])
	}
	b = b[j:]
	j = bytes.IndexAny(b, "\n")
	if j < 0 {
		return string(buf[:n])
	}
	b = b[j+1:]
	buf[m] = '\n' // put a newline between s and stack frames
	n = copy(buf[m+1:], b)
	n += m + 1
	if buf[n] != '\n' {
		buf[n] = '\n' // always end with newline
	}
	return string(buf[:n])
}
<commit_msg>fix call depth of vlog.I etc<commit_after>// Packet vlog provides package level verbose logging.
// The actual logging is done with the standard log package.
//
// To define a package level logging variable,
//   var v = vlog.New()
//
// To log a message at INFO level,
//   v.I("a")
//
// The logging level can be set with either the flag -vlog or
// the environment variable GO_VLOG.
//
// The -vlog or GO_VLOG format is,
//  k=v(,k=v)*
//  k can be exact match like "foo/bar" or prefix match like "foo/*".
//  v can be w|i|v1|v2
// Default level can be set with prefix match "*".
package vlog

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync/atomic"
)

//go:generate stringer -type=Level
type Level int32

const (
	v2 Level = -2 + iota
	v1
	info
	err
)

// E logs error message.
// If args[0] is a format string, args is formatted with Printf,
// otherwise args is formatted with Println.
func (v *Level) E(args ...interface{}) {
	if *v <= err {
		lg.Log("E " + Format(args...))
	}
}

func E(args ...interface{}) {
	if levelVars[0].Level <= err {
		lg.Log("E " + Format(args...))
	}
}

// I logs info message.
func (v *Level) I(args ...interface{}) {
	if *v <= info {
		lg.Log(Format(args...))
	}
}

func I(args ...interface{}) {
	if levelVars[0].Level <= info {
		lg.Log(Format(args...))
	}
}

// V1 logs verbose level 1 message.
func (v *Level) V1(args ...interface{}) {
	if *v <= v1 {
		lg.Log(Format(args...))
	}
}

func V1(args ...interface{}) {
	if levelVars[0].Level <= v1 {
		lg.Log(Format(args...))
	}
}

// V2 logs verbose level 2 message.
func (v *Level) V2(args ...interface{}) {
	if *v <= v2 {
		lg.Log(Format(args...))
	}
}

func V2(args ...interface{}) {
	if levelVars[0].Level <= v2 {
		lg.Log(Format(args...))
	}
}

// Vstack logs the message and the stacktrace of this goroutine.
// It is noop when verbose logging is not enabled.
func (v *Level) Vstack(args ...interface{}) {
	if *v >= info {
		return
	}
	s := Format(args...)
	lg.Log(stackTrace(s))
}

func Vstack(args ...interface{}) {
	levelVars[0].Level.Vstack(args...)
}

// On returns true if the specific verbose level 1-3 is enabled.
func (v *Level) On(l int) bool {
	lv := Level(-l)
	return *v <= lv
}

func On(l int) bool {
	return levelVars[0].Level.On(l)
}

// Vset sets the verbose logging level.
func (v *Level) Vset(l int) Level {
	lv := Level(-l)
	if lv < v2 || lv >= info {
		lg.Log(Format("invalid verbose level=%d", l))
		return *v
	}
	old := *v
	atomic.StoreInt32((*int32)(v), int32(lv))
	return old
}

func Vset(l int) Level {
	return levelVars[0].Level.Vset(l)
}

// Error returns an error. The message of the error is formatted
// from args. If verbose level 1 is enabled, the error messag includes
// the caller, and if verbose level 2 is enabled, the error message
// includes the call stack.
func (v *Level) Error(args ...interface{}) error {
	return v.newError(Format(args...))
}

func Error(args ...interface{}) error {
	return levelVars[0].Level.Error(args...)
}

// newError is necessary to get the correct call stack
func (v *Level) newError(s string) error {
	switch *v {
	default:
		return errors.New(s)

	case v1:
		_, fn, ln, ok := runtime.Caller(2)
		if !ok {
			return errors.New("???: " + s)
		}
		return errors.New(fn + ":" + strconv.Itoa(ln) + " " + s)

	case v2:
		return errors.New(stackTrace(s))
	}
}

func parseLevel(lvs string) Level {
	switch strings.ToLower(lvs) {
	case "2", "v2":
		return v2
	case "1", "v1":
		return v1
	case "i", "info":
		return info
	case "e", "err":
		return err
	default:
		lg.Log(Format("ignore invalid logging level=%s", lvs))
		return info
	}
}

// New returns a vlog Level variable.
// The name of the Level variable is inferred with these rules,
//  - file name of the caller must be under "/src/", to follow go path convention
//  - if the file is ".../src/<foo_pkg>/{main,cmd}/bar.go", name is "foo_pkg/bar".
//  - if the file is ".../src/<foo_pkg>/bar.go", name is "foo_pkg".
//
// New must be called before Parse() is callled.
func New() *Level {
	// Note: 1 to skip New
	_, fn, _, ok := runtime.Caller(1)
	if !ok {
		lg.Log("fail to get file from runtime.caller")
		return &levelVars[0].Level // [0] is default
	}
	name := inferName(fn)
	return newVar(name, fn)
}

func newVar(name, fn string) *Level {
	if name == "" {
		lg.Log(Format("fail to infer name from file=%s", fn))
		return &levelVars[0].Level // [0] is default
	}
	for _, lv := range levelVars {
		if lv.Name == name {
			lg.Log(Format("dup level name=%s inferred from file=%s", name, fn))
			return &lv.Level
		}
	}
	lv := &levelVar{
		Name: name,
		File: fn,
	}
	levelVars = append(levelVars, lv)
	return &lv.Level
}

func inferName(fn string) string {
	fn = strings.ToLower(fn)
	i := strings.LastIndex(fn, "/src/")
	if i < 0 {
		return ""
	}
	dn, fn := path.Split(fn[i+len("/src/"):])
	if !strings.HasSuffix(fn, ".go") {
		return ""
	}
	for _, p := range []string{"/main/", "/cmd/"} {
		if strings.HasSuffix(dn, p) {
			dn = dn[:len(dn)-len(p)]
			fn = fn[:len(fn)-len(".go")]
			return path.Join(dn, fn)
		}
	}
	return strings.TrimRight(dn, "/")
}

type levelVar struct {
	Name  string
	File  string
	Level Level
}

func (lv *levelVar) String() string {
	return fmt.Sprintf("%s@%s", lv.Name, lv.File)
}

var levelVars = []*levelVar{&levelVar{}} // default level

func Parse() {
	flag.Parse()
	setLevels(*vlogFlag)
	if *vlogHelp {
		lg.Log("vlog setting:" + printLevelVars())
		flag.Usage()
		os.Exit(2)
	}
	if *vlogFile != "" {
		lg = newRotateLogger(*vlogFile)
	}
}

func ParseEnv() {
	if val := os.Getenv("GO_VLOG"); val != "" { // for testing
		setLevels(val)
		lg.Log(printLevelVars())
	}
}

// TODO: set level from a string at runtime
func setLevels(value string) {
	exact, prefix := parseFlag(value)
	if v, ok := prefix["/"]; ok {
		levelVars[0].Level = v // default level
	}
	def := levelVars[0].Level
	for _, lv := range levelVars[1:] {
		lv.Level = def
	}
	if len(prefix) > 0 {
		prefixes := make([]string, 0, len(prefix))
		for k := range prefix {
			prefixes = append(prefixes, k)
		}
		sort.Strings(prefixes)

		for _, lv := range levelVars[1:] {
			for i := len(prefixes) - 1; i >= 0; i-- {
				k := prefixes[i]
				// Match "foo" with "foo/" and "foo/bar" with "foo/"
				if lv.Name == k[:len(k)-1] || strings.HasPrefix(lv.Name, k) {
					lv.Level = prefix[k]
					break
				}
			}
		}
	}
	for _, lv := range levelVars[1:] {
		if i, ok := exact[lv.Name]; ok {
			lv.Level = i
		}
	}
}

func parseFlag(value string) (exact, prefix map[string]Level) {
	exact = make(map[string]Level)
	prefix = make(map[string]Level)
	s := value
	for s != "" {
		k := s
		if i := strings.Index(s, ","); i >= 0 {
			k, s = s[:i], s[i+1:]
		} else {
			s = ""
		}
		j := strings.Index(k, "=")
		if j < 0 {
			panic(Format("malformed: no level", value))
		}
		k, v := k[:j], k[j+1:]
		lv := parseLevel(v)
		k = strings.ToLower(k)
		pre := false
		if k == "*" || strings.HasSuffix(k, "/*") {
			k = k[:len(k)-1]
			if strings.Contains(k, "*") {
				panic(Format("malformed: multiple star", s))
			}
			pre = true
		} else if strings.Contains(k, "*") {
			panic(Format("malformed: star in middle", s))
		}
		k = strings.TrimRight(k, "/")
		if pre {
			prefix[k+"/"] = lv
		} else {
			exact[k] = lv
		}
	}
	return exact, prefix
}

func printLevelVars() string {
	var b bytes.Buffer
	fmt.Fprintf(&b, "*=%v", levelVars[0].Level)
	for _, lv := range levelVars[1:] {
		fmt.Fprintf(&b, ",%s=%v", lv.Name, lv.Level)
	}
	return b.String()
}

var (
	vlogFlag = flag.String("vlog", "", "vlog settings, k=v(,k=v)*")
	vlogFile = flag.String("vlogfile", "", "vlog file prefix")
	vlogHelp = flag.Bool("vloghelp", false, "show vlog setting and flag help")
)

type Logger interface {
	Log(s string)
	Flush()
}

type stderrLogger struct {
	lg *log.Logger
}

func (l *stderrLogger) Log(s string) {
	l.lg.Output(3, s)
}

func (l *stderrLogger) Flush() {}

const logPrefix = log.Ldate | log.Lmicroseconds | log.Lshortfile

// lg should always be available
var lg Logger = &stderrLogger{lg: log.New(os.Stderr, "", logPrefix)}

var stackTraceBegin = []byte("/vlog.go:")

func stackTrace(s string) string {
	var buf [4 << 10]byte
	m := copy(buf[:], s)
	n := runtime.Stack(buf[m:], false)
	n += m

	// Trim the frames in vlog.go, any line that contains "/vlog.go:"
	b := buf[m:n]
	j := bytes.LastIndex(b, stackTraceBegin)
	if j < 0 {
		return string(buf[:n])
	}
	b = b[j:]
	j = bytes.IndexAny(b, "\n")
	if j < 0 {
		return string(buf[:n])
	}
	b = b[j+1:]
	buf[m] = '\n' // put a newline between s and stack frames
	n = copy(buf[m+1:], b)
	n += m + 1
	if buf[n] != '\n' {
		buf[n] = '\n' // always end with newline
	}
	return string(buf[:n])
}
<|endoftext|>


<commit_before>package kafka

import (
	"crypto/tls"
	"fmt"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/outputs"
	"github.com/influxdata/telegraf/plugins/serializers"

	"github.com/Shopify/sarama"
)

type Kafka struct {
	// Kafka brokers to send metrics to
	Brokers []string
	// Kafka topic
	Topic string
	// Routing Key Tag
	RoutingTag string `toml:"routing_tag"`

	// Legacy SSL config options
	// TLS client certificate
	Certificate string
	// TLS client key
	Key string
	// TLS certificate authority
	CA string

	// Path to CA file
	SSLCA string `toml:"ssl_ca"`
	// Path to host cert file
	SSLCert string `toml:"ssl_cert"`
	// Path to cert key file
	SSLKey string `toml:"ssl_key"`

	// Skip SSL verification
	InsecureSkipVerify bool

	tlsConfig tls.Config
	producer  sarama.SyncProducer

	serializer serializers.Serializer
}

var sampleConfig = `
  ## URLs of kafka brokers
  brokers = ["localhost:9092"]
  ## Kafka topic for producer messages
  topic = "telegraf"
  ## Telegraf tag to use as a routing key
  ##  ie, if this tag exists, it's value will be used as the routing key
  routing_tag = "host"

  ## Optional SSL Config
  # ssl_ca = "/etc/telegraf/ca.pem"
  # ssl_cert = "/etc/telegraf/cert.pem"
  # ssl_key = "/etc/telegraf/key.pem"
  ## Use SSL but skip chain & host verification
  # insecure_skip_verify = false

  ## Data format to output. This can be "influx" or "graphite"
  ## Each data format has it's own unique set of configuration options, read
  ## more about them here:
  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
  data_format = "influx"
`

func (k *Kafka) SetSerializer(serializer serializers.Serializer) {
	k.serializer = serializer
}

func (k *Kafka) Connect() error {
	config := sarama.NewConfig()
	// Wait for all in-sync replicas to ack the message
	config.Producer.RequiredAcks = sarama.WaitForAll
	// Retry up to 10 times to produce the message
	config.Producer.Retry.Max = 10

	// Legacy support ssl config
	if k.Certificate != "" {
		k.SSLCert = k.Certificate
		k.SSLCA = k.CA
		k.SSLKey = k.Key
	}

	tlsConfig, err := internal.GetTLSConfig(
		k.SSLCert, k.SSLKey, k.SSLCA, k.InsecureSkipVerify)
	if err != nil {
		return err
	}

	if tlsConfig != nil {
		config.Net.TLS.Config = tlsConfig
		config.Net.TLS.Enable = true
	}

	producer, err := sarama.NewSyncProducer(k.Brokers, config)
	if err != nil {
		return err
	}
	k.producer = producer
	return nil
}

func (k *Kafka) Close() error {
	return k.producer.Close()
}

func (k *Kafka) SampleConfig() string {
	return sampleConfig
}

func (k *Kafka) Description() string {
	return "Configuration for the Kafka server to send metrics to"
}

func (k *Kafka) Write(metrics []telegraf.Metric) error {
	if len(metrics) == 0 {
		return nil
	}

	for _, metric := range metrics {
		values, err := k.serializer.Serialize(metric)
		if err != nil {
			return err
		}

		var pubErr error
		for _, value := range values {
			m := &sarama.ProducerMessage{
				Topic: k.Topic,
				Value: sarama.StringEncoder(value),
			}
			if h, ok := metric.Tags()[k.RoutingTag]; ok {
				m.Key = sarama.StringEncoder(h)
			}

			_, _, pubErr = k.producer.SendMessage(m)
		}

		if pubErr != nil {
			return fmt.Errorf("FAILED to send kafka message: %s\n", pubErr)
		}
	}
	return nil
}

func init() {
	outputs.Add("kafka", func() telegraf.Output {
		return &Kafka{}
	})
}
<commit_msg>Add compression/acks/retry conf to Kafka output plugin<commit_after>package kafka

import (
	"crypto/tls"
	"fmt"
	"strconv"
	"strings"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/outputs"
	"github.com/influxdata/telegraf/plugins/serializers"

	"github.com/Shopify/sarama"
)

type Kafka struct {
	// Kafka brokers to send metrics to
	Brokers []string
	// Kafka topic
	Topic string
	// Routing Key Tag
	RoutingTag string `toml:"routing_tag"`
	// Compression Codec Tag
	CompressionCodec string
	// RequiredAcks Tag
	RequiredAcks string
	// MaxRetry Tag
	MaxRetry string

	// Legacy SSL config options
	// TLS client certificate
	Certificate string
	// TLS client key
	Key string
	// TLS certificate authority
	CA string

	// Path to CA file
	SSLCA string `toml:"ssl_ca"`
	// Path to host cert file
	SSLCert string `toml:"ssl_cert"`
	// Path to cert key file
	SSLKey string `toml:"ssl_key"`

	// Skip SSL verification
	InsecureSkipVerify bool

	tlsConfig tls.Config
	producer  sarama.SyncProducer

	serializer serializers.Serializer
}

var sampleConfig = `
  ## URLs of kafka brokers
  brokers = ["localhost:9092"]
  ## Kafka topic for producer messages
  topic = "telegraf"
  ## Telegraf tag to use as a routing key
  ##  ie, if this tag exists, it's value will be used as the routing key
  routing_tag = "host"

	## CompressionCodec represents the various compression codecs recognized by Kafka in messages.
	##  "none" : No compression
	##  "gzip" : Gzip compression
	##  "snappy" : Snappy compression
	# compression_codec = "none"

	##  RequiredAcks is used in Produce Requests to tell the broker how many replica acknowledgements it must see before responding
	##  "none" : the producer never waits for an acknowledgement from the broker. This option provides the lowest latency but the weakest durability guarantees (some data will be lost when a server fails).
	##  "leader" : the producer gets an acknowledgement after the leader replica has received the data. This option provides better durability as the client waits until the server acknowledges the request as successful (only messages that were written to the now-dead leader but not yet replicated will be lost).
	##  "leader_and_replicas" : the producer gets an acknowledgement after all in-sync replicas have received the data. This option provides the best durability, we guarantee that no messages will be lost as long as at least one in sync replica remains.
	# required_acks = "leader_and_replicas"

	##  The total number of times to retry sending a message
	# max_retry = "3"

  ## Optional SSL Config
  # ssl_ca = "/etc/telegraf/ca.pem"
  # ssl_cert = "/etc/telegraf/cert.pem"
  # ssl_key = "/etc/telegraf/key.pem"
  ## Use SSL but skip chain & host verification
  # insecure_skip_verify = false

  ## Data format to output. This can be "influx" or "graphite"
  ## Each data format has it's own unique set of configuration options, read
  ## more about them here:
  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
  data_format = "influx"
`

func (k *Kafka) SetSerializer(serializer serializers.Serializer) {
	k.serializer = serializer
}

func requiredAcks(value string) (sarama.RequiredAcks, error) {
	switch strings.ToLower(value) {
	case "none":
		return sarama.NoResponse, nil
	case "leader":
		return sarama.WaitForLocal, nil
	case "", "leader_and_replicas":
		return sarama.WaitForAll, nil
	default:
		return 0, fmt.Errorf("Failed to recognize required_acks: %s", value)
	}
}

func compressionCodec(value string) (sarama.CompressionCodec, error) {
	switch strings.ToLower(value) {
	case "gzip":
		return sarama.CompressionGZIP, nil
	case "snappy":
		return sarama.CompressionSnappy, nil
	case "", "none":
		return sarama.CompressionNone, nil
	default:
		return 0, fmt.Errorf("Failed to recognize compression_codec: %s", value)
	}
}

func maxRetry(value string) (int, error) {
	if value == "" {
		return 3, nil
	}
	maxRetry, err := strconv.Atoi(value)
	if err != nil {
		return -1, fmt.Errorf("Failed to parse max_retry: %s", value)
	}
	if maxRetry < 0 {
		return -1, fmt.Errorf("max_retry is %s but it should not be negative", value)
	}
	return maxRetry, nil
}

func (k *Kafka) Connect() error {
	config := sarama.NewConfig()

	requiredAcks, err := requiredAcks(k.RequiredAcks)
	if err != nil {
		return err
	}
	config.Producer.RequiredAcks = requiredAcks

	compressionCodec, err := compressionCodec(k.CompressionCodec)
	if err != nil {
		return err
	}
	config.Producer.Compression = compressionCodec

	maxRetry, err := maxRetry(k.MaxRetry)
	if err != nil {
		return err
	}
	config.Producer.Retry.Max = maxRetry

	// Legacy support ssl config
	if k.Certificate != "" {
		k.SSLCert = k.Certificate
		k.SSLCA = k.CA
		k.SSLKey = k.Key
	}

	tlsConfig, err := internal.GetTLSConfig(
		k.SSLCert, k.SSLKey, k.SSLCA, k.InsecureSkipVerify)
	if err != nil {
		return err
	}

	if tlsConfig != nil {
		config.Net.TLS.Config = tlsConfig
		config.Net.TLS.Enable = true
	}

	producer, err := sarama.NewSyncProducer(k.Brokers, config)
	if err != nil {
		return err
	}
	k.producer = producer
	return nil
}

func (k *Kafka) Close() error {
	return k.producer.Close()
}

func (k *Kafka) SampleConfig() string {
	return sampleConfig
}

func (k *Kafka) Description() string {
	return "Configuration for the Kafka server to send metrics to"
}

func (k *Kafka) Write(metrics []telegraf.Metric) error {
	if len(metrics) == 0 {
		return nil
	}

	for _, metric := range metrics {
		values, err := k.serializer.Serialize(metric)
		if err != nil {
			return err
		}

		var pubErr error
		for _, value := range values {
			m := &sarama.ProducerMessage{
				Topic: k.Topic,
				Value: sarama.StringEncoder(value),
			}
			if h, ok := metric.Tags()[k.RoutingTag]; ok {
				m.Key = sarama.StringEncoder(h)
			}

			_, _, pubErr = k.producer.SendMessage(m)
		}

		if pubErr != nil {
			return fmt.Errorf("FAILED to send kafka message: %s\n", pubErr)
		}
	}
	return nil
}

func init() {
	outputs.Add("kafka", func() telegraf.Output {
		return &Kafka{}
	})
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/laher/uggo"
	"io"
	"math"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)
func main() {
	err := Wget(os.Args)
	if err != nil {
		fmt.Printf("Error: %v\n", err)
		os.Exit(1)
	}
}

//TODO
// ftp
// no-scheme
// clobber behaviour
// progress
// limit-rate
// timeouts - connect,read,dns?
// wait,waitretry
// proxies/?
// quota/?
// user/password/ask-password
// certificates/no-check-certificate ...
// exit statuses
// recursive downloads
// timestamping
// wgetrc
type WgetOptions struct {
	IsContinue *bool
	Filename *string
	Timeout *int //TODO timeout
	Retries *int //TODO retries
}

const (
	FILEMODE os.FileMode = 0660
)

func Wget(call []string) error {

	options := WgetOptions{}
	flagSet := flag.NewFlagSet("wget", flag.ContinueOnError)
	options.IsContinue = flagSet.Bool("c", false, "continue")
	options.Filename = flagSet.String("o", "", "output filename")
	helpFlag := flagSet.Bool("help", false, "Show this help")

	err := flagSet.Parse(uggo.Gnuify(call[1:]))
	if err != nil {
		return err
	}
	if *helpFlag {
		println("wget [options] URL")
		flagSet.PrintDefaults()
		return nil
	}
	args := flagSet.Args()
	if len(args) < 1 {
		flagSet.PrintDefaults()
		return errors.New("Not enough args")
	}
	if len(args) > 0 {
		links := args
		return wget(links, options)
	} else {
		if uggo.IsPipingStdin() {
			//check STDIN
			return wget([]string{}, options)
		} else {
			//NOT piping.
			return errors.New("Not enough args")
		}
	}
}

func wget(links []string, options WgetOptions) error {
	for _, link := range links {
		err := wgetOne(link, options)
		if err != nil {
			return err
		}
	}
	return nil
}

func tidyFilename(filename string) string {
	//invalid filenames ...
	if filename == "" || filename == "/" ||filename == "\\" || filename == "." {
		filename = "index"
	}
	return filename
}

func wgetOne(link string, options WgetOptions) error {
	if !strings.Contains(link, ":") {
		link = "http://" + link
	}
	startTime := time.Now()
	request, err := http.NewRequest("GET", link, nil)
	//resp, err := http.Get(link)
	if err != nil {
		return err
	}

	filename := ""
	if *options.Filename != "" {
		filename = *options.Filename
	}
	client := &http.Client{}
	//continue from where we left off ...
	if *options.IsContinue {
		if filename == "" {
			filename = filepath.Base(request.URL.Path)
			filename = tidyFilename(filename)
			if !strings.Contains(filename, ".") {
				filename = filename + ".html"
			}
		}
		fi, err := os.Stat(filename)
		if err != nil {
			return err
		}
		from := fi.Size()
		headRequest, err := http.NewRequest("HEAD", link, nil)
		if err != nil {
			return err
		}
		headResp, err := client.Do(headRequest)
		if err != nil {
			return err
		}
		cl := headResp.Header.Get("Content-Length")
		if cl != "" {
		rangeHeader := fmt.Sprintf("bytes %d-%s", from, cl)
		fmt.Printf("Adding range header: %s\n", rangeHeader)
		request.Header.Add("Range", rangeHeader)
		} else {
			fmt.Println("Could not find file length using HEAD request")
		}
	}
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	fmt.Printf("Http response: %s\n", resp.Status)

	lenS := resp.Header.Get("Content-Length")
	len := int64(-1)
	if lenS != "" {
		len, err = strconv.ParseInt(lenS, 10, 32)
		if err != nil {
			return err
		}
	}
	typ := resp.Header.Get("Content-Type")
	fmt.Printf("Length: %v [%s]\n", len, typ)

	if filename == "" {
		filename, err = getFilename(resp)
		if err != nil {
			return err
		}
	}
	fmt.Printf("Saving to: '%v'\n\n", filename)
	out, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer out.Close()

	buf := make([]byte, 4068)
	tot := int64(0)
	i := 0

	for {
        // read a chunk
        n, err := resp.Body.Read(buf)
        if err != nil && err != io.EOF {
			return err
		}
        if n == 0 { break }
		tot += int64(n)

        // write a chunk
        if _, err := out.Write(buf[:n]); err != nil {
            return err
        }
		i+=1
		if len > -1 {
			if len < 1 {
				fmt.Printf("\r     [ <=>                                  ] %d\t-.--KB/s eta ?s             ", tot)
			} else {
				//show percentage
				perc := (100 * tot) / len
				prog := progress(perc)
				nowTime := time.Now()
				totTime := nowTime.Sub(startTime)
				spd := float64(tot / 1000) / totTime.Seconds()
				remKb := float64(len - tot) / float64(1000)
				eta :=  remKb / spd
				fmt.Printf("\r%3d%% [%s] %d\t%0.2fKB/s eta %0.1fs             ", perc, prog, tot, spd, eta)
			}
		} else {
			//show dots
			if math.Mod(float64(i), 20) == 0 {
				fmt.Print(".")
			}
		}
    }
	perc := (100 * tot) / len
	prog := progress(perc)
	nowTime := time.Now()
	totTime := nowTime.Sub(startTime)
	spd := float64(tot / 1000) / totTime.Seconds()
	if len < 1 {
		fmt.Printf("\r     [ <=>                                  ] %d\t-.--KB/s in %0.1fs             ", tot, totTime.Seconds())
		fmt.Printf("\n (%0.2fKB/s) - '%v' saved [%v]\n", spd, filename, tot)
	} else {
		fmt.Printf("\r%3d%% [%s] %d\t%0.2fKB/s in %0.1fs             ", perc, prog, tot, spd, totTime.Seconds())
		fmt.Printf("\n '%v' saved [%v/%v]\n", filename, tot, len)
	}
	if err != nil {
		return err
	}
	err = out.Close()
	return err
}

func progress(perc int64) string {
	equalses := perc * 38 / 100
	if equalses < 0 {
		equalses = 0
	}
	spaces := 38 - equalses
	if spaces < 0 {
		spaces = 0
	}
	prog := strings.Repeat("=", int(equalses)) + ">" + strings.Repeat(" ", int(spaces))
	return prog
}

func getFilename(resp *http.Response) (string, error) {
	filename := filepath.Base(resp.Request.URL.Path)
	filename = tidyFilename(filename)

	if !strings.Contains(filename, ".") {
		ct := resp.Header.Get("Content-Type")
		//println(ct)
		ext := "htm"
		mediatype, _, err := mime.ParseMediaType(ct)
		if err != nil {
			fmt.Printf("mime error: %v\n", err)
		} else {
			fmt.Printf("mime type: %v (from Content-Type %v)\n", mediatype, ct)
			slash := strings.Index(mediatype, "/")
			if slash != -1 {
				_, sub := mediatype[:slash], mediatype[slash+1:]
				if sub != "" {
					ext = sub
				}
			}
		}
		filename = filename + "." + ext
	}
	_, err := os.Stat(filename)
	if err != nil {
		if os.IsNotExist(err) {
			return filename, nil
		} else {
			return "", err
		}
	} else {
		num := 1
		//just stop after 100
		for num < 100 {
			filenameNew := filename + "." + strconv.Itoa(num)
			_, err := os.Stat(filenameNew)
			if err != nil {
				if os.IsNotExist(err) {
					return filenameNew, nil
				} else {
					return "", err
				}
			}
			num += 1
		}
		return filename, errors.New("Stopping after trying 100 filename variants")
	}
}
<commit_msg>fixed Range header (continue) support<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/laher/uggo"
	"io"
	"math"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)
func main() {
	err := Wget(os.Args)
	if err != nil {
		fmt.Printf("Error: %v\n", err)
		os.Exit(1)
	}
}

//TODO
// ftp (3rd party supp?)
// clobber behaviour
// limit-rate
// timeouts - connect,read,dns?
// wait,waitretry
// proxies/?
// quota/?
// user/password/ask-password
// certificates/no-check-certificate ...
// exit statuses
// recursive downloads
// timestamping
// wgetrc
type WgetOptions struct {
	IsContinue *bool
	OutputFilename *string
	Timeout *int //TODO timeout
	Retries *int //TODO retries
	IsVerbose *bool
}

const (
	FILEMODE os.FileMode = 0660
)

func Wget(call []string) error {

	options := WgetOptions{}
	flagSet := flag.NewFlagSet("wget", flag.ContinueOnError)
	options.IsContinue = flagSet.Bool("c", false, "continue")
	options.OutputFilename = flagSet.String("o", "", "output filename")
	options.IsVerbose = flagSet.Bool("v", false, "verbose")
	helpFlag := flagSet.Bool("help", false, "Show this help")

	err := flagSet.Parse(uggo.Gnuify(call[1:]))
	if err != nil {
		return err
	}
	if *helpFlag {
		println("wget [options] URL")
		flagSet.PrintDefaults()
		return nil
	}
	args := flagSet.Args()
	if len(args) < 1 {
		flagSet.PrintDefaults()
		return errors.New("Not enough args")
	}
	if len(args) > 0 {
		links := args
		return wget(links, options)
	} else {
		if uggo.IsPipingStdin() {
			//check STDIN
			return wget([]string{}, options)
		} else {
			//NOT piping.
			return errors.New("Not enough args")
		}
	}
}

func wget(links []string, options WgetOptions) error {
	for _, link := range links {
		err := wgetOne(link, options)
		if err != nil {
			return err
		}
	}
	return nil
}

func tidyFilename(filename string) string {
	//invalid filenames ...
	if filename == "" || filename == "/" ||filename == "\\" || filename == "." {
		filename = "index"
	}
	return filename
}

func wgetOne(link string, options WgetOptions) error {
	if !strings.Contains(link, ":") {
		link = "http://" + link
	}
	startTime := time.Now()
	request, err := http.NewRequest("GET", link, nil)
	//resp, err := http.Get(link)
	if err != nil {
		return err
	}

	filename := ""
	if *options.OutputFilename != "" {
		filename = *options.OutputFilename
	}
	client := &http.Client{}
	//continue from where we left off ...
	if *options.IsContinue {
		if filename == "" {
			filename = filepath.Base(request.URL.Path)
			filename = tidyFilename(filename)
			if !strings.Contains(filename, ".") {
				filename = filename + ".html"
			}
		}
		fi, err := os.Stat(filename)
		if err != nil {
			return err
		}
		from := fi.Size()
		rangeHeader := fmt.Sprintf("bytes=%d-", from)
		/*
		NOTE: this could be added as an option later. ...
		//not needed
		headRequest, err := http.NewRequest("HEAD", link, nil)
		if err != nil {
			return err
		}
		headResp, err := client.Do(headRequest)
		if err != nil {
			return err
		}

		cl := headResp.Header.Get("Content-Length")
		if cl != "" {
			rangeHeader = fmt.Sprintf("bytes=%d-%s", from, cl)
			if *options.IsVerbose {
				fmt.Printf("Adding range header: %s\n", rangeHeader)
			}
		} else {
			fmt.Println("Could not find file length using HEAD request")
		}
		*/
		request.Header.Add("Range", rangeHeader)
	}
	if *options.IsVerbose {
		for headerName, headerValue := range request.Header {
			fmt.Printf("Request header %s: %s\n", headerName, headerValue)
		}
	}
	resp, err := client.Do(request)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	fmt.Printf("Http response status: %s\n", resp.Status)
	if *options.IsVerbose {
		for headerName, headerValue := range resp.Header {
			fmt.Printf("Response header %s: %s\n", headerName, headerValue)
		}
	}
	lenS := resp.Header.Get("Content-Length")
	length := int64(-1)
	if lenS != "" {
		length, err = strconv.ParseInt(lenS, 10, 32)
		if err != nil {
			return err
		}
	}

	typ := resp.Header.Get("Content-Type")
	fmt.Printf("Content-Length: %v Content-Type: %s\n", lenS, typ)

	if filename == "" {
		filename, err = getFilename(request, resp)
		if err != nil {
			return err
		}
	}

	contentRange := resp.Header.Get("Content-Range")
	rangeEffective := false
	if contentRange != "" {
		//TODO parse it?
		rangeEffective = true
	} else if *options.IsContinue {
		fmt.Printf("Range request did not produce a Content-Range response\n")
	}
	fmt.Printf("Saving to: '%v'\n\n", filename)
	var openFlags int
	if *options.IsContinue && rangeEffective {
		openFlags = os.O_WRONLY | os.O_APPEND
	} else {
		openFlags = os.O_WRONLY | os.O_CREATE | os.O_TRUNC
	}
	out, err := os.OpenFile(filename, openFlags, FILEMODE)
	if err != nil {
		return err
	}
	defer out.Close()

	buf := make([]byte, 4068)
	tot := int64(0)
	i := 0

	for {
        // read a chunk
        n, err := resp.Body.Read(buf)
        if err != nil && err != io.EOF {
			return err
		}
        if n == 0 { break }
		tot += int64(n)

        // write a chunk
        if _, err := out.Write(buf[:n]); err != nil {
            return err
        }
		i+=1
		if length > -1 {
			if length < 1 {
				fmt.Printf("\r     [ <=>                                  ] %d\t-.--KB/s eta ?s             ", tot)
			} else {
				//show percentage
				perc := (100 * tot) / length
				prog := progress(perc)
				nowTime := time.Now()
				totTime := nowTime.Sub(startTime)
				spd := float64(tot / 1000) / totTime.Seconds()
				remKb := float64(length - tot) / float64(1000)
				eta :=  remKb / spd
				fmt.Printf("\r%3d%% [%s] %d\t%0.2fKB/s eta %0.1fs             ", perc, prog, tot, spd, eta)
			}
		} else {
			//show dots
			if math.Mod(float64(i), 20) == 0 {
				fmt.Print(".")
			}
		}
    }
	perc := (100 * tot) / length
	prog := progress(perc)
	nowTime := time.Now()
	totTime := nowTime.Sub(startTime)
	spd := float64(tot / 1000) / totTime.Seconds()
	if length < 1 {
		fmt.Printf("\r     [ <=>                                  ] %d\t-.--KB/s in %0.1fs             ", tot, totTime.Seconds())
		fmt.Printf("\n (%0.2fKB/s) - '%v' saved [%v]\n", spd, filename, tot)
	} else {
		fmt.Printf("\r%3d%% [%s] %d\t%0.2fKB/s in %0.1fs             ", perc, prog, tot, spd, totTime.Seconds())
		fmt.Printf("\n '%v' saved [%v/%v]\n", filename, tot, length)
	}
	if err != nil {
		return err
	}
	err = out.Close()
	return err
}

func progress(perc int64) string {
	equalses := perc * 38 / 100
	if equalses < 0 {
		equalses = 0
	}
	spaces := 38 - equalses
	if spaces < 0 {
		spaces = 0
	}
	prog := strings.Repeat("=", int(equalses)) + ">" + strings.Repeat(" ", int(spaces))
	return prog
}

func getFilename(request *http.Request, resp *http.Response) (string, error) {
	filename := filepath.Base(request.URL.Path)

	if !strings.Contains(filename, ".") {
		//original link didnt represent the file type. Try using the response url (after redirects)
		filename = filepath.Base(resp.Request.URL.Path)
	}
	filename = tidyFilename(filename)

	if !strings.Contains(filename, ".") {
		ct := resp.Header.Get("Content-Type")
		//println(ct)
		ext := "htm"
		mediatype, _, err := mime.ParseMediaType(ct)
		if err != nil {
			fmt.Printf("mime error: %v\n", err)
		} else {
			fmt.Printf("mime type: %v (from Content-Type %v)\n", mediatype, ct)
			slash := strings.Index(mediatype, "/")
			if slash != -1 {
				_, sub := mediatype[:slash], mediatype[slash+1:]
				if sub != "" {
					ext = sub
				}
			}
		}
		filename = filename + "." + ext
	}
	_, err := os.Stat(filename)
	if err != nil {
		if os.IsNotExist(err) {
			return filename, nil
		} else {
			return "", err
		}
	} else {
		num := 1
		//just stop after 100
		for num < 100 {
			filenameNew := filename + "." + strconv.Itoa(num)
			_, err := os.Stat(filenameNew)
			if err != nil {
				if os.IsNotExist(err) {
					return filenameNew, nil
				} else {
					return "", err
				}
			}
			num += 1
		}
		return filename, errors.New("Stopping after trying 100 filename variants")
	}
}
<|endoftext|>


<commit_before>package persist

import (
	"log"
	"os"
)

// FileLogger is a wrapper for the standard library logger that enforces logging
// into a file with the Sia-standard settings.
type FileLogger struct {
	*log.Logger
	logFile *os.File
}

// CreateLogger returns a logger that can be closed.
func CreateFileLogger(logFilename string) (*FileLogger, error) {
	logFile, err := os.OpenFile(logFilename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0660)
	if err != nil {
		return nil, err
	}
	logger := log.New(logFile, "", log.Ldate|log.Ltime|log.Lmicroseconds|log.Lshortfile|log.LUTC)
	logger.Println("STARTUP: Logging has started.")
	return &FileLogger{Logger: logger, logFile: logFile}, nil
}

// Close terminates the Logger.
func (l *FileLogger) Close() error {
	return l.logFile.Close()
}
<commit_msg>Add a shutdown message for when the logger closes.<commit_after>package persist

import (
	"log"
	"os"
)

// FileLogger is a wrapper for the standard library logger that enforces logging
// into a file with the Sia-standard settings.
type FileLogger struct {
	*log.Logger
	logFile *os.File
}

// CreateLogger returns a logger that can be closed.
func CreateFileLogger(logFilename string) (*FileLogger, error) {
	logFile, err := os.OpenFile(logFilename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0660)
	if err != nil {
		return nil, err
	}
	logger := log.New(logFile, "", log.Ldate|log.Ltime|log.Lmicroseconds|log.Lshortfile|log.LUTC)
	logger.Println("STARTUP: Logging has started.")
	return &FileLogger{Logger: logger, logFile: logFile}, nil
}

// Close terminates the Logger.
func (fl *FileLogger) Close() error {
	fl.Println("SHUTDOWN: Logging has terminated.")
	return fl.logFile.Close()
}
<|endoftext|>