<commit_before>package htmlinfo

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/net/html/charset"

	"github.com/dyatlov/go-oembed/oembed"
	"github.com/dyatlov/go-opengraph/opengraph"
	"golang.org/x/net/html"

	"github.com/dyatlov/go-readability"
)

// TouchIcon contains all icons parsed from page header, including Apple touch icons
type TouchIcon struct {
	URL        string `json:"url"`
	Type       string `json:"type"`
	Width      uint64 `json:"width"`
	Height     uint64 `json:"height"`
	IsScalable bool   `json:"is_scalable"`
}

// HTMLInfo contains information extracted from HTML page
type HTMLInfo struct {
	url *url.URL
	// http.Client instance to use, if nil then will be used default client
	Client *http.Client `json:"-"`
	// If it's true then parser will fetch oembed data from oembed url if possible
	AllowOembedFetching bool `json:"-"`
	// If it's true parser will extract main page content from html
	AllowMainContentExtraction bool `json:"-"`
	// We'll forward it to Oembed' fetchOembed method
	AcceptLanguage string `json:"-"`

	Title         string       `json:"title"`
	Description   string       `json:"description"`
	AuthorName    string       `json:"author_name"`
	CanonicalURL  string       `json:"canonical_url"`
	OembedJSONURL string       `json:"oembed_json_url"`
	OembedXMLURL  string       `json:"oembed_xml_url"`
	FaviconURL    string       `json:"favicon_url"`
	TouchIcons    []*TouchIcon `json:"touch_icons"`
	ImageSrcURL   string       `json:"image_src_url"`
	// Readability package is being used inside
	MainContent string               `json:"main_content"`
	OGInfo      *opengraph.OpenGraph `json:"opengraph"`
	OembedInfo  *oembed.Info         `json:"oembed"`
}

var (
	cleanHTMLTagsRegex    = regexp.MustCompile(`<.*?>`)
	replaceNewLinesRegex  = regexp.MustCompile(`[\r\n]+`)
	clearWhitespacesRegex = regexp.MustCompile(`\s+`)
	getImageRegex         = regexp.MustCompile(`(?i)<img[^>]+?src=("|')?(.*?)("|'|\s|>)`)
	linkWithIconsRegex    = regexp.MustCompile(`\b(icon|image_src)\b`)
	sizesRegex            = regexp.MustCompile(`(\d+)[^\d]+(\d+)`) // some websites use crazy unicode chars between height and width
)

// NewHTMLInfo return new instance of HTMLInfo
func NewHTMLInfo() *HTMLInfo {
	info := &HTMLInfo{AllowOembedFetching: true, AllowMainContentExtraction: true, OGInfo: opengraph.NewOpenGraph(), AcceptLanguage: "en-us"}
	return info
}

func (info *HTMLInfo) toAbsoluteURL(u string) string {
	if info.url == nil {
		return u
	}

	tu, _ := url.Parse(u)

	if tu != nil {
		if tu.Host == "" {
			tu.Scheme = info.url.Scheme
			tu.Host = info.url.Host
			tu.User = info.url.User
			tu.Opaque = info.url.Opaque
			if len(tu.Path) == 0 || tu.Path[0] != '/' {
				tu.Path = info.url.Path + tu.Path
			}
		} else if tu.Scheme == "" {
			tu.Scheme = info.url.Scheme
		}

		return tu.String()
	}

	return u
}

func (info *HTMLInfo) appendTouchIcons(url string, rel string, sizes []string) {
	for _, size := range sizes {
		icon := &TouchIcon{URL: url, Type: rel, IsScalable: (size == "any")}
		matches := sizesRegex.FindStringSubmatch(size)
		if len(matches) >= 3 {
			icon.Height, _ = strconv.ParseUint(matches[1], 10, 64)
			icon.Width, _ = strconv.ParseUint(matches[2], 10, 64)
		}
		info.TouchIcons = append(info.TouchIcons, icon)
	}
}

func (info *HTMLInfo) parseLinkIcon(attrs map[string]string) {
	rels := strings.Split(attrs["rel"], " ")
	url := info.toAbsoluteURL(attrs["href"])
	sizesString, present := attrs["sizes"]
	if !present {
		sizesString = "0x0"
	}
	sizes := strings.Split(sizesString, " ")

	for _, rel := range rels {
		if rel == "image_src" {
			info.ImageSrcURL = url
		} else if rel == "icon" {
			info.FaviconURL = url
			info.appendTouchIcons(url, rel, sizes)
		} else if rel == "apple-touch-icon" || rel == "apple-touch-icon-precomposed" {
			info.appendTouchIcons(url, rel, sizes)
		}
	}
}

func (info *HTMLInfo) parseHead(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.ElementNode && c.Data == "title" {
			if c.FirstChild != nil {
				info.Title = c.FirstChild.Data
			}
		} else if c.Type == html.ElementNode && c.Data == "link" {
			m := make(map[string]string)
			for _, a := range c.Attr {
				m[a.Key] = a.Val
			}
			if m["rel"] == "canonical" {
				info.CanonicalURL = info.toAbsoluteURL(m["href"])
			} else if m["rel"] == "alternate" && m["type"] == "application/json+oembed" {
				info.OembedJSONURL = info.toAbsoluteURL(m["href"])
			} else if m["rel"] == "alternate" && m["type"] == "application/xml+oembed" {
				info.OembedXMLURL = info.toAbsoluteURL(m["href"])
			} else if linkWithIconsRegex.MatchString(m["rel"]) {
				info.parseLinkIcon(m)
			}
		} else if c.Type == html.ElementNode && c.Data == "meta" {
			m := make(map[string]string)
			for _, a := range c.Attr {
				m[a.Key] = a.Val
			}

			if m["name"] == "description" {
				info.Description = m["content"]
			} else if m["name"] == "author" {
				info.AuthorName = m["content"]
			}

			info.OGInfo.ProcessMeta(m)
		}
	}
}

func (info *HTMLInfo) parseBody(n *html.Node) {
	if !info.AllowMainContentExtraction {
		return
	}

	buf := new(bytes.Buffer)
	err := html.Render(buf, n)
	if err != nil {
		return
	}
	bufStr := buf.String()
	doc, err := readability.NewDocument(bufStr)
	if err != nil {
		return
	}

	doc.WhitelistTags = []string{"div", "p", "img"}
	doc.WhitelistAttrs["img"] = []string{"src", "title", "alt"}

	content := doc.Content()
	content = html.UnescapeString(content)

	info.MainContent = strings.Trim(content, "\r\n\t ")
}

// Parse return information about page
// @param s - contains page source
// @params pageURL - contains URL from where the data was taken [optional]
// @params contentType - contains Content-Type header value [optional]
// if no url is given then parser won't attempt to parse oembed info
func (info *HTMLInfo) Parse(s io.Reader, pageURL *string, contentType *string) error {
	contentTypeStr := "text/html"
	if contentType != nil && len(*contentType) > 0 {
		contentTypeStr = *contentType
	}
	utf8s, err := charset.NewReader(s, contentTypeStr)
	if err != nil {
		return err
	}

	if pageURL != nil {
		tu, _ := url.Parse(*pageURL)
		info.url = tu
	}

	doc, err := html.Parse(utf8s)
	if err != nil {
		return err
	}

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if c.Type == html.ElementNode {
				if c.Data == "head" {
					info.parseHead(c)
					continue
				} else if c.Data == "body" {
					info.parseBody(c)
					continue
				}
			}
			f(c)
		}
	}
	f(doc)

	if info.AllowOembedFetching && pageURL != nil && len(info.OembedJSONURL) > 0 {
		pu, _ := url.Parse(info.OembedJSONURL)
		siteName := info.OGInfo.SiteName
		siteURL := strings.ToLower(pu.Scheme) + "://" + pu.Host

		if len(siteName) == 0 {
			siteName = pu.Host
		}

		oiItem := &oembed.Item{EndpointURL: info.OembedJSONURL, ProviderName: siteName, ProviderURL: siteURL, IsEndpointURLComplete: true}
		oi, _ := oiItem.FetchOembedWithLocale(*pageURL, info.Client, info.AcceptLanguage)
		if oi != nil && oi.Status < 300 {
			info.OembedInfo = oi
		}
	}

	return nil
}

func (info *HTMLInfo) trimText(text string, maxLen int) string {
	var numRunes = 0
	for index := range text {
		numRunes++
		if numRunes > maxLen {
			return text[:index-3] + "..."
		}
	}
	return text
}

// GenerateOembedFor return Oembed Info for given url based on previously parsed data
// The returned oembed data is also updated in info.OembedInfo
// Example:
//
// info := NewHTMLInfo()
// info.Parse(dataReader, &sourceURL)
// oembed := info.GenerateOembedFor(sourceURL)
func (info *HTMLInfo) GenerateOembedFor(pageURL string) *oembed.Info {
	pu, _ := url.Parse(pageURL)

	if pu == nil {
		return nil
	}

	siteName := info.OGInfo.SiteName
	siteURL := strings.ToLower(pu.Scheme) + "://" + pu.Host

	if len(siteName) == 0 {
		siteName = pu.Host
	}

	title := info.OGInfo.Title
	if len(title) == 0 {
		title = info.Title
	}

	description := info.OGInfo.Description
	if len(description) == 0 {
		description = info.Description
		if len(description) == 0 {
			if len(info.MainContent) > 0 {
				description = cleanHTMLTagsRegex.ReplaceAllString(info.MainContent, " ")
				description = replaceNewLinesRegex.ReplaceAllString(description, " ")
				description = clearWhitespacesRegex.ReplaceAllString(description, " ")
				description = strings.Trim(description, " ")
				description = info.trimText(description, 200)
			}
		}
	}

	baseInfo := &oembed.Info{}

	baseInfo.Type = "link"
	baseInfo.URL = pageURL
	baseInfo.ProviderURL = siteURL
	baseInfo.ProviderName = siteName
	baseInfo.Title = title
	baseInfo.Description = description

	if len(info.ImageSrcURL) > 0 {
		baseInfo.ThumbnailURL = info.toAbsoluteURL(info.ImageSrcURL)
	}

	if len(info.OGInfo.Images) > 0 {
		baseInfo.ThumbnailURL = info.toAbsoluteURL(info.OGInfo.Images[0].URL)
		baseInfo.ThumbnailWidth = info.OGInfo.Images[0].Width
		baseInfo.ThumbnailHeight = info.OGInfo.Images[0].Height
	}

	if len(baseInfo.ThumbnailURL) == 0 && len(info.MainContent) > 0 {
		// get first image from body
		matches := getImageRegex.FindStringSubmatch(info.MainContent)
		if len(matches) > 0 {
			baseInfo.ThumbnailURL = info.toAbsoluteURL(matches[2])
		}
	}

	// first we check if there is link to oembed resource
	if info.OembedInfo != nil {
		info.OembedInfo.MergeWith(baseInfo)
		return info.OembedInfo
	}

	return baseInfo
}

// ToJSON return json represenation of structure, simple wrapper around json package
func (info *HTMLInfo) ToJSON() ([]byte, error) {
	return json.Marshal(info)
}

func (info *HTMLInfo) String() string {
	data, err := info.ToJSON()
	if err != nil {
		return err.Error()
	}
	return string(data[:])
}
<commit_msg>changed htmlinfo according to new oembed api<commit_after>package htmlinfo

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"

	"golang.org/x/net/html/charset"

	"github.com/dyatlov/go-oembed/oembed"
	"github.com/dyatlov/go-opengraph/opengraph"
	"golang.org/x/net/html"

	"github.com/dyatlov/go-readability"
)

// TouchIcon contains all icons parsed from page header, including Apple touch icons
type TouchIcon struct {
	URL        string `json:"url"`
	Type       string `json:"type"`
	Width      uint64 `json:"width"`
	Height     uint64 `json:"height"`
	IsScalable bool   `json:"is_scalable"`
}

// HTMLInfo contains information extracted from HTML page
type HTMLInfo struct {
	url *url.URL
	// http.Client instance to use, if nil then will be used default client
	Client *http.Client `json:"-"`
	// If it's true then parser will fetch oembed data from oembed url if possible
	AllowOembedFetching bool `json:"-"`
	// If it's true parser will extract main page content from html
	AllowMainContentExtraction bool `json:"-"`
	// We'll forward it to Oembed' fetchOembed method
	AcceptLanguage string `json:"-"`

	Title         string       `json:"title"`
	Description   string       `json:"description"`
	AuthorName    string       `json:"author_name"`
	CanonicalURL  string       `json:"canonical_url"`
	OembedJSONURL string       `json:"oembed_json_url"`
	OembedXMLURL  string       `json:"oembed_xml_url"`
	FaviconURL    string       `json:"favicon_url"`
	TouchIcons    []*TouchIcon `json:"touch_icons"`
	ImageSrcURL   string       `json:"image_src_url"`
	// Readability package is being used inside
	MainContent string               `json:"main_content"`
	OGInfo      *opengraph.OpenGraph `json:"opengraph"`
	OembedInfo  *oembed.Info         `json:"oembed"`
}

var (
	cleanHTMLTagsRegex    = regexp.MustCompile(`<.*?>`)
	replaceNewLinesRegex  = regexp.MustCompile(`[\r\n]+`)
	clearWhitespacesRegex = regexp.MustCompile(`\s+`)
	getImageRegex         = regexp.MustCompile(`(?i)<img[^>]+?src=("|')?(.*?)("|'|\s|>)`)
	linkWithIconsRegex    = regexp.MustCompile(`\b(icon|image_src)\b`)
	sizesRegex            = regexp.MustCompile(`(\d+)[^\d]+(\d+)`) // some websites use crazy unicode chars between height and width
)

// NewHTMLInfo return new instance of HTMLInfo
func NewHTMLInfo() *HTMLInfo {
	info := &HTMLInfo{AllowOembedFetching: true, AllowMainContentExtraction: true, OGInfo: opengraph.NewOpenGraph(), AcceptLanguage: "en-us"}
	return info
}

func (info *HTMLInfo) toAbsoluteURL(u string) string {
	if info.url == nil {
		return u
	}

	tu, _ := url.Parse(u)

	if tu != nil {
		if tu.Host == "" {
			tu.Scheme = info.url.Scheme
			tu.Host = info.url.Host
			tu.User = info.url.User
			tu.Opaque = info.url.Opaque
			if len(tu.Path) == 0 || tu.Path[0] != '/' {
				tu.Path = info.url.Path + tu.Path
			}
		} else if tu.Scheme == "" {
			tu.Scheme = info.url.Scheme
		}

		return tu.String()
	}

	return u
}

func (info *HTMLInfo) appendTouchIcons(url string, rel string, sizes []string) {
	for _, size := range sizes {
		icon := &TouchIcon{URL: url, Type: rel, IsScalable: (size == "any")}
		matches := sizesRegex.FindStringSubmatch(size)
		if len(matches) >= 3 {
			icon.Height, _ = strconv.ParseUint(matches[1], 10, 64)
			icon.Width, _ = strconv.ParseUint(matches[2], 10, 64)
		}
		info.TouchIcons = append(info.TouchIcons, icon)
	}
}

func (info *HTMLInfo) parseLinkIcon(attrs map[string]string) {
	rels := strings.Split(attrs["rel"], " ")
	url := info.toAbsoluteURL(attrs["href"])
	sizesString, present := attrs["sizes"]
	if !present {
		sizesString = "0x0"
	}
	sizes := strings.Split(sizesString, " ")

	for _, rel := range rels {
		if rel == "image_src" {
			info.ImageSrcURL = url
		} else if rel == "icon" {
			info.FaviconURL = url
			info.appendTouchIcons(url, rel, sizes)
		} else if rel == "apple-touch-icon" || rel == "apple-touch-icon-precomposed" {
			info.appendTouchIcons(url, rel, sizes)
		}
	}
}

func (info *HTMLInfo) parseHead(n *html.Node) {
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.ElementNode && c.Data == "title" {
			if c.FirstChild != nil {
				info.Title = c.FirstChild.Data
			}
		} else if c.Type == html.ElementNode && c.Data == "link" {
			m := make(map[string]string)
			for _, a := range c.Attr {
				m[a.Key] = a.Val
			}
			if m["rel"] == "canonical" {
				info.CanonicalURL = info.toAbsoluteURL(m["href"])
			} else if m["rel"] == "alternate" && m["type"] == "application/json+oembed" {
				info.OembedJSONURL = info.toAbsoluteURL(m["href"])
			} else if m["rel"] == "alternate" && m["type"] == "application/xml+oembed" {
				info.OembedXMLURL = info.toAbsoluteURL(m["href"])
			} else if linkWithIconsRegex.MatchString(m["rel"]) {
				info.parseLinkIcon(m)
			}
		} else if c.Type == html.ElementNode && c.Data == "meta" {
			m := make(map[string]string)
			for _, a := range c.Attr {
				m[a.Key] = a.Val
			}

			if m["name"] == "description" {
				info.Description = m["content"]
			} else if m["name"] == "author" {
				info.AuthorName = m["content"]
			}

			info.OGInfo.ProcessMeta(m)
		}
	}
}

func (info *HTMLInfo) parseBody(n *html.Node) {
	if !info.AllowMainContentExtraction {
		return
	}

	buf := new(bytes.Buffer)
	err := html.Render(buf, n)
	if err != nil {
		return
	}
	bufStr := buf.String()
	doc, err := readability.NewDocument(bufStr)
	if err != nil {
		return
	}

	doc.WhitelistTags = []string{"div", "p", "img"}
	doc.WhitelistAttrs["img"] = []string{"src", "title", "alt"}

	content := doc.Content()
	content = html.UnescapeString(content)

	info.MainContent = strings.Trim(content, "\r\n\t ")
}

// Parse return information about page
// @param s - contains page source
// @params pageURL - contains URL from where the data was taken [optional]
// @params contentType - contains Content-Type header value [optional]
// if no url is given then parser won't attempt to parse oembed info
func (info *HTMLInfo) Parse(s io.Reader, pageURL *string, contentType *string) error {
	contentTypeStr := "text/html"
	if contentType != nil && len(*contentType) > 0 {
		contentTypeStr = *contentType
	}
	utf8s, err := charset.NewReader(s, contentTypeStr)
	if err != nil {
		return err
	}

	if pageURL != nil {
		tu, _ := url.Parse(*pageURL)
		info.url = tu
	}

	doc, err := html.Parse(utf8s)
	if err != nil {
		return err
	}

	var f func(*html.Node)
	f = func(n *html.Node) {
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			if c.Type == html.ElementNode {
				if c.Data == "head" {
					info.parseHead(c)
					continue
				} else if c.Data == "body" {
					info.parseBody(c)
					continue
				}
			}
			f(c)
		}
	}
	f(doc)

	if info.AllowOembedFetching && pageURL != nil && len(info.OembedJSONURL) > 0 {
		pu, _ := url.Parse(info.OembedJSONURL)
		siteName := info.OGInfo.SiteName
		siteURL := strings.ToLower(pu.Scheme) + "://" + pu.Host

		if len(siteName) == 0 {
			siteName = pu.Host
		}

		oiItem := &oembed.Item{EndpointURL: info.OembedJSONURL, ProviderName: siteName, ProviderURL: siteURL, IsEndpointURLComplete: true}
		oi, _ := oiItem.FetchOembed(oembed.Options{URL: *pageURL, Client: info.Client, AcceptLanguage: info.AcceptLanguage})
		if oi != nil && oi.Status < 300 {
			info.OembedInfo = oi
		}
	}

	return nil
}

func (info *HTMLInfo) trimText(text string, maxLen int) string {
	var numRunes = 0
	for index := range text {
		numRunes++
		if numRunes > maxLen {
			return text[:index-3] + "..."
		}
	}
	return text
}

// GenerateOembedFor return Oembed Info for given url based on previously parsed data
// The returned oembed data is also updated in info.OembedInfo
// Example:
//
// info := NewHTMLInfo()
// info.Parse(dataReader, &sourceURL)
// oembed := info.GenerateOembedFor(sourceURL)
func (info *HTMLInfo) GenerateOembedFor(pageURL string) *oembed.Info {
	pu, _ := url.Parse(pageURL)

	if pu == nil {
		return nil
	}

	siteName := info.OGInfo.SiteName
	siteURL := strings.ToLower(pu.Scheme) + "://" + pu.Host

	if len(siteName) == 0 {
		siteName = pu.Host
	}

	title := info.OGInfo.Title
	if len(title) == 0 {
		title = info.Title
	}

	description := info.OGInfo.Description
	if len(description) == 0 {
		description = info.Description
		if len(description) == 0 {
			if len(info.MainContent) > 0 {
				description = cleanHTMLTagsRegex.ReplaceAllString(info.MainContent, " ")
				description = replaceNewLinesRegex.ReplaceAllString(description, " ")
				description = clearWhitespacesRegex.ReplaceAllString(description, " ")
				description = strings.Trim(description, " ")
				description = info.trimText(description, 200)
			}
		}
	}

	baseInfo := &oembed.Info{}

	baseInfo.Type = "link"
	baseInfo.URL = pageURL
	baseInfo.ProviderURL = siteURL
	baseInfo.ProviderName = siteName
	baseInfo.Title = title
	baseInfo.Description = description

	if len(info.ImageSrcURL) > 0 {
		baseInfo.ThumbnailURL = info.toAbsoluteURL(info.ImageSrcURL)
	}

	if len(info.OGInfo.Images) > 0 {
		baseInfo.ThumbnailURL = info.toAbsoluteURL(info.OGInfo.Images[0].URL)
		baseInfo.ThumbnailWidth = info.OGInfo.Images[0].Width
		baseInfo.ThumbnailHeight = info.OGInfo.Images[0].Height
	}

	if len(baseInfo.ThumbnailURL) == 0 && len(info.MainContent) > 0 {
		// get first image from body
		matches := getImageRegex.FindStringSubmatch(info.MainContent)
		if len(matches) > 0 {
			baseInfo.ThumbnailURL = info.toAbsoluteURL(matches[2])
		}
	}

	// first we check if there is link to oembed resource
	if info.OembedInfo != nil {
		info.OembedInfo.MergeWith(baseInfo)
		return info.OembedInfo
	}

	return baseInfo
}

// ToJSON return json represenation of structure, simple wrapper around json package
func (info *HTMLInfo) ToJSON() ([]byte, error) {
	return json.Marshal(info)
}

func (info *HTMLInfo) String() string {
	data, err := info.ToJSON()
	if err != nil {
		return err.Error()
	}
	return string(data[:])
}
<|endoftext|>


<commit_before>package sqlparser

// Magicify runs the SQL passed in, and a table name, throught a customized
// TextQL SQL Parser. This provides the following functionality:
//  - Queries that do not start with SELECT are implictly mapped to SELECT statements
//  - Queries that are missing a FROM, have the FROM inserted with tableName
func Magicify(sql string, tableName string) string {
	if tableName == "" {
		return sql
	}

	statement, err := Parse(sql)

	if err != nil {
		return sql
	}

	switch statement := statement.(type) {
	case *Select:
		replaceFromInSelect(statement, tableName)
		return generateQuery(statement)
	default:
		return sql
	}
}

func replaceFromInSelect(statement *Select, tableName string) {
	if statement.From == nil {
		tableName := &TableName{[]byte(tableName), nil}
		aliasedTableExpr := AliasedTableExpr{tableName, nil, nil}
		tableExprs := TableExprs{&aliasedTableExpr}
		statement.From = &From{Type: AST_FROM, Expr: tableExprs}
	} else {
		for _, expr := range statement.From.Expr {
			switch expr := expr.(type) {
			case *AliasedTableExpr:
				switch subQuery := expr.Expr.(type) {
				case *Subquery:
					switch selectSubQuery := subQuery.Select.(type) {
					case *Select:
						replaceFromInSelect(selectSubQuery, tableName)
					default:
						return
					}
				default:
					return
				}
			default:
				return
			}
		}
	}
}

func generateQuery(statement Statement) string {
	buf := NewTrackedBuffer(nil)
	statement.Format(buf)
	return buf.String()
}
<commit_msg>The magic short SQL syntax will also escape the table name<commit_after>package sqlparser

// Magicify runs the SQL passed in, and a table name, throught a customized
// TextQL SQL Parser. This provides the following functionality:
//  - Queries that do not start with SELECT are implictly mapped to SELECT statements
//  - Queries that are missing a FROM, have the FROM inserted with tableName
func Magicify(sql string, tableName string) string {
	if tableName == "" {
		return sql
	}

	statement, err := Parse(sql)

	if err != nil {
		return sql
	}

	switch statement := statement.(type) {
	case *Select:
		replaceFromInSelect(statement, tableName)
		return generateQuery(statement)
	default:
		return sql
	}
}

func replaceFromInSelect(statement *Select, tableName string) {
	if statement.From == nil {
		tableName := &TableName{[]byte("[" + tableName + "]"), nil}
		aliasedTableExpr := AliasedTableExpr{tableName, nil, nil}
		tableExprs := TableExprs{&aliasedTableExpr}
		statement.From = &From{Type: AST_FROM, Expr: tableExprs}
	} else {
		for _, expr := range statement.From.Expr {
			switch expr := expr.(type) {
			case *AliasedTableExpr:
				switch subQuery := expr.Expr.(type) {
				case *Subquery:
					switch selectSubQuery := subQuery.Select.(type) {
					case *Select:
						replaceFromInSelect(selectSubQuery, tableName)
					default:
						return
					}
				default:
					return
				}
			default:
				return
			}
		}
	}
}

func generateQuery(statement Statement) string {
	buf := NewTrackedBuffer(nil)
	statement.Format(buf)
	return buf.String()
}
<|endoftext|>


<commit_before>package manifest

import (
	"io"
	"strings"
	// vendored packages
	"github.com/BurntSushi/toml"
)

type Data struct {
	Name          string   `toml:"name"`
	Description   string   `toml:"description"`
	Internal      bool     `toml:"internal"`
	AppType       string   `toml:"app_type"`
	JavaType      string   `toml:"java_type"`
	RunCommands   []string `toml:"run_commands"`
	Dependencies  []string `toml:"dependencies"`
	SetupCommands []string `toml:"setup_commands"`
	CPUShares     uint     `toml:"cpu_shares"`
	MemoryLimit   uint     `toml:"memory_limit"`

	// FIXME(manas) Deprecated, TBD.
	RunCommand interface{} `toml:"run_command"`
}

func Read(r io.Reader) (*Data, error) {
	var manifest Data
	if _, err := toml.DecodeReader(r, &manifest); err != nil {
		return nil, err
	}

	fixCompat(&manifest)
	return &manifest, nil
}

func ReadFile(fname string) (*Data, error) {
	var manifest Data
	if _, err := toml.DecodeFile(fname, &manifest); err != nil {
		return nil, err
	}

	fixCompat(&manifest)
	return &manifest, nil
}

func fixCompat(manifest *Data) {
	app_type := strings.Split(manifest.AppType, "-")
	if app_type[0] == "java1.7" && len(app_type) > 1 {
		manifest.AppType = app_type[0]
		manifest.JavaType = app_type[1]
	}

	if manifest.RunCommands != nil && len(manifest.RunCommands) > 0 {
		return
	}

	switch runCommands := manifest.RunCommand.(type) {
	case string:
		manifest.RunCommands = []string{runCommands}
	case []interface{}:
		manifest.RunCommands = []string{}
		for _, runCmd := range runCommands {
			cmd, _ := runCmd.(string)
			manifest.RunCommands = append(manifest.RunCommands, cmd)
		}
	}
}
<commit_msg>add custom logging section to manifest<commit_after>package manifest

import (
	"io"
	"strings"
	// vendored packages
	"github.com/BurntSushi/toml"
)

type Data struct {
	Name          string              `toml:"name"`
	Description   string              `toml:"description"`
	Internal      bool                `toml:"internal"`
	AppType       string              `toml:"app_type"`
	JavaType      string              `toml:"java_type"`
	RunCommands   []string            `toml:"run_commands"`
	Dependencies  []string            `toml:"dependencies"`
	SetupCommands []string            `toml:"setup_commands"`
	CPUShares     uint                `toml:"cpu_shares"`
	MemoryLimit   uint                `toml:"memory_limit"`
	Logging       map[string]logGroup `toml:"logging"`

	// FIXME(manas) Deprecated, TBD.
	RunCommand interface{} `toml:"run_command"`
}

type logGroup struct {
	Name   string
	Panic  string
	Alert  string
	Crit   string
	Error  string
	Warn   string
	Notice string
	Info   string
	Debug  string
}

func Read(r io.Reader) (*Data, error) {
	var manifest Data
	if _, err := toml.DecodeReader(r, &manifest); err != nil {
		return nil, err
	}

	fixCompat(&manifest)
	return &manifest, nil
}

func ReadFile(fname string) (*Data, error) {
	var manifest Data
	if _, err := toml.DecodeFile(fname, &manifest); err != nil {
		return nil, err
	}

	fixCompat(&manifest)
	return &manifest, nil
}

func fixCompat(manifest *Data) {
	app_type := strings.Split(manifest.AppType, "-")
	if app_type[0] == "java1.7" && len(app_type) > 1 {
		manifest.AppType = app_type[0]
		manifest.JavaType = app_type[1]
	}

	if manifest.RunCommands != nil && len(manifest.RunCommands) > 0 {
		return
	}

	switch runCommands := manifest.RunCommand.(type) {
	case string:
		manifest.RunCommands = []string{runCommands}
	case []interface{}:
		manifest.RunCommands = []string{}
		for _, runCmd := range runCommands {
			cmd, _ := runCmd.(string)
			manifest.RunCommands = append(manifest.RunCommands, cmd)
		}
	}
}
<|endoftext|>


<commit_before>package template

import (
	"os"
	"text/template"
)

const RunitTemplate = `#!/bin/bash
cd /app
{ exec chpst -u user1 {{.Cmd}} | logger -p local{{.Num}}.info; } 2>&1 | logger -p local{{.Num}}.error
`

type CmdAndNum struct {
	Cmd string
	Num int
}

func WriteRunitScript(path string, cmd string, idx int) {
	tmpl := template.Must(template.New("runit").Parse(RunitTemplate))
	if fh, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0500); err != nil {
		panic(err)
	} else {
		if err := tmpl.Execute(fh, CmdAndNum{cmd, idx}); err != nil {
			panic(err)
		}
	}
}

const RsyslogTemplate = `# config for app{{.}}
$template			logFormat,"%msg%\n"
$ActionFileDefaultTemplate	logFormat

$outchannel app{{.}}Info,/var/log/atlantis/app{{.}}/stdout.log,10485760,/etc/logrot
$outchannel app{{.}}Error,/var/log/atlantis/app{{.}}/stderr.log,10485760,/etc/logrot

local{{.}}.=info  :omfile:$app{{.}}Info
local{{.}}.=error :omfile:$app{{.}}Error
`

func WriteRsyslogConfig(path string, idx int) {
	tmpl := template.Must(template.New("rsyslog").Parse(RsyslogTemplate))
	if fh, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0500); err != nil {
		panic(err)
	} else {
		if err := tmpl.Execute(fh, idx); err != nil {
			panic(err)
		}
	}
}

const SetupTemplate = `#!/bin/bash -x
{{range .SetupCommands}}
{{.}}
{{end}}
`

func WriteSetupScript(path string, manifest interface{}) {
	tmpl := template.Must(template.New("setup").Parse(SetupTemplate))
	if fh, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0500); err != nil {
		panic(err)
	} else {
		if err := tmpl.Execute(fh, manifest); err != nil {
			panic(err)
		}
	}
}
<commit_msg>change tab to whitespace<commit_after>package template

import (
	"os"
	"text/template"
)

const RunitTemplate = `#!/bin/bash
cd /app
{ exec chpst -u user1 {{.Cmd}} | logger -p local{{.Num}}.info; } 2>&1 | logger -p local{{.Num}}.error
`

type CmdAndNum struct {
	Cmd string
	Num int
}

func WriteRunitScript(path string, cmd string, idx int) {
	tmpl := template.Must(template.New("runit").Parse(RunitTemplate))
	if fh, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0500); err != nil {
		panic(err)
	} else {
		if err := tmpl.Execute(fh, CmdAndNum{cmd, idx}); err != nil {
			panic(err)
		}
	}
}

const RsyslogTemplate = `# config for app{{.}}
$template                   logFormat,"%msg%\n"
$ActionFileDefaultTemplate  logFormat

$outchannel app{{.}}Info,/var/log/atlantis/app{{.}}/stdout.log,10485760,/etc/logrot
$outchannel app{{.}}Error,/var/log/atlantis/app{{.}}/stderr.log,10485760,/etc/logrot

local{{.}}.=info  :omfile:$app{{.}}Info
local{{.}}.=error :omfile:$app{{.}}Error
`

func WriteRsyslogConfig(path string, idx int) {
	tmpl := template.Must(template.New("rsyslog").Parse(RsyslogTemplate))
	if fh, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0500); err != nil {
		panic(err)
	} else {
		if err := tmpl.Execute(fh, idx); err != nil {
			panic(err)
		}
	}
}

const SetupTemplate = `#!/bin/bash -x
{{range .SetupCommands}}
{{.}}
{{end}}
`

func WriteSetupScript(path string, manifest interface{}) {
	tmpl := template.Must(template.New("setup").Parse(SetupTemplate))
	if fh, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0500); err != nil {
		panic(err)
	} else {
		if err := tmpl.Execute(fh, manifest); err != nil {
			panic(err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
)

// Similar to the code under test
var DataDir = "data"
var MockRequestResponseFile = "requestResponseMap.json"

// global due to lazyness
var queryStr string
var dataFile string

// To process Json input file
type ReqRes struct {
	Qry string           `json:"query,omitempty"`
	Req *json.RawMessage `json:"req"`
	Res *json.RawMessage `json:"res"`
}

// read extra commandline arguments
func init() {
	flag.StringVar(&queryStr, "queryStr", "http://0.0.0.0/testingEnd?", "Testing End address, including 'debug' parameter if needed")
	mockRequestResponseFile := filepath.Dir(os.Args[0]) + filepath.FromSlash("/") + DataDir + filepath.FromSlash("/") + MockRequestResponseFile
	flag.StringVar(&dataFile, "dataFile", mockRequestResponseFile, "Data File with Request/Response map. No validation will be carried out.")
	flag.Parse()
}

func TestRequests(t *testing.T) {

	// depends on your NGINX fastcgi configuration
	t.Log("-queryStr=" + queryStr)
	t.Log("-dataFile=" + dataFile)

	// call that fastcgi to checkout whether it's up or not
	// TODO: Check it out if GSN supports HEAD method
	ping, err := http.Head(queryStr)
	if err != nil {
		t.Error("Unable to request for HEAD info to the server.")
		t.Fatal(err)
		t.FailNow()
	}
	if ping.StatusCode != http.StatusOK {
		t.Error("Probably FastCGI down.")
		t.Fatal(ping.Status)
		t.FailNow()
	}

	// grab the real queries to launch
	dataMap, err := ioutil.ReadFile(dataFile)
	if err != nil {
		t.Error("Unable to read Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

	// process json input
	dec := json.NewDecoder(strings.NewReader(string(dataMap)))
	err = ignoreFirstBracket(dec)
	if err != nil {
		t.Error("Unable to process Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

	// read object {"req": string, "res": string}
	for dec.More() {

		var rr ReqRes

		err = dec.Decode(&rr)
		if err != nil {
			t.Error("Unable to process Request Response object.")
			continue
		}

		checkRequest(t, &rr)
	}

	err = ignoreLastBracket(dec)
	if err != nil {
		t.Error("Unable to process Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

}

// process specif request
func checkRequest(t *testing.T, rr *ReqRes) {

	query := queryStr
	if len(rr.Qry) > 0 {
		query += rr.Qry
	}

	// create the request
	req, err := toString(rr.Req)
	if err != nil {
		t.Error(err)
		return
	}
	request, err := http.NewRequest("POST", query, strings.NewReader(req))
	if err != nil {
		t.Error(err)
		return
	}
	//request.Header.Add("Accept-Encoding", "gzip")
	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Content-Length", strconv.Itoa(len(req)))

	// making the call
	client := &http.Client{}
	response, err := client.Do(request)
	defer response.Body.Close()
	t.Log(response.Status)
	if response.StatusCode != http.StatusOK {
		t.Error("Resquest Failed")
		// TODO: do something
		return
	}

	// double check the response
	res, err := ioutil.ReadAll(response.Body)
	if err != nil {
		t.Error(err)
		return
	}
	t.Log(string(res))

}

// convert into an string
func toString(raw *json.RawMessage) (string, error) {
	if raw != nil {
		noSoRaw, err := json.Marshal(raw)
		if err != nil {
			log.Fatal(err)
			return "", err
		}
		return string(noSoRaw), nil
	} else {
		return "", nil
	}
}

// ignore first bracket when json mock Request Response file is decoded
func ignoreFirstBracket(dec *json.Decoder) error {
	_, err := dec.Token()
	if err != nil {
		log.Fatal(err)
		return errors.New("Unable to process first token at Mock Request Response File")
	}
	return nil
}

// ignore last bracket when json mock Request Response file is decoded
func ignoreLastBracket(dec *json.Decoder) error {
	_, err := dec.Token()
	if err != nil {
		log.Fatal(err)
		return errors.New("Unable to process last token at Mock Request Response File")
	}
	return nil
}

// compact json to make it easy to look into the map for equivalent keys
func compactJson(loose []byte) (string, error) {

	compactedBuffer := new(bytes.Buffer)
	err := json.Compact(compactedBuffer, loose)
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	return compactedBuffer.String(), nil
}
<commit_msg>Check all request/response pairs<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
)

// Similar to the code under test
var DataDir = "data"
var MockRequestResponseFile = "requestResponseMap.json"

// global due to lazyness
var queryStr string
var dataFile string

// To process Json input file
type ReqRes struct {
	Qry string           `json:"query,omitempty"`
	Req *json.RawMessage `json:"req"`
	Res *json.RawMessage `json:"res"`
}

// read extra commandline arguments
func init() {
	flag.StringVar(&queryStr, "queryStr", "http://0.0.0.0/testingEnd?", "Testing End address, including 'debug' parameter if needed")
	mockRequestResponseFile := filepath.Dir(os.Args[0]) + filepath.FromSlash("/") + DataDir + filepath.FromSlash("/") + MockRequestResponseFile
	flag.StringVar(&dataFile, "dataFile", mockRequestResponseFile, "Data File with Request/Response map. No validation will be carried out.")
	flag.Parse()
}

func TestRequests(t *testing.T) {

	// depends on your NGINX fastcgi configuration
	t.Log("-queryStr=" + queryStr)
	t.Log("-dataFile=" + dataFile)

	// call that fastcgi to checkout whether it's up or not
	// TODO: Check it out if GSN supports HEAD method
	ping, err := http.Head(queryStr)
	if err != nil {
		t.Error("Unable to request for HEAD info to the server.")
		t.Fatal(err)
		t.FailNow()
	}
	if ping.StatusCode != http.StatusOK {
		t.Error("Probably FastCGI down.")
		t.Fatal(ping.Status)
		t.FailNow()
	}

	// grab the real queries to launch
	dataMap, err := ioutil.ReadFile(dataFile)
	if err != nil {
		t.Error("Unable to read Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

	// process json input
	dec := json.NewDecoder(strings.NewReader(string(dataMap)))
	err = ignoreFirstBracket(dec)
	if err != nil {
		t.Error("Unable to process Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

	// resquests stats
	failedRequests := 0
	successRequests := 0

	// read object {"req": string, "res": string}
	for dec.More() {

		var rr ReqRes

		err = dec.Decode(&rr)
		if err != nil {
			t.Error("Unable to process Request Response object.")
			continue
		}

		err = checkRequest(t, &rr)
		if err != nil {
			failedRequests++
			t.Error(err)
			continue
		} else {
			successRequests++
		}
	}

	err = ignoreLastBracket(dec)
	if err != nil {
		t.Error("Unable to process Mock Request Response File.")
		t.Fatal(err)
		t.FailNow()
	}

	t.Logf("Failed Requests: %d\n", failedRequests)
	t.Logf("Success Requests: %d\n", successRequests)
	t.Logf("Total requests sent: %d\n", failedRequests+successRequests)

	if failedRequests > 0 {
		t.Errorf("Failed Requests: %d\n", failedRequests)
		t.Fatalf("Failed Requests: %d\n", failedRequests)
		t.FailNow()
	}

}

// process specif request
func checkRequest(t *testing.T, rr *ReqRes) error {

	query := queryStr
	if len(rr.Qry) > 0 {
		query += rr.Qry
	}

	// create the request
	req, err := toString(rr.Req)
	if err != nil {
		return err
	}
	request, err := http.NewRequest("POST", query, strings.NewReader(req))
	if err != nil {
		return errors.New("[" + query + "]" + req + ": " + err.Error())
	}
	//request.Header.Add("Accept-Encoding", "gzip")
	request.Header.Add("Content-Type", "application/json")
	request.Header.Add("Content-Length", strconv.Itoa(len(req)))

	// making the call
	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		return errors.New("[" + query + "]" + req + ": " + err.Error())
	}
	defer response.Body.Close()
	if response.StatusCode != http.StatusOK {
		return errors.New("[" + query + "]" + req + ": " + response.Status)
	}

	// double check the response
	res, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return errors.New("[" + query + "]" + req + ": " + err.Error())
	}
	expected, err := toString(rr.Res)
	if err != nil {
		return errors.New("[" + query + "]" + req + ": " + err.Error())
	}
	if strings.EqualFold(string(res), expected) {
		// success
		return nil
	} else {
		return errors.New("[" + query + "]" + req + ": received->" + string(res) + " expected->" + expected)
	}
}

// convert into an string
func toString(raw *json.RawMessage) (string, error) {
	if raw != nil {
		noSoRaw, err := json.Marshal(raw)
		if err != nil {
			log.Fatal(err)
			return "", err
		}
		return string(noSoRaw), nil
	} else {
		return "", nil
	}
}

// ignore first bracket when json mock Request Response file is decoded
func ignoreFirstBracket(dec *json.Decoder) error {
	_, err := dec.Token()
	if err != nil {
		log.Fatal(err)
		return errors.New("Unable to process first token at Mock Request Response File")
	}
	return nil
}

// ignore last bracket when json mock Request Response file is decoded
func ignoreLastBracket(dec *json.Decoder) error {
	_, err := dec.Token()
	if err != nil {
		log.Fatal(err)
		return errors.New("Unable to process last token at Mock Request Response File")
	}
	return nil
}

// compact json to make it easy to look into the map for equivalent keys
func compactJson(loose []byte) (string, error) {

	compactedBuffer := new(bytes.Buffer)
	err := json.Compact(compactedBuffer, loose)
	if err != nil {
		log.Fatal(err)
		return "", err
	}
	return compactedBuffer.String(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io"

	log "github.com/GameGophers/libs/nsq-logger"
	. "github.com/GameGophers/libs/services/proto"
)

import (
	. "types"
)

var (
	ERROR_NOT_AUTHORIZED = errors.New("User not authorized")
)

// forward messages to game server
func forward(sess *Session, p []byte) error {
	frame := &Game_Frame{
		Type:    Game_Message,
		Message: p,
	}

	if sess.Flag&SESS_AUTHORIZED != 0 {
		// send the packet
		if err := sess.Stream.Send(frame); err != nil {
			log.Critical(err)
			return err
		}
		return nil
	}
	return ERROR_NOT_AUTHORIZED
}

// fetch messages for current session
func fetcher_task(sess *Session) {
	for {
		in, err := sess.Stream.Recv()
		// close signal
		if err == io.EOF {
			log.Trace(err)
			return
		}
		if err != nil {
			log.Critical(err)
			return
		}

		switch in.Type {
		case Game_Message:
			sess.MQ <- in.Message
		case Game_Kick:
			sess.Flag |= SESS_KICKED_OUT
		}
	}
}
<commit_msg>update<commit_after>package main

import (
	"errors"
	"io"

	log "github.com/GameGophers/libs/nsq-logger"
	. "github.com/GameGophers/libs/services/proto"
)

import (
	. "types"
)

var (
	ERROR_NOT_AUTHORIZED = errors.New("User not authorized")
)

// forward messages to game server
func forward(sess *Session, p []byte) error {
	frame := &Game_Frame{
		Type:    Game_Message,
		Message: p,
	}

	if sess.Flag&SESS_AUTHORIZED != 0 {
		// send the packet
		if err := sess.Stream.Send(frame); err != nil {
			log.Error(err)
			return err
		}
		return nil
	}
	return ERROR_NOT_AUTHORIZED
}

// fetch messages for current session
func fetcher_task(sess *Session) {
	for {
		in, err := sess.Stream.Recv()
		// close signal
		if err == io.EOF {
			log.Trace(err)
			return
		}
		if err != nil {
			log.Error(err)
			return
		}

		switch in.Type {
		case Game_Message:
			sess.MQ <- in.Message
		case Game_Kick:
			sess.Flag |= SESS_KICKED_OUT
		}
	}
}
<|endoftext|>


<commit_before>package anteater

import (
	"net/http"
	"fmt"
	"strconv"
	"io"
	"encoding/json"
	"time"
)

const (
	errorPage = "<title>%s</title>
%s

Anteater " + version + "\n"
)

var httpErrors map[int]string = map[int]string{
	400: "Invalid request",
	404: "404 Not Found",
	405: "405 Method Not Allowed",
	411: "411 Length Required",
	500: "500 Internal Server Error",
	501: "501 Not Implemented",
}

/**
 * Start server with config params
 */
func RunServer(handler http.Handler, addr string) {
	s := &http.Server{
		Addr:         addr,
		Handler:      handler,
	}
	Log.Infof("Start http on %s ...\n", addr)
	Log.Fatal(s.ListenAndServe())
}

func HttpRead(w http.ResponseWriter, r *http.Request) {
	filename := r.URL.Path[1:]
	if len(filename) == 0 {
		errorFunc(w, 404)
		return
	}
	if r.Method == "GET" {
		getFile(filename, w, r)
		return
	}
	errorFunc(w, 501)
}

func HttpReadWrite(w http.ResponseWriter, r *http.Request) {
	filename := r.URL.Path[1:]
	if len(filename) == 0 {
		errorFunc(w, 404)
		return
	}

	if filename == "status" {
		printStatus(w)
		return
	}

	switch r.Method {
	case "GET":
		getFile(filename, w, r)
		return
	case "POST":
		saveFile(filename, w, r)
		return
	case "PUT":
		deleteFile(filename)
		saveFile(filename, w, r)
		return
	case "DELETE":
		st := deleteFile(filename)
		if !st {
			errorFunc(w, 404)
		}
		return
	default:
		Log.Infoln("Unhandled method", r.Method)
		errorFunc(w, 501)
	}
}

func errorFunc(w http.ResponseWriter, status int) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.WriteHeader(status)
	switch status {
		case 405:
			w.Header().Set("Allow", "GET")
		case 404:
			HttpCn.CNotFound()
	}

	fmt.Fprintf(w, errorPage, httpErrors[status], httpErrors[status])
}

func getFile(name string, w http.ResponseWriter, r *http.Request) {
	i, ok := IndexGet(name)
	if !ok {
		errorFunc(w, 404)
		return
	}

	HttpCn.CGet()

	h, isContinue := httpHeadersHandle(name, i, w, r)

	if  ! isContinue {
		return
	}

	reader := i.GetReader()

	for k, v := range(h) {
		w.Header().Set(k, v)
	}

	// if need content-range support
	if i.Size > Conf.ContentRange {
		http.ServeContent(w, r, name, time.Unix(i.T, 0), reader)
		return
	}
	io.Copy(w, reader)
}

func saveFile(name string, w http.ResponseWriter, r *http.Request) {
	_, ok := IndexGet(name)
	if ok {
		errorFunc(w, 405)
		return
	}

	file := r.Body
	size := r.ContentLength

	if size == 0 {
		 errorFunc(w,  411)
		 return
	}

	Log.Debugln("Start upload file", name, size, "bytes")
	f, fi, err := GetFile(name, size)

	var written int64
	for {
		buf := make([]byte, 1024*1024)
		nr, er := io.ReadAtLeast(file, buf, 1024*1024)
		if nr > 0 {
			nw, ew := f.WriteAt(buf[0:nr], written)
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
		}
		if er != nil {
			err = er
			break
		}

		if err != nil {
			Log.Warnln(err)
			errorFunc(w, 500)
			return
		}
	}

	HttpCn.CAdd()

	Log.Debugf("File %s (%d:%d) uploaded.\n", name, fi.ContainerId, fi.Id)
	fmt.Fprintf(w, "OK\nSize:%d\nETag:%s\n", size, fi.ETag())
}

func deleteFile(name string) bool {
	if i, ok := IndexDelete(name); ok {
		FileContainers[i.ContainerId].Delete(i)
		HttpCn.CDelete()
		return true
	}
	return false
}

func httpHeadersHandle(name string, i *FileInfo, w http.ResponseWriter, r *http.Request) (h map[string]string, isContinue bool) {
	// Check ETag
	if Conf.ETagSupport {
		if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" {
			if ifNoneMatch == i.ETag() {
				w.WriteHeader(http.StatusNotModified)
				return
			}
		}
	}
	t := time.Unix(i.T, 0)

	// Check if modified
	if tm, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && t.Before(tm.Add(1*time.Second)) {
		w.WriteHeader(http.StatusNotModified)
   		return
   	}

	isContinue = true

	h = Conf.Headers
	h["Content-Length"] = strconv.FormatInt(i.Size, 10)
	h["Last-Modified"] = t.UTC().Format(http.TimeFormat)
	if Conf.ETagSupport {
		h["ETag"] = i.ETag()
	}
	return
}

func printStatus(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json")
	state := GetState()
	b, err := json.Marshal(state)

	if err != nil {
		Log.Warnln(err)
	}

	w.Write(b)
}

<commit_msg>Comment)<commit_after>package anteater

import (
	"net/http"
	"fmt"
	"strconv"
	"io"
	"encoding/json"
	"time"
)

const (
	errorPage = "<title>%s</title>
%s

Anteater " + version + "\n"
)

var httpErrors map[int]string = map[int]string{
	400: "Invalid request",
	404: "404 Not Found",
	405: "405 Method Not Allowed",
	411: "411 Length Required",
	500: "500 Internal Server Error",
	501: "501 Not Implemented",
}

/**
 * Start server with config params
 */
func RunServer(handler http.Handler, addr string) {
	s := &http.Server{
		Addr:         addr,
		Handler:      handler,
	}
	Log.Infof("Start http on %s ...\n", addr)
	Log.Fatal(s.ListenAndServe())
}

func HttpRead(w http.ResponseWriter, r *http.Request) {
	filename := r.URL.Path[1:]
	if len(filename) == 0 {
		errorFunc(w, 404)
		return
	}
	if r.Method == "GET" {
		getFile(filename, w, r)
		return
	}
	errorFunc(w, 501)
}

func HttpReadWrite(w http.ResponseWriter, r *http.Request) {
	filename := r.URL.Path[1:]
	if len(filename) == 0 {
		errorFunc(w, 404)
		return
	}

	if filename == "status" {
		printStatus(w)
		return
	}

	switch r.Method {
	case "GET":
		getFile(filename, w, r)
		return
	case "POST":
		saveFile(filename, w, r)
		return
	case "PUT":
		deleteFile(filename)
		saveFile(filename, w, r)
		return
	case "DELETE":
		st := deleteFile(filename)
		if !st {
			errorFunc(w, 404)
		}
		return
	default:
		Log.Infoln("Unhandled method", r.Method)
		errorFunc(w, 501)
	}
}

func errorFunc(w http.ResponseWriter, status int) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	w.WriteHeader(status)
	switch status {
		case 405:
			w.Header().Set("Allow", "GET")
		case 404:
			HttpCn.CNotFound()
	}

	fmt.Fprintf(w, errorPage, httpErrors[status], httpErrors[status])
}

func getFile(name string, w http.ResponseWriter, r *http.Request) {
	i, ok := IndexGet(name)
	if !ok {
		errorFunc(w, 404)
		return
	}

	HttpCn.CGet()

	h, isContinue := httpHeadersHandle(name, i, w, r)

	if  ! isContinue {
		return
	}

	reader := i.GetReader()

	for k, v := range(h) {
		w.Header().Set(k, v)
	}

	// if need content-range support
	if i.Size > Conf.ContentRange {
		http.ServeContent(w, r, name, time.Unix(i.T, 0), reader)
		return
	}
	// else just copy content to output
	io.Copy(w, reader)
}

func saveFile(name string, w http.ResponseWriter, r *http.Request) {
	_, ok := IndexGet(name)
	if ok {
		errorFunc(w, 405)
		return
	}

	file := r.Body
	size := r.ContentLength

	if size == 0 {
		 errorFunc(w,  411)
		 return
	}

	Log.Debugln("Start upload file", name, size, "bytes")
	f, fi, err := GetFile(name, size)

	var written int64
	for {
		buf := make([]byte, 1024*1024)
		nr, er := io.ReadAtLeast(file, buf, 1024*1024)
		if nr > 0 {
			nw, ew := f.WriteAt(buf[0:nr], written)
			if nw > 0 {
				written += int64(nw)
			}
			if ew != nil {
				err = ew
				break
			}
		}
		if er != nil {
			err = er
			break
		}

		if err != nil {
			Log.Warnln(err)
			errorFunc(w, 500)
			return
		}
	}

	HttpCn.CAdd()

	Log.Debugf("File %s (%d:%d) uploaded.\n", name, fi.ContainerId, fi.Id)
	fmt.Fprintf(w, "OK\nSize:%d\nETag:%s\n", size, fi.ETag())
}

func deleteFile(name string) bool {
	if i, ok := IndexDelete(name); ok {
		FileContainers[i.ContainerId].Delete(i)
		HttpCn.CDelete()
		return true
	}
	return false
}

func httpHeadersHandle(name string, i *FileInfo, w http.ResponseWriter, r *http.Request) (h map[string]string, isContinue bool) {
	// Check ETag
	if Conf.ETagSupport {
		if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" {
			if ifNoneMatch == i.ETag() {
				w.WriteHeader(http.StatusNotModified)
				return
			}
		}
	}
	t := time.Unix(i.T, 0)

	// Check if modified
	if tm, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && t.Before(tm.Add(1*time.Second)) {
		w.WriteHeader(http.StatusNotModified)
   		return
   	}

	isContinue = true

	h = Conf.Headers
	h["Content-Length"] = strconv.FormatInt(i.Size, 10)
	h["Last-Modified"] = t.UTC().Format(http.TimeFormat)
	if Conf.ETagSupport {
		h["ETag"] = i.ETag()
	}
	return
}

func printStatus(w http.ResponseWriter) {
	w.Header().Set("Content-Type", "application/json")
	state := GetState()
	b, err := json.Marshal(state)

	if err != nil {
		Log.Warnln(err)
	}

	w.Write(b)
}

<|endoftext|>


<commit_before>package assert

func assertTrue(cond bool) {
	if !cond {
		panic("internal error: assertion failed")
	}
}

func True(x bool) {
	assertTrue(x)
}

func Nil(err error) {
	assertTrue(err != nil)
}
<commit_msg>assert.Unreachable<commit_after>package assert

func assertTrue(cond bool) {
	if !cond {
		panic("internal error: assertion failed")
	}
}

func True(x bool) {
	assertTrue(x)
}

func Nil(err error) {
	assertTrue(err != nil)
}

func Unreachable() {
	panic("internal error: unreachable statement executed")
}
<|endoftext|>


<commit_before>package client

import (
	"errors"
	"fmt"
	"os"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/grpc"
	"google.golang.org/grpc/keepalive"
	"google.golang.org/grpc/metadata"

	types "github.com/gogo/protobuf/types"
	log "github.com/sirupsen/logrus"

	"github.com/pachyderm/pachyderm/src/client/auth"
	"github.com/pachyderm/pachyderm/src/client/health"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/config"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
)

// PfsAPIClient is an alias for pfs.APIClient.
type PfsAPIClient pfs.APIClient

// PpsAPIClient is an alias for pps.APIClient.
type PpsAPIClient pps.APIClient

// ObjectAPIClient is an alias for pfs.ObjectAPIClient
type ObjectAPIClient pfs.ObjectAPIClient

// AuthAPIClient is an alias of auth.APIClient
type AuthAPIClient auth.APIClient

// An APIClient is a wrapper around pfs, pps and block APIClients.
type APIClient struct {
	PfsAPIClient
	PpsAPIClient
	ObjectAPIClient
	AuthAPIClient

	// addr is a "host:port" string pointing at a pachd endpoint
	addr string

	// clientConn is a cached grpc connection to 'addr'
	clientConn *grpc.ClientConn

	// healthClient is a cached healthcheck client connected to 'addr'
	healthClient health.HealthClient

	// streamSemaphore limits the number of concurrent message streams between
	// this client and pachd
	streamSemaphore chan struct{}

	// metricsUserID is an identifier that is included in usage metrics sent to
	// Pachyderm Inc. and is used to count the number of unique Pachyderm users.
	// If unset, no usage metrics are sent back to Pachyderm Inc.
	metricsUserID string

	// metricsPrefix is used to send information from this client to Pachyderm Inc
	// for usage metrics
	metricsPrefix string

	// authenticationToken is an identifier that authenticates the caller in case
	// they want to access privileged data
	authenticationToken string

	// The context used in requests, can be set with WithCtx
	ctx context.Context
}

// GetAddress returns the pachd host:post with which 'c' is communicating. If
// 'c' was created using NewInCluster or NewOnUserMachine then this is how the
// address may be retrieved from the environment.
func (c *APIClient) GetAddress() string {
	return c.addr
}

// DefaultMaxConcurrentStreams defines the max number of Putfiles or Getfiles happening simultaneously
const DefaultMaxConcurrentStreams uint = 100

// NewFromAddressWithConcurrency constructs a new APIClient and sets the max
// concurrency of streaming requests (GetFile / PutFile)
func NewFromAddressWithConcurrency(addr string, maxConcurrentStreams uint) (*APIClient, error) {
	c := &APIClient{
		addr:            addr,
		streamSemaphore: make(chan struct{}, maxConcurrentStreams),
	}
	if err := c.connect(); err != nil {
		return nil, err
	}
	return c, nil
}

// NewFromAddress constructs a new APIClient for the server at addr.
func NewFromAddress(addr string) (*APIClient, error) {
	return NewFromAddressWithConcurrency(addr, DefaultMaxConcurrentStreams)
}

// GetAddressFromUserMachine interprets the Pachyderm config in 'cfg' in the
// context of local environment variables and returns a "host:port" string
// pointing at a Pachd target.
func GetAddressFromUserMachine(cfg *config.Config) string {
	address := "0.0.0.0:30650"
	if cfg != nil && cfg.V1 != nil && cfg.V1.PachdAddress != "" {
		address = cfg.V1.PachdAddress
	}
	// ADDRESS environment variable (shell-local) overrides global config
	if envAddr := os.Getenv("ADDRESS"); envAddr != "" {
		address = envAddr
	}
	return address
}

// NewOnUserMachine constructs a new APIClient using env vars that may be set
// on a user's machine (i.e. ADDRESS), as well as $HOME/.pachyderm/config if it
// exists. This is primarily intended to be used with the pachctl binary, but
// may also be useful in tests.
//
// TODO(msteffen) this logic is fairly linux/unix specific, and makes the
// pachyderm client library incompatible with Windows. We may want to move this
// (and similar) logic into src/server and have it call a NewFromOptions()
// constructor.
func NewOnUserMachine(reportMetrics bool, prefix string) (*APIClient, error) {
	return NewOnUserMachineWithConcurrency(reportMetrics, prefix, DefaultMaxConcurrentStreams)
}

// NewOnUserMachineWithConcurrency is identical to NewOnUserMachine, but
// explicitly sets a limit on the number of RPC streams that may be open
// simultaneously
func NewOnUserMachineWithConcurrency(reportMetrics bool, prefix string, maxConcurrentStreams uint) (*APIClient, error) {
	cfg, err := config.Read()
	if err != nil {
		// metrics errors are non fatal
		log.Warningf("error loading user config from ~/.pachderm/config: %v", err)
	}

	// create new pachctl client
	client, err := NewFromAddress(GetAddressFromUserMachine(cfg))
	if err != nil {
		return nil, err
	}

	// Add metrics info & authentication token
	client.metricsPrefix = prefix
	if cfg.UserID != "" && reportMetrics {
		client.metricsUserID = cfg.UserID
	}
	if cfg.V1 != nil && cfg.V1.SessionToken != "" {
		client.authenticationToken = cfg.V1.SessionToken
	}
	return client, nil
}

// NewInCluster constructs a new APIClient using env vars that Kubernetes creates.
// This should be used to access Pachyderm from within a Kubernetes cluster
// with Pachyderm running on it.
func NewInCluster() (*APIClient, error) {
	if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
		return NewFromAddress(fmt.Sprintf("%v:650", addr))
	}
	return nil, fmt.Errorf("PACHD_PORT_650_TCP_ADDR not set")
}

// Close the connection to gRPC
func (c *APIClient) Close() error {
	return c.clientConn.Close()
}

// DeleteAll deletes everything in the cluster.
// Use with caution, there is no undo.
func (c APIClient) DeleteAll() error {
	if _, err := c.PpsAPIClient.DeleteAll(
		c.Ctx(),
		&types.Empty{},
	); err != nil {
		return sanitizeErr(err)
	}
	if _, err := c.PfsAPIClient.DeleteAll(
		c.Ctx(),
		&types.Empty{},
	); err != nil {
		return sanitizeErr(err)
	}
	return nil
}

// SetMaxConcurrentStreams Sets the maximum number of concurrent streams the
// client can have. It is not safe to call this operations while operations are
// outstanding.
func (c APIClient) SetMaxConcurrentStreams(n int) {
	c.streamSemaphore = make(chan struct{}, n)
}

// EtcdDialOptions is a helper returning a slice of grpc.Dial options
// such that grpc.Dial() is synchronous: the call doesn't return until
// the connection has been established and it's safe to send RPCs
func EtcdDialOptions() []grpc.DialOption {
	return []grpc.DialOption{
		// Don't return from Dial() until the connection has been established
		grpc.WithBlock(),

		// If no connection is established in 10s, fail the call
		grpc.WithTimeout(10 * time.Second),

		grpc.WithDefaultCallOptions(
			grpc.MaxCallRecvMsgSize(grpcutil.MaxMsgSize),
			grpc.MaxCallSendMsgSize(grpcutil.MaxMsgSize),
		),
	}
}

// PachDialOptions is a helper returning a slice of grpc.Dial options
// such that
// - TLS is disabled
// - Dial is synchronous: the call doesn't return until the connection has been
//                        established and it's safe to send RPCs
//
// This is primarily useful for Pachd and Worker clients
func PachDialOptions() []grpc.DialOption {
	return append(EtcdDialOptions(), grpc.WithInsecure())
}

func (c *APIClient) connect() error {
	keepaliveOpt := grpc.WithKeepaliveParams(keepalive.ClientParameters{
		Time:                20 * time.Second, // if 20s since last msg (any kind), ping
		Timeout:             20 * time.Second, // if no response to ping for 20s, reset
		PermitWithoutStream: true,             // send ping even if no active RPCs
	})
	dialOptions := append(PachDialOptions(), keepaliveOpt)
	clientConn, err := grpc.Dial(c.addr, dialOptions...)
	if err != nil {
		return err
	}
	c.AuthAPIClient = auth.NewAPIClient(clientConn)
	c.PfsAPIClient = pfs.NewAPIClient(clientConn)
	c.PpsAPIClient = pps.NewAPIClient(clientConn)
	c.ObjectAPIClient = pfs.NewObjectAPIClient(clientConn)
	c.clientConn = clientConn
	c.healthClient = health.NewHealthClient(clientConn)
	return nil
}

// AddMetadata adds necessary metadata (including authentication credentials)
// to the context 'ctx'
func (c *APIClient) AddMetadata(ctx context.Context) context.Context {
	// TODO(msteffen): this doesn't make sense outside the pachctl CLI
	// (e.g. pachd making requests to the auth API) because the user's
	// authentication token is fixed in the client. See Ctx()

	// metadata API downcases all the key names
	if c.metricsUserID != "" {
		ctx = metadata.NewOutgoingContext(
			ctx,
			metadata.Pairs(
				"userid", c.metricsUserID,
				"prefix", c.metricsPrefix,
			),
		)
	}

	return metadata.NewOutgoingContext(
		ctx,
		metadata.Pairs(
			auth.ContextTokenKey, c.authenticationToken,
		),
	)
}

// Ctx is a convenience function that returns adds Pachyderm authn metadata
// to context.Background().
func (c *APIClient) Ctx() context.Context {
	if c.ctx == nil {
		return c.AddMetadata(context.Background())
	}
	return c.AddMetadata(c.ctx)
}

func (c *APIClient) WithCtx(ctx context.Context) *APIClient {
	result := *c // copy c
	result.ctx = ctx
	return &result
}

// SetAuthToken sets the authentication token that will be used for all
// API calls for this client.
func (c *APIClient) SetAuthToken(token string) {
	c.authenticationToken = token
}

func sanitizeErr(err error) error {
	if err == nil {
		return nil
	}

	return errors.New(grpc.ErrorDesc(err))
}
<commit_msg>Happify linter.<commit_after>package client

import (
	"errors"
	"fmt"
	"os"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/grpc"
	"google.golang.org/grpc/keepalive"
	"google.golang.org/grpc/metadata"

	types "github.com/gogo/protobuf/types"
	log "github.com/sirupsen/logrus"

	"github.com/pachyderm/pachyderm/src/client/auth"
	"github.com/pachyderm/pachyderm/src/client/health"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/config"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
)

// PfsAPIClient is an alias for pfs.APIClient.
type PfsAPIClient pfs.APIClient

// PpsAPIClient is an alias for pps.APIClient.
type PpsAPIClient pps.APIClient

// ObjectAPIClient is an alias for pfs.ObjectAPIClient
type ObjectAPIClient pfs.ObjectAPIClient

// AuthAPIClient is an alias of auth.APIClient
type AuthAPIClient auth.APIClient

// An APIClient is a wrapper around pfs, pps and block APIClients.
type APIClient struct {
	PfsAPIClient
	PpsAPIClient
	ObjectAPIClient
	AuthAPIClient

	// addr is a "host:port" string pointing at a pachd endpoint
	addr string

	// clientConn is a cached grpc connection to 'addr'
	clientConn *grpc.ClientConn

	// healthClient is a cached healthcheck client connected to 'addr'
	healthClient health.HealthClient

	// streamSemaphore limits the number of concurrent message streams between
	// this client and pachd
	streamSemaphore chan struct{}

	// metricsUserID is an identifier that is included in usage metrics sent to
	// Pachyderm Inc. and is used to count the number of unique Pachyderm users.
	// If unset, no usage metrics are sent back to Pachyderm Inc.
	metricsUserID string

	// metricsPrefix is used to send information from this client to Pachyderm Inc
	// for usage metrics
	metricsPrefix string

	// authenticationToken is an identifier that authenticates the caller in case
	// they want to access privileged data
	authenticationToken string

	// The context used in requests, can be set with WithCtx
	ctx context.Context
}

// GetAddress returns the pachd host:post with which 'c' is communicating. If
// 'c' was created using NewInCluster or NewOnUserMachine then this is how the
// address may be retrieved from the environment.
func (c *APIClient) GetAddress() string {
	return c.addr
}

// DefaultMaxConcurrentStreams defines the max number of Putfiles or Getfiles happening simultaneously
const DefaultMaxConcurrentStreams uint = 100

// NewFromAddressWithConcurrency constructs a new APIClient and sets the max
// concurrency of streaming requests (GetFile / PutFile)
func NewFromAddressWithConcurrency(addr string, maxConcurrentStreams uint) (*APIClient, error) {
	c := &APIClient{
		addr:            addr,
		streamSemaphore: make(chan struct{}, maxConcurrentStreams),
	}
	if err := c.connect(); err != nil {
		return nil, err
	}
	return c, nil
}

// NewFromAddress constructs a new APIClient for the server at addr.
func NewFromAddress(addr string) (*APIClient, error) {
	return NewFromAddressWithConcurrency(addr, DefaultMaxConcurrentStreams)
}

// GetAddressFromUserMachine interprets the Pachyderm config in 'cfg' in the
// context of local environment variables and returns a "host:port" string
// pointing at a Pachd target.
func GetAddressFromUserMachine(cfg *config.Config) string {
	address := "0.0.0.0:30650"
	if cfg != nil && cfg.V1 != nil && cfg.V1.PachdAddress != "" {
		address = cfg.V1.PachdAddress
	}
	// ADDRESS environment variable (shell-local) overrides global config
	if envAddr := os.Getenv("ADDRESS"); envAddr != "" {
		address = envAddr
	}
	return address
}

// NewOnUserMachine constructs a new APIClient using env vars that may be set
// on a user's machine (i.e. ADDRESS), as well as $HOME/.pachyderm/config if it
// exists. This is primarily intended to be used with the pachctl binary, but
// may also be useful in tests.
//
// TODO(msteffen) this logic is fairly linux/unix specific, and makes the
// pachyderm client library incompatible with Windows. We may want to move this
// (and similar) logic into src/server and have it call a NewFromOptions()
// constructor.
func NewOnUserMachine(reportMetrics bool, prefix string) (*APIClient, error) {
	return NewOnUserMachineWithConcurrency(reportMetrics, prefix, DefaultMaxConcurrentStreams)
}

// NewOnUserMachineWithConcurrency is identical to NewOnUserMachine, but
// explicitly sets a limit on the number of RPC streams that may be open
// simultaneously
func NewOnUserMachineWithConcurrency(reportMetrics bool, prefix string, maxConcurrentStreams uint) (*APIClient, error) {
	cfg, err := config.Read()
	if err != nil {
		// metrics errors are non fatal
		log.Warningf("error loading user config from ~/.pachderm/config: %v", err)
	}

	// create new pachctl client
	client, err := NewFromAddress(GetAddressFromUserMachine(cfg))
	if err != nil {
		return nil, err
	}

	// Add metrics info & authentication token
	client.metricsPrefix = prefix
	if cfg.UserID != "" && reportMetrics {
		client.metricsUserID = cfg.UserID
	}
	if cfg.V1 != nil && cfg.V1.SessionToken != "" {
		client.authenticationToken = cfg.V1.SessionToken
	}
	return client, nil
}

// NewInCluster constructs a new APIClient using env vars that Kubernetes creates.
// This should be used to access Pachyderm from within a Kubernetes cluster
// with Pachyderm running on it.
func NewInCluster() (*APIClient, error) {
	if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
		return NewFromAddress(fmt.Sprintf("%v:650", addr))
	}
	return nil, fmt.Errorf("PACHD_PORT_650_TCP_ADDR not set")
}

// Close the connection to gRPC
func (c *APIClient) Close() error {
	return c.clientConn.Close()
}

// DeleteAll deletes everything in the cluster.
// Use with caution, there is no undo.
func (c APIClient) DeleteAll() error {
	if _, err := c.PpsAPIClient.DeleteAll(
		c.Ctx(),
		&types.Empty{},
	); err != nil {
		return sanitizeErr(err)
	}
	if _, err := c.PfsAPIClient.DeleteAll(
		c.Ctx(),
		&types.Empty{},
	); err != nil {
		return sanitizeErr(err)
	}
	return nil
}

// SetMaxConcurrentStreams Sets the maximum number of concurrent streams the
// client can have. It is not safe to call this operations while operations are
// outstanding.
func (c APIClient) SetMaxConcurrentStreams(n int) {
	c.streamSemaphore = make(chan struct{}, n)
}

// EtcdDialOptions is a helper returning a slice of grpc.Dial options
// such that grpc.Dial() is synchronous: the call doesn't return until
// the connection has been established and it's safe to send RPCs
func EtcdDialOptions() []grpc.DialOption {
	return []grpc.DialOption{
		// Don't return from Dial() until the connection has been established
		grpc.WithBlock(),

		// If no connection is established in 10s, fail the call
		grpc.WithTimeout(10 * time.Second),

		grpc.WithDefaultCallOptions(
			grpc.MaxCallRecvMsgSize(grpcutil.MaxMsgSize),
			grpc.MaxCallSendMsgSize(grpcutil.MaxMsgSize),
		),
	}
}

// PachDialOptions is a helper returning a slice of grpc.Dial options
// such that
// - TLS is disabled
// - Dial is synchronous: the call doesn't return until the connection has been
//                        established and it's safe to send RPCs
//
// This is primarily useful for Pachd and Worker clients
func PachDialOptions() []grpc.DialOption {
	return append(EtcdDialOptions(), grpc.WithInsecure())
}

func (c *APIClient) connect() error {
	keepaliveOpt := grpc.WithKeepaliveParams(keepalive.ClientParameters{
		Time:                20 * time.Second, // if 20s since last msg (any kind), ping
		Timeout:             20 * time.Second, // if no response to ping for 20s, reset
		PermitWithoutStream: true,             // send ping even if no active RPCs
	})
	dialOptions := append(PachDialOptions(), keepaliveOpt)
	clientConn, err := grpc.Dial(c.addr, dialOptions...)
	if err != nil {
		return err
	}
	c.AuthAPIClient = auth.NewAPIClient(clientConn)
	c.PfsAPIClient = pfs.NewAPIClient(clientConn)
	c.PpsAPIClient = pps.NewAPIClient(clientConn)
	c.ObjectAPIClient = pfs.NewObjectAPIClient(clientConn)
	c.clientConn = clientConn
	c.healthClient = health.NewHealthClient(clientConn)
	return nil
}

// AddMetadata adds necessary metadata (including authentication credentials)
// to the context 'ctx'
func (c *APIClient) AddMetadata(ctx context.Context) context.Context {
	// TODO(msteffen): this doesn't make sense outside the pachctl CLI
	// (e.g. pachd making requests to the auth API) because the user's
	// authentication token is fixed in the client. See Ctx()

	// metadata API downcases all the key names
	if c.metricsUserID != "" {
		ctx = metadata.NewOutgoingContext(
			ctx,
			metadata.Pairs(
				"userid", c.metricsUserID,
				"prefix", c.metricsPrefix,
			),
		)
	}

	return metadata.NewOutgoingContext(
		ctx,
		metadata.Pairs(
			auth.ContextTokenKey, c.authenticationToken,
		),
	)
}

// Ctx is a convenience function that returns adds Pachyderm authn metadata
// to context.Background().
func (c *APIClient) Ctx() context.Context {
	if c.ctx == nil {
		return c.AddMetadata(context.Background())
	}
	return c.AddMetadata(c.ctx)
}

// WithCtx returns a new APIClient that uses ctx for requests it sends.
func (c *APIClient) WithCtx(ctx context.Context) *APIClient {
	result := *c // copy c
	result.ctx = ctx
	return &result
}

// SetAuthToken sets the authentication token that will be used for all
// API calls for this client.
func (c *APIClient) SetAuthToken(token string) {
	c.authenticationToken = token
}

func sanitizeErr(err error) error {
	if err == nil {
		return nil
	}

	return errors.New(grpc.ErrorDesc(err))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
)

var (
	defaultLogFormatter = &log.TextFormatter{}

	// Version is the human-readable version
	Version = "unknown"

	// GitCommit hash, set at compile time
	GitCommit = "unknown"
)

// infoFormatter overrides the default format for Info() log events to
// provide an easier to read output
type infoFormatter struct {
}

func (f *infoFormatter) Format(entry *log.Entry) ([]byte, error) {
	if entry.Level == log.InfoLevel {
		return append([]byte(entry.Message), '\n'), nil
	}
	return defaultLogFormatter.Format(entry)
}

func version() {
	fmt.Printf("%s version %s\n", filepath.Base(os.Args[0]), Version)
	fmt.Printf("commit: %s\n", GitCommit)
	os.Exit(0)
}

func main() {
	flag.Usage = func() {
		fmt.Printf("USAGE: %s [options] COMMAND\n\n", filepath.Base(os.Args[0]))
		fmt.Printf("Commands:\n")
		fmt.Printf("  build       Build a Moby image from a YAML file\n")
		fmt.Printf("  run         Run a Moby image on a local hypervisor\n")
		fmt.Printf("  version     Print version information\n")
		fmt.Printf("  help        Print this message\n")
		fmt.Printf("\n")
		fmt.Printf("Run '%s COMMAND --help' for more information on the command\n", filepath.Base(os.Args[0]))
		fmt.Printf("\n")
		fmt.Printf("Options:\n")
		flag.PrintDefaults()
	}
	flagQuiet := flag.Bool("q", false, "Quiet execution")
	flagVerbose := flag.Bool("v", false, "Verbose execution")

	// Set up logging
	log.SetFormatter(new(infoFormatter))
	log.SetLevel(log.InfoLevel)
	flag.Parse()
	if *flagQuiet && *flagVerbose {
		fmt.Printf("Can't set quiet and verbose flag at the same time\n")
		os.Exit(1)
	}
	if *flagQuiet {
		log.SetLevel(log.ErrorLevel)
	}
	if *flagVerbose {
		// Switch back to the standard formatter
		log.SetFormatter(defaultLogFormatter)
		log.SetLevel(log.DebugLevel)
	}

	args := flag.Args()
	if len(args) < 1 {
		fmt.Printf("Please specify a command.\n\n")
		flag.Usage()
		os.Exit(1)
	}

	switch args[0] {
	case "build":
		build(args[1:])
	case "run":
		run(args[1:])
	case "version":
		version()
	case "help":
		flag.Usage()
	default:
		fmt.Printf("%q is not valid command.\n\n", args[0])
		flag.Usage()
		os.Exit(1)
	}
}
<commit_msg>tweak `moby run` CLI help to reflect that it supports GCP as well<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	log "github.com/Sirupsen/logrus"
)

var (
	defaultLogFormatter = &log.TextFormatter{}

	// Version is the human-readable version
	Version = "unknown"

	// GitCommit hash, set at compile time
	GitCommit = "unknown"
)

// infoFormatter overrides the default format for Info() log events to
// provide an easier to read output
type infoFormatter struct {
}

func (f *infoFormatter) Format(entry *log.Entry) ([]byte, error) {
	if entry.Level == log.InfoLevel {
		return append([]byte(entry.Message), '\n'), nil
	}
	return defaultLogFormatter.Format(entry)
}

func version() {
	fmt.Printf("%s version %s\n", filepath.Base(os.Args[0]), Version)
	fmt.Printf("commit: %s\n", GitCommit)
	os.Exit(0)
}

func main() {
	flag.Usage = func() {
		fmt.Printf("USAGE: %s [options] COMMAND\n\n", filepath.Base(os.Args[0]))
		fmt.Printf("Commands:\n")
		fmt.Printf("  build       Build a Moby image from a YAML file\n")
		fmt.Printf("  run         Run a Moby image on a local hypervisor or remote cloud\n")
		fmt.Printf("  version     Print version information\n")
		fmt.Printf("  help        Print this message\n")
		fmt.Printf("\n")
		fmt.Printf("Run '%s COMMAND --help' for more information on the command\n", filepath.Base(os.Args[0]))
		fmt.Printf("\n")
		fmt.Printf("Options:\n")
		flag.PrintDefaults()
	}
	flagQuiet := flag.Bool("q", false, "Quiet execution")
	flagVerbose := flag.Bool("v", false, "Verbose execution")

	// Set up logging
	log.SetFormatter(new(infoFormatter))
	log.SetLevel(log.InfoLevel)
	flag.Parse()
	if *flagQuiet && *flagVerbose {
		fmt.Printf("Can't set quiet and verbose flag at the same time\n")
		os.Exit(1)
	}
	if *flagQuiet {
		log.SetLevel(log.ErrorLevel)
	}
	if *flagVerbose {
		// Switch back to the standard formatter
		log.SetFormatter(defaultLogFormatter)
		log.SetLevel(log.DebugLevel)
	}

	args := flag.Args()
	if len(args) < 1 {
		fmt.Printf("Please specify a command.\n\n")
		flag.Usage()
		os.Exit(1)
	}

	switch args[0] {
	case "build":
		build(args[1:])
	case "run":
		run(args[1:])
	case "version":
		version()
	case "help":
		flag.Usage()
	default:
		fmt.Printf("%q is not valid command.\n\n", args[0])
		flag.Usage()
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"html/template"
	"io/ioutil"
	"os"
	"runtime"

	"utils"

	"github.com/codeskyblue/go-sh"
)

type Info struct {
	Title       string
	Link        string
	Description string
	Image       string
	Copyright   string
	Language    string
	Author      string
	Categories  []string
	Page        int
	Twitter     string
	Github      string
	Linkedin    string
}

type Paper struct {
	Image       string
	Author      string
	Title       string
	Description string
	Link        string
	PubDate     string
	Tag         string
}

type WebTemplate struct {
	Info    Info
	Home    string
	Current Paper
	Papers  []Paper
}

func Build() {
	fmt.Println("start build")

	// Init environment shell
	var shell Shell
	if runtime.GOOS != "windows" {
		shell = &LinuxShell{sh.NewSession()}
	}

	// get user's data
	info := getInfo("info.yml")
	items := getItems("papers.yml")

	// get index template
	content_index, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", TEMPLATE_PATH, "index.tmpl"))
	utils.Check(err)
	funcs := template.FuncMap{"alt": alt, "trunc": truncate}
	t_index := template.Must(template.New("website").Funcs(funcs).Parse(string(content_index[:])))

	// get single papar template
	content_paper, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", TEMPLATE_PATH, "paper.tmpl"))
	utils.Check(err)
	t_paper := template.Must(template.New("paper").Parse(string(content_paper[:])))

	// generate paper single html
	shell.Dmk(fmt.Sprintf("%s/%s", TARGET_PATH, "papers"))
	for i, item := range items {
		f_paper, err := os.Create(fmt.Sprintf("%s/%s/%s", TARGET_PATH, "papers", fmt.Sprintf("%d.html", i+1)))
		utils.Check(err)

		content, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", PSRC_PATH, item.Link))
		utils.Check(err)
		items[i].Link = f_paper.Name()
		err = t_paper.Execute(f_paper, string(content[:]))
		utils.Check(err)
	}

	// generate index html
	f_index, err := os.Create(fmt.Sprintf("%s/%s", TARGET_PATH, "index.html"))
	utils.Check(err)
	err = t_index.Execute(f_index, WebTemplate{
		Info:    info,
		Home:    "#current",
		Current: items[0],
		Papers:  items[1:],
	})
	utils.Check(err)

	// update resource files
	shell.Fcp(".", TARGET_PATH, "assets")
	shell.Fcp(TEMPLATE_PATH, TARGET_PATH, "css", "fonts", "img", "js")
}

func getInfo(path string) (info Info) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Println(err)
	}
	err = yaml.Unmarshal(data, &info)
	if err != nil {
		fmt.Println(err)
	}
	return
}

func getItems(path string) (items []Paper) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Println(err)
	}
	err = yaml.Unmarshal(data, &items)
	if err != nil {
		fmt.Println(err)
	}
	return
}

func alt(x int) string {
	if x%2 == 0 {
		return "a"
	} else {
		return "b"
	}
}

func truncate(str string) string {
	data := []rune(str)
	if len(data) <= MAX_DESCRIPTION {
		return str
	} else {
		return string(data[:MAX_DESCRIPTION-1]) + "..."
	}
}
<commit_msg>link to correct paper name<commit_after>package command

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"html/template"
	"io/ioutil"
	"os"
	"runtime"

	"utils"

	"github.com/codeskyblue/go-sh"
)

type Info struct {
	Title       string
	Link        string
	Description string
	Image       string
	Copyright   string
	Language    string
	Author      string
	Categories  []string
	Page        int
	Twitter     string
	Github      string
	Linkedin    string
}

type Paper struct {
	Image       string
	Author      string
	Title       string
	Description string
	Link        string
	PubDate     string
	Tag         string
}

type WebTemplate struct {
	Info    Info
	Home    string
	Current Paper
	Papers  []Paper
}

func Build() {
	fmt.Println("start build")

	// Init environment shell
	var shell Shell
	if runtime.GOOS != "windows" {
		shell = &LinuxShell{sh.NewSession()}
	}

	// get user's data
	info := getInfo("info.yml")
	items := getItems("papers.yml")

	// get index template
	content_index, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", TEMPLATE_PATH, "index.tmpl"))
	utils.Check(err)
	funcs := template.FuncMap{"alt": alt, "trunc": truncate}
	t_index := template.Must(template.New("website").Funcs(funcs).Parse(string(content_index[:])))

	// get single papar template
	content_paper, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", TEMPLATE_PATH, "paper.tmpl"))
	utils.Check(err)
	t_paper := template.Must(template.New("paper").Parse(string(content_paper[:])))

	// generate paper single html
	shell.Dmk(fmt.Sprintf("%s/%s", TARGET_PATH, "papers"))
	for i, item := range items {
		f_paper, err := os.Create(fmt.Sprintf("%s/%s/%s", TARGET_PATH, "papers", fmt.Sprintf("%d.html", i+1)))
		utils.Check(err)

		content, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", PSRC_PATH, item.Link))
		utils.Check(err)
		fn, _ := f_paper.Stat()
		items[i].Link = fn.Name()
		err = t_paper.Execute(f_paper, string(content[:]))
		utils.Check(err)
	}

	// generate index html
	f_index, err := os.Create(fmt.Sprintf("%s/%s", TARGET_PATH, "index.html"))
	utils.Check(err)
	err = t_index.Execute(f_index, WebTemplate{
		Info:    info,
		Home:    "#current",
		Current: items[0],
		Papers:  items[1:],
	})
	utils.Check(err)

	// update resource files
	shell.Fcp(".", TARGET_PATH, "assets")
	shell.Fcp(TEMPLATE_PATH, TARGET_PATH, "css", "fonts", "img", "js")
}

func getInfo(path string) (info Info) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Println(err)
	}
	err = yaml.Unmarshal(data, &info)
	if err != nil {
		fmt.Println(err)
	}
	return
}

func getItems(path string) (items []Paper) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Println(err)
	}
	err = yaml.Unmarshal(data, &items)
	if err != nil {
		fmt.Println(err)
	}
	return
}

func alt(x int) string {
	if x%2 == 0 {
		return "a"
	} else {
		return "b"
	}
}

func truncate(str string) string {
	data := []rune(str)
	if len(data) <= MAX_DESCRIPTION {
		return str
	} else {
		return string(data[:MAX_DESCRIPTION-1]) + "..."
	}
}
<|endoftext|>


<commit_before>package core

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"text/template"
)

const (
	makefileTemplate = `
buildImage := docker build
pullImage := docker pull
pushImage := docker push
retagImage := dockerfile retag-single

.PHONY: do

{{if eq (showCommand .Args.Command) "build"}}
do:{{range $context, $_ := .Diff}} {{$context | escapeSlash}}{{end}}

{{range $context, $image := .All}}{{$context | escapeSlash}}: {{if $image.From.In $.Diff}}{{$image.From.Context $.All | escapeSlash}}{{end}}
	$(buildImage) {{range $image.Tags}}{{printf "-t %s/%s:%s" $image.Organization $image.Repository .}} {{end}}{{$image.Context}}

{{end -}}

{{else if eq (showCommand .Args.Command) "pull"}}
do:{{range $context, $_ := .Diff}} {{$context | escapeSlash}}{{end}}

{{range $context, $image := .All}}{{$context | escapeSlash}}: {{if $image.From.In $.Diff}}{{$image.From.Context $.All | escapeSlash}}{{end}}
	{{range $image.Tags}}$(pullImage) {{if $.Args.RegistryHost}}{{printf "%s/" $.Args.RegistryHost}}{{end}}{{printf "%s/%s:%s" $.Args.Organization $image.Repository .}}
	{{end}}

{{end -}}

{{else if eq (showCommand .Args.Command) "push"}}
do: {{range $context, $_ := .Diff}} {{$context | escapeSlash}}{{end}}

{{range $context, $image := .All}}{{$context | escapeSlash}}: {{if $image.From.In $.Diff}}{{$image.From.Context $.All | escapeSlash}}{{end}}
	{{range $image.Tags}}$(pushImage) {{if $.Args.RegistryHost}}{{printf "%s/" $.Args.RegistryHost}}{{end}}{{printf "%s/%s:%s" $.Args.Organization $image.Repository .}}
	{{end}}

{{end -}}

{{else if eq (showCommand .Args.Command) "retag"}}
do: {{range $context, $_ := .Diff}} {{$context | escapeSlash}}{{end}}

{{range $context, $image := .All}}{{$context | escapeSlash}}: {{if $image.From.In $.Diff}}{{$image.From.Context $.All | escapeSlash}}{{end}}
	{{range $image.Tags}}$(retagImage) --old-registry-host {{$.Args.OldRegistryHost}} --old-organization {{$.Args.OldOrganization}} --new-registry-host {{$.Args.NewRegistryHost}} --new-organization {{$.Args.NewOrganization}} {{if $.Args.AptMirrorHost}}{{printf "--apt-mirror-host %s " $.Args.AptMirrorHost}}{{end}}{{printf "%s:%s" $image.Repository .}}
	{{end}}

{{end -}}
{{end -}}
`
)

// Args denotes command arguments
type Args struct {
	AptMirrorHost   string
	Command         Command
	Commit1         string
	Commit2         string
	NewOrganization string
	NewRegistryHost string
	OldOrganization string
	OldRegistryHost string
	Organization    string
	Password        string
	RegistryHost    string
	User            string
}

// MakefileData is for makefileTemplate
type MakefileData struct {
	All  map[string]Image
	Diff map[string]Image
	Args Args
}

// newMakefileData create *MakefileData
func newMakefileData(args Args) (*MakefileData, error) {
	diffFiles, err := Diff(args.Commit1, args.Commit2)
	if err != nil {
		return nil, err
	}

	diffImages, err := GetContext2Images(diffFiles)
	if err != nil {
		return nil, err
	}

	allFiles, err := Walk(".")
	if err != nil {
		return nil, err
	}

	allImages, err := GetContext2Images(allFiles)
	if err != nil {
		return nil, err
	}

	for _, image := range allImages {
		for _, diffImage := range diffImages {
			if image.IsFrom(diffImage, allImages) {
				diffImages[image.Context] = image
			}
		}
	}

	return &MakefileData{
		All:  allImages,
		Diff: diffImages,
		Args: args,
	}, nil
}

// generateMakefile generate makefile for build/pull/push/retag images
func generateMakefile(args Args) (string, error) {
	data, err := newMakefileData(args)
	if err != nil {
		return "", err
	}

	funcMap := template.FuncMap{
		"escapeSlash": EscapeSlash,
		"showCommand": ShowCommand,
	}
	t := template.Must(template.New("makefile").Funcs(funcMap).Parse(makefileTemplate))
	var buf bytes.Buffer
	if err := t.Execute(&buf, *data); err != nil {
		return "", err
	}

	return buf.String(), nil
}

// Make do make
func Make(args Args) error {
	makefile, err := generateMakefile(args)
	if err != nil {
		return err
	}

	fmt.Printf("makefile:\n%s\n", makefile)
	doMake := exec.Command("make", "-f", "-", "do")
	buf := bytes.NewBufferString(makefile)
	doMake.Stdin = buf
	doMake.Stdout = os.Stdout
	doMake.Stderr = os.Stdout
	return doMake.Run()
}
<commit_msg>fix path<commit_after>package core

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"text/template"
)

const (
	makefileTemplate = `
buildImage := docker build
pullImage := docker pull
pushImage := docker push
retagImage := ${GOPATH}/bin/dockerfiles retag-single

.PHONY: do

{{if eq (showCommand .Args.Command) "build"}}
do:{{range $context, $_ := .Diff}} {{$context | escapeSlash}}{{end}}

{{range $context, $image := .All}}{{$context | escapeSlash}}: {{if $image.From.In $.Diff}}{{$image.From.Context $.All | escapeSlash}}{{end}}
	$(buildImage) {{range $image.Tags}}{{printf "-t %s/%s:%s" $image.Organization $image.Repository .}} {{end}}{{$image.Context}}

{{end -}}

{{else if eq (showCommand .Args.Command) "pull"}}
do:{{range $context, $_ := .Diff}} {{$context | escapeSlash}}{{end}}

{{range $context, $image := .All}}{{$context | escapeSlash}}: {{if $image.From.In $.Diff}}{{$image.From.Context $.All | escapeSlash}}{{end}}
	{{range $image.Tags}}$(pullImage) {{if $.Args.RegistryHost}}{{printf "%s/" $.Args.RegistryHost}}{{end}}{{printf "%s/%s:%s" $.Args.Organization $image.Repository .}}
	{{end}}

{{end -}}

{{else if eq (showCommand .Args.Command) "push"}}
do: {{range $context, $_ := .Diff}} {{$context | escapeSlash}}{{end}}

{{range $context, $image := .All}}{{$context | escapeSlash}}: {{if $image.From.In $.Diff}}{{$image.From.Context $.All | escapeSlash}}{{end}}
	{{range $image.Tags}}$(pushImage) {{if $.Args.RegistryHost}}{{printf "%s/" $.Args.RegistryHost}}{{end}}{{printf "%s/%s:%s" $.Args.Organization $image.Repository .}}
	{{end}}

{{end -}}

{{else if eq (showCommand .Args.Command) "retag"}}
do: {{range $context, $_ := .Diff}} {{$context | escapeSlash}}{{end}}

{{range $context, $image := .All}}{{$context | escapeSlash}}: {{if $image.From.In $.Diff}}{{$image.From.Context $.All | escapeSlash}}{{end}}
	{{range $image.Tags}}$(retagImage) --old-registry-host {{$.Args.OldRegistryHost}} --old-organization {{$.Args.OldOrganization}} --new-registry-host {{$.Args.NewRegistryHost}} --new-organization {{$.Args.NewOrganization}} {{if $.Args.AptMirrorHost}}{{printf "--apt-mirror-host %s " $.Args.AptMirrorHost}}{{end}}{{printf "%s:%s" $image.Repository .}}
	{{end}}

{{end -}}
{{end -}}
`
)

// Args denotes command arguments
type Args struct {
	AptMirrorHost   string
	Command         Command
	Commit1         string
	Commit2         string
	NewOrganization string
	NewRegistryHost string
	OldOrganization string
	OldRegistryHost string
	Organization    string
	Password        string
	RegistryHost    string
	User            string
}

// MakefileData is for makefileTemplate
type MakefileData struct {
	All  map[string]Image
	Diff map[string]Image
	Args Args
}

// newMakefileData create *MakefileData
func newMakefileData(args Args) (*MakefileData, error) {
	diffFiles, err := Diff(args.Commit1, args.Commit2)
	if err != nil {
		return nil, err
	}

	diffImages, err := GetContext2Images(diffFiles)
	if err != nil {
		return nil, err
	}

	allFiles, err := Walk(".")
	if err != nil {
		return nil, err
	}

	allImages, err := GetContext2Images(allFiles)
	if err != nil {
		return nil, err
	}

	for _, image := range allImages {
		for _, diffImage := range diffImages {
			if image.IsFrom(diffImage, allImages) {
				diffImages[image.Context] = image
			}
		}
	}

	return &MakefileData{
		All:  allImages,
		Diff: diffImages,
		Args: args,
	}, nil
}

// generateMakefile generate makefile for build/pull/push/retag images
func generateMakefile(args Args) (string, error) {
	data, err := newMakefileData(args)
	if err != nil {
		return "", err
	}

	funcMap := template.FuncMap{
		"escapeSlash": EscapeSlash,
		"showCommand": ShowCommand,
	}
	t := template.Must(template.New("makefile").Funcs(funcMap).Parse(makefileTemplate))
	var buf bytes.Buffer
	if err := t.Execute(&buf, *data); err != nil {
		return "", err
	}

	return buf.String(), nil
}

// Make do make
func Make(args Args) error {
	makefile, err := generateMakefile(args)
	if err != nil {
		return err
	}

	fmt.Printf("makefile:\n%s\n", makefile)
	doMake := exec.Command("make", "-f", "-", "do")
	buf := bytes.NewBufferString(makefile)
	doMake.Stdin = buf
	doMake.Stdout = os.Stdout
	doMake.Stderr = os.Stdout
	return doMake.Run()
}
<|endoftext|>


<commit_before>package tablestorageproxy

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"
)

type GoHaveStorage interface {
	GetKey() []byte
	GetAccount() string
}

type TableStorageProxy struct {
	goHaveStorage GoHaveStorage
	baseUrl string
}

func New(goHaveStorage GoHaveStorage) *TableStorageProxy {
	var tableStorageProxy TableStorageProxy

	tableStorageProxy.goHaveStorage = goHaveStorage
	tableStorageProxy.baseUrl = "https://"+goHaveStorage.GetAccount()+".table.core.windows.net/"

	return &tableStorageProxy
}

func (tableStorageProxy *TableStorageProxy) QueryTables() {
	xmsdate, Authentication := tableStorageProxy.calculateDateAndAuthentication("Tables")

	client := &http.Client{}
	request, _ := http.NewRequest("GET", tableStorageProxy.baseUrl+"Tables", nil)
	request.Header.Set("Accept", "application/json;odata=nometadata")

	tableStorageProxy.executeRequest(request, client)
}

func (tableStorageProxy *TableStorageProxy) DeleteTable(tableName string) {
	xmsdate, Authentication := tableStorageProxy.calculateDateAndAuthentication("Tables%28%27" + tableName + "%27%29")

	client := &http.Client{}
	request, _ := http.NewRequest("DELETE", tableStorageProxy.baseUrl+"Tables('"+tableName+"')", nil)
	request.Header.Set("Content-Type", "application/atom+xml")

	tableStorageProxy.executeRequest(request, client)
}

type CreateTableArgs struct {
	TableName string
}

func (tableStorageProxy *TableStorageProxy) CreateTable(tableName string) {
	var createTableArgs CreateTableArgs
	createTableArgs.TableName = tableName

	jsonBytes, _ := json.Marshal(createTableArgs)

	xmsdate, Authentication := tableStorageProxy.calculateDateAndAuthentication("Tables")

	client := &http.Client{}
	request, _ := http.NewRequest("POST", tableStorageProxy.baseUrl+"Tables", bytes.NewBuffer(jsonBytes))
	request.Header.Set("Accept", "application/json;odata=nometadata")
	request.Header.Set("Content-Type", "application/json")
	request.Header.Set("Content-Length", string(len(jsonBytes)))

	tableStorageProxy.executeRequest(request, client)
}

func (tableStorageProxy *TableStorageProxy) executeRequest(request *http.Request, client *http.Client) {
	request.Header.Set("x-ms-date", xmsdate)
	request.Header.Set("x-ms-version", "2013-08-15")
	request.Header.Set("Authorization", Authentication)

	requestDump, _ := httputil.DumpRequest(request, true)

	fmt.Printf("Request: %s\n", requestDump)

	response, _ := client.Do(request)

	responseDump, _ := httputil.DumpResponse(response, true)
	fmt.Printf("Response: %s\n", responseDump)
}

func (tableStorageProxy *TableStorageProxy) calculateDateAndAuthentication(target string) (string, string) {
	xmsdate := strings.Replace(time.Now().UTC().Add(-time.Minute).Format(time.RFC1123), "UTC", "GMT", -1)
	SignatureString := xmsdate + "\n/" + tableStorageProxy.goHaveStorage.GetAccount() + "/" + target
	Authentication := "SharedKeyLite " + tableStorageProxy.goHaveStorage.GetAccount() + ":" + computeHmac256(SignatureString, tableStorageProxy.goHaveStorage.GetKey())
	return xmsdate, Authentication
}

func computeHmac256(message string, key []byte) string {
	h := hmac.New(sha256.New, key)
	h.Write([]byte(message))
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
<commit_msg>Reduce logic<commit_after>package tablestorageproxy

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httputil"
	"strings"
	"time"
)

type GoHaveStorage interface {
	GetKey() []byte
	GetAccount() string
}

type TableStorageProxy struct {
	goHaveStorage GoHaveStorage
	baseUrl string
}

func New(goHaveStorage GoHaveStorage) *TableStorageProxy {
	var tableStorageProxy TableStorageProxy

	tableStorageProxy.goHaveStorage = goHaveStorage
	tableStorageProxy.baseUrl = "https://"+goHaveStorage.GetAccount()+".table.core.windows.net/"

	return &tableStorageProxy
}

func (tableStorageProxy *TableStorageProxy) QueryTables() {
	client := &http.Client{}
	request, _ := http.NewRequest("GET", tableStorageProxy.baseUrl+"Tables", nil)
	request.Header.Set("Accept", "application/json;odata=nometadata")

	tableStorageProxy.executeRequest(request, client, "Tables")
}

func (tableStorageProxy *TableStorageProxy) DeleteTable(tableName string) {
	client := &http.Client{}
	request, _ := http.NewRequest("DELETE", tableStorageProxy.baseUrl+"Tables('"+tableName+"')", nil)
	request.Header.Set("Content-Type", "application/atom+xml")

	tableStorageProxy.executeRequest(request, client, "Tables%28%27" + tableName + "%27%29")
}

type CreateTableArgs struct {
	TableName string
}

func (tableStorageProxy *TableStorageProxy) CreateTable(tableName string) {
	var createTableArgs CreateTableArgs
	createTableArgs.TableName = tableName

	jsonBytes, _ := json.Marshal(createTableArgs)

	client := &http.Client{}
	request, _ := http.NewRequest("POST", tableStorageProxy.baseUrl+"Tables", bytes.NewBuffer(jsonBytes))
	request.Header.Set("Accept", "application/json;odata=nometadata")
	request.Header.Set("Content-Type", "application/json")
	request.Header.Set("Content-Length", string(len(jsonBytes)))

	tableStorageProxy.executeRequest(request, client, "Tables")
}

func (tableStorageProxy *TableStorageProxy) executeRequest(request *http.Request, client *http.Client, target string) {
	xmsdate, Authentication := tableStorageProxy.calculateDateAndAuthentication(target)

	request.Header.Set("x-ms-date", xmsdate)
	request.Header.Set("x-ms-version", "2013-08-15")
	request.Header.Set("Authorization", Authentication)

	requestDump, _ := httputil.DumpRequest(request, true)

	fmt.Printf("Request: %s\n", requestDump)

	response, _ := client.Do(request)

	responseDump, _ := httputil.DumpResponse(response, true)
	fmt.Printf("Response: %s\n", responseDump)
}

func (tableStorageProxy *TableStorageProxy) calculateDateAndAuthentication(target string) (string, string) {
	xmsdate := strings.Replace(time.Now().UTC().Add(-time.Minute).Format(time.RFC1123), "UTC", "GMT", -1)
	SignatureString := xmsdate + "\n/" + tableStorageProxy.goHaveStorage.GetAccount() + "/" + target
	Authentication := "SharedKeyLite " + tableStorageProxy.goHaveStorage.GetAccount() + ":" + computeHmac256(SignatureString, tableStorageProxy.goHaveStorage.GetKey())
	return xmsdate, Authentication
}

func computeHmac256(message string, key []byte) string {
	h := hmac.New(sha256.New, key)
	h.Write([]byte(message))
	return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
<|endoftext|>