<commit_before>/* Copyright 2014 Ooyala, Inc. All rights reserved.
 *
 * This file is licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is
 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */

package dns

import (
	"bufio"
	"io"
	"os"
)

const fileHeaderString = "## The following records are managed by Atlantis ##\n"
const fileFooterString = "## The preceding records are managed by Atlantis ##\n"

type DnsmasqProvider struct {
	file  string
	hosts map[string]int
}

func readUntil(marker string, from *bufio.Reader, to *os.File) error {
	line, err := from.ReadString('\n')
	for ; err == nil; line, err = from.ReadString('\n') {
		if line == marker {
			break
		}
		if to != nil {
			io.WriteString(to, line)
		}
	}
	return err
}

func (d *DnsmasqProvider) rewriteHosts() error {
	oldFile, err := os.Open(d.file)
	if err != nil {
		return err
	}
	defer oldFile.Close()
	oldReader := bufio.NewReader(oldFile)

	newFile, err := os.Create(d.file + ".new")
	if err != nil {
		return err
	}
	defer newFile.Close()

	readUntil(fileHeaderString, oldReader, newFile)
	readUntil(fileFooterString, oldReader, nil)
	readUntil("eof", oldReader, newFile)

	io.WriteString(newFile, fileHeaderString)
	io.WriteString(newFile, "127.0.0.1 magic\n")
	io.WriteString(newFile, fileFooterString)

	if err := os.Rename(d.file+".new", d.file); err != nil {
		return err
	}

	return nil
}

func (d *DnsmasqProvider) CreateRecords(region, comment string, records []Record) error {

	return nil
}

func (d *DnsmasqProvider) GetRecordsForValue(region, value string) ([]string, error) {
	return nil, nil
}

func (d *DnsmasqProvider) DeleteRecords(region, comment string, ids ...string) (error, chan error) {
	return nil, nil
}

func (d *DnsmasqProvider) Suffix(region string) (string, error) {
	return "", nil
}

func NewDnsmasqProvider(file string) (*DnsmasqProvider, error) {
	return &DnsmasqProvider{file: file}, nil
}
<commit_msg>Dnsmasq provider: Add hosts.<commit_after>/* Copyright 2014 Ooyala, Inc. All rights reserved.
 *
 * This file is licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
 * except in compliance with the License. You may obtain a copy of the License at
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is
 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and limitations under the License.
 */

package dns

import (
	"bufio"
	"errors"
	"io"
	"os"
)

const fileHeaderString = "## The following records are managed by Atlantis ##\n"
const fileFooterString = "## The preceding records are managed by Atlantis ##\n"

// TODO(edanaher): The records should be stored (as comments) in the hosts file to persist across master
// restarts.
type DnsmasqProvider struct {
	file    string
	records []Record
}

func readUntil(marker string, from *bufio.Reader, to *os.File) error {
	line, err := from.ReadString('\n')
	for ; err == nil; line, err = from.ReadString('\n') {
		if line == marker {
			break
		}
		if to != nil {
			io.WriteString(to, line)
		}
	}
	return err
}

func (d *DnsmasqProvider) rewriteHosts() error {
	oldFile, err := os.Open(d.file)
	if err != nil {
		return err
	}
	defer oldFile.Close()
	oldReader := bufio.NewReader(oldFile)

	newFile, err := os.Create(d.file + ".new")
	if err != nil {
		return err
	}
	defer newFile.Close()

	readUntil(fileHeaderString, oldReader, newFile)
	readUntil(fileFooterString, oldReader, nil)
	readUntil("eof", oldReader, newFile)

	io.WriteString(newFile, fileHeaderString)
	hosts, err := d.getHosts()
	if err != nil {
		return err
	}
	for host, ip := range hosts {
		io.WriteString(newFile, ip+" "+host+"\n")
	}
	io.WriteString(newFile, fileFooterString)

	if err := os.Rename(d.file+".new", d.file); err != nil {
		return err
	}

	return nil
}

func (d *DnsmasqProvider) getHosts() (map[string]string, error) {
	// TODO(edanaher): This code is copied from route53.  Is it worth pulling out?
	aliases := []*Alias{}
	cnames := []*CName{}
	arecords := []*ARecord{}
	for _, record := range d.records {
		switch typedRecord := record.(type) {
		case *Alias:
			aliases = append(aliases, typedRecord)
		case *CName:
			cnames = append(cnames, typedRecord)
		case *ARecord:
			arecords = append(arecords, typedRecord)
		default:
			return nil, errors.New("Unsupported record type")
		}
	}
	hosts := map[string]string{}
	for _, arecord := range arecords {
		hosts[arecord.Name] = arecord.IP
	}
	// Loop until fixed point to handle recursive cnames and aliases.
	for len(cnames) > 0 {
		unknownNames := []*CName{}
		for _, cname := range cnames {
			if target, ok := hosts[cname.Original]; ok {
				hosts[cname.Name] = target
			} else {
				unknownNames = append(unknownNames, cname)
			}
		}
		if len(cnames) == len(unknownNames) {
			break // Should this be an error?  Unclear.
		}
		cnames = unknownNames
	}
	for len(aliases) > 0 {
		unknownAliases := []*Alias{}
		for _, alias := range aliases {
			if target, ok := hosts[alias.Original]; ok {
				hosts[alias.Alias] = target
			} else {
				unknownAliases = append(unknownAliases, alias)
			}
		}
		if len(aliases) == len(unknownAliases) {
			break // Should this be an error?  Unclear.
		}
		aliases = unknownAliases
	}
	return hosts, nil
}

func (d *DnsmasqProvider) CreateRecords(region, comment string, records []Record) error {
	d.records = append(d.records, records...)
	d.rewriteHosts()
	return nil
}

func (d *DnsmasqProvider) GetRecordsForValue(region, value string) ([]string, error) {
	return nil, nil
}

func (d *DnsmasqProvider) DeleteRecords(region, comment string, ids ...string) (error, chan error) {
	return nil, nil
}

func (d *DnsmasqProvider) Suffix(region string) (string, error) {
	return "", nil
}

func NewDnsmasqProvider(file string) (*DnsmasqProvider, error) {
	return &DnsmasqProvider{file: file}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"encoding/json"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/tealeg/xlsx"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

type message struct {
	User, Type, Subtype, Text, Ts string
	Timestamp                     time.Time
}

// Implement the sort interface for type messages
type messages []message

func (m messages) Len() int {
	return len(m)
}

func (m messages) Less(i, j int) bool {
	return m[i].Timestamp.Before(m[j].Timestamp)
}

func (m messages) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
}

type meta struct {
	ID, Name string
	RealName string `json:"real_name,omitempty"`
}

// Set global CLI vars
var filename, outputDir, timezone string
var metadata = make(map[string][]meta)

func main() {

	app := cli.NewApp()
	app.Usage = "An app for exporting Slack history to Excel (.xlsx)"
	app.HideVersion = true

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "name, n",
			Value:       time.Now().Format("2006-Jan-02") + "_SlackExport.xlsx",
			Usage:       "Set the name of the exported spreadsheet",
			Destination: &filename,
		},
		cli.StringFlag{
			Name:        "destination, d",
			Value:       "./",
			Usage:       "Specify the output directory for\n\t  the exported xlsx workbook",
			Destination: &outputDir,
		},
		cli.StringFlag{
			Name:        "timezone, t",
			Value:       "Local",
			Usage:       "Specify an alternate timezone.\n\t  See here for available options https://goo.gl/Tmq0oR",
			Destination: &timezone,
		},
	}

	app.Action = func(c *cli.Context) {
		if len(c.Args()) != 1 {
			fmt.Printf("\n*** Error: The location of the target zip file must be the last and only argument!\n***\n" +
				"*** For additional help, enter \"slackhist help\"\n\n")
			os.Exit(1)
		}

		if suffix := strings.HasSuffix(filename, ".xlsx"); suffix == false {
			filename = filename + ".xlsx"
		}

		messages := processData(c.Args()[0], metadata)
		createWorkbook(messages)
	}

	app.Run(os.Args)
}

func processData(f string, md map[string][]meta) map[string]messages {

	payload := make(map[string]messages)
	var dirname string

	// Open a readable stream from the zip file
	r, err := zip.OpenReader(f)
	if err != nil {
		panic(err)
	}
	defer r.Close()

	// Collect the required meta to process the information HACK
	for _, file := range r.File {
		switch file.FileHeader.Name {
		case "channels.json", "users.json":
			getMeta(file, md)
		}
	}

	// For every item in the entire zip folder (recursive)
	for _, file := range r.File {

		isDirectory := file.FileInfo().IsDir()

		// Switch on file = directory
		switch isDirectory {
		case true:

			// Retreive the current directory's base name
			path, err := filepath.Abs(file.FileHeader.Name)
			if err != nil {
				panic(err)
			}
			dirname = filepath.Base(path)

			// Make the directory's base name (thus, the channel name) the key
			// and instantiate an empty slice within payload for the messages to
			// be stored
			payload[dirname] = make([]message, 0)

		case false:
			var thisJSON []message

			// If the file is one that we've already processed, break
			// from this particular loop cycle
			switch file.FileHeader.Name {
			case "integration_logs.json", "channels.json", "users.json":
				continue
			}

			// Grab the parent directory's name
			dirname = filepath.Base(filepath.Dir(file.FileHeader.Name))

			// Open a readable stream from the current file
			rc, err := file.Open()
			if err != nil {
				panic(err)
			}
			defer rc.Close()

			// Create JSON decoder and decode the readable stream to "thisJSON"
			decoder := json.NewDecoder(rc)
			for decoder.More() {
				if err := decoder.Decode(&thisJSON); err != nil {
					panic(err)
				}
			}

			// After a full file decode, iterate through the decoded JSON for
			// messages or attachments, and parse the timezone according to the
			// current user's local time zone format
			for _, value := range thisJSON {
				if value.Type == "message" && value.Subtype == "" || value.Subtype == "file_share" {

					value.Timestamp = parseTimestamp(value.Ts)
					_, value.User = parseUser(value.User, md)

					re := regexp.MustCompile(`(<@[a-zA-Z0-9]{9}(\|[a-zA-Z0-9._]+)?>)`)
					if matches := re.FindAllString(value.Text, -1); matches != nil {

						value.Text = re.ReplaceAllStringFunc(value.Text, func(match string) string {
							uid := match[2 : len(match)-1]
							if value.Subtype == "file_share" {
								uid = strings.Split(uid, "|")[0]
							}
							username, _ := parseUser(uid, md)
							return "@" + username
						})

					}

					payload[dirname] = append(payload[dirname], value)
				}
			}

		}

	}

	return payload

}

func createWorkbook(m map[string]messages) {

	var channelNames sort.StringSlice

	workbook := xlsx.NewFile()
	fullPath := filepath.Clean(filepath.Join(outputDir, filename))

	// First, sort the map by channel name
	for channel := range m {
		channelNames = append(channelNames, channel)
	}

	channelNames.Sort()

	for _, channel := range channelNames {
		sheet, err := workbook.AddSheet(channel)
		if err != nil {
			panic(err)
		}

		sort.Sort(sort.Reverse(m[channel]))

		headingRow := sheet.AddRow()
		headingRow.AddCell().SetString("Timestamp")
		headingRow.AddCell().SetString("User")
		headingRow.AddCell().SetString("Message")

		if err := sheet.SetColWidth(0, 1, 20); err != nil {
			panic(err)
		}

		if err := sheet.SetColWidth(2, 2, 200); err != nil {
			panic(err)
		}

		for _, message := range m[channel] {
			messageRow := sheet.AddRow()
			messageRow.AddCell().SetString(message.Timestamp.Format("Jan 02, 2006 | 15:04"))
			messageRow.AddCell().SetString(message.User)
			messageRow.AddCell().SetString(message.Text)
		}

	}

	if err := os.MkdirAll(filepath.Dir(fullPath), os.ModePerm); err != nil {
		panic(err)
	}

	if err := workbook.Save(fullPath); err != nil {
		panic(err)
	}

}

/**************************************************************************
 *                         Utility functions                              *
 **************************************************************************/

func getMeta(f *zip.File, m map[string][]meta) {
	filename := f.FileHeader.Name

	var thisJSON []meta

	rc, err := f.Open()
	if err != nil {
		panic(err)
	}
	defer rc.Close()

	decoder := json.NewDecoder(rc)
	for decoder.More() {
		if err := decoder.Decode(&thisJSON); err != nil {
			panic(err)
		}
	}

	switch filename {
	case "users.json":
		m["users"] = thisJSON
	case "channels.json":
		m["channels"] = thisJSON
	default:
		panic(filename)
	}

}

func parseTimestamp(ts string) time.Time {

	timestring, err := strconv.ParseInt(strings.Split(ts, ".")[0], 10, 64)
	if err != nil {
		panic(err)
	}
	tm := time.Unix(timestring, 0)

	loc, err := time.LoadLocation(timezone)
	if err != nil {
		panic(err)
	}

	return tm.In(loc)
}

func parseUser(uid string, m map[string][]meta) (string, string) {
	var username, realname string
	for _, value := range m["users"] {
		if value.ID == uid {
			username = value.Name
			realname = value.RealName
			break
		}
	}
	return username, realname
}
<commit_msg>Update usage message<commit_after>package main

import (
	"archive/zip"
	"encoding/json"
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/tealeg/xlsx"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"
)

type message struct {
	User, Type, Subtype, Text, Ts string
	Timestamp                     time.Time
}

// Implement the sort interface for type messages
type messages []message

func (m messages) Len() int {
	return len(m)
}

func (m messages) Less(i, j int) bool {
	return m[i].Timestamp.Before(m[j].Timestamp)
}

func (m messages) Swap(i, j int) {
	m[i], m[j] = m[j], m[i]
}

type meta struct {
	ID, Name string
	RealName string `json:"real_name,omitempty"`
}

// Set global CLI vars
var filename, outputDir, timezone string
var metadata = make(map[string][]meta)

func main() {

	app := cli.NewApp()
	app.Usage = "A command-line utility for exporting Slack history to Excel (.xlsx)"
	app.HideVersion = true

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "name, n",
			Value:       time.Now().Format("2006-Jan-02") + "_SlackExport.xlsx",
			Usage:       "Set the name of the exported spreadsheet",
			Destination: &filename,
		},
		cli.StringFlag{
			Name:        "destination, d",
			Value:       "./",
			Usage:       "Specify the output directory for\n\t  the exported xlsx workbook",
			Destination: &outputDir,
		},
		cli.StringFlag{
			Name:        "timezone, t",
			Value:       "Local",
			Usage:       "Specify an alternate timezone.\n\t  See here for available options https://goo.gl/Tmq0oR",
			Destination: &timezone,
		},
	}

	app.Action = func(c *cli.Context) {
		if len(c.Args()) != 1 {
			fmt.Printf("\n*** Error: The location of the target zip file must be the last and only argument!\n***\n" +
				"*** For additional help, enter \"slackhist help\"\n\n")
			os.Exit(1)
		}

		if suffix := strings.HasSuffix(filename, ".xlsx"); suffix == false {
			filename = filename + ".xlsx"
		}

		messages := processData(c.Args()[0], metadata)
		createWorkbook(messages)
	}

	app.Run(os.Args)
}

func processData(f string, md map[string][]meta) map[string]messages {

	payload := make(map[string]messages)
	var dirname string

	// Open a readable stream from the zip file
	r, err := zip.OpenReader(f)
	if err != nil {
		panic(err)
	}
	defer r.Close()

	// Collect the required meta to process the information HACK
	for _, file := range r.File {
		switch file.FileHeader.Name {
		case "channels.json", "users.json":
			getMeta(file, md)
		}
	}

	// For every item in the entire zip folder (recursive)
	for _, file := range r.File {

		isDirectory := file.FileInfo().IsDir()

		// Switch on file = directory
		switch isDirectory {
		case true:

			// Retreive the current directory's base name
			path, err := filepath.Abs(file.FileHeader.Name)
			if err != nil {
				panic(err)
			}
			dirname = filepath.Base(path)

			// Make the directory's base name (thus, the channel name) the key
			// and instantiate an empty slice within payload for the messages to
			// be stored
			payload[dirname] = make([]message, 0)

		case false:
			var thisJSON []message

			// If the file is one that we've already processed, break
			// from this particular loop cycle
			switch file.FileHeader.Name {
			case "integration_logs.json", "channels.json", "users.json":
				continue
			}

			// Grab the parent directory's name
			dirname = filepath.Base(filepath.Dir(file.FileHeader.Name))

			// Open a readable stream from the current file
			rc, err := file.Open()
			if err != nil {
				panic(err)
			}
			defer rc.Close()

			// Create JSON decoder and decode the readable stream to "thisJSON"
			decoder := json.NewDecoder(rc)
			for decoder.More() {
				if err := decoder.Decode(&thisJSON); err != nil {
					panic(err)
				}
			}

			// After a full file decode, iterate through the decoded JSON for
			// messages or attachments, and parse the timezone according to the
			// current user's local time zone format
			for _, value := range thisJSON {
				if value.Type == "message" && value.Subtype == "" || value.Subtype == "file_share" {

					value.Timestamp = parseTimestamp(value.Ts)
					_, value.User = parseUser(value.User, md)

					re := regexp.MustCompile(`(<@[a-zA-Z0-9]{9}(\|[a-zA-Z0-9._]+)?>)`)
					if matches := re.FindAllString(value.Text, -1); matches != nil {

						value.Text = re.ReplaceAllStringFunc(value.Text, func(match string) string {
							uid := match[2 : len(match)-1]
							if value.Subtype == "file_share" {
								uid = strings.Split(uid, "|")[0]
							}
							username, _ := parseUser(uid, md)
							return "@" + username
						})

					}

					payload[dirname] = append(payload[dirname], value)
				}
			}

		}

	}

	return payload

}

func createWorkbook(m map[string]messages) {

	var channelNames sort.StringSlice

	workbook := xlsx.NewFile()
	fullPath := filepath.Clean(filepath.Join(outputDir, filename))

	// First, sort the map by channel name
	for channel := range m {
		channelNames = append(channelNames, channel)
	}

	channelNames.Sort()

	for _, channel := range channelNames {
		sheet, err := workbook.AddSheet(channel)
		if err != nil {
			panic(err)
		}

		sort.Sort(sort.Reverse(m[channel]))

		headingRow := sheet.AddRow()
		headingRow.AddCell().SetString("Timestamp")
		headingRow.AddCell().SetString("User")
		headingRow.AddCell().SetString("Message")

		if err := sheet.SetColWidth(0, 1, 20); err != nil {
			panic(err)
		}

		if err := sheet.SetColWidth(2, 2, 200); err != nil {
			panic(err)
		}

		for _, message := range m[channel] {
			messageRow := sheet.AddRow()
			messageRow.AddCell().SetString(message.Timestamp.Format("Jan 02, 2006 | 15:04"))
			messageRow.AddCell().SetString(message.User)
			messageRow.AddCell().SetString(message.Text)
		}

	}

	if err := os.MkdirAll(filepath.Dir(fullPath), os.ModePerm); err != nil {
		panic(err)
	}

	if err := workbook.Save(fullPath); err != nil {
		panic(err)
	}

}

/**************************************************************************
 *                         Utility functions                              *
 **************************************************************************/

func getMeta(f *zip.File, m map[string][]meta) {
	filename := f.FileHeader.Name

	var thisJSON []meta

	rc, err := f.Open()
	if err != nil {
		panic(err)
	}
	defer rc.Close()

	decoder := json.NewDecoder(rc)
	for decoder.More() {
		if err := decoder.Decode(&thisJSON); err != nil {
			panic(err)
		}
	}

	switch filename {
	case "users.json":
		m["users"] = thisJSON
	case "channels.json":
		m["channels"] = thisJSON
	default:
		panic(filename)
	}

}

func parseTimestamp(ts string) time.Time {

	timestring, err := strconv.ParseInt(strings.Split(ts, ".")[0], 10, 64)
	if err != nil {
		panic(err)
	}
	tm := time.Unix(timestring, 0)

	loc, err := time.LoadLocation(timezone)
	if err != nil {
		panic(err)
	}

	return tm.In(loc)
}

func parseUser(uid string, m map[string][]meta) (string, string) {
	var username, realname string
	for _, value := range m["users"] {
		if value.ID == uid {
			username = value.Name
			realname = value.RealName
			break
		}
	}
	return username, realname
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
	"fmt"
	"github.com/fatih/color"
	"regexp"
)

var (
	// Colors
	tagColor     *color.Color = color.New(color.FgYellow).Add(color.Bold)
	tokenColor                = color.New(color.FgCyan).Add(color.Bold)
	attrKeyColor              = color.New(color.FgRed)
	quoteColor                = color.New(color.FgBlue)

	// Regexp helpers
	whitespaceRegexp *regexp.Regexp = regexp.MustCompile(`^\s*$`)
	preWhitespace                   = regexp.MustCompile(`^\s+`)
	postWhitespace                  = regexp.MustCompile(`\s+$`)
)

func printIndent(level int) {
	for ; level > 0; level-- {
		fmt.Print(indentString)
	}
}

// Is this node a tag with no end tag such as <meta> or
?
// http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
func isVoidElement(n *html.Node) bool {
	switch n.DataAtom {
	case atom.Area, atom.Base, atom.Br, atom.Col, atom.Command, atom.Embed,
		atom.Hr, atom.Img, atom.Input, atom.Keygen, atom.Link,
		atom.Meta, atom.Param, atom.Source, atom.Track, atom.Wbr:
		return true
	}
	return false

}

func printChildren(n *html.Node, level int) {
	if maxPrintLevel > -1 {
		if level >= maxPrintLevel {
			printIndent(level)
			fmt.Println("...")
			return
		}
	}
	child := n.FirstChild
	for child != nil {
		PrintNode(child, level)
		child = child.NextSibling
	}
}

// Print a node and all of it's children to `maxlevel`.
func PrintNode(n *html.Node, level int) {
	switch n.Type {
	case html.TextNode:
		s := n.Data
		if !whitespaceRegexp.MatchString(s) {
			s = preWhitespace.ReplaceAllString(s, "")
			s = postWhitespace.ReplaceAllString(s, "")
			printIndent(level)
			fmt.Println(s)
		}
	case html.ElementNode:
		printIndent(level)
		if printColor {
			tokenColor.Print("<")
			tagColor.Printf("%s", n.Data)
		} else {
			fmt.Printf("<%s", n.Data)
		}
		for _, a := range n.Attr {
			if printColor {
				fmt.Print(" ")
				attrKeyColor.Printf("%s", a.Key)
				tokenColor.Print("=")
				quoteColor.Printf(`"%s"`, a.Val)
			} else {
				fmt.Printf(` %s="%s"`, a.Key, a.Val)
			}
		}
		if printColor {
			tokenColor.Println(">")
		} else {
			fmt.Print(">\n")
		}
		if !isVoidElement(n) {
			printChildren(n, level+1)
			printIndent(level)
			if printColor {
				tokenColor.Print("</")
				tagColor.Printf("%s", n.Data)
				tokenColor.Println(">")
			} else {
				fmt.Printf("</%s>\n", n.Data)
			}
		}
	case html.CommentNode, html.DoctypeNode, html.DocumentNode:
		printChildren(n, level)
	}
}
<commit_msg>Support windows<commit_after>package main

import (
	"code.google.com/p/go.net/html"
	"code.google.com/p/go.net/html/atom"
	"fmt"
	"github.com/fatih/color"
	"github.com/mattn/go-colorable"
	"regexp"
)

var (
	// Colors
	tagColor     *color.Color = color.New(color.FgYellow).Add(color.Bold)
	tokenColor                = color.New(color.FgCyan).Add(color.Bold)
	attrKeyColor              = color.New(color.FgRed)
	quoteColor                = color.New(color.FgBlue)

	// Regexp helpers
	whitespaceRegexp *regexp.Regexp = regexp.MustCompile(`^\s*$`)
	preWhitespace                   = regexp.MustCompile(`^\s+`)
	postWhitespace                  = regexp.MustCompile(`\s+$`)
)

func init() {
	color.Output = colorable.NewColorableStdout()
}

func printIndent(level int) {
	for ; level > 0; level-- {
		fmt.Print(indentString)
	}
}

// Is this node a tag with no end tag such as <meta> or
?
// http://www.w3.org/TR/html-markup/syntax.html#syntax-elements
func isVoidElement(n *html.Node) bool {
	switch n.DataAtom {
	case atom.Area, atom.Base, atom.Br, atom.Col, atom.Command, atom.Embed,
		atom.Hr, atom.Img, atom.Input, atom.Keygen, atom.Link,
		atom.Meta, atom.Param, atom.Source, atom.Track, atom.Wbr:
		return true
	}
	return false

}

func printChildren(n *html.Node, level int) {
	if maxPrintLevel > -1 {
		if level >= maxPrintLevel {
			printIndent(level)
			fmt.Println("...")
			return
		}
	}
	child := n.FirstChild
	for child != nil {
		PrintNode(child, level)
		child = child.NextSibling
	}
}

// Print a node and all of it's children to `maxlevel`.
func PrintNode(n *html.Node, level int) {
	switch n.Type {
	case html.TextNode:
		s := n.Data
		if !whitespaceRegexp.MatchString(s) {
			s = preWhitespace.ReplaceAllString(s, "")
			s = postWhitespace.ReplaceAllString(s, "")
			printIndent(level)
			fmt.Println(s)
		}
	case html.ElementNode:
		printIndent(level)
		if printColor {
			tokenColor.Print("<")
			tagColor.Printf("%s", n.Data)
		} else {
			fmt.Printf("<%s", n.Data)
		}
		for _, a := range n.Attr {
			if printColor {
				fmt.Print(" ")
				attrKeyColor.Printf("%s", a.Key)
				tokenColor.Print("=")
				quoteColor.Printf(`"%s"`, a.Val)
			} else {
				fmt.Printf(` %s="%s"`, a.Key, a.Val)
			}
		}
		if printColor {
			tokenColor.Println(">")
		} else {
			fmt.Print(">\n")
		}
		if !isVoidElement(n) {
			printChildren(n, level+1)
			printIndent(level)
			if printColor {
				tokenColor.Print("</")
				tagColor.Printf("%s", n.Data)
				tokenColor.Println(">")
			} else {
				fmt.Printf("</%s>\n", n.Data)
			}
		}
	case html.CommentNode, html.DoctypeNode, html.DocumentNode:
		printChildren(n, level)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/log"
)

const (
	nameSpace = "selenium_grid"
	subSystem = "hub"
)

var (
	listenAddress = flag.String("listen-address", ":8080", "Address on which to expose metrics.")
	metricsPath   = flag.String("telemetry-path", "/metrics", "Path under which to expose metrics.")
	scrapeURI     = flag.String("scrape-uri", "http://grid.local", "URI on which to scrape Selenium Grid.")
)

type Exporter struct {
	URI                       string
	mutex                     sync.RWMutex
	up, slotsTotal, slotsFree, newSessionRequestCount prometheus.Gauge
}

type hubResponse struct {
	Success      bool       `json:"success"`
	Debug        bool       `json:"debug"`
	CleanUpCycle int        `json:"cleanUpCycle"`
	Slots        slotCounts `json:"slotCounts"`
	newSession   float64    `json:"newSessionRequestCount"`
}

type slotCounts struct {
	Free  float64 `json:"free"`
	Total float64 `json:"total"`
}

func NewExporter(uri string) *Exporter {
	log.Infoln("Collecting data from:", uri)

	return &Exporter{
		URI: uri,
		up: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: nameSpace,
			Name:      "up",
			Help:      "was the last scrape of Selenium Grid successful.",
		}),
		slotsTotal: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: nameSpace,
			Subsystem: subSystem,
			Name:      "slotsTotal",
			Help:      "total number of slots",
		}),
		slotsFree: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: nameSpace,
			Subsystem: subSystem,
			Name:      "slotsFree",
			Help:      "number of free slots",
		}),
		newSessionRequestCount: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: nameSpace,
			Subsystem: subSystem,
			Name:      "sessions_backlog",
			Help:      "number of sessions waiting for a slot",
		}),
	}
}

func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.up.Describe(ch)
	e.slotsTotal.Describe(ch)
	e.slotsFree.Describe(ch)
	e.newSessionRequestCount.Describe(ch)
}

func (e *Exporter) Collect(ch chan<- prometheus.Metric) {

	e.mutex.Lock()
	defer e.mutex.Unlock()

	e.scrape()

	ch <- e.up
	ch <- e.slotsTotal
	ch <- e.slotsFree
	ch <- e.newSessionRequestCount

	return
}

func (e *Exporter) scrape() {

	e.slotsTotal.Set(0)
	e.slotsFree.Set(0)

	body, err := e.fetch()
	if err != nil {
		e.up.Set(0)

		log.Errorf("Can't scrape Selenium Grid: %v", err)
		return
	}

	e.up.Set(1)

	var hResponse hubResponse
	if err := json.Unmarshal(body, &hResponse); err != nil {

		log.Errorf("Can't decode Selenium Grid response: %v", err)
		return
	}

	e.slotsTotal.Set(hResponse.Slots.Total)
	e.slotsFree.Set(hResponse.Slots.Free)
	e.newSessionRequestCount.Set(hResponse.newSession)

}

func (e Exporter) fetch() (output []byte, err error) {

	client := http.Client{
		Timeout: 3 * time.Second,
	}

	response, err := client.Get(e.URI + "/grid/api/hub")
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	output, err = ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	return
}

func main() {
	flag.Parse()

	log.Infoln("Starting selenium_grid_exporter")

	prometheus.MustRegister(NewExporter(*scrapeURI))
	prometheus.Unregister(prometheus.NewGoCollector())
	prometheus.Unregister(prometheus.NewProcessCollector(os.Getegid(), ""))

	http.Handle(*metricsPath, promhttp.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently)
	})

	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<commit_msg>renamed newSession to NewSession in struct<commit_after>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/common/log"
)

const (
	nameSpace = "selenium_grid"
	subSystem = "hub"
)

var (
	listenAddress = flag.String("listen-address", ":8080", "Address on which to expose metrics.")
	metricsPath   = flag.String("telemetry-path", "/metrics", "Path under which to expose metrics.")
	scrapeURI     = flag.String("scrape-uri", "http://grid.local", "URI on which to scrape Selenium Grid.")
)

type Exporter struct {
	URI                       string
	mutex                     sync.RWMutex
	up, slotsTotal, slotsFree, newSessionRequestCount prometheus.Gauge
}

type hubResponse struct {
	Success      bool       `json:"success"`
	Debug        bool       `json:"debug"`
	CleanUpCycle int        `json:"cleanUpCycle"`
	Slots        slotCounts `json:"slotCounts"`
	NewSession   float64    `json:"newSessionRequestCount"`
}

type slotCounts struct {
	Free  float64 `json:"free"`
	Total float64 `json:"total"`
}

func NewExporter(uri string) *Exporter {
	log.Infoln("Collecting data from:", uri)

	return &Exporter{
		URI: uri,
		up: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: nameSpace,
			Name:      "up",
			Help:      "was the last scrape of Selenium Grid successful.",
		}),
		slotsTotal: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: nameSpace,
			Subsystem: subSystem,
			Name:      "slotsTotal",
			Help:      "total number of slots",
		}),
		slotsFree: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: nameSpace,
			Subsystem: subSystem,
			Name:      "slotsFree",
			Help:      "number of free slots",
		}),
		newSessionRequestCount: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: nameSpace,
			Subsystem: subSystem,
			Name:      "sessions_backlog",
			Help:      "number of sessions waiting for a slot",
		}),
	}
}

func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
	e.up.Describe(ch)
	e.slotsTotal.Describe(ch)
	e.slotsFree.Describe(ch)
	e.newSessionRequestCount.Describe(ch)
}

func (e *Exporter) Collect(ch chan<- prometheus.Metric) {

	e.mutex.Lock()
	defer e.mutex.Unlock()

	e.scrape()

	ch <- e.up
	ch <- e.slotsTotal
	ch <- e.slotsFree
	ch <- e.newSessionRequestCount

	return
}

func (e *Exporter) scrape() {

	e.slotsTotal.Set(0)
	e.slotsFree.Set(0)

	body, err := e.fetch()
	if err != nil {
		e.up.Set(0)

		log.Errorf("Can't scrape Selenium Grid: %v", err)
		return
	}

	e.up.Set(1)
	var hResponse hubResponse
	if err := json.Unmarshal(body, &hResponse); err != nil {

		log.Errorf("Can't decode Selenium Grid response: %v", err)
		return
	}
	e.slotsTotal.Set(hResponse.Slots.Total)
	e.slotsFree.Set(hResponse.Slots.Free)
	e.newSessionRequestCount.Set(hResponse.NewSession)

}

func (e Exporter) fetch() (output []byte, err error) {

	client := http.Client{
		Timeout: 3 * time.Second,
	}

	response, err := client.Get(e.URI + "/grid/api/hub")
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()

	output, err = ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, err
	}

	return
}

func main() {
	flag.Parse()

	log.Infoln("Starting selenium_grid_exporter")

	prometheus.MustRegister(NewExporter(*scrapeURI))
	prometheus.Unregister(prometheus.NewGoCollector())
	prometheus.Unregister(prometheus.NewProcessCollector(os.Getegid(), ""))

	http.Handle(*metricsPath, promhttp.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, *metricsPath, http.StatusMovedPermanently)
	})

	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<|endoftext|>


<commit_before>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     October 26, 2022
;       related version of root zone:     2022102601
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<commit_msg>Automatic update for Thu Nov 10 08:21:15 UTC 2022<commit_after>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     November 09, 2022
;       related version of root zone:     2022110902
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<|endoftext|>


<commit_before>// Package fatherhood offers a fast JSON decoder with a terrible API.
package fatherhood

import (
	"errors"
	"fmt"
	"github.com/benbjohnson/megajson/scanner"
	"io"
)

// Some docs shamelessly stolen from encoding/json

// Decoder reads and decodes JSON objects from an input stream.
type Decoder struct {
	scan scanner.Scanner
}

// NewDecoder returns a new decoder that reads from r.
//
// Alike encoding/json, the decoder introduces its own buffering and may read
// data from r beyond the JSON values requested.
//
// Unlike encoding/json, there's no Buffered method to retrieve the data
// buffered. Sorry.
func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{scan: scanner.NewScanner(r)}
}

func (d *Decoder) ReadString(target *string) error              { return d.scan.ReadString(target) }
func (d *Decoder) ReadInt(target *int) error                    { return d.scan.ReadInt(target) }
func (d *Decoder) ReadInt64(target *int64) error                { return d.scan.ReadInt64(target) }
func (d *Decoder) ReadUint(target *uint) error                  { return d.scan.ReadUint(target) }
func (d *Decoder) ReadUint64(target *uint64) error              { return d.scan.ReadUint64(target) }
func (d *Decoder) ReadFloat32(target *float32) error            { return d.scan.ReadFloat32(target) }
func (d *Decoder) ReadFloat64(target *float64) error            { return d.scan.ReadFloat64(target) }
func (d *Decoder) ReadBool(target *bool) error                  { return d.scan.ReadBool(target) }
func (d *Decoder) ReadMap(target *map[string]interface{}) error { return d.scan.ReadMap(target) }

// Discard will read the next token out of the reader. You should discard any
// member in an object that you do not use, to avoid leaving the decoder in an
// unexpected state in future calls.
func (d *Decoder) Discard() error {
	_, _, err := d.scan.Scan()
	return err
}

// Disclaimer: The code that follows is almost verbatim copy/pasted from a
// codegenerated megajson decoder: https://github.com/benbjohnson/megajson/blob/master/.bench/code_decoder.go#L22:L132

// EachMember iterates over the members of an object. Invoke the proper Read
// function to get the value back.
func (d *Decoder) EachMember(dst interface{}, visitFunc func(*Decoder, interface{}, string) error) error {

	if tok, tokval, err := d.scan.Scan(); err != nil {
		return err
	} else if tok == scanner.TNULL {
		return nil
	} else if tok != scanner.TLBRACE {
		return fmt.Errorf("unexpected %s at %d: %s; expected '{'", scanner.TokenName(tok), d.scan.Pos(), string(tokval))
	}

	index := 0
	for {
		// Read in key.
		var key string
		tok, tokval, err := d.scan.Scan()
		if err != nil {
			return err
		} else if tok == scanner.TRBRACE {
			return nil
		} else if tok == scanner.TCOMMA {
			if index == 0 {
				return fmt.Errorf("unexpected comma at %d", d.scan.Pos())
			}
			if tok, tokval, err = d.scan.Scan(); err != nil {
				return err
			}
		}

		if tok != scanner.TSTRING {
			return fmt.Errorf("unexpected %s at %d: %s; expected '{' or string", scanner.TokenName(tok), d.scan.Pos(), string(tokval))
		}

		key = string(tokval)

		// Read in the colon.
		if tok, tokval, err := d.scan.Scan(); err != nil {
			return err
		} else if tok != scanner.TCOLON {
			return fmt.Errorf("unexpected %s at %d (member %s): %s; expected colon", scanner.TokenName(tok), d.scan.Pos(), key, string(tokval))
		}

		if err := visitFunc(d, dst, key); err != nil {
			return fmt.Errorf("visiting member %s, type %s, %v", key, scanner.TokenName(tok), err)
		}
		index++
	}
}

// EachValue iterates over and invokes visitFunc at each value of an array. Invoke
// the proper Read function to get the value. The type of the value to read is
// specified by the JSONType given as argument.
func (d *Decoder) EachValue(dst interface{}, visitFunc func(*Decoder, interface{}, JSONType) error) error {
	if tok, _, err := d.scan.Scan(); err != nil {
		return err
	} else if tok != scanner.TLBRACKET {
		return errors.New("expected '['")
	}

	// Loop over items.
	index := 0
	for {
		tok, tokval, err := d.scan.Scan()
		if err != nil {
			return err
		} else if tok == scanner.TRBRACKET {
			return nil
		} else if tok == scanner.TCOMMA {
			if index == 0 {
				return fmt.Errorf("unexpected comma in array at %d", d.scan.Pos())
			}
			if tok, tokval, err = d.scan.Scan(); err != nil {
				return err
			}
		}
		d.scan.Unscan(tok, tokval)

		if err := visitFunc(d, dst, toJSONType(tok)); err != nil {
			return fmt.Errorf("visiting index %d, type %s, %v", index, scanner.TokenName(tok), err)
		}
		index++
	}
}

// JSONType that can be received by EachValue funcs.
type JSONType uint8

// Types recognized by the decoder.
const (
	String = iota
	Number
	Bool
	Null
	Object
	Array
)

func toJSONType(token int) JSONType {
	switch token {
	case scanner.TSTRING:
		return String
	case scanner.TNUMBER:
		return Number
	case scanner.TTRUE:
		return Bool
	case scanner.TFALSE:
		return Bool
	case scanner.TNULL:
		return Null
	case scanner.TLBRACE:
		return Object
	case scanner.TLBRACKET:
		return Array
	}
	panic(fmt.Sprintf("unexpected token type, %s (%d). This is likely a bug, please open an issue with the stacktrace at '%s'",
		scanner.TokenName(token), token, "https://github.com/aybabtme/fatherhood/issues/new"))
}
<commit_msg>Temporarly use own fork of megajson to support large strings.<commit_after>// Package fatherhood offers a fast JSON decoder with a terrible API.
package fatherhood

import (
	"errors"
	"fmt"
	// "github.com/benbjohnson/megajson/scanner"
	"github.com/aybabtme/megajson/scanner" // until long-strings can be used
	"io"
)

// Some docs shamelessly stolen from encoding/json

// Decoder reads and decodes JSON objects from an input stream.
type Decoder struct {
	scan scanner.Scanner
}

// NewDecoder returns a new decoder that reads from r.
//
// Alike encoding/json, the decoder introduces its own buffering and may read
// data from r beyond the JSON values requested.
//
// Unlike encoding/json, there's no Buffered method to retrieve the data
// buffered. Sorry.
func NewDecoder(r io.Reader) *Decoder {
	return &Decoder{scan: scanner.NewScanner(r)}
}

func (d *Decoder) ReadString(target *string) error              { return d.scan.ReadString(target) }
func (d *Decoder) ReadInt(target *int) error                    { return d.scan.ReadInt(target) }
func (d *Decoder) ReadInt64(target *int64) error                { return d.scan.ReadInt64(target) }
func (d *Decoder) ReadUint(target *uint) error                  { return d.scan.ReadUint(target) }
func (d *Decoder) ReadUint64(target *uint64) error              { return d.scan.ReadUint64(target) }
func (d *Decoder) ReadFloat32(target *float32) error            { return d.scan.ReadFloat32(target) }
func (d *Decoder) ReadFloat64(target *float64) error            { return d.scan.ReadFloat64(target) }
func (d *Decoder) ReadBool(target *bool) error                  { return d.scan.ReadBool(target) }
func (d *Decoder) ReadMap(target *map[string]interface{}) error { return d.scan.ReadMap(target) }

// Discard will read the next token out of the reader. You should discard any
// member in an object that you do not use, to avoid leaving the decoder in an
// unexpected state in future calls.
func (d *Decoder) Discard() error {
	_, _, err := d.scan.Scan()
	return err
}

// Disclaimer: The code that follows is almost verbatim copy/pasted from a
// codegenerated megajson decoder: https://github.com/benbjohnson/megajson/blob/master/.bench/code_decoder.go#L22:L132

// EachMember iterates over the members of an object. Invoke the proper Read
// function to get the value back.
func (d *Decoder) EachMember(dst interface{}, visitFunc func(*Decoder, interface{}, string) error) error {

	if tok, tokval, err := d.scan.Scan(); err != nil {
		return err
	} else if tok == scanner.TNULL {
		return nil
	} else if tok != scanner.TLBRACE {
		return fmt.Errorf("unexpected %s at %d: %s; expected '{'", scanner.TokenName(tok), d.scan.Pos(), string(tokval))
	}

	index := 0
	for {
		// Read in key.
		var key string
		tok, tokval, err := d.scan.Scan()
		if err != nil {
			return err
		} else if tok == scanner.TRBRACE {
			return nil
		} else if tok == scanner.TCOMMA {
			if index == 0 {
				return fmt.Errorf("unexpected comma at %d", d.scan.Pos())
			}
			if tok, tokval, err = d.scan.Scan(); err != nil {
				return err
			}
		}

		if tok != scanner.TSTRING {
			return fmt.Errorf("unexpected %s at %d: %s; expected '{' or string", scanner.TokenName(tok), d.scan.Pos(), string(tokval))
		}

		key = string(tokval)

		// Read in the colon.
		if tok, tokval, err := d.scan.Scan(); err != nil {
			return err
		} else if tok != scanner.TCOLON {
			return fmt.Errorf("unexpected %s at %d (member %s): %s; expected colon", scanner.TokenName(tok), d.scan.Pos(), key, string(tokval))
		}

		if err := visitFunc(d, dst, key); err != nil {
			return fmt.Errorf("visiting member %s, type %s, %v", key, scanner.TokenName(tok), err)
		}
		index++
	}
}

// EachValue iterates over and invokes visitFunc at each value of an array. Invoke
// the proper Read function to get the value. The type of the value to read is
// specified by the JSONType given as argument.
func (d *Decoder) EachValue(dst interface{}, visitFunc func(*Decoder, interface{}, JSONType) error) error {
	if tok, _, err := d.scan.Scan(); err != nil {
		return err
	} else if tok != scanner.TLBRACKET {
		return errors.New("expected '['")
	}

	// Loop over items.
	index := 0
	for {
		tok, tokval, err := d.scan.Scan()
		if err != nil {
			return err
		} else if tok == scanner.TRBRACKET {
			return nil
		} else if tok == scanner.TCOMMA {
			if index == 0 {
				return fmt.Errorf("unexpected comma in array at %d", d.scan.Pos())
			}
			if tok, tokval, err = d.scan.Scan(); err != nil {
				return err
			}
		}
		d.scan.Unscan(tok, tokval)

		if err := visitFunc(d, dst, toJSONType(tok)); err != nil {
			return fmt.Errorf("visiting index %d, type %s, %v", index, scanner.TokenName(tok), err)
		}
		index++
	}
}

// JSONType that can be received by EachValue funcs.
type JSONType uint8

// Types recognized by the decoder.
const (
	String = iota
	Number
	Bool
	Null
	Object
	Array
)

func toJSONType(token int) JSONType {
	switch token {
	case scanner.TSTRING:
		return String
	case scanner.TNUMBER:
		return Number
	case scanner.TTRUE:
		return Bool
	case scanner.TFALSE:
		return Bool
	case scanner.TNULL:
		return Null
	case scanner.TLBRACE:
		return Object
	case scanner.TLBRACKET:
		return Array
	}
	panic(fmt.Sprintf("unexpected token type, %s (%d). This is likely a bug, please open an issue with the stacktrace at '%s'",
		scanner.TokenName(token), token, "https://github.com/aybabtme/fatherhood/issues/new"))
}
<|endoftext|>


<commit_before>package qb

import (
	"fmt"
	"strings"
)

// NewColumn creates and returns a table column
func NewColumn(name string, t *Type, constraints []Constraint) Column {
	return Column{
		Name:        name,
		Type:        t,
		Constraints: constraints,
	}
}

// Column is the base abstraction for table struct columns
type Column struct {
	Name        string
	Type        *Type
	Constraints []Constraint
}

// SQL returns column as an sql statement
func (c *Column) SQL(driver string) string {

	constraints := []string{}
	for _, v := range c.Constraints {
		constraints = append(constraints, v.Name)
	}

	colPieces := []string{
		fmt.Sprintf("%s", c.Name),
		c.Type.SQL(),
	}

	if driver == "postgres" {
		colPieces[0] = fmt.Sprintf("\"%s\"", colPieces[0])
	} else {
		colPieces[0] = fmt.Sprintf("`%s`", colPieces[0])
	}

	if len(constraints) > 0 {
		colPieces = append(colPieces, strings.Join(constraints, " "))
	}

	return fmt.Sprintf("%s", strings.Join(colPieces, " "))
}
<commit_msg>disable table ddl column name additions<commit_after>package qb

import (
	"fmt"
	"strings"
)

// NewColumn creates and returns a table column
func NewColumn(name string, t *Type, constraints []Constraint) Column {
	return Column{
		Name:        name,
		Type:        t,
		Constraints: constraints,
	}
}

// Column is the base abstraction for table struct columns
type Column struct {
	Name        string
	Type        *Type
	Constraints []Constraint
}

// SQL returns column as an sql statement
func (c *Column) SQL(driver string) string {

	constraints := []string{}
	for _, v := range c.Constraints {
		constraints = append(constraints, v.Name)
	}

	colPieces := []string{
		fmt.Sprintf("%s", c.Name),
		c.Type.SQL(),
	}

	// TODO: Find a way to properly do `` and "" for columns
	//if driver == "postgres" {
	//	colPieces[0] = fmt.Sprintf("\"%s\"", colPieces[0])
	//} else {
	//	colPieces[0] = fmt.Sprintf("`%s`", colPieces[0])
	//}

	if len(constraints) > 0 {
		colPieces = append(colPieces, strings.Join(constraints, " "))
	}

	return fmt.Sprintf("%s", strings.Join(colPieces, " "))
}
<|endoftext|>


<commit_before>package irckit

import (
	"html"
	"regexp"
	"strings"
	"time"

	"github.com/nlopes/slack"
)

type SlackInfo struct {
	Token     string
	sc        *slack.Client
	rtm       *slack.RTM
	sinfo     *slack.Info
	susers    map[string]slack.User
	connected bool
}

func (u *User) loginToSlack() (*slack.Client, error) {
	u.sc = slack.New(u.Token)
	u.rtm = u.sc.NewRTM()
	u.susers = make(map[string]slack.User)
	go u.rtm.ManageConnection()
	//time.Sleep(time.Second * 2)
	u.sinfo = u.rtm.GetInfo()
	for u.sinfo == nil {
		time.Sleep(time.Millisecond * 500)
		logger.Debug("still waiting for sinfo")
		u.sinfo = u.rtm.GetInfo()
	}
	go u.handleSlack()
	u.addSlackUsersToChannels()
	u.connected = true
	return u.sc, nil
}

func (u *User) logoutFromSlack() error {
	logger.Debug("calling logout from slack")
	err := u.rtm.Disconnect()
	if err != nil {
		logger.Debug("logoutfrom slack", err)
		return err
	}
	u.Srv.Logout(u)
	u.sc = nil
	logger.Info("logout succeeded")
	u.connected = false
	return nil
}

func (u *User) createSlackUser(slackuser *slack.User) *User {
	if slackuser == nil {
		return nil
	}
	if ghost, ok := u.Srv.HasUser(slackuser.Name); ok {
		return ghost
	}
	ghost := &User{Nick: slackuser.Name, User: slackuser.ID, Real: slackuser.RealName, Host: "host", Roles: "", channels: map[Channel]struct{}{}}
	ghost.MmGhostUser = true
	u.Srv.Add(ghost)
	return ghost
}

func (u *User) addSlackUserToChannel(user *slack.User, channel string, channelId string) {
	if user == nil {
		return
	}
	ghost := u.createSlackUser(user)
	if ghost == nil {
		logger.Warnf("Cannot join %v into %s", user, channel)
		return
	}
	logger.Debugf("adding %s to %s (%s)", ghost.Nick, channel, channelId)
	ch := u.Srv.Channel(channelId)
	logger.Debugf("channel: %#v %#v", ch.String(), ch.ID())
	ch.Join(ghost)
}

func (u *User) addSlackUsersToChannels() {
	srv := u.Srv
	throttle := time.Tick(time.Millisecond * 100)
	logger.Debug("in addUsersToChannels()")
	// add all users, also who are not on channels
	ch := srv.Channel("&users")
	users, _ := u.sc.GetUsers()
	for _, mmuser := range users {
		// do not add our own nick
		if mmuser.ID == u.sinfo.User.ID {
			continue
		}
		u.createSlackUser(&mmuser)
		u.addSlackUserToChannel(&mmuser, "&users", "&users")
		u.susers[mmuser.ID] = mmuser
	}
	ch.Join(u)

	channels := make(chan interface{}, 10)
	for i := 0; i < 10; i++ {
		go u.addSlackUserToChannelWorker(channels, throttle)
	}
	groups, _ := u.sc.GetGroups(true)
	mmchannels, _ := u.sc.GetChannels(true)
	for _, mmchannel := range mmchannels {
		if mmchannel.IsMember {
			logger.Debug("Adding channel", mmchannel)
			channels <- mmchannel
		}
	}
	for _, mmchannel := range groups {
		logger.Debug("Adding private channel", mmchannel)
		channels <- mmchannel
	}
	close(channels)
}

func (u *User) addSlackUserToChannelWorker(channels <-chan interface{}, throttle <-chan time.Time) {
	var ID, name string
	for {
		mmchannel, ok := <-channels
		if !ok {
			logger.Debug("Done adding user to channels")
			return
		}
		<-throttle
		switch mmchannel.(type) {
		case slack.Channel:
			ID = mmchannel.(slack.Channel).ID
			name = mmchannel.(slack.Channel).Name
			u.syncSlackChannel(ID, name)
		case slack.Group:
			ID = mmchannel.(slack.Group).ID
			name = mmchannel.(slack.Group).Name
			logger.Debugf("GROUP %#v", mmchannel.(slack.Group))
			u.syncSlackGroup(ID, name)

		}
		// exclude direct messages
		//var spoof func(string, string)
		//ch := u.Srv.Channel(mmchannel.ID)
		// post everything to the channel you haven't seen yet
	}
}

func (u *User) handleSlack() {
	for {
		/*
			if u.mc.WsQuit {
				logger.Debug("exiting handleWsMessage")
				return
			}
		*/
		logger.Debug("in handleSlack")
		for msg := range u.rtm.IncomingEvents {
			switch ev := msg.Data.(type) {
			case *slack.MessageEvent:
				u.handleSlackActionPost(ev)
			case *slack.DisconnectedEvent:
				logger.Debugf("disconnecting..")
				return
			}
		}
	}
	/*
			logger.Debugf("MMUser WsReceiver: %#v", message.Raw)
			// check if we have the users/channels in our cache. If not update
			u.checkWsActionMessage(message.Raw, updateChannelsThrottle)
			switch message.Raw.Event {
			case model.WEBSOCKET_EVENT_POSTED:
				u.handleWsActionPost(message.Raw)
			case model.WEBSOCKET_EVENT_POST_EDITED:
				u.handleWsActionPost(message.Raw)
			case model.WEBSOCKET_EVENT_USER_REMOVED:
				u.handleWsActionUserRemoved(message.Raw)
			case model.WEBSOCKET_EVENT_USER_ADDED:
				u.handleWsActionUserAdded(message.Raw)
			}
		}
	*/
}

func (u *User) handleSlackActionPost(rmsg *slack.MessageEvent) {
	var ch Channel
	logger.Debugf("handleSlackActionPost() receiving msg %#v", rmsg)
	if len(rmsg.Attachments) > 0 {
		// skip messages we made ourselves
		if rmsg.Attachments[0].CallbackID == "matterircd" {
			return
		}
	}

	user, err := u.rtm.GetUserInfo(rmsg.User)
	if err != nil {
		return
	}

	// create new "ghost" user
	ghost := u.createSlackUser(user)

	spoofUsername := user.ID
	if ghost != nil {
		spoofUsername = ghost.Nick
	}

	msgs := strings.Split(rmsg.Text, "\n")
	// direct message

	if ghost != nil {
		ch = u.Srv.Channel(rmsg.Channel)
		// join if not in channel
		if !ch.HasUser(ghost) {
			ch.Join(ghost)
		}
	}

	for _, m := range msgs {
		// cleanup the message
		m = u.replaceMention(m)
		m = u.replaceVariable(m)
		m = u.replaceChannel(m)
		m = u.replaceURL(m)
		m = html.UnescapeString(m)

		if m == "" {
			continue
		}
		if strings.HasPrefix(rmsg.Channel, "D") {
			u.MsgSpoofUser(spoofUsername, m)
		} else {
			ch.SpoofMessage(spoofUsername, m)
		}
	}
}

// sync IRC with mattermost channel state
func (u *User) syncSlackChannel(id string, name string) {
	srv := u.Srv
	info, err := u.sc.GetChannelInfo(id)
	if err != nil {
		logger.Info(err)
	}

	for _, user := range info.Members {
		if u.sinfo.User.ID != user {
			//slackuser, _ := u.sc.GetUserInfo(user)
			if slackuser, ok := u.susers[user]; ok {
				u.addSlackUserToChannel(&slackuser, "#"+name, id)
			}
		}
	}
	// before joining ourself
	for _, user := range info.Members {
		// join all the channels we're on on MM
		if user == u.sinfo.User.ID {
			ch := srv.Channel(id)
			// only join when we're not yet on the channel
			if !ch.HasUser(u) {
				logger.Debugf("syncSlackchannel adding myself to %s (id: %s)", name, id)
				ch.Join(u)
				//ch.Topic(u, u.mc.GetChannelHeader(id))
			}
			break
		}
	}
}

// sync IRC with mattermost channel state
func (u *User) syncSlackGroup(id string, name string) {
	srv := u.Srv
	info, err := u.sc.GetGroupInfo(id)
	if err != nil {
		logger.Info(err)
	}

	for _, user := range info.Members {
		if u.sinfo.User.ID != user {
			//slackuser, _ := u.sc.GetUserInfo(user)
			if slackuser, ok := u.susers[user]; ok {
				u.addSlackUserToChannel(&slackuser, "#"+name, id)
			}
		}
	}
	// before joining ourself
	for _, user := range info.Members {
		// join all the channels we're on on MM
		if user == u.sinfo.User.ID {
			ch := srv.Channel(id)
			// only join when we're not yet on the channel
			if !ch.HasUser(u) {
				logger.Debugf("syncSlackgroup adding myself to %s (id: %s)", name, id)
				ch.Join(u)
				//ch.Topic(u, u.mc.GetChannelHeader(id))
			}
			break
		}
	}
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (u *User) replaceMention(text string) string {
	results := regexp.MustCompile(`<@([a-zA-z0-9]+)>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, "<@"+r[1]+">", "@"+u.userName(r[1]), -1)
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (u *User) replaceChannel(text string) string {
	results := regexp.MustCompile(`<#[a-zA-Z0-9]+\|(.+?)>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, r[0], "#"+r[1], -1)
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#variables
func (u *User) replaceVariable(text string) string {
	results := regexp.MustCompile(`<!((?:subteam\^)?[a-zA-Z0-9]+)(?:\|@?(.+?))?>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		if r[2] != "" {
			text = strings.Replace(text, r[0], "@"+r[2], -1)
		} else {
			text = strings.Replace(text, r[0], "@"+r[1], -1)
		}
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#linking_to_urls
func (u *User) replaceURL(text string) string {
	results := regexp.MustCompile(`<(.*?)(\|.*?)?>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, r[0], r[1], -1)
	}
	return text
}

func (u *User) userName(id string) string {
	for _, us := range u.susers {
		if us.ID == id {
			if us.Profile.DisplayName != "" {
				return us.Profile.DisplayName
			}
			return us.Name
		}
	}
	return ""
}

func (u *User) isConnected() bool {
	return u.connected
}
<commit_msg>Allow to mention yourself.. Closes 42wim/matterircd#157<commit_after>package irckit

import (
	"html"
	"regexp"
	"strings"
	"time"

	"github.com/nlopes/slack"
)

type SlackInfo struct {
	Token     string
	sc        *slack.Client
	rtm       *slack.RTM
	sinfo     *slack.Info
	susers    map[string]slack.User
	connected bool
}

func (u *User) loginToSlack() (*slack.Client, error) {
	u.sc = slack.New(u.Token)
	u.rtm = u.sc.NewRTM()
	u.susers = make(map[string]slack.User)
	go u.rtm.ManageConnection()
	//time.Sleep(time.Second * 2)
	u.sinfo = u.rtm.GetInfo()
	for u.sinfo == nil {
		time.Sleep(time.Millisecond * 500)
		logger.Debug("still waiting for sinfo")
		u.sinfo = u.rtm.GetInfo()
	}
	go u.handleSlack()
	u.addSlackUsersToChannels()
	u.connected = true
	return u.sc, nil
}

func (u *User) logoutFromSlack() error {
	logger.Debug("calling logout from slack")
	err := u.rtm.Disconnect()
	if err != nil {
		logger.Debug("logoutfrom slack", err)
		return err
	}
	u.Srv.Logout(u)
	u.sc = nil
	logger.Info("logout succeeded")
	u.connected = false
	return nil
}

func (u *User) createSlackUser(slackuser *slack.User) *User {
	if slackuser == nil {
		return nil
	}
	if ghost, ok := u.Srv.HasUser(slackuser.Name); ok {
		return ghost
	}
	ghost := &User{Nick: slackuser.Name, User: slackuser.ID, Real: slackuser.RealName, Host: "host", Roles: "", channels: map[Channel]struct{}{}}
	ghost.MmGhostUser = true
	u.Srv.Add(ghost)
	return ghost
}

func (u *User) addSlackUserToChannel(user *slack.User, channel string, channelId string) {
	if user == nil {
		return
	}
	ghost := u.createSlackUser(user)
	if ghost == nil {
		logger.Warnf("Cannot join %v into %s", user, channel)
		return
	}
	logger.Debugf("adding %s to %s (%s)", ghost.Nick, channel, channelId)
	ch := u.Srv.Channel(channelId)
	logger.Debugf("channel: %#v %#v", ch.String(), ch.ID())
	ch.Join(ghost)
}

func (u *User) addSlackUsersToChannels() {
	srv := u.Srv
	throttle := time.Tick(time.Millisecond * 100)
	logger.Debug("in addUsersToChannels()")
	// add all users, also who are not on channels
	ch := srv.Channel("&users")
	users, _ := u.sc.GetUsers()
	for _, mmuser := range users {
		// do not add our own nick
		if mmuser.ID == u.sinfo.User.ID {
			continue
		}
		u.createSlackUser(&mmuser)
		u.addSlackUserToChannel(&mmuser, "&users", "&users")
		u.susers[mmuser.ID] = mmuser
	}
	ch.Join(u)

	channels := make(chan interface{}, 10)
	for i := 0; i < 10; i++ {
		go u.addSlackUserToChannelWorker(channels, throttle)
	}
	groups, _ := u.sc.GetGroups(true)
	mmchannels, _ := u.sc.GetChannels(true)
	for _, mmchannel := range mmchannels {
		if mmchannel.IsMember {
			logger.Debug("Adding channel", mmchannel)
			channels <- mmchannel
		}
	}
	for _, mmchannel := range groups {
		logger.Debug("Adding private channel", mmchannel)
		channels <- mmchannel
	}
	close(channels)
}

func (u *User) addSlackUserToChannelWorker(channels <-chan interface{}, throttle <-chan time.Time) {
	var ID, name string
	for {
		mmchannel, ok := <-channels
		if !ok {
			logger.Debug("Done adding user to channels")
			return
		}
		<-throttle
		switch mmchannel.(type) {
		case slack.Channel:
			ID = mmchannel.(slack.Channel).ID
			name = mmchannel.(slack.Channel).Name
			u.syncSlackChannel(ID, name)
		case slack.Group:
			ID = mmchannel.(slack.Group).ID
			name = mmchannel.(slack.Group).Name
			logger.Debugf("GROUP %#v", mmchannel.(slack.Group))
			u.syncSlackGroup(ID, name)

		}
		// exclude direct messages
		//var spoof func(string, string)
		//ch := u.Srv.Channel(mmchannel.ID)
		// post everything to the channel you haven't seen yet
	}
}

func (u *User) handleSlack() {
	for {
		/*
			if u.mc.WsQuit {
				logger.Debug("exiting handleWsMessage")
				return
			}
		*/
		logger.Debug("in handleSlack")
		for msg := range u.rtm.IncomingEvents {
			switch ev := msg.Data.(type) {
			case *slack.MessageEvent:
				u.handleSlackActionPost(ev)
			case *slack.DisconnectedEvent:
				logger.Debugf("disconnecting..")
				return
			}
		}
	}
	/*
			logger.Debugf("MMUser WsReceiver: %#v", message.Raw)
			// check if we have the users/channels in our cache. If not update
			u.checkWsActionMessage(message.Raw, updateChannelsThrottle)
			switch message.Raw.Event {
			case model.WEBSOCKET_EVENT_POSTED:
				u.handleWsActionPost(message.Raw)
			case model.WEBSOCKET_EVENT_POST_EDITED:
				u.handleWsActionPost(message.Raw)
			case model.WEBSOCKET_EVENT_USER_REMOVED:
				u.handleWsActionUserRemoved(message.Raw)
			case model.WEBSOCKET_EVENT_USER_ADDED:
				u.handleWsActionUserAdded(message.Raw)
			}
		}
	*/
}

func (u *User) handleSlackActionPost(rmsg *slack.MessageEvent) {
	var ch Channel
	logger.Debugf("handleSlackActionPost() receiving msg %#v", rmsg)
	if len(rmsg.Attachments) > 0 {
		// skip messages we made ourselves
		if rmsg.Attachments[0].CallbackID == "matterircd" {
			return
		}
	}

	user, err := u.rtm.GetUserInfo(rmsg.User)
	if err != nil {
		return
	}

	// create new "ghost" user
	ghost := u.createSlackUser(user)

	spoofUsername := user.ID
	if ghost != nil {
		spoofUsername = ghost.Nick
	}

	msgs := strings.Split(rmsg.Text, "\n")
	// direct message

	if ghost != nil {
		ch = u.Srv.Channel(rmsg.Channel)
		// join if not in channel
		if !ch.HasUser(ghost) {
			ch.Join(ghost)
		}
	}

	for _, m := range msgs {
		// cleanup the message
		m = u.replaceMention(m)
		m = u.replaceVariable(m)
		m = u.replaceChannel(m)
		m = u.replaceURL(m)
		m = html.UnescapeString(m)

		if m == "" {
			continue
		}
		if strings.HasPrefix(rmsg.Channel, "D") {
			u.MsgSpoofUser(spoofUsername, m)
		} else {
			ch.SpoofMessage(spoofUsername, m)
		}
	}
}

// sync IRC with mattermost channel state
func (u *User) syncSlackChannel(id string, name string) {
	srv := u.Srv
	info, err := u.sc.GetChannelInfo(id)
	if err != nil {
		logger.Info(err)
	}

	for _, user := range info.Members {
		if u.sinfo.User.ID != user {
			//slackuser, _ := u.sc.GetUserInfo(user)
			if slackuser, ok := u.susers[user]; ok {
				u.addSlackUserToChannel(&slackuser, "#"+name, id)
			}
		}
	}
	// before joining ourself
	for _, user := range info.Members {
		// join all the channels we're on on MM
		if user == u.sinfo.User.ID {
			ch := srv.Channel(id)
			// only join when we're not yet on the channel
			if !ch.HasUser(u) {
				logger.Debugf("syncSlackchannel adding myself to %s (id: %s)", name, id)
				ch.Join(u)
				//ch.Topic(u, u.mc.GetChannelHeader(id))
			}
			break
		}
	}
}

// sync IRC with mattermost channel state
func (u *User) syncSlackGroup(id string, name string) {
	srv := u.Srv
	info, err := u.sc.GetGroupInfo(id)
	if err != nil {
		logger.Info(err)
	}

	for _, user := range info.Members {
		if u.sinfo.User.ID != user {
			//slackuser, _ := u.sc.GetUserInfo(user)
			if slackuser, ok := u.susers[user]; ok {
				u.addSlackUserToChannel(&slackuser, "#"+name, id)
			}
		}
	}
	// before joining ourself
	for _, user := range info.Members {
		// join all the channels we're on on MM
		if user == u.sinfo.User.ID {
			ch := srv.Channel(id)
			// only join when we're not yet on the channel
			if !ch.HasUser(u) {
				logger.Debugf("syncSlackgroup adding myself to %s (id: %s)", name, id)
				ch.Join(u)
				//ch.Topic(u, u.mc.GetChannelHeader(id))
			}
			break
		}
	}
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (u *User) replaceMention(text string) string {
	results := regexp.MustCompile(`<@([a-zA-z0-9]+)>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, "<@"+r[1]+">", "@"+u.userName(r[1]), -1)
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#linking_to_channels_and_users
func (u *User) replaceChannel(text string) string {
	results := regexp.MustCompile(`<#[a-zA-Z0-9]+\|(.+?)>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, r[0], "#"+r[1], -1)
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#variables
func (u *User) replaceVariable(text string) string {
	results := regexp.MustCompile(`<!((?:subteam\^)?[a-zA-Z0-9]+)(?:\|@?(.+?))?>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		if r[2] != "" {
			text = strings.Replace(text, r[0], "@"+r[2], -1)
		} else {
			text = strings.Replace(text, r[0], "@"+r[1], -1)
		}
	}
	return text
}

// @see https://api.slack.com/docs/message-formatting#linking_to_urls
func (u *User) replaceURL(text string) string {
	results := regexp.MustCompile(`<(.*?)(\|.*?)?>`).FindAllStringSubmatch(text, -1)
	for _, r := range results {
		text = strings.Replace(text, r[0], r[1], -1)
	}
	return text
}

func (u *User) userName(id string) string {
	// TODO dynamically update when new users are joining slack
	for _, us := range u.susers {
		if us.ID == id {
			if us.Profile.DisplayName != "" {
				return us.Profile.DisplayName
			}
			return us.Name
		}
	}
	if id == u.sinfo.User.ID {
		return u.sinfo.User.Name
	}
	return ""
}

func (u *User) isConnected() bool {
	return u.connected
}
<|endoftext|>


<commit_before>package rope

import (
	"bytes"
	"math"
	"unicode/utf8"
)

type Rope struct {
	height   int
	weight   int
	left     *Rope
	right    *Rope
	content  []byte
	balanced bool
}

var MaxLengthPerNode = 128

func NewFromBytes(bs []byte) (ret *Rope) {
	if len(bs) == 0 {
		return nil
	}
	slots := make([]*Rope, 32)
	var slotIndex int
	var r *Rope
	for blockIndex := 0; blockIndex < len(bs)/MaxLengthPerNode; blockIndex++ {
		r = &Rope{
			height:   1,
			weight:   MaxLengthPerNode,
			content:  bs[blockIndex*MaxLengthPerNode : (blockIndex+1)*MaxLengthPerNode],
			balanced: true,
		}
		slotIndex = 0
		for slots[slotIndex] != nil {
			r = &Rope{
				height:   slotIndex + 2,
				weight:   (1 << uint(slotIndex)) * MaxLengthPerNode,
				left:     slots[slotIndex],
				right:    r,
				balanced: true,
			}
			slots[slotIndex] = nil
			slotIndex++
		}
		slots[slotIndex] = r
	}
	tailStart := len(bs) / MaxLengthPerNode * MaxLengthPerNode
	if tailStart < len(bs) {
		ret = &Rope{
			height:   1,
			weight:   len(bs) - tailStart,
			content:  bs[tailStart:],
			balanced: false,
		}
	}
	for _, c := range slots {
		if c != nil {
			if ret == nil {
				ret = c
			} else {
				ret = c.Concat(ret)
			}
		}
	}
	return
}

func (r *Rope) Index(i int) byte {
	if i >= r.weight {
		return r.right.Index(i - r.weight)
	}
	if r.left != nil { // non leaf
		return r.left.Index(i)
	}
	// leaf
	return r.content[i]
}

func (r *Rope) Len() int {
	if r == nil {
		return 0
	}
	return r.weight + r.right.Len()
}

func (r *Rope) Bytes() []byte {
	ret := make([]byte, r.Len())
	i := 0
	r.Iter(0, func(bs []byte) bool {
		copy(ret[i:], bs)
		i += len(bs)
		return true
	})
	return ret
}

func (r *Rope) Concat(r2 *Rope) (ret *Rope) {
	ret = &Rope{
		weight: r.Len(),
		left:   r,
		right:  r2,
	}
	if ret.left != nil {
		ret.height = ret.left.height
	}
	if ret.right != nil && ret.right.height > ret.height {
		ret.height = ret.right.height
	}
	if ret.left != nil && ret.left.balanced &&
		ret.right != nil && ret.right.balanced &&
		ret.left.height == ret.right.height {
		ret.balanced = true
	}
	ret.height++
	// check and rebalance
	if !ret.balanced {
		l := int((math.Ceil(math.Log2(float64((ret.Len()/MaxLengthPerNode)+1))) + 1) * 1.5)
		if ret.height > l {
			ret = ret.rebalance()
		}
	}
	return
}

func (r *Rope) rebalance() (ret *Rope) {
	var currentBytes []byte
	slots := make([]*Rope, 32)
	r.iterNodes(func(node *Rope) bool {
		var balancedNode *Rope
		iterSubNodes := true
		if len(currentBytes) == 0 && node.balanced { // balanced, insert to slots
			balancedNode = node
			iterSubNodes = false
		} else { // collect bytes
			currentBytes = append(currentBytes, node.content...)
			if len(currentBytes) >= MaxLengthPerNode { // a full leaf
				balancedNode = &Rope{
					height:   1,
					weight:   MaxLengthPerNode,
					balanced: true,
					content:  currentBytes[:MaxLengthPerNode],
				}
				currentBytes = currentBytes[MaxLengthPerNode:]
			}
		}
		if balancedNode != nil {
			slotIndex := balancedNode.height - 1
			for slots[slotIndex] != nil {
				balancedNode = &Rope{
					height:   balancedNode.height + 1,
					weight:   slots[slotIndex].Len(),
					left:     slots[slotIndex],
					right:    balancedNode,
					balanced: true,
				}
				slots[slotIndex] = nil
				slotIndex++
			}
			slots[slotIndex] = balancedNode
		}
		return iterSubNodes
	})
	if len(currentBytes) > 0 {
		ret = &Rope{
			height:   1,
			weight:   len(currentBytes),
			balanced: false,
			content:  currentBytes,
		}
	}
	for _, c := range slots {
		if c != nil {
			if ret == nil {
				ret = c
			} else {
				ret = c.Concat(ret)
			}
		}
	}
	return
}

func (r *Rope) Split(n int) (out1, out2 *Rope) {
	if r == nil {
		return
	}
	if len(r.content) > 0 { // leaf
		if n > len(r.content) { // offset overflow
			n = len(r.content)
		}
		out1 = NewFromBytes(r.content[:n])
		out2 = NewFromBytes(r.content[n:])
	} else { // non leaf
		var r1 *Rope
		if n >= r.weight { // at right subtree
			r1, out2 = r.right.Split(n - r.weight)
			out1 = r.left.Concat(r1)
		} else { // at left subtree
			out1, r1 = r.left.Split(n)
			out2 = r1.Concat(r.right)
		}
	}
	return
}

func (r *Rope) Insert(n int, bs []byte) *Rope {
	r1, r2 := r.Split(n)
	return r1.Concat(NewFromBytes(bs)).Concat(r2)
}

func (r *Rope) Delete(n, l int) *Rope {
	r1, r2 := r.Split(n)
	_, r2 = r2.Split(l)
	return r1.Concat(r2)
}

func (r *Rope) Sub(n, l int) []byte {
	buf := new(bytes.Buffer)
	r.Iter(n, func(bs []byte) bool {
		if l >= len(bs) {
			buf.Write(bs)
			l -= len(bs)
			return true
		} else {
			buf.Write(bs[:l])
			return false
		}
	})
	return buf.Bytes()
}

func (r *Rope) Iter(offset int, fn func([]byte) bool) bool {
	if r == nil {
		return true
	}
	if len(r.content) > 0 { // leaf
		if offset < len(r.content) {
			if !fn(r.content[offset:]) {
				return false
			}
		}
	} else { // non leaf
		if offset >= r.weight { // start at right subtree
			if !r.right.Iter(offset-r.weight, fn) {
				return false
			}
		} else { // start at left subtree
			if !r.left.Iter(offset, fn) {
				return false
			}
			if !r.right.Iter(0, fn) {
				return false
			}
		}
	}
	return true
}

func (r *Rope) IterBackward(offset int, fn func([]byte) bool) bool {
	if r == nil {
		return true
	}
	if len(r.content) > 0 { // leaf
		content := r.content[:offset]
		if len(content) == 0 {
			return true
		}
		bs := reversedBytes(content)
		if !fn(bs) {
			return false
		}
	} else { // non leaf
		if offset >= r.weight { // start at right subtree
			if !r.right.IterBackward(offset-r.weight, fn) {
				return false
			}
			if !r.left.IterBackward(r.weight, fn) {
				return false
			}
		} else { // start at left subtree
			if !r.left.IterBackward(offset, fn) {
				return false
			}
		}
	}
	return true
}

func (r *Rope) iterNodes(fn func(*Rope) bool) {
	if r == nil {
		return
	}
	if fn(r) {
		r.left.iterNodes(fn)
		r.right.iterNodes(fn)
	}
}

func (r *Rope) IterRune(offset int, fn func(rune, int) bool) {
	var bs []byte
	stopped := false
	r.Iter(offset, func(slice []byte) bool {
		bs = append(bs, slice...)
		for len(bs) >= 4 {
			ru, l := utf8.DecodeRune(bs)
			bs = bs[l:]
			if ru == utf8.RuneError {
				return false
			}
			if !fn(ru, l) {
				stopped = true
				return false
			}
		}
		return true
	})
	if !stopped && len(bs) > 0 {
		for {
			ru, l := utf8.DecodeRune(bs)
			bs = bs[l:]
			if ru != utf8.RuneError {
				fn(ru, l)
			} else {
				break
			}
		}
	}
}
<commit_msg>optimize Sub<commit_after>package rope

import (
	"math"
	"unicode/utf8"
)

type Rope struct {
	height   int
	weight   int
	left     *Rope
	right    *Rope
	content  []byte
	balanced bool
}

var MaxLengthPerNode = 128

func NewFromBytes(bs []byte) (ret *Rope) {
	if len(bs) == 0 {
		return nil
	}
	slots := make([]*Rope, 32)
	var slotIndex int
	var r *Rope
	for blockIndex := 0; blockIndex < len(bs)/MaxLengthPerNode; blockIndex++ {
		r = &Rope{
			height:   1,
			weight:   MaxLengthPerNode,
			content:  bs[blockIndex*MaxLengthPerNode : (blockIndex+1)*MaxLengthPerNode],
			balanced: true,
		}
		slotIndex = 0
		for slots[slotIndex] != nil {
			r = &Rope{
				height:   slotIndex + 2,
				weight:   (1 << uint(slotIndex)) * MaxLengthPerNode,
				left:     slots[slotIndex],
				right:    r,
				balanced: true,
			}
			slots[slotIndex] = nil
			slotIndex++
		}
		slots[slotIndex] = r
	}
	tailStart := len(bs) / MaxLengthPerNode * MaxLengthPerNode
	if tailStart < len(bs) {
		ret = &Rope{
			height:   1,
			weight:   len(bs) - tailStart,
			content:  bs[tailStart:],
			balanced: false,
		}
	}
	for _, c := range slots {
		if c != nil {
			if ret == nil {
				ret = c
			} else {
				ret = c.Concat(ret)
			}
		}
	}
	return
}

func (r *Rope) Index(i int) byte {
	if i >= r.weight {
		return r.right.Index(i - r.weight)
	}
	if r.left != nil { // non leaf
		return r.left.Index(i)
	}
	// leaf
	return r.content[i]
}

func (r *Rope) Len() int {
	if r == nil {
		return 0
	}
	return r.weight + r.right.Len()
}

func (r *Rope) Bytes() []byte {
	ret := make([]byte, r.Len())
	i := 0
	r.Iter(0, func(bs []byte) bool {
		copy(ret[i:], bs)
		i += len(bs)
		return true
	})
	return ret
}

func (r *Rope) Concat(r2 *Rope) (ret *Rope) {
	ret = &Rope{
		weight: r.Len(),
		left:   r,
		right:  r2,
	}
	if ret.left != nil {
		ret.height = ret.left.height
	}
	if ret.right != nil && ret.right.height > ret.height {
		ret.height = ret.right.height
	}
	if ret.left != nil && ret.left.balanced &&
		ret.right != nil && ret.right.balanced &&
		ret.left.height == ret.right.height {
		ret.balanced = true
	}
	ret.height++
	// check and rebalance
	if !ret.balanced {
		l := int((math.Ceil(math.Log2(float64((ret.Len()/MaxLengthPerNode)+1))) + 1) * 1.5)
		if ret.height > l {
			ret = ret.rebalance()
		}
	}
	return
}

func (r *Rope) rebalance() (ret *Rope) {
	var currentBytes []byte
	slots := make([]*Rope, 32)
	r.iterNodes(func(node *Rope) bool {
		var balancedNode *Rope
		iterSubNodes := true
		if len(currentBytes) == 0 && node.balanced { // balanced, insert to slots
			balancedNode = node
			iterSubNodes = false
		} else { // collect bytes
			currentBytes = append(currentBytes, node.content...)
			if len(currentBytes) >= MaxLengthPerNode { // a full leaf
				balancedNode = &Rope{
					height:   1,
					weight:   MaxLengthPerNode,
					balanced: true,
					content:  currentBytes[:MaxLengthPerNode],
				}
				currentBytes = currentBytes[MaxLengthPerNode:]
			}
		}
		if balancedNode != nil {
			slotIndex := balancedNode.height - 1
			for slots[slotIndex] != nil {
				balancedNode = &Rope{
					height:   balancedNode.height + 1,
					weight:   slots[slotIndex].Len(),
					left:     slots[slotIndex],
					right:    balancedNode,
					balanced: true,
				}
				slots[slotIndex] = nil
				slotIndex++
			}
			slots[slotIndex] = balancedNode
		}
		return iterSubNodes
	})
	if len(currentBytes) > 0 {
		ret = &Rope{
			height:   1,
			weight:   len(currentBytes),
			balanced: false,
			content:  currentBytes,
		}
	}
	for _, c := range slots {
		if c != nil {
			if ret == nil {
				ret = c
			} else {
				ret = c.Concat(ret)
			}
		}
	}
	return
}

func (r *Rope) Split(n int) (out1, out2 *Rope) {
	if r == nil {
		return
	}
	if len(r.content) > 0 { // leaf
		if n > len(r.content) { // offset overflow
			n = len(r.content)
		}
		out1 = NewFromBytes(r.content[:n])
		out2 = NewFromBytes(r.content[n:])
	} else { // non leaf
		var r1 *Rope
		if n >= r.weight { // at right subtree
			r1, out2 = r.right.Split(n - r.weight)
			out1 = r.left.Concat(r1)
		} else { // at left subtree
			out1, r1 = r.left.Split(n)
			out2 = r1.Concat(r.right)
		}
	}
	return
}

func (r *Rope) Insert(n int, bs []byte) *Rope {
	r1, r2 := r.Split(n)
	return r1.Concat(NewFromBytes(bs)).Concat(r2)
}

func (r *Rope) Delete(n, l int) *Rope {
	r1, r2 := r.Split(n)
	_, r2 = r2.Split(l)
	return r1.Concat(r2)
}

func (r *Rope) Sub(n, l int) []byte {
	ret := make([]byte, l)
	i := 0
	r.Iter(n, func(bs []byte) bool {
		if l >= len(bs) {
			copy(ret[i:], bs)
			i += len(bs)
			l -= len(bs)
			return true
		} else {
			copy(ret[i:], bs[:l])
			i += l
			return false
		}
	})
	return ret[:i]
}

func (r *Rope) Iter(offset int, fn func([]byte) bool) bool {
	if r == nil {
		return true
	}
	if len(r.content) > 0 { // leaf
		if offset < len(r.content) {
			if !fn(r.content[offset:]) {
				return false
			}
		}
	} else { // non leaf
		if offset >= r.weight { // start at right subtree
			if !r.right.Iter(offset-r.weight, fn) {
				return false
			}
		} else { // start at left subtree
			if !r.left.Iter(offset, fn) {
				return false
			}
			if !r.right.Iter(0, fn) {
				return false
			}
		}
	}
	return true
}

func (r *Rope) IterBackward(offset int, fn func([]byte) bool) bool {
	if r == nil {
		return true
	}
	if len(r.content) > 0 { // leaf
		content := r.content[:offset]
		if len(content) == 0 {
			return true
		}
		bs := reversedBytes(content)
		if !fn(bs) {
			return false
		}
	} else { // non leaf
		if offset >= r.weight { // start at right subtree
			if !r.right.IterBackward(offset-r.weight, fn) {
				return false
			}
			if !r.left.IterBackward(r.weight, fn) {
				return false
			}
		} else { // start at left subtree
			if !r.left.IterBackward(offset, fn) {
				return false
			}
		}
	}
	return true
}

func (r *Rope) iterNodes(fn func(*Rope) bool) {
	if r == nil {
		return
	}
	if fn(r) {
		r.left.iterNodes(fn)
		r.right.iterNodes(fn)
	}
}

func (r *Rope) IterRune(offset int, fn func(rune, int) bool) {
	var bs []byte
	stopped := false
	r.Iter(offset, func(slice []byte) bool {
		bs = append(bs, slice...)
		for len(bs) >= 4 {
			ru, l := utf8.DecodeRune(bs)
			bs = bs[l:]
			if ru == utf8.RuneError {
				return false
			}
			if !fn(ru, l) {
				stopped = true
				return false
			}
		}
		return true
	})
	if !stopped && len(bs) > 0 {
		for {
			ru, l := utf8.DecodeRune(bs)
			bs = bs[l:]
			if ru != utf8.RuneError {
				fn(ru, l)
			} else {
				break
			}
		}
	}
}
<|endoftext|>


<commit_before>package fbb

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/textproto"
	"strings"

	"github.com/paulrosania/go-charset/charset"
	_ "github.com/paulrosania/go-charset/data"
)

// This file contains code from from net/http/header.go

// Common Winlink 2000 Message headers
const (
	HEADER_MID     = `Mid`
	HEADER_TO      = `To`
	HEADER_DATE    = `Date`
	HEADER_TYPE    = `Type`
	HEADER_FROM    = `From`
	HEADER_CC      = `Cc`
	HEADER_SUBJECT = `Subject`
	HEADER_MBO     = `Mbo`
	HEADER_BODY    = `Body`
	HEADER_FILE    = `File`

	// These headers are stripped by the winlink system, but let's
	// include it anyway... just in case the winlink team one day
	// starts taking encoding seriously.
	HEADER_CONTENT_TYPE              = `Content-Type`
	HEADER_CONTENT_TRANSFER_ENCODING = `Content-Transfer-Encoding`

	// The default body charset seems to be ISO-8859-1
	//
	// The Winlink Message Structure docs says that the body should
	// be ASCII-only, but RMS Express seems to encode the body as
	// ISO-8859-1. This is also the charset set (Content-Type header)
	// when a message reaches an SMTP server.
	DefaultCharset = "ISO-8859-1"

	// Mails going out over SMTP from the Winlink system is sent
	// with the header 'Content-Transfer-Encoding: 7bit', but
	// let's be reasonable... we don't send ASCII-only body.
	DefaultTransferEncoding = "8bit"

	// The date (in UTC) format as described in the Winlink
	// Message Structure docs (YYYY/MM/DD HH:MM).
	DateLayout = `2006/01/02 15:04`
)

// A Header represents the key-value pairs in a Winlink 2000 Message header.
type Header map[string][]string

// Add adds the key, value pair to the header.
// It appends to any existing values associated with key.
func (h Header) Add(key, value string) {
	textproto.MIMEHeader(h).Add(key, value)
}

// Set sets the header entries associated with key to
// the single element value.  It replaces any existing
// values associated with key.
func (h Header) Set(key, value string) {
	textproto.MIMEHeader(h).Set(key, value)
}

// Get gets the first value associated with the given key.
// If there are no values associated with the key, Get returns "".
// To access multiple values of a key, access the map directly
// with CanonicalHeaderKey.
func (h Header) Get(key string) string {
	return textproto.MIMEHeader(h).Get(key)
}

// get is like Get, but key must already be in CanonicalHeaderKey form.
func (h Header) get(key string) string {
	if v := h[key]; len(v) > 0 {
		return v[0]
	}
	return ""
}

// Del deletes the values associated with key.
func (h Header) Del(key string) {
	textproto.MIMEHeader(h).Del(key)
}

// Write writes a header in wire format.
func (h Header) Write(w io.Writer) error {
	var err error

	// Mid is required
	if h.get(HEADER_MID) == "" {
		return errors.New("Missing MID in header")
	}

	// Write mid, this is defined to be the first value
	_, err = fmt.Fprintf(w, "Mid: %s\r\n", h.get(HEADER_MID))
	if err != nil {
		return err
	}

	for k, slice := range h {
		if strings.EqualFold(k, HEADER_MID) {
			continue
		}

		for _, v := range slice {
			v = textproto.TrimString(v)
			_, err = fmt.Fprintf(w, "%s: %s\r\n", k, v)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// WordDecoder decodes MIME headers containing RFC 2047 encoded-words.
//
// (See DecodeHeader for mime.WordDecoder differences).
type WordDecoder struct{ mime.WordDecoder }

// Decode decodes an encoded-word.
//
// If word is not a valid RFC 2047 encoded-word, word is decoded as raw ISO-8859-1 as a work-around for RMS Express' non-conforming encoding of the Subject header.
func (d *WordDecoder) DecodeHeader(header string) (string, error) {
	i := strings.Index(header, "=?")
	if i > -1 {
		return d.WordDecoder.DecodeHeader(header)
	}

	// If there is no encoded-word, decode as ISO-8859-1 (RMS Express compatibility hack)
	r, err := charset.NewReader(DefaultCharset, bytes.NewReader([]byte(header)))
	if err != nil {
		return header, err
	}

	utf8, err := ioutil.ReadAll(r)
	return string(utf8), err
}

func toCharset(set, s string) (string, error) {
	buf := new(bytes.Buffer)
	w, err := charset.NewWriter(set, buf)
	if err != nil {
		return s, err
	}

	fmt.Fprint(w, s)
	w.Close()
	return buf.String(), nil
}
<commit_msg>Handle utf8-encoded headers<commit_after>package fbb

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime"
	"net/textproto"
	"strings"
	"unicode/utf8"

	"github.com/paulrosania/go-charset/charset"
	_ "github.com/paulrosania/go-charset/data"
)

// This file contains code from from net/http/header.go

// Common Winlink 2000 Message headers
const (
	HEADER_MID     = `Mid`
	HEADER_TO      = `To`
	HEADER_DATE    = `Date`
	HEADER_TYPE    = `Type`
	HEADER_FROM    = `From`
	HEADER_CC      = `Cc`
	HEADER_SUBJECT = `Subject`
	HEADER_MBO     = `Mbo`
	HEADER_BODY    = `Body`
	HEADER_FILE    = `File`

	// These headers are stripped by the winlink system, but let's
	// include it anyway... just in case the winlink team one day
	// starts taking encoding seriously.
	HEADER_CONTENT_TYPE              = `Content-Type`
	HEADER_CONTENT_TRANSFER_ENCODING = `Content-Transfer-Encoding`

	// The default body charset seems to be ISO-8859-1
	//
	// The Winlink Message Structure docs says that the body should
	// be ASCII-only, but RMS Express seems to encode the body as
	// ISO-8859-1. This is also the charset set (Content-Type header)
	// when a message reaches an SMTP server.
	DefaultCharset = "ISO-8859-1"

	// Mails going out over SMTP from the Winlink system is sent
	// with the header 'Content-Transfer-Encoding: 7bit', but
	// let's be reasonable... we don't send ASCII-only body.
	DefaultTransferEncoding = "8bit"

	// The date (in UTC) format as described in the Winlink
	// Message Structure docs (YYYY/MM/DD HH:MM).
	DateLayout = `2006/01/02 15:04`
)

// A Header represents the key-value pairs in a Winlink 2000 Message header.
type Header map[string][]string

// Add adds the key, value pair to the header.
// It appends to any existing values associated with key.
func (h Header) Add(key, value string) {
	textproto.MIMEHeader(h).Add(key, value)
}

// Set sets the header entries associated with key to
// the single element value.  It replaces any existing
// values associated with key.
func (h Header) Set(key, value string) {
	textproto.MIMEHeader(h).Set(key, value)
}

// Get gets the first value associated with the given key.
// If there are no values associated with the key, Get returns "".
// To access multiple values of a key, access the map directly
// with CanonicalHeaderKey.
func (h Header) Get(key string) string {
	return textproto.MIMEHeader(h).Get(key)
}

// get is like Get, but key must already be in CanonicalHeaderKey form.
func (h Header) get(key string) string {
	if v := h[key]; len(v) > 0 {
		return v[0]
	}
	return ""
}

// Del deletes the values associated with key.
func (h Header) Del(key string) {
	textproto.MIMEHeader(h).Del(key)
}

// Write writes a header in wire format.
func (h Header) Write(w io.Writer) error {
	var err error

	// Mid is required
	if h.get(HEADER_MID) == "" {
		return errors.New("Missing MID in header")
	}

	// Write mid, this is defined to be the first value
	_, err = fmt.Fprintf(w, "Mid: %s\r\n", h.get(HEADER_MID))
	if err != nil {
		return err
	}

	for k, slice := range h {
		if strings.EqualFold(k, HEADER_MID) {
			continue
		}

		for _, v := range slice {
			v = textproto.TrimString(v)
			_, err = fmt.Fprintf(w, "%s: %s\r\n", k, v)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

// WordDecoder decodes MIME headers containing RFC 2047 encoded-words.
//
// (See DecodeHeader for mime.WordDecoder differences).
type WordDecoder struct{ mime.WordDecoder }

// Decode decodes an encoded-word.
//
// If word is not a valid RFC 2047 encoded-word, word is decoded as raw ISO-8859-1 as a work-around for RMS Express' non-conforming encoding of the Subject header.
func (d *WordDecoder) DecodeHeader(header string) (string, error) {
	i := strings.Index(header, "=?")
	if i > -1 {
		return d.WordDecoder.DecodeHeader(header)
	}

	// If there is no encoded-word, the data may be ISO-8859-1 or UTF-8 depending on how CMS decoded it.
	//
	// It turns out that if CMS receives a Q-encoded subject it decodes it and forwards it as UTF-8.
	// If CMS on the other hand receives an SMTP email from gmail, it is enocded as ISO-8859-1.
	if utf8.Valid([]byte(header)) {
		return header, nil
	}

	r, err := charset.NewReader(DefaultCharset, bytes.NewReader([]byte(header)))
	if err != nil {
		return header, err
	}

	decoded, _ := ioutil.ReadAll(r)
	return string(decoded), nil
}

func toCharset(set, s string) (string, error) {
	buf := new(bytes.Buffer)
	w, err := charset.NewWriter(set, buf)
	if err != nil {
		return s, err
	}

	fmt.Fprint(w, s)
	w.Close()
	return buf.String(), nil
}
<|endoftext|>


<commit_before>package xkcd

import (
	"encoding/json"
	"net/http"
	"strconv"
	"time"
)

type comicResponse struct {
	Alt        string `json:"alt"`
	Day        string `json:"day"`
	ImageURL   string `json:"img"`
	URL        string `json:"link"`
	Month      string `json:"month"`
	Number     int    `json:"num"`
	SafeTitle  string `json:"safe_title"`
	Title      string `json:"title"`
	Transcript string `json:"transcript"`
	Year       string `json:"year"`
}

type Comic struct {
	Alt         string
	PublishDate time.Time
	ImageURL    string
	URL         string
	Number      int
	SafeTitle   string
	Title       string
	Transcript  string
}

func (comic *Comic) UnmarshalJSON(data []byte) error {
	var aux comicResponse
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}

	date, err := time.Parse("2006/1/2", aux.Year+"/"+aux.Month+"/"+aux.Day)
	if err != nil {
		return err
	}

	comic.Alt = aux.Alt
	comic.PublishDate = date
	comic.ImageURL = aux.ImageURL
	comic.URL = aux.URL
	comic.Number = aux.Number
	comic.SafeTitle = aux.SafeTitle
	comic.Title = aux.Title
	comic.Transcript = aux.Transcript

	return nil
}

func (c *Client) doComicRequest(path string) (Comic, error) {
	var comic Comic

	req, err := http.NewRequest("GET", c.baseURL()+path, nil)
	if err != nil {
		return comic, err
	}

	body, err := c.do(req)
	if err != nil {
		return comic, err
	}
	defer body.Close()

	if err := json.NewDecoder(body).Decode(&comic); err != nil {
		return comic, err
	}
	return comic, nil
}

func (c *Client) GetLatest() (Comic, error) {
	return c.doComicRequest("/info.0.json")
}

func (c *Client) Get(number int) (Comic, error) {
	numStr := strconv.Itoa(number)
	return c.doComicRequest("/" + numStr + "/info.0.json")
}
<commit_msg>Add News field to Comic<commit_after>package xkcd

import (
	"encoding/json"
	"net/http"
	"strconv"
	"time"
)

type comicResponse struct {
	Alt        string `json:"alt"`
	Day        string `json:"day"`
	ImageURL   string `json:"img"`
	URL        string `json:"link"`
	Month      string `json:"month"`
	News       string `json:"news"`
	Number     int    `json:"num"`
	SafeTitle  string `json:"safe_title"`
	Title      string `json:"title"`
	Transcript string `json:"transcript"`
	Year       string `json:"year"`
}

type Comic struct {
	Alt         string
	PublishDate time.Time
	ImageURL    string
	URL         string
	News        string
	Number      int
	SafeTitle   string
	Title       string
	Transcript  string
}

func (comic *Comic) UnmarshalJSON(data []byte) error {
	var aux comicResponse
	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}

	date, err := time.Parse("2006/1/2", aux.Year+"/"+aux.Month+"/"+aux.Day)
	if err != nil {
		return err
	}

	comic.Alt = aux.Alt
	comic.PublishDate = date
	comic.ImageURL = aux.ImageURL
	comic.URL = aux.URL
	comic.Number = aux.Number
	comic.News = aux.News
	comic.SafeTitle = aux.SafeTitle
	comic.Title = aux.Title
	comic.Transcript = aux.Transcript

	return nil
}

func (c *Client) doComicRequest(path string) (Comic, error) {
	var comic Comic

	req, err := http.NewRequest("GET", c.baseURL()+path, nil)
	if err != nil {
		return comic, err
	}

	body, err := c.do(req)
	if err != nil {
		return comic, err
	}
	defer body.Close()

	if err := json.NewDecoder(body).Decode(&comic); err != nil {
		return comic, err
	}
	return comic, nil
}

func (c *Client) GetLatest() (Comic, error) {
	return c.doComicRequest("/info.0.json")
}

func (c *Client) Get(number int) (Comic, error) {
	numStr := strconv.Itoa(number)
	return c.doComicRequest("/" + numStr + "/info.0.json")
}
<|endoftext|>