<commit_before>package httpd

// Config represents a configuration for a HTTP service.
type Config struct {
	Enabled          bool   `toml:"enabled"`
	BindAddress      string `toml:"bind-address"`
	AuthEnabled      bool   `toml:"auth-enabled"`
	LogEnabled       bool   `toml:"log-enabled"`
	WriteTracing     bool   `toml:"write-tracing"`
	PprofEnabled     bool   `toml:"pprof-enabled"`
	HTTPSEnabled     bool   `toml:"https-enabled"`
	HTTPSCertificate string `toml:"https-certificate"`
	JSONWriteEnabled bool   `toml:"json-write-enabled"`
}

// NewConfig returns a new Config with default settings.
func NewConfig() Config {
	return Config{
		Enabled:          true,
		BindAddress:      ":8086",
		LogEnabled:       true,
		HTTPSEnabled:     false,
		HTTPSCertificate: "/etc/ssl/influxdb.pem",
		JSONWriteEnabled: false,
	}
}
<commit_msg>Add default port<commit_after>package httpd

// DefaultBindAddress is the default address to bind to.
const DefaultBindAddress = ":8086"

// Config represents a configuration for a HTTP service.
type Config struct {
	Enabled          bool   `toml:"enabled"`
	BindAddress      string `toml:"bind-address"`
	AuthEnabled      bool   `toml:"auth-enabled"`
	LogEnabled       bool   `toml:"log-enabled"`
	WriteTracing     bool   `toml:"write-tracing"`
	PprofEnabled     bool   `toml:"pprof-enabled"`
	HTTPSEnabled     bool   `toml:"https-enabled"`
	HTTPSCertificate string `toml:"https-certificate"`
	JSONWriteEnabled bool   `toml:"json-write-enabled"`
}

// NewConfig returns a new Config with default settings.
func NewConfig() Config {
	return Config{
		Enabled:          true,
		BindAddress:      ":8086",
		LogEnabled:       true,
		HTTPSEnabled:     false,
		HTTPSCertificate: "/etc/ssl/influxdb.pem",
		JSONWriteEnabled: false,
	}
}
<|endoftext|>


<commit_before>package services

import (
	"log"
	"strings"
	"sync"

	"github.com/simongui/fastlane/storage"
	"github.com/tidwall/redcon"
)

// RedisServer Represents an instance of a redis protocol server.
type RedisServer struct {
	store *storage.BoltDBStore
}

// NewRedisServer Returns a new RedisServer instance.
func NewRedisServer(store *storage.BoltDBStore) *RedisServer {
	server := &RedisServer{store: store}
	return server
}

// ListenAndServe Starts the Redis protocol server.
func (server *RedisServer) ListenAndServe(address string) {
	var mu sync.RWMutex
	var items = make(map[string]string)

	err := redcon.ListenAndServe(address,
		func(conn redcon.Conn, commands [][]string) {
			for _, args := range commands {
				switch strings.ToLower(args[0]) {
				default:
					conn.WriteError("ERR unknown command '" + args[0] + "'")
				case "ping":
					conn.WriteString("PONG")
				case "quit":
					conn.WriteString("OK")
					conn.Close()
				case "set":
					if len(args) != 3 {
						conn.WriteError("ERR wrong number of arguments for '" + args[0] + "' command")
						continue
					}

					// mu.Lock()
					// items[args[1]] = args[2]
					// mu.Unlock()
					go server.store.Set([]byte(args[1]), []byte(args[2]))
					conn.WriteString("OK")
				case "get":
					if len(args) != 2 {
						conn.WriteError("ERR wrong number of arguments for '" + args[0] + "' command")
						continue
					}
					// mu.RLock()
					// val, ok := items[args[1]]
					// mu.RUnlock()
					// go func(con redcon.Conn) {
					val, err := server.store.Get([]byte(args[1]))
					if err != nil {
						conn.WriteNull()
					} else {
						conn.WriteBulk(string(val))
					}
					// }(conn)
				case "del":
					if len(args) != 2 {
						conn.WriteError("ERR wrong number of arguments for '" + args[0] + "' command")
						continue
					}
					mu.Lock()
					_, ok := items[args[1]]
					delete(items, args[1])
					mu.Unlock()
					if !ok {
						conn.WriteInt(0)
					} else {
						conn.WriteInt(1)
					}
				}
			}
		},
		func(conn redcon.Conn) bool {
			// use this function to accept or deny the connection.
			// log.Printf("accept: %s", conn.RemoteAddr())
			return true
		},
		func(conn redcon.Conn, err error) {
			// this is called when the connection has been closed
			// log.Printf("closed: %s, err: %v", conn.RemoteAddr(), err)
		},
	)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Update redis_server.go<commit_after>package services

import (
	"log"
	"strings"
	"sync"

	"github.com/simongui/fastlane/storage"
	"github.com/tidwall/redcon"
)

// RedisServer Represents an instance of a redis protocol server.
type RedisServer struct {
	started bool
	store   *storage.BoltDBStore
}

// NewRedisServer Returns a new RedisServer instance.
func NewRedisServer(store *storage.BoltDBStore) *RedisServer {
	server := &RedisServer{store: store}
	return server
}

// IsStarted Returns whether the Redis protocol server is started.
func (server *RedisServer) IsStarted() bool {
	return server.started
}

// ListenAndServe Starts the Redis protocol server.
func (server *RedisServer) ListenAndServe(address string) {
	var mu sync.RWMutex
	var items = make(map[string]string)

	server.started = true

	err := redcon.ListenAndServe(address,
		func(conn redcon.Conn, commands [][]string) {

			for _, args := range commands {
				switch strings.ToLower(args[0]) {
				default:
					conn.WriteError("ERR unknown command '" + args[0] + "'")
				case "ping":
					conn.WriteString("PONG")
				case "quit":
					conn.WriteString("OK")
					conn.Close()
				case "set":
					if len(args) != 3 {
						conn.WriteError("ERR wrong number of arguments for '" + args[0] + "' command")
						continue
					}

					// mu.Lock()
					// items[args[1]] = args[2]
					// mu.Unlock()
					go server.store.Set([]byte(args[1]), []byte(args[2]))
					conn.WriteString("OK")
				case "get":
					if len(args) != 2 {
						conn.WriteError("ERR wrong number of arguments for '" + args[0] + "' command")
						continue
					}
					// mu.RLock()
					// val, ok := items[args[1]]
					// mu.RUnlock()
					// go func(con redcon.Conn) {
					val, err := server.store.Get([]byte(args[1]))
					if err != nil {
						conn.WriteNull()
					} else {
						conn.WriteBulk(string(val))
					}
					// }(conn)
				case "del":
					if len(args) != 2 {
						conn.WriteError("ERR wrong number of arguments for '" + args[0] + "' command")
						continue
					}
					mu.Lock()
					_, ok := items[args[1]]
					delete(items, args[1])
					mu.Unlock()
					if !ok {
						conn.WriteInt(0)
					} else {
						conn.WriteInt(1)
					}
				}
			}
		},
		func(conn redcon.Conn) bool {
			// use this function to accept or deny the connection.
			// log.Printf("accept: %s", conn.RemoteAddr())
			return true
		},
		func(conn redcon.Conn, err error) {
			// this is called when the connection has been closed
			// log.Printf("closed: %s, err: %v", conn.RemoteAddr(), err)
		},
	)
	if err != nil {
		server.started = false
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package premailer

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestBasicHTML(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

There

Hello

Yes!

Green color

        `

	p := NewPremailerFromString(html, nil)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
There
")
	assert.Contains(t, result_html, "
Hello
")
	assert.Contains(t, result_html, "
Yes!
")
	assert.Contains(t, result_html, "
Green color
")
}

func TestDataPremailerIgnore(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

        `

	p := NewPremailerFromString(html, nil)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")
}

func TestWithInline(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

        `

	p := NewPremailerFromString(html, nil)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")
	assert.NotContains(t, result_html, "

Hi!

        	Yes!
        	No!

        `

	p := NewPremailerFromString(html, nil)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "Yes!")
	assert.Contains(t, result_html, "

Hi!

Yes!

        `

	options := &Options{}
	options.RemoveClasses = true
	p := NewPremailerFromString(html, options)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")
}

func TestCssToAttributesFalse(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

        `

	options := &Options{}
	options.CssToAttributes = false
	p := NewPremailerFromString(html, options)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")
}

func TestWithImportant(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

        `

	p := NewPremailerFromString(html, NewOptions())
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")
}

func TestWithMediaRule(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

        `

	p := NewPremailerFromString(html, NewOptions())
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")

	assert.Contains(t, result_html, "@media all and (min-width: 62em){")
	assert.Contains(t, result_html, "font-size: 55px !important;")
	assert.Contains(t, result_html, "line-height: 60px !important;")
	assert.Contains(t, result_html, "padding-bottom: 5px !important;")
	assert.Contains(t, result_html, "padding-top: 0 !important")
}

func TestWithMediaAttribute(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

        `

	p := NewPremailerFromString(html, NewOptions())
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")

	assert.Contains(t, result_html, "

Hi!

Yes!

        `

	p := NewPremailerFromString(html, NewOptions())
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")

	assert.Contains(t, result_html, "

1

2

        `

	p := NewPremailerFromString(html, NewOptions())
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, `<tr><td style="padding:5px">1</td></tr>`)
	assert.Contains(t, result_html, `<tr><td class="bar-area" style="padding:10px">2</td></tr>`)
}
<commit_msg>collect rules from style which has media="all"<commit_after>package premailer

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestBasicHTML(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

There

Hello

Yes!

Green color

        `

	p := NewPremailerFromString(html, nil)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
There
")
	assert.Contains(t, result_html, "
Hello
")
	assert.Contains(t, result_html, "
Yes!
")
	assert.Contains(t, result_html, "
Green color
")
}

func TestDataPremailerIgnore(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

        `

	p := NewPremailerFromString(html, nil)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")
}

func TestWithInline(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

        `

	p := NewPremailerFromString(html, nil)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")
	assert.NotContains(t, result_html, "

Hi!

        	Yes!
        	No!

        `

	p := NewPremailerFromString(html, nil)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "Yes!")
	assert.Contains(t, result_html, "

Hi!

Yes!

        `

	options := &Options{}
	options.RemoveClasses = true
	p := NewPremailerFromString(html, options)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")
}

func TestCssToAttributesFalse(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

        `

	options := &Options{}
	options.CssToAttributes = false
	p := NewPremailerFromString(html, options)
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")
}

func TestWithImportant(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

        `

	p := NewPremailerFromString(html, NewOptions())
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")
}

func TestWithMediaRule(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

        `

	p := NewPremailerFromString(html, NewOptions())
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")

	assert.Contains(t, result_html, "@media all and (min-width: 62em){")
	assert.Contains(t, result_html, "font-size: 55px !important;")
	assert.Contains(t, result_html, "line-height: 60px !important;")
	assert.Contains(t, result_html, "padding-bottom: 5px !important;")
	assert.Contains(t, result_html, "padding-top: 0 !important")
}

func TestWithMediaAttribute(t *testing.T) {
	html := `

        <title>Title</title>

Hi!

Yes!

Hi, all media style!

        `

	p := NewPremailerFromString(html, NewOptions())
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")
	assert.Contains(t, result_html, "
Hi, all media style!
")

	assert.Contains(t, result_html, "

Hi!

Yes!

        `

	p := NewPremailerFromString(html, NewOptions())
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, "
Hi!
")
	assert.Contains(t, result_html, "
Yes!
")

	assert.Contains(t, result_html, "

1

2

        `

	p := NewPremailerFromString(html, NewOptions())
	result_html, err := p.Transform()
	assert.Nil(t, err)

	assert.Contains(t, result_html, `<tr><td style="padding:5px">1</td></tr>`)
	assert.Contains(t, result_html, `<tr><td class="bar-area" style="padding:10px">2</td></tr>`)
}
<|endoftext|>


<commit_before>package rpc

import (
	"bufio"
	"bytes"
	"encoding/json"
	"io"
	"log"
)

const LF = 0xA // Line-feed

type RequestHandler func(*Message)

// Object represents a JSON object.
type Object map[string]interface{}

// Array represents a JSON array.
type Array []interface{}

type Request struct {
	Method string      `json:"method"`
	Params interface{} `json:"params"`
	ViewID string      `json:"view_id,omitempty"`
}

// incomingMessage is a representation of an incoming RPC message.
type incomingMessage struct {
	Id     int             `json:"id"`
	Method string          `json:"method"`
	Params json.RawMessage `json:"params"`
	Error  string          `json:"error,omitempty"`
	Result interface{}     `json:"result,omitempty"`
}

// outgoingMessage is a representation of an outgoing RPC message.
type outgoingMessage struct {
	*Request
	ID int `json:"id,omitempty"`
}

// Message is a deserialized message which will be passed to the `Messages` channel.
type Message struct {
	Method string
	Value  interface{}
}

// Connection represents the connection to the backend.
// The underlying protocol doesn't matter as long as it is writeable and readable.
type Connection struct {
	Messages chan *Message
	rw       io.ReadWriter
	rpcIndex int
	pending  map[int]func(*Message)
}

func NewConnection(rw io.ReadWriter) *Connection {
	c := &Connection{
		Messages: make(chan *Message, 1),
		rw:       rw,
		pending:  make(map[int]func(*Message)),
	}

	go c.recv()

	return c
}

func (c *Connection) recv() {
	var buffer bytes.Buffer
	reader := bufio.NewReader(c.rw)
	for {
		line, pre, err := reader.ReadLine()
		if err != nil && err == io.EOF {
			break
		}
		buffer.Write(line)
		for pre {
			lineNext, p, err := reader.ReadLine()
			pre = p
			if err != nil && err == io.EOF {
				break
			}
			buffer.Write(lineNext)
		}
		line = buffer.Bytes()
		buffer.Reset()
		//log.Printf("<<< %s\n", string(line))
		//log.Println("<<< ")
		var msg incomingMessage
		err = json.Unmarshal(line, &msg)
		if err != nil {
			log.Fatal(err)
		}
		if msg.Id != 0 {
			// response
			if fn, ok := c.pending[msg.Id]; ok {
				fn(&Message{msg.Method, msg.Result})
			} else {
				log.Println("unhandled response: ", msg)
			}
		} else {
			// request
			switch msg.Method {
			case "update":
				var update Update
				json.Unmarshal(msg.Params, &update)
				c.Messages <- &Message{msg.Method, &update}
			case "def_style":
				var defStyle DefineStyle
				json.Unmarshal(msg.Params, &defStyle)
				c.Messages <- &Message{msg.Method, &defStyle}
			case "theme_changed":
				var themeChanged ThemeChanged
				json.Unmarshal(msg.Params, &themeChanged)
				c.Messages <- &Message{msg.Method, &themeChanged}
			case "scroll_to":
				var scrollTo ScrollTo
				json.Unmarshal(msg.Params, &scrollTo)
				c.Messages <- &Message{msg.Method, &scrollTo}
			case "language_changed":
				var languageChanged LanguageChanged
				json.Unmarshal(msg.Params, &languageChanged)
				c.Messages <- &Message{msg.Method, &languageChanged}
			case "find_status":
				var findStatus FindStatus
				json.Unmarshal(msg.Params, &findStatus)
				c.Messages <- &Message{msg.Method, &findStatus}
			default:
				log.Println("unhandled request: " + msg.Method)
			}
		}
	}
}

func (c *Connection) send(msg *outgoingMessage) int {
	b, _ := json.Marshal(msg)
	log.Printf(">>> %s\n", b)
	c.rw.Write(b)
	c.rw.Write([]byte{LF})
	return c.rpcIndex
}

func (c *Connection) Request(r *Request) (*Message, error) {
	ch := make(chan *Message, 1)
	c.RequestAsync(r, func(m *Message) {
		ch <- m
	})

	select {
	case m := <-ch:
		return m, nil
	}
}

func (c *Connection) RequestAsync(r *Request, callback func(*Message)) int {
	c.rpcIndex++
	c.pending[c.rpcIndex] = callback

	c.send(&outgoingMessage{
		ID:      c.rpcIndex,
		Request: r,
	})
	return c.rpcIndex
}

func (c *Connection) Notify(r *Request) {
	c.send(&outgoingMessage{
		Request: r,
	})
}

func (c *Connection) Start(configPath string) {
	c.Notify(&Request{
		Method: "client_started",
		Params: Object{
			"config_dir": configPath,
		},
	})
}

func (c *Connection) SetTheme(theme string) {
	c.Notify(&Request{
		Method: "set_theme",
		Params: Object{"theme_name": theme},
	})
}
<commit_msg>Debug<commit_after>package rpc

import (
	"bufio"
	"bytes"
	"encoding/json"
	"io"
	"log"
)

const LF = 0xA // Line-feed

type RequestHandler func(*Message)

// Object represents a JSON object.
type Object map[string]interface{}

// Array represents a JSON array.
type Array []interface{}

type Request struct {
	Method string      `json:"method"`
	Params interface{} `json:"params"`
	ViewID string      `json:"view_id,omitempty"`
}

// incomingMessage is a representation of an incoming RPC message.
type incomingMessage struct {
	Id     int             `json:"id"`
	Method string          `json:"method"`
	Params json.RawMessage `json:"params"`
	Error  string          `json:"error,omitempty"`
	Result interface{}     `json:"result,omitempty"`
}

// outgoingMessage is a representation of an outgoing RPC message.
type outgoingMessage struct {
	*Request
	ID int `json:"id,omitempty"`
}

// Message is a deserialized message which will be passed to the `Messages` channel.
type Message struct {
	Method string
	Value  interface{}
}

// Connection represents the connection to the backend.
// The underlying protocol doesn't matter as long as it is writeable and readable.
type Connection struct {
	Messages chan *Message
	rw       io.ReadWriter
	rpcIndex int
	pending  map[int]func(*Message)
}

func NewConnection(rw io.ReadWriter) *Connection {
	c := &Connection{
		Messages: make(chan *Message, 1),
		rw:       rw,
		pending:  make(map[int]func(*Message)),
	}

	go c.recv()

	return c
}

func (c *Connection) recv() {
	var buffer bytes.Buffer
	reader := bufio.NewReader(c.rw)
	for {
		line, pre, err := reader.ReadLine()
		if err != nil && err == io.EOF {
			break
		}
		buffer.Write(line)
		for pre {
			lineNext, p, err := reader.ReadLine()
			pre = p
			if err != nil && err == io.EOF {
				break
			}
			buffer.Write(lineNext)
		}
		line = buffer.Bytes()
		buffer.Reset()
		//log.Printf("<<< %s\n", string(line))
		//log.Println("<<< ")
		var msg incomingMessage
		err = json.Unmarshal(line, &msg)
		if err != nil {
			log.Fatal(err)
		}
		if msg.Id != 0 {
			// response
			if fn, ok := c.pending[msg.Id]; ok {
				fn(&Message{msg.Method, msg.Result})
			} else {
				log.Println("unhandled response: ", msg)
			}
		} else {
			// request
			switch msg.Method {
			case "update":
				var update Update
				json.Unmarshal(msg.Params, &update)
				c.Messages <- &Message{msg.Method, &update}
			case "def_style":
				var defStyle DefineStyle
				json.Unmarshal(msg.Params, &defStyle)
				c.Messages <- &Message{msg.Method, &defStyle}
			case "theme_changed":
				var themeChanged ThemeChanged
				json.Unmarshal(msg.Params, &themeChanged)
				c.Messages <- &Message{msg.Method, &themeChanged}
			case "scroll_to":
				var scrollTo ScrollTo
				json.Unmarshal(msg.Params, &scrollTo)
				c.Messages <- &Message{msg.Method, &scrollTo}
			case "language_changed":
				var languageChanged LanguageChanged
				json.Unmarshal(msg.Params, &languageChanged)
				c.Messages <- &Message{msg.Method, &languageChanged}
			case "find_status":
				var findStatus FindStatus
				json.Unmarshal(msg.Params, &findStatus)
				c.Messages <- &Message{msg.Method, &findStatus}
			default:
				log.Println("unhandled request: " + msg.Method)
				log.Println("unhandled request: " + string(line))
			}
		}
	}
}

func (c *Connection) send(msg *outgoingMessage) int {
	b, _ := json.Marshal(msg)
	log.Printf(">>> %s\n", b)
	c.rw.Write(b)
	c.rw.Write([]byte{LF})
	return c.rpcIndex
}

func (c *Connection) Request(r *Request) (*Message, error) {
	ch := make(chan *Message, 1)
	c.RequestAsync(r, func(m *Message) {
		ch <- m
	})

	select {
	case m := <-ch:
		return m, nil
	}
}

func (c *Connection) RequestAsync(r *Request, callback func(*Message)) int {
	c.rpcIndex++
	c.pending[c.rpcIndex] = callback

	c.send(&outgoingMessage{
		ID:      c.rpcIndex,
		Request: r,
	})
	return c.rpcIndex
}

func (c *Connection) Notify(r *Request) {
	c.send(&outgoingMessage{
		Request: r,
	})
}

func (c *Connection) Start(configPath string) {
	c.Notify(&Request{
		Method: "client_started",
		Params: Object{
			"config_dir": configPath,
		},
	})
}

func (c *Connection) SetTheme(theme string) {
	c.Notify(&Request{
		Method: "set_theme",
		Params: Object{"theme_name": theme},
	})
}
<|endoftext|>


<commit_before>// Copyright 2012 Ric Szopa (Ryszard) <ryszard.szopa@gmail.com> All
// rights reserved.  Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

// Package skiplist implements skip list based maps and sets. For more
// information about skip lists take a look at
// http://en.wikipedia.org/wiki/Skip_list.

package skiplist

import "math/rand"

// TODO(ryszard):
//   - A separately seeded source of randomness
//   - Make maxLevel configurable
//   - Sets.

const p = 0.5
const maxLevel = 32 // maybe this should be configurable

type node struct {
	forward    []*node
	key, value interface{}
}

func (n *node) Next() *node {
	return n.forward[0]
}

func (n node) HasNext() bool {
	return n.Next() != nil
}

func (n *node) Key() interface{} {
	return n.key
}

func (n *node) Value() interface{} {
	return n.value
}

type SkipList struct {
	lessThan func(l, r interface{}) bool
	header   *node
}

// Len returns the length of s.
func (s *SkipList) Len() (i int) {
	for node := s.Front(); node != nil; node = node.Next() {
		i++
	}
	return
}

// Front returns the first element of s in a way suitable for
// iteration.
func (s *SkipList) Front() *node {
	return s.header.Next()
}

func (s *SkipList) level() int {
	return len(s.header.forward) - 1
}

func (s *SkipList) LessThan(l, r interface{}) bool {
	// nil is the maximum
	if l == nil {
		return false
	}
	return s.lessThan(l, r)
}

// Returns a new random level.
func (s SkipList) randomLevel() (n int) {
	for n = 0; n < maxLevel && rand.Float64() < p; n++ {
	}
	return
}

// Get returns the value associated with key from s (nil if the key is
// not present in s). The second return value is true when the key is
// present.
func (s *SkipList) Get(key interface{}) (value interface{}, present bool) {
	candidate := s.getPath(nil, key)

	if candidate != nil && candidate.key == key {
		return candidate.value, true
	}

	return nil, false
}

// getPath populates update with nodes that constitute the path to the
// node that may contain key. The candidate node will be returned. If
// update is nil, it will be left alone (the candidate node will still
// be returned). If update is not nil, but it doesn't have enough
// slots for all the nodes in the path, getPath will panic.
func (s *SkipList) getPath(update []*node, key interface{}) *node {
	current := s.header
	for i := s.level(); i >= 0; i-- {
		for current.forward[i] != nil && s.LessThan(current.forward[i].key, key) {
			current = current.forward[i]
		}
		if update != nil {
			update[i] = current
		}
	}
	return current.Next()
}

// Sets set the value associated with key in s.
func (s *SkipList) Set(key, value interface{}) {

	// s.level starts from 0, so we need to allocate one
	update := make([]*node, s.level()+1, maxLevel)
	candidate := s.getPath(update, key)

	if candidate != nil && candidate.key == key {
		candidate.value = value
		return
	}

	newLevel := s.randomLevel()

	if currentLevel := s.level(); newLevel > currentLevel {
		// there are no pointers for the higher levels in
		// update. Header should be there. Also add higher
		// level links to the header.
		for i := currentLevel + 1; i <= newLevel; i++ {
			update = append(update, s.header)
			s.header.forward = append(s.header.forward, nil)
		}
	}

	newNode := &node{make([]*node, newLevel+1, maxLevel), key, value}

	for i := 0; i <= newLevel; i++ {
		newNode.forward[i] = update[i].forward[i]
		update[i].forward[i] = newNode
	}
}

// Delete removes node with key key from s and returns its value. A
// second value, a boolean, is return to indicate if key was present
// in s.
func (s *SkipList) Delete(key interface{}) (value interface{}, present bool) {
	update := make([]*node, s.level()+1, maxLevel)
	candidate := s.getPath(update, key)

	if candidate.key != key {
		return nil, false
	}

	for i := 0; i <= s.level() && update[i].forward[i] == candidate; i++ {
		update[i].forward[i] = candidate.forward[i]
	}

	for s.level() > 0 && s.header.forward[s.level()] == nil {
		s.header.forward = s.header.forward[:s.level()-1]
	}

	return candidate.Value(), true
}

// New returns a new SkipList that will use lessThan as the comparison
// function. lessThan should be linear order on keys you intend to use
// with the SkipList.
func NewMap(lessThan func(l, r interface{}) bool) *SkipList {
	return &SkipList{lessThan, &node{[]*node{nil}, nil, nil}}
}

type Comparable interface {
	LessThan(Comparable) bool
}

// NewComparable returns a SkipList that accepts skiplist.Comparable
// objects as keys.
func NewComparableMap() (s *SkipList) {
	comparator := func(left, right interface{}) bool {
		return left.(Comparable).LessThan(right.(Comparable))
	}
	return NewMap(comparator)

}

// NewIntKey returns a SkipList that accepts int keys.
func NewIntMap() *SkipList {
	return NewMap(func(l, r interface{}) bool {
		return l.(int) < r.(int)
	})
}

// NewStringMap returns a SkipList accepting strings as keys.
func NewStringMap() *SkipList {
	return NewMap(func(l, r interface{}) bool {
		return l.(string) < r.(string)
	})
}
<commit_msg>HasNext takes a pointer.<commit_after>// Copyright 2012 Ric Szopa (Ryszard) <ryszard.szopa@gmail.com> All
// rights reserved.  Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

// Package skiplist implements skip list based maps and sets. For more
// information about skip lists take a look at
// http://en.wikipedia.org/wiki/Skip_list.

package skiplist

import "math/rand"

// TODO(ryszard):
//   - A separately seeded source of randomness
//   - Make maxLevel configurable
//   - Sets.

const p = 0.5
const maxLevel = 32 // maybe this should be configurable

type node struct {
	forward    []*node
	key, value interface{}
}

func (n *node) Next() *node {
	return n.forward[0]
}

func (n *node) HasNext() bool {
	return n.Next() != nil
}

func (n *node) Key() interface{} {
	return n.key
}

func (n *node) Value() interface{} {
	return n.value
}

type SkipList struct {
	lessThan func(l, r interface{}) bool
	header   *node
}

// Len returns the length of s.
func (s *SkipList) Len() (i int) {
	for node := s.Front(); node != nil; node = node.Next() {
		i++
	}
	return
}

// Front returns the first element of s in a way suitable for
// iteration.
func (s *SkipList) Front() *node {
	return s.header.Next()
}

func (s *SkipList) level() int {
	return len(s.header.forward) - 1
}

func (s *SkipList) LessThan(l, r interface{}) bool {
	// nil is the maximum
	if l == nil {
		return false
	}
	return s.lessThan(l, r)
}

// Returns a new random level.
func (s SkipList) randomLevel() (n int) {
	for n = 0; n < maxLevel && rand.Float64() < p; n++ {
	}
	return
}

// Get returns the value associated with key from s (nil if the key is
// not present in s). The second return value is true when the key is
// present.
func (s *SkipList) Get(key interface{}) (value interface{}, present bool) {
	candidate := s.getPath(nil, key)

	if candidate != nil && candidate.key == key {
		return candidate.value, true
	}

	return nil, false
}

// getPath populates update with nodes that constitute the path to the
// node that may contain key. The candidate node will be returned. If
// update is nil, it will be left alone (the candidate node will still
// be returned). If update is not nil, but it doesn't have enough
// slots for all the nodes in the path, getPath will panic.
func (s *SkipList) getPath(update []*node, key interface{}) *node {
	current := s.header
	for i := s.level(); i >= 0; i-- {
		for current.forward[i] != nil && s.LessThan(current.forward[i].key, key) {
			current = current.forward[i]
		}
		if update != nil {
			update[i] = current
		}
	}
	return current.Next()
}

// Sets set the value associated with key in s.
func (s *SkipList) Set(key, value interface{}) {

	// s.level starts from 0, so we need to allocate one
	update := make([]*node, s.level()+1, maxLevel)
	candidate := s.getPath(update, key)

	if candidate != nil && candidate.key == key {
		candidate.value = value
		return
	}

	newLevel := s.randomLevel()

	if currentLevel := s.level(); newLevel > currentLevel {
		// there are no pointers for the higher levels in
		// update. Header should be there. Also add higher
		// level links to the header.
		for i := currentLevel + 1; i <= newLevel; i++ {
			update = append(update, s.header)
			s.header.forward = append(s.header.forward, nil)
		}
	}

	newNode := &node{make([]*node, newLevel+1, maxLevel), key, value}

	for i := 0; i <= newLevel; i++ {
		newNode.forward[i] = update[i].forward[i]
		update[i].forward[i] = newNode
	}
}

// Delete removes node with key key from s and returns its value. A
// second value, a boolean, is return to indicate if key was present
// in s.
func (s *SkipList) Delete(key interface{}) (value interface{}, present bool) {
	update := make([]*node, s.level()+1, maxLevel)
	candidate := s.getPath(update, key)

	if candidate.key != key {
		return nil, false
	}

	for i := 0; i <= s.level() && update[i].forward[i] == candidate; i++ {
		update[i].forward[i] = candidate.forward[i]
	}

	for s.level() > 0 && s.header.forward[s.level()] == nil {
		s.header.forward = s.header.forward[:s.level()-1]
	}

	return candidate.Value(), true
}

// New returns a new SkipList that will use lessThan as the comparison
// function. lessThan should be linear order on keys you intend to use
// with the SkipList.
func NewMap(lessThan func(l, r interface{}) bool) *SkipList {
	return &SkipList{lessThan, &node{[]*node{nil}, nil, nil}}
}

type Comparable interface {
	LessThan(Comparable) bool
}

// NewComparable returns a SkipList that accepts skiplist.Comparable
// objects as keys.
func NewComparableMap() (s *SkipList) {
	comparator := func(left, right interface{}) bool {
		return left.(Comparable).LessThan(right.(Comparable))
	}
	return NewMap(comparator)

}

// NewIntKey returns a SkipList that accepts int keys.
func NewIntMap() *SkipList {
	return NewMap(func(l, r interface{}) bool {
		return l.(int) < r.(int)
	})
}

// NewStringMap returns a SkipList accepting strings as keys.
func NewStringMap() *SkipList {
	return NewMap(func(l, r interface{}) bool {
		return l.(string) < r.(string)
	})
}
<|endoftext|>


<commit_before>package slack

import (
	"crypto/sha256"
	"encoding/hex"
)

type ID interface {
	String() string
	Equals(ID) bool
	Valid() bool
}

type UserID struct {
	UserID string
	TeamID string
}

// SecureID is an opaque, deterministic representation of a Slack user identity
// which can be used in place of UserID to minimize the consequences
// of accidentally logging or otherwise compromising a user's ID.
//
// A SecureID can be constructed from a UserID
// by calling UserID.Secure()
type SecureID struct {
	HashSum string
}

// Equals indicates if id and o represents the same user identity.
func (id UserID) Equals(o ID) bool {
	switch o := o.(type) {
	case UserID:
		if !(id.Valid() && o.Valid()) {
			return false
		}

		// Notice that user IDs are not guaranteed to be globally unique across all Slack users.
		// The combination of user ID and team ID, on the other hand, is guaranteed to be globally unique.
		//
		// - Slack API documentation
		return id.UserID == o.UserID && id.TeamID == o.TeamID
	case SecureID:
		return id.Secure().Equals(o)
	}

	return false
}

func (id UserID) Valid() bool {
	return id.UserID != "" && id.TeamID != ""
}

func (id UserID) String() string {
	return id.TeamID + "." + id.UserID
}

// Secure converts id into a SecureID.
func (id UserID) Secure() SecureID {
	if !id.Valid() {
		return SecureID{}
	}

	concat := id.TeamID + "." + id.UserID
	h := sha256.New()
	h.Write([]byte(concat))
	s := hex.EncodeToString(h.Sum(nil))

	return SecureID{
		HashSum: s,
	}
}

// Equals indicates if id and o represents the same user identity.
func (id SecureID) Equals(o ID) bool {
	switch o := o.(type) {
	case SecureID:
		if !(id.Valid() && o.Valid()) {
			return false
		}

		return id.HashSum == o.HashSum
	case UserID:
		return o.Secure().Equals(id)
	}
	return false
}

func (id SecureID) Valid() bool {
	return id.HashSum != ""
}

func (id SecureID) String() string {
	return id.HashSum
}
<commit_msg>Optimize SecureID Equals() validity check<commit_after>package slack

import (
	"crypto/sha256"
	"encoding/hex"
)

type ID interface {
	String() string
	Equals(ID) bool
	Valid() bool
}

type UserID struct {
	UserID string
	TeamID string
}

// SecureID is an opaque, deterministic representation of a Slack user identity
// which can be used in place of UserID to minimize the consequences
// of accidentally logging or otherwise compromising a user's ID.
//
// A SecureID can be constructed from a UserID
// by calling UserID.Secure()
type SecureID struct {
	HashSum string
}

// Equals indicates if id and o represents the same user identity.
func (id UserID) Equals(o ID) bool {
	switch o := o.(type) {
	case UserID:
		if !(id.Valid() && o.Valid()) {
			return false
		}

		// Notice that user IDs are not guaranteed to be globally unique across all Slack users.
		// The combination of user ID and team ID, on the other hand, is guaranteed to be globally unique.
		//
		// - Slack API documentation
		return id.UserID == o.UserID && id.TeamID == o.TeamID
	case SecureID:
		return id.Secure().Equals(o)
	}

	return false
}

func (id UserID) Valid() bool {
	return id.UserID != "" && id.TeamID != ""
}

func (id UserID) String() string {
	return id.TeamID + "." + id.UserID
}

// Secure converts id into a SecureID.
func (id UserID) Secure() SecureID {
	if !id.Valid() {
		return SecureID{}
	}

	concat := id.TeamID + "." + id.UserID
	h := sha256.New()
	h.Write([]byte(concat))
	s := hex.EncodeToString(h.Sum(nil))

	return SecureID{
		HashSum: s,
	}
}

// Equals indicates if id and o represents the same user identity.
func (id SecureID) Equals(o ID) bool {
	switch o := o.(type) {
	case SecureID:
		if !id.Valid() || !o.Valid() {
			return false
		}

		return id.HashSum == o.HashSum
	case UserID:
		return o.Secure().Equals(id)
	}
	return false
}

func (id SecureID) Valid() bool {
	return id.HashSum != ""
}

func (id SecureID) String() string {
	return id.HashSum
}
<|endoftext|>


<commit_before>// Package api contains Go structs for all LXD API objects
//
// Overview
//
// This package has Go structs for every API object, all the various
// structs are named after the object they represent and some variations of
// those structs exist for initial object creation, object update and
// object retrieval.
//
// A few convenience functions are also tied to those structs which let
// you convert between the various strucs for a given object and also query
// some of the more complex metadata that LXD can export.
package api
<commit_msg>shared/api: Update to new godoc comment<commit_after>// Package api contains Go structs for all LXD API objects
//
// # Overview
//
// This package has Go structs for every API object, all the various
// structs are named after the object they represent and some variations of
// those structs exist for initial object creation, object update and
// object retrieval.
//
// A few convenience functions are also tied to those structs which let
// you convert between the various strucs for a given object and also query
// some of the more complex metadata that LXD can export.
package api
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package grumpy

import (
	"fmt"
	"os"
	"reflect"
	"runtime/pprof"
	"strings"
	"sync"
)

type moduleState int

const (
	moduleStateNew moduleState = iota
	moduleStateInitializing
	moduleStateReady
)

var (
	importMutex sync.Mutex
	// ModuleType is the object representing the Python 'module' type.
	ModuleType = newBasisType("module", reflect.TypeOf(Module{}), toModuleUnsafe, ObjectType)
	// SysModules is the global dict of imported modules, aka sys.modules.
	SysModules = NewDict()
)

// Module represents Python 'module' objects.
type Module struct {
	Object
	mutex recursiveMutex
	state moduleState
}

// ModuleInit functions are called when importing Grumpy modules to execute the
// top level code for that module.
type ModuleInit func(f *Frame, m *Module) *BaseException

// ImportModule takes a fully qualified module name (e.g. a.b.c) and a slice of
// code objects where the name of the i'th module is the prefix of name
// ending in the i'th dot. The number of dot delimited parts of name must be the
// same as the number of code objects. For each successive prefix, ImportModule
// looks in sys.modules for an existing module with that name and if not
// present creates a new module object, adds it to sys.modules and initializes
// it with the corresponding code object. If the module was already present in
// sys.modules, it is not re-initialized. The returned slice contains each
// package and module initialized in this way in order.
//
// For example, ImportModule(f, "a.b", []*Code{a.Code, b.Code})
// causes the initialization and entry into sys.modules of Grumpy module a and
// then Grumpy module b. The two initialized modules are returned.
//
// If ImportModule is called in two threads concurrently to import the same
// module, both invocations will produce the same module object and the module
// is guaranteed to only be initialized once. The second invocation will not
// return the module until it is fully initialized.
func ImportModule(f *Frame, name string, codeObjs []*Code) ([]*Object, *BaseException) {
	parts := strings.Split(name, ".")
	numParts := len(parts)
	if numParts != len(codeObjs) {
		return nil, f.RaiseType(SystemErrorType, fmt.Sprintf("invalid import: %s", name))
	}
	result := make([]*Object, numParts)
	var prev *Object
	for i := 0; i < numParts; i++ {
		name := strings.Join(parts[:i+1], ".")
		// We do very limited locking here resulting in some
		// sys.modules consistency gotchas.
		importMutex.Lock()
		o, raised := SysModules.GetItemString(f, name)
		if raised == nil && o == nil {
			o = newModule(name, codeObjs[i].filename).ToObject()
			raised = SysModules.SetItemString(f, name, o)
		}
		importMutex.Unlock()
		if raised != nil {
			return nil, raised
		}
		if o.isInstance(ModuleType) {
			var raised *BaseException
			m := toModuleUnsafe(o)
			m.mutex.Lock(f)
			if m.state == moduleStateNew {
				m.state = moduleStateInitializing
				if _, raised = codeObjs[i].Eval(f, m.Dict(), nil, nil); raised == nil {
					m.state = moduleStateReady
				} else {
					// If the module failed to initialize
					// then before we relinquish the module
					// lock, remove it from sys.modules.
					// Threads waiting on this module will
					// fail when they don't find it in
					// sys.modules below.
					e, tb := f.ExcInfo()
					if _, raised := SysModules.DelItemString(f, name); raised != nil {
						f.RestoreExc(e, tb)
					}
				}
			}
			m.mutex.Unlock(f)
			if raised != nil {
				return nil, raised
			}
			// The result should be what's in sys.modules, not
			// necessarily the originally created module since this
			// is CPython's behavior.
			o, raised = SysModules.GetItemString(f, name)
			if raised != nil {
				return nil, raised
			}
			if o == nil {
				// This can happen in the pathological case
				// where the module clears itself from
				// sys.modules during execution and is handled
				// by CPython in PyImport_ExecCodeModuleEx in
				// import.c.
				format := "Loaded module %s not found in sys.modules"
				return nil, f.RaiseType(ImportErrorType, fmt.Sprintf(format, name))
			}
		}
		if prev != nil {
			if raised := SetAttr(f, prev, NewStr(parts[i]), o); raised != nil {
				return nil, raised
			}
		}
		result[i] = o
		prev = o
	}
	return result, nil
}

// ImportNativeModule takes a fully qualified native module name (e.g.
// grumpy.native.fmt) and a mapping of module members that will be used to
// populate the module. The same logic is used as ImportModule for looking in
// sys.modules first. The last module created in this way is populated with the
// given members and returned.
func ImportNativeModule(f *Frame, name string, members map[string]*Object) (*Object, *BaseException) {
	parts := strings.Split(name, ".")
	numParts := len(parts)
	var prev *Object
	for i := 0; i < numParts; i++ {
		name := strings.Join(parts[:i+1], ".")
		importMutex.Lock()
		o, raised := SysModules.GetItemString(f, name)
		if raised == nil && o == nil {
			o = newModule(name, "<native>").ToObject()
			raised = SysModules.SetItemString(f, name, o)
		}
		importMutex.Unlock()
		if raised != nil {
			return nil, raised
		}
		if prev != nil {
			if raised := SetAttr(f, prev, NewStr(parts[i]), o); raised != nil {
				return nil, raised
			}
		}
		prev = o
	}
	for k, v := range members {
		if raised := SetAttr(f, prev, NewStr(k), v); raised != nil {
			return nil, raised
		}
	}
	return prev, nil
}

// LoadMembers scans over all the members in module
// and populates globals with them, taking __all__ into
// account.
func LoadMembers(f *Frame, module *Object) *BaseException {
	allAttr, raised := GetAttr(f, module, NewStr("__all__"), nil)
	if raised != nil && !raised.isInstance(AttributeErrorType) {
		return raised
	}
	f.RestoreExc(nil, nil)

	if raised == nil {
		raised = loadMembersFromIterable(f, module, allAttr, nil)
		if raised != nil {
			return raised
		}
		return nil
	}

	// Fall back on __dict__
	dictAttr := module.dict.ToObject()
	raised = loadMembersFromIterable(f, module, dictAttr, func(key *Object) bool {
		return strings.HasPrefix(toStrUnsafe(key).value, "_")
	})
	if raised != nil {
		return raised
	}
	return nil
}

func loadMembersFromIterable(f *Frame, module, iterable *Object, filterF func(*Object) bool) *BaseException {
	globals := f.Globals()
	raised := seqForEach(f, iterable, func(memberName *Object) *BaseException {
		if !memberName.isInstance(StrType) {
			error_message := fmt.Sprintf("attribute name must be string, not '%v'", memberName.typ.Name())
			return f.RaiseType(AttributeErrorType, error_message)
		}
		member, raised := GetAttr(f, module, toStrUnsafe(memberName), nil)
		if raised != nil {
			return raised
		}
		if filterF != nil && filterF(memberName) {
			return nil
		}
		raised = globals.SetItem(f, memberName, member)
		if raised != nil {
			return raised
		}
		return nil
	})
	return raised
}

// newModule creates a new Module object with the given fully qualified name
// (e.g a.b.c) and its corresponding Python filename.
func newModule(name, filename string) *Module {
	d := newStringDict(map[string]*Object{
		"__file__": NewStr(filename).ToObject(),
		"__name__": NewStr(name).ToObject(),
	})
	return &Module{Object: Object{typ: ModuleType, dict: d}}
}

func toModuleUnsafe(o *Object) *Module {
	return (*Module)(o.toPointer())
}

// GetFilename returns the __file__ attribute of m, raising SystemError if it
// does not exist.
func (m *Module) GetFilename(f *Frame) (*Str, *BaseException) {
	fileAttr, raised := GetAttr(f, m.ToObject(), NewStr("__file__"), None)
	if raised != nil {
		return nil, raised
	}
	if !fileAttr.isInstance(StrType) {
		return nil, f.RaiseType(SystemErrorType, "module filename missing")
	}
	return toStrUnsafe(fileAttr), nil
}

// GetName returns the __name__ attribute of m, raising SystemError if it does
// not exist.
func (m *Module) GetName(f *Frame) (*Str, *BaseException) {
	nameAttr, raised := GetAttr(f, m.ToObject(), NewStr("__name__"), None)
	if raised != nil {
		return nil, raised
	}
	if !nameAttr.isInstance(StrType) {
		return nil, f.RaiseType(SystemErrorType, "nameless module")
	}
	return toStrUnsafe(nameAttr), nil
}

// ToObject upcasts m to an Object.
func (m *Module) ToObject() *Object {
	return &m.Object
}

func moduleInit(f *Frame, o *Object, args Args, _ KWArgs) (*Object, *BaseException) {
	expectedTypes := []*Type{StrType, ObjectType}
	argc := len(args)
	if argc == 1 {
		expectedTypes = expectedTypes[:1]
	}
	if raised := checkFunctionArgs(f, "__init__", args, expectedTypes...); raised != nil {
		return nil, raised
	}
	if raised := SetAttr(f, o, NewStr("__name__"), args[0]); raised != nil {
		return nil, raised
	}
	if argc > 1 {
		if raised := SetAttr(f, o, NewStr("__doc__"), args[1]); raised != nil {
			return nil, raised
		}
	}
	return None, nil
}

func moduleRepr(f *Frame, o *Object) (*Object, *BaseException) {
	m := toModuleUnsafe(o)
	name := "?"
	nameAttr, raised := m.GetName(f)
	if raised == nil {
		name = nameAttr.Value()
	} else {
		f.RestoreExc(nil, nil)
	}
	file := "(built-in)"
	fileAttr, raised := m.GetFilename(f)
	if raised == nil {
		file = fmt.Sprintf("from '%s'", fileAttr.Value())
	} else {
		f.RestoreExc(nil, nil)
	}
	return NewStr(fmt.Sprintf("<module '%s' %s>", name, file)).ToObject(), nil
}

func initModuleType(map[string]*Object) {
	ModuleType.slots.Init = &initSlot{moduleInit}
	ModuleType.slots.Repr = &unaryOpSlot{moduleRepr}
}

// RunMain execs the given code object as a module under the name "__main__".
// It handles any exceptions raised during module execution. If no exceptions
// were raised then the return value is zero. If a SystemExit was raised then
// the return value depends on its code attribute: None -> zero, integer values
// are returned as-is. Other code values and exception types produce a return
// value of 1.
func RunMain(code *Code) int {
	if file := os.Getenv("GRUMPY_PROFILE"); file != "" {
		f, err := os.Create(file)
		if err != nil {
			logFatal(err.Error())
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			logFatal(err.Error())
		}
		defer pprof.StopCPUProfile()
	}
	m := newModule("__main__", code.filename)
	m.state = moduleStateInitializing
	f := NewRootFrame()
	if raised := SysModules.SetItemString(f, "__main__", m.ToObject()); raised != nil {
		Stderr.writeString(raised.String())
	}
	_, e := code.Eval(f, m.Dict(), nil, nil)
	if e == nil {
		return 0
	}
	if !e.isInstance(SystemExitType) {
		s, raised := FormatException(f, e)
		if raised != nil {
			s = e.String()
		}
		Stderr.writeString(s)
		return 1
	}
	f.RestoreExc(nil, nil)
	o, raised := GetAttr(f, e.ToObject(), NewStr("code"), nil)
	if raised != nil {
		return 1
	}
	if o.isInstance(IntType) {
		return toIntUnsafe(o).Value()
	}
	if o == None {
		return 0
	}
	if s, raised := ToStr(f, o); raised == nil {
		Stderr.writeString(s.Value() + "\n")
	}
	return 1
}
<commit_msg>Remove underscore from variable name<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package grumpy

import (
	"fmt"
	"os"
	"reflect"
	"runtime/pprof"
	"strings"
	"sync"
)

type moduleState int

const (
	moduleStateNew moduleState = iota
	moduleStateInitializing
	moduleStateReady
)

var (
	importMutex sync.Mutex
	// ModuleType is the object representing the Python 'module' type.
	ModuleType = newBasisType("module", reflect.TypeOf(Module{}), toModuleUnsafe, ObjectType)
	// SysModules is the global dict of imported modules, aka sys.modules.
	SysModules = NewDict()
)

// Module represents Python 'module' objects.
type Module struct {
	Object
	mutex recursiveMutex
	state moduleState
}

// ModuleInit functions are called when importing Grumpy modules to execute the
// top level code for that module.
type ModuleInit func(f *Frame, m *Module) *BaseException

// ImportModule takes a fully qualified module name (e.g. a.b.c) and a slice of
// code objects where the name of the i'th module is the prefix of name
// ending in the i'th dot. The number of dot delimited parts of name must be the
// same as the number of code objects. For each successive prefix, ImportModule
// looks in sys.modules for an existing module with that name and if not
// present creates a new module object, adds it to sys.modules and initializes
// it with the corresponding code object. If the module was already present in
// sys.modules, it is not re-initialized. The returned slice contains each
// package and module initialized in this way in order.
//
// For example, ImportModule(f, "a.b", []*Code{a.Code, b.Code})
// causes the initialization and entry into sys.modules of Grumpy module a and
// then Grumpy module b. The two initialized modules are returned.
//
// If ImportModule is called in two threads concurrently to import the same
// module, both invocations will produce the same module object and the module
// is guaranteed to only be initialized once. The second invocation will not
// return the module until it is fully initialized.
func ImportModule(f *Frame, name string, codeObjs []*Code) ([]*Object, *BaseException) {
	parts := strings.Split(name, ".")
	numParts := len(parts)
	if numParts != len(codeObjs) {
		return nil, f.RaiseType(SystemErrorType, fmt.Sprintf("invalid import: %s", name))
	}
	result := make([]*Object, numParts)
	var prev *Object
	for i := 0; i < numParts; i++ {
		name := strings.Join(parts[:i+1], ".")
		// We do very limited locking here resulting in some
		// sys.modules consistency gotchas.
		importMutex.Lock()
		o, raised := SysModules.GetItemString(f, name)
		if raised == nil && o == nil {
			o = newModule(name, codeObjs[i].filename).ToObject()
			raised = SysModules.SetItemString(f, name, o)
		}
		importMutex.Unlock()
		if raised != nil {
			return nil, raised
		}
		if o.isInstance(ModuleType) {
			var raised *BaseException
			m := toModuleUnsafe(o)
			m.mutex.Lock(f)
			if m.state == moduleStateNew {
				m.state = moduleStateInitializing
				if _, raised = codeObjs[i].Eval(f, m.Dict(), nil, nil); raised == nil {
					m.state = moduleStateReady
				} else {
					// If the module failed to initialize
					// then before we relinquish the module
					// lock, remove it from sys.modules.
					// Threads waiting on this module will
					// fail when they don't find it in
					// sys.modules below.
					e, tb := f.ExcInfo()
					if _, raised := SysModules.DelItemString(f, name); raised != nil {
						f.RestoreExc(e, tb)
					}
				}
			}
			m.mutex.Unlock(f)
			if raised != nil {
				return nil, raised
			}
			// The result should be what's in sys.modules, not
			// necessarily the originally created module since this
			// is CPython's behavior.
			o, raised = SysModules.GetItemString(f, name)
			if raised != nil {
				return nil, raised
			}
			if o == nil {
				// This can happen in the pathological case
				// where the module clears itself from
				// sys.modules during execution and is handled
				// by CPython in PyImport_ExecCodeModuleEx in
				// import.c.
				format := "Loaded module %s not found in sys.modules"
				return nil, f.RaiseType(ImportErrorType, fmt.Sprintf(format, name))
			}
		}
		if prev != nil {
			if raised := SetAttr(f, prev, NewStr(parts[i]), o); raised != nil {
				return nil, raised
			}
		}
		result[i] = o
		prev = o
	}
	return result, nil
}

// ImportNativeModule takes a fully qualified native module name (e.g.
// grumpy.native.fmt) and a mapping of module members that will be used to
// populate the module. The same logic is used as ImportModule for looking in
// sys.modules first. The last module created in this way is populated with the
// given members and returned.
func ImportNativeModule(f *Frame, name string, members map[string]*Object) (*Object, *BaseException) {
	parts := strings.Split(name, ".")
	numParts := len(parts)
	var prev *Object
	for i := 0; i < numParts; i++ {
		name := strings.Join(parts[:i+1], ".")
		importMutex.Lock()
		o, raised := SysModules.GetItemString(f, name)
		if raised == nil && o == nil {
			o = newModule(name, "<native>").ToObject()
			raised = SysModules.SetItemString(f, name, o)
		}
		importMutex.Unlock()
		if raised != nil {
			return nil, raised
		}
		if prev != nil {
			if raised := SetAttr(f, prev, NewStr(parts[i]), o); raised != nil {
				return nil, raised
			}
		}
		prev = o
	}
	for k, v := range members {
		if raised := SetAttr(f, prev, NewStr(k), v); raised != nil {
			return nil, raised
		}
	}
	return prev, nil
}

// LoadMembers scans over all the members in module
// and populates globals with them, taking __all__ into
// account.
func LoadMembers(f *Frame, module *Object) *BaseException {
	allAttr, raised := GetAttr(f, module, NewStr("__all__"), nil)
	if raised != nil && !raised.isInstance(AttributeErrorType) {
		return raised
	}
	f.RestoreExc(nil, nil)

	if raised == nil {
		raised = loadMembersFromIterable(f, module, allAttr, nil)
		if raised != nil {
			return raised
		}
		return nil
	}

	// Fall back on __dict__
	dictAttr := module.dict.ToObject()
	raised = loadMembersFromIterable(f, module, dictAttr, func(key *Object) bool {
		return strings.HasPrefix(toStrUnsafe(key).value, "_")
	})
	if raised != nil {
		return raised
	}
	return nil
}

func loadMembersFromIterable(f *Frame, module, iterable *Object, filterF func(*Object) bool) *BaseException {
	globals := f.Globals()
	raised := seqForEach(f, iterable, func(memberName *Object) *BaseException {
		if !memberName.isInstance(StrType) {
			errorMessage := fmt.Sprintf("attribute name must be string, not '%v'", memberName.typ.Name())
			return f.RaiseType(AttributeErrorType, errorMessage)
		}
		member, raised := GetAttr(f, module, toStrUnsafe(memberName), nil)
		if raised != nil {
			return raised
		}
		if filterF != nil && filterF(memberName) {
			return nil
		}
		raised = globals.SetItem(f, memberName, member)
		if raised != nil {
			return raised
		}
		return nil
	})
	return raised
}

// newModule creates a new Module object with the given fully qualified name
// (e.g a.b.c) and its corresponding Python filename.
func newModule(name, filename string) *Module {
	d := newStringDict(map[string]*Object{
		"__file__": NewStr(filename).ToObject(),
		"__name__": NewStr(name).ToObject(),
	})
	return &Module{Object: Object{typ: ModuleType, dict: d}}
}

func toModuleUnsafe(o *Object) *Module {
	return (*Module)(o.toPointer())
}

// GetFilename returns the __file__ attribute of m, raising SystemError if it
// does not exist.
func (m *Module) GetFilename(f *Frame) (*Str, *BaseException) {
	fileAttr, raised := GetAttr(f, m.ToObject(), NewStr("__file__"), None)
	if raised != nil {
		return nil, raised
	}
	if !fileAttr.isInstance(StrType) {
		return nil, f.RaiseType(SystemErrorType, "module filename missing")
	}
	return toStrUnsafe(fileAttr), nil
}

// GetName returns the __name__ attribute of m, raising SystemError if it does
// not exist.
func (m *Module) GetName(f *Frame) (*Str, *BaseException) {
	nameAttr, raised := GetAttr(f, m.ToObject(), NewStr("__name__"), None)
	if raised != nil {
		return nil, raised
	}
	if !nameAttr.isInstance(StrType) {
		return nil, f.RaiseType(SystemErrorType, "nameless module")
	}
	return toStrUnsafe(nameAttr), nil
}

// ToObject upcasts m to an Object.
func (m *Module) ToObject() *Object {
	return &m.Object
}

func moduleInit(f *Frame, o *Object, args Args, _ KWArgs) (*Object, *BaseException) {
	expectedTypes := []*Type{StrType, ObjectType}
	argc := len(args)
	if argc == 1 {
		expectedTypes = expectedTypes[:1]
	}
	if raised := checkFunctionArgs(f, "__init__", args, expectedTypes...); raised != nil {
		return nil, raised
	}
	if raised := SetAttr(f, o, NewStr("__name__"), args[0]); raised != nil {
		return nil, raised
	}
	if argc > 1 {
		if raised := SetAttr(f, o, NewStr("__doc__"), args[1]); raised != nil {
			return nil, raised
		}
	}
	return None, nil
}

func moduleRepr(f *Frame, o *Object) (*Object, *BaseException) {
	m := toModuleUnsafe(o)
	name := "?"
	nameAttr, raised := m.GetName(f)
	if raised == nil {
		name = nameAttr.Value()
	} else {
		f.RestoreExc(nil, nil)
	}
	file := "(built-in)"
	fileAttr, raised := m.GetFilename(f)
	if raised == nil {
		file = fmt.Sprintf("from '%s'", fileAttr.Value())
	} else {
		f.RestoreExc(nil, nil)
	}
	return NewStr(fmt.Sprintf("<module '%s' %s>", name, file)).ToObject(), nil
}

func initModuleType(map[string]*Object) {
	ModuleType.slots.Init = &initSlot{moduleInit}
	ModuleType.slots.Repr = &unaryOpSlot{moduleRepr}
}

// RunMain execs the given code object as a module under the name "__main__".
// It handles any exceptions raised during module execution. If no exceptions
// were raised then the return value is zero. If a SystemExit was raised then
// the return value depends on its code attribute: None -> zero, integer values
// are returned as-is. Other code values and exception types produce a return
// value of 1.
func RunMain(code *Code) int {
	if file := os.Getenv("GRUMPY_PROFILE"); file != "" {
		f, err := os.Create(file)
		if err != nil {
			logFatal(err.Error())
		}
		if err := pprof.StartCPUProfile(f); err != nil {
			logFatal(err.Error())
		}
		defer pprof.StopCPUProfile()
	}
	m := newModule("__main__", code.filename)
	m.state = moduleStateInitializing
	f := NewRootFrame()
	if raised := SysModules.SetItemString(f, "__main__", m.ToObject()); raised != nil {
		Stderr.writeString(raised.String())
	}
	_, e := code.Eval(f, m.Dict(), nil, nil)
	if e == nil {
		return 0
	}
	if !e.isInstance(SystemExitType) {
		s, raised := FormatException(f, e)
		if raised != nil {
			s = e.String()
		}
		Stderr.writeString(s)
		return 1
	}
	f.RestoreExc(nil, nil)
	o, raised := GetAttr(f, e.ToObject(), NewStr("code"), nil)
	if raised != nil {
		return 1
	}
	if o.isInstance(IntType) {
		return toIntUnsafe(o).Value()
	}
	if o == None {
		return 0
	}
	if s, raised := ToStr(f, o); raised == nil {
		Stderr.writeString(s.Value() + "\n")
	}
	return 1
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"
	"sync"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/iterator"
	ldb_util "github.com/syndtr/goleveldb/leveldb/util"
)

// for Database
type DataAccess interface {
	Abort()
	Begin() error
	Commit() error
	Delete([]byte)
	DumpTx() []byte
	Get([]byte) ([]byte, error)
	Has([]byte) (bool, error)
	InUse() bool
	Iterator(*ldb_util.Range) iterator.Iterator
	Put([]byte, []byte)
}

type DataAccessImpl struct {
	sync.Mutex
	inUse bool
	db    *leveldb.DB
	batch *leveldb.Batch
	cache Cache
}

func newDA(db *leveldb.DB, trx *leveldb.Batch, cache Cache) DataAccess {
	return &DataAccessImpl{
		inUse: false,
		db:    db,
		batch: trx,
		cache: cache,
	}
}

func (d *DataAccessImpl) Begin() error {
	d.Lock()
	defer d.Unlock()

	if d.inUse {
		return fmt.Errorf("Error, batch already in use")
	}

	d.inUse = true
	return nil
}

func (d *DataAccessImpl) Put(key []byte, value []byte) {
	d.cache.Set(dbPut, string(key), value)
	d.batch.Put(key, value)
}

func (d *DataAccessImpl) Delete(key []byte) {
	d.cache.Set(dbDelete, string(key), []byte{})
	d.batch.Delete(key)
}

func (d *DataAccessImpl) Commit() error {
	err := d.db.Write(d.batch, nil)
	if nil != err {
		return err
	}
	return nil
}

func (d *DataAccessImpl) DumpTx() []byte {
	return d.batch.Dump()
}

func (d *DataAccessImpl) Get(key []byte) ([]byte, error) {
	val, found := d.getFromCache(key)
	if found {
		return val, nil
	}
	return d.getFromDB(key)
}

func (d *DataAccessImpl) getFromCache(key []byte) ([]byte, bool) {
	return d.cache.Get(string(key))
}

func (d *DataAccessImpl) getFromDB(key []byte) ([]byte, error) {
	return d.db.Get(key, nil)
}

func (d *DataAccessImpl) Iterator(searchRange *ldb_util.Range) iterator.Iterator {
	return d.db.NewIterator(searchRange, nil)
}

func (d *DataAccessImpl) Has(key []byte) (bool, error) {
	_, found := d.getFromCache(key)
	if found {
		return true, nil
	}
	return d.db.Has(key, nil)
}

func (d *DataAccessImpl) InUse() bool {
	return d.inUse
}

func (d *DataAccessImpl) Abort() {
	d.Lock()
	defer d.Unlock()

	d.batch.Reset()
	d.cache.Clear()
	d.inUse = false
}
<commit_msg>[storage] - Fixes error strings should not be capitalized (ST1005)<commit_after>package storage

import (
	"fmt"
	"sync"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/iterator"
	ldb_util "github.com/syndtr/goleveldb/leveldb/util"
)

// for Database
type DataAccess interface {
	Abort()
	Begin() error
	Commit() error
	Delete([]byte)
	DumpTx() []byte
	Get([]byte) ([]byte, error)
	Has([]byte) (bool, error)
	InUse() bool
	Iterator(*ldb_util.Range) iterator.Iterator
	Put([]byte, []byte)
}

type DataAccessImpl struct {
	sync.Mutex
	inUse bool
	db    *leveldb.DB
	batch *leveldb.Batch
	cache Cache
}

func newDA(db *leveldb.DB, trx *leveldb.Batch, cache Cache) DataAccess {
	return &DataAccessImpl{
		inUse: false,
		db:    db,
		batch: trx,
		cache: cache,
	}
}

func (d *DataAccessImpl) Begin() error {
	d.Lock()
	defer d.Unlock()

	if d.inUse {
		return fmt.Errorf("batch already in use")
	}

	d.inUse = true
	return nil
}

func (d *DataAccessImpl) Put(key []byte, value []byte) {
	d.cache.Set(dbPut, string(key), value)
	d.batch.Put(key, value)
}

func (d *DataAccessImpl) Delete(key []byte) {
	d.cache.Set(dbDelete, string(key), []byte{})
	d.batch.Delete(key)
}

func (d *DataAccessImpl) Commit() error {
	err := d.db.Write(d.batch, nil)
	if nil != err {
		return err
	}
	return nil
}

func (d *DataAccessImpl) DumpTx() []byte {
	return d.batch.Dump()
}

func (d *DataAccessImpl) Get(key []byte) ([]byte, error) {
	val, found := d.getFromCache(key)
	if found {
		return val, nil
	}
	return d.getFromDB(key)
}

func (d *DataAccessImpl) getFromCache(key []byte) ([]byte, bool) {
	return d.cache.Get(string(key))
}

func (d *DataAccessImpl) getFromDB(key []byte) ([]byte, error) {
	return d.db.Get(key, nil)
}

func (d *DataAccessImpl) Iterator(searchRange *ldb_util.Range) iterator.Iterator {
	return d.db.NewIterator(searchRange, nil)
}

func (d *DataAccessImpl) Has(key []byte) (bool, error) {
	_, found := d.getFromCache(key)
	if found {
		return true, nil
	}
	return d.db.Has(key, nil)
}

func (d *DataAccessImpl) InUse() bool {
	return d.inUse
}

func (d *DataAccessImpl) Abort() {
	d.Lock()
	defer d.Unlock()

	d.batch.Reset()
	d.cache.Clear()
	d.inUse = false
}
<|endoftext|>


<commit_before>// PromHouse
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package storages

import (
	"context"
	"database/sql"
	"fmt"
	"strings"

	_ "github.com/kshvakov/clickhouse" // register SQL driver
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/model"
	"github.com/sirupsen/logrus"

	"github.com/Percona-Lab/PromHouse/util"
)

// ClickHouse implements storage interface for the ClickHouse.
type ClickHouse struct {
	db       *sql.DB
	l        *logrus.Entry
	database string
	// labels   []string

	mSamplesCurrent prometheus.Gauge
	mLabels         *prometheus.CounterVec
}

func NewClickHouse(dsn string, database string, init bool) (*ClickHouse, error) {
	db, err := sql.Open("clickhouse", dsn)
	if err != nil {
		return nil, err
	}
	if err = db.Ping(); err != nil {
		return nil, err
	}

	var queries []string
	if init {
		queries = append(queries, fmt.Sprintf(`DROP DATABASE IF EXISTS %s`, database))
	}
	queries = append(queries, fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %s`, database))

	// TODO use GraphiteMergeTree?
	// TODO remove fingerprint?
	// TODO add __name__, instance, job as separate columns?
	// labels := []string{"instance", "job"}
	queries = append(queries, fmt.Sprintf(`
		CREATE TABLE IF NOT EXISTS %s.samples (
			__date__ Date,
			__labels__ String,
			__fingerprint__ UInt64,
			__ts__ Int64,
			__value__ Float64
		)
		ENGINE = MergeTree(__date__, (__fingerprint__, __ts__), 8192)`, database))

	for _, q := range queries {
		if _, err = db.Exec(q); err != nil {
			return nil, err
		}
	}

	return &ClickHouse{
		db:       db,
		l:        logrus.WithField("component", "clickhouse"),
		database: database,
		// labels:   labels,

		mSamplesCurrent: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: "promhouse",
			Subsystem: "clickhouse",
			Name:      "samples_current",
			Help:      "Current number of stored samples.",
		}),
		mLabels: prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: "promhouse",
			Subsystem: "clickhouse",
			Name:      "labels",
			Help:      "Written labels by name.",
		}, []string{"name"}),
	}, nil
}

func (ch *ClickHouse) Describe(c chan<- *prometheus.Desc) {
	ch.mSamplesCurrent.Describe(c)
	ch.mLabels.Describe(c)
}

func (ch *ClickHouse) Collect(c chan<- prometheus.Metric) {
	query := fmt.Sprintf("SELECT COUNT(*) FROM %s.samples", ch.database)
	var count uint64
	if err := ch.db.QueryRow(query).Scan(&count); err != nil {
		ch.l.Error(err)
		return
	}

	ch.mSamplesCurrent.Set(float64(count))

	ch.mSamplesCurrent.Collect(c)
	ch.mLabels.Collect(c)
}

func (ch *ClickHouse) Read(ctx context.Context, queries []Query) ([]model.Matrix, error) {
	res := make([]model.Matrix, len(queries))
	for i, q := range queries {
		query := fmt.Sprintf("SELECT __labels__, __fingerprint__, __ts__, __value__ FROM %s.samples WHERE __ts__ >= ? AND __ts__ <= ?", ch.database)
		args := []interface{}{int64(q.Start), int64(q.End)}
		for _, matcher := range q.Matchers {
			// if matcher.Name == model.MetricNameLabel {
			// 	query += fmt.Sprintf(" AND %s", matcher.Name)
			// } else {

			switch matcher.Type {
			case MatchEqual:
				query += fmt.Sprintf(" AND visitParamExtractString(__labels__, '%s') = ?", matcher.Name)
				args = append(args, matcher.Value)
			case MatchNotEqual:
				query += fmt.Sprintf(" AND visitParamExtractString(__labels__, '%s') != ?", matcher.Name)
				args = append(args, matcher.Value)
			case MatchRegexp:
				query += fmt.Sprintf(" AND match(visitParamExtractString(__labels__, '%s'), ?)", matcher.Name)
				args = append(args, "^(?:"+matcher.Value+")$")
			case MatchNotRegexp:
				query += fmt.Sprintf(" AND NOT match(visitParamExtractString(__labels__, '%s'), ?)", matcher.Name)
				args = append(args, "^(?:"+matcher.Value+")$")
			default:
				panic("unknown match type")
			}
		}
		query += " ORDER BY __fingerprint__, __ts__"

		rows, err := ch.db.Query(query, args...)
		if err != nil {
			return nil, err
		}

		var ss *model.SampleStream
		var b []byte
		var fingerprint, prevFingerprint uint64
		var ts int64
		var value float64
		for rows.Next() {
			if err = rows.Scan(&b, &fingerprint, &ts, &value); err != nil {
				return nil, err
			}
			if fingerprint != prevFingerprint {
				prevFingerprint = fingerprint
				if ss != nil {
					res[i] = append(res[i], ss)
				}

				var ls model.LabelSet
				if err = ls.UnmarshalJSON(b); err != nil {
					return nil, err
				}
				ss = &model.SampleStream{
					Metric: model.Metric(ls),
				}
			}
			ss.Values = append(ss.Values, model.SamplePair{
				Timestamp: model.Time(ts),
				Value:     model.SampleValue(value),
			})
		}
		if err = rows.Close(); err != nil {
			return nil, err
		}
		if ss != nil {
			res[i] = append(res[i], ss)
		}
	}

	return res, nil
}

func (ch *ClickHouse) Write(ctx context.Context, data model.Matrix) (err error) {
	var commited bool
	var tx *sql.Tx
	tx, err = ch.db.BeginTx(ctx, nil)
	if err != nil {
		return
	}
	defer func() {
		if !commited {
			e := tx.Rollback()
			if err == nil {
				err = e
			}
		}
	}()

	// columns := 4 + len(ch.labels)
	const columns = 5
	placeholders := strings.Repeat("?, ", columns)
	query := fmt.Sprintf(
		`INSERT INTO %s.samples (__labels__, __fingerprint__, __ts__, __value__, __date__) VALUES (%s)`,
		ch.database, placeholders[:len(placeholders)-2], // cut last ", "
	)

	var stmt *sql.Stmt
	if stmt, err = tx.PrepareContext(ctx, query); err != nil {
		return err
	}
	defer func() {
		e := stmt.Close()
		if err == nil {
			err = e
		}
	}()

	for _, ss := range data {
		args := make([]interface{}, columns)

		labels := make(map[string]string, len(ss.Metric))
		for n, v := range ss.Metric {
			// if n == model.MetricNameLabel {
			// 	args[0] = string(v)
			// } else {
			labels[string(n)] = string(v)
			ch.mLabels.WithLabelValues(string(n)).Inc()
		}
		args[0] = util.MarshalLabels(labels)
		args[1] = uint64(ss.Metric.Fingerprint())

		// for i, l := range ch.labels {
		// 	args[i+1] = string(ss.Metric[model.LabelName(l)])
		// }

		for _, sp := range ss.Values {
			args[len(args)-3] = int64(sp.Timestamp)
			args[len(args)-2] = float64(sp.Value)
			args[len(args)-1] = sp.Timestamp.Time()
			_, err = stmt.ExecContext(ctx, args...)
			if err != nil {
				return
			}
		}
	}

	if err = tx.Commit(); err != nil {
		return
	}
	commited = true
	return
}

// check interface
var _ Storage = (*ClickHouse)(nil)
<commit_msg>Fix typos.<commit_after>// PromHouse
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package storages

import (
	"context"
	"database/sql"
	"fmt"
	"strings"

	_ "github.com/kshvakov/clickhouse" // register SQL driver
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/model"
	"github.com/sirupsen/logrus"

	"github.com/Percona-Lab/PromHouse/util"
)

// ClickHouse implements storage interface for the ClickHouse.
type ClickHouse struct {
	db       *sql.DB
	l        *logrus.Entry
	database string
	// labels   []string

	mSamplesCurrent prometheus.Gauge
	mLabels         *prometheus.CounterVec
}

func NewClickHouse(dsn string, database string, init bool) (*ClickHouse, error) {
	db, err := sql.Open("clickhouse", dsn)
	if err != nil {
		return nil, err
	}
	if err = db.Ping(); err != nil {
		return nil, err
	}

	var queries []string
	if init {
		queries = append(queries, fmt.Sprintf(`DROP DATABASE IF EXISTS %s`, database))
	}
	queries = append(queries, fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %s`, database))

	// TODO use GraphiteMergeTree?
	// TODO remove fingerprint?
	// TODO add __name__, instance, job as separate columns?
	// labels := []string{"instance", "job"}
	queries = append(queries, fmt.Sprintf(`
		CREATE TABLE IF NOT EXISTS %s.samples (
			__date__ Date,
			__labels__ String,
			__fingerprint__ UInt64,
			__ts__ Int64,
			__value__ Float64
		)
		ENGINE = MergeTree(__date__, (__fingerprint__, __ts__), 8192)`, database))

	for _, q := range queries {
		if _, err = db.Exec(q); err != nil {
			return nil, err
		}
	}

	return &ClickHouse{
		db:       db,
		l:        logrus.WithField("component", "clickhouse"),
		database: database,
		// labels:   labels,

		mSamplesCurrent: prometheus.NewGauge(prometheus.GaugeOpts{
			Namespace: "promhouse",
			Subsystem: "clickhouse",
			Name:      "samples_current",
			Help:      "Current number of stored samples.",
		}),
		mLabels: prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: "promhouse",
			Subsystem: "clickhouse",
			Name:      "labels",
			Help:      "Written labels by name.",
		}, []string{"name"}),
	}, nil
}

func (ch *ClickHouse) Describe(c chan<- *prometheus.Desc) {
	ch.mSamplesCurrent.Describe(c)
	ch.mLabels.Describe(c)
}

func (ch *ClickHouse) Collect(c chan<- prometheus.Metric) {
	query := fmt.Sprintf("SELECT COUNT(*) FROM %s.samples", ch.database)
	var count uint64
	if err := ch.db.QueryRow(query).Scan(&count); err != nil {
		ch.l.Error(err)
		return
	}

	ch.mSamplesCurrent.Set(float64(count))

	ch.mSamplesCurrent.Collect(c)
	ch.mLabels.Collect(c)
}

func (ch *ClickHouse) Read(ctx context.Context, queries []Query) ([]model.Matrix, error) {
	res := make([]model.Matrix, len(queries))
	for i, q := range queries {
		query := fmt.Sprintf("SELECT __labels__, __fingerprint__, __ts__, __value__ FROM %s.samples WHERE __ts__ >= ? AND __ts__ <= ?", ch.database)
		args := []interface{}{int64(q.Start), int64(q.End)}
		for _, matcher := range q.Matchers {
			// if matcher.Name == model.MetricNameLabel {
			// 	query += fmt.Sprintf(" AND %s", matcher.Name)
			// } else {

			switch matcher.Type {
			case MatchEqual:
				query += fmt.Sprintf(" AND visitParamExtractString(__labels__, '%s') = ?", matcher.Name)
				args = append(args, matcher.Value)
			case MatchNotEqual:
				query += fmt.Sprintf(" AND visitParamExtractString(__labels__, '%s') != ?", matcher.Name)
				args = append(args, matcher.Value)
			case MatchRegexp:
				query += fmt.Sprintf(" AND match(visitParamExtractString(__labels__, '%s'), ?)", matcher.Name)
				args = append(args, "^(?:"+matcher.Value+")$")
			case MatchNotRegexp:
				query += fmt.Sprintf(" AND NOT match(visitParamExtractString(__labels__, '%s'), ?)", matcher.Name)
				args = append(args, "^(?:"+matcher.Value+")$")
			default:
				panic("unknown match type")
			}
		}
		query += " ORDER BY __fingerprint__, __ts__"

		rows, err := ch.db.Query(query, args...)
		if err != nil {
			return nil, err
		}

		var ss *model.SampleStream
		var b []byte
		var fingerprint, prevFingerprint uint64
		var ts int64
		var value float64
		for rows.Next() {
			if err = rows.Scan(&b, &fingerprint, &ts, &value); err != nil {
				return nil, err
			}
			if fingerprint != prevFingerprint {
				prevFingerprint = fingerprint
				if ss != nil {
					res[i] = append(res[i], ss)
				}

				var ls model.LabelSet
				if err = ls.UnmarshalJSON(b); err != nil {
					return nil, err
				}
				ss = &model.SampleStream{
					Metric: model.Metric(ls),
				}
			}
			ss.Values = append(ss.Values, model.SamplePair{
				Timestamp: model.Time(ts),
				Value:     model.SampleValue(value),
			})
		}
		if err = rows.Close(); err != nil {
			return nil, err
		}
		if ss != nil {
			res[i] = append(res[i], ss)
		}
	}

	return res, nil
}

func (ch *ClickHouse) Write(ctx context.Context, data model.Matrix) (err error) {
	var committed bool
	var tx *sql.Tx
	tx, err = ch.db.BeginTx(ctx, nil)
	if err != nil {
		return
	}
	defer func() {
		if !committed {
			e := tx.Rollback()
			if err == nil {
				err = e
			}
		}
	}()

	// columns := 4 + len(ch.labels)
	const columns = 5
	placeholders := strings.Repeat("?, ", columns)
	query := fmt.Sprintf(
		`INSERT INTO %s.samples (__labels__, __fingerprint__, __ts__, __value__, __date__) VALUES (%s)`,
		ch.database, placeholders[:len(placeholders)-2], // cut last ", "
	)

	var stmt *sql.Stmt
	if stmt, err = tx.PrepareContext(ctx, query); err != nil {
		return err
	}
	defer func() {
		e := stmt.Close()
		if err == nil {
			err = e
		}
	}()

	for _, ss := range data {
		args := make([]interface{}, columns)

		labels := make(map[string]string, len(ss.Metric))
		for n, v := range ss.Metric {
			// if n == model.MetricNameLabel {
			// 	args[0] = string(v)
			// } else {
			labels[string(n)] = string(v)
			ch.mLabels.WithLabelValues(string(n)).Inc()
		}
		args[0] = util.MarshalLabels(labels)
		args[1] = uint64(ss.Metric.Fingerprint())

		// for i, l := range ch.labels {
		// 	args[i+1] = string(ss.Metric[model.LabelName(l)])
		// }

		for _, sp := range ss.Values {
			args[len(args)-3] = int64(sp.Timestamp)
			args[len(args)-2] = float64(sp.Value)
			args[len(args)-1] = sp.Timestamp.Time()
			_, err = stmt.ExecContext(ctx, args...)
			if err != nil {
				return
			}
		}
	}

	if err = tx.Commit(); err != nil {
		return
	}
	committed = true
	return
}

// check interface
var _ Storage = (*ClickHouse)(nil)
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package surfacers is the base package for creating Surfacer objects that are
used for writing metics data to different monitoring services.

Any Surfacer that is created for writing metrics data to a monitor system
should implement the below Surfacer interface and should accept
metrics.EventMetrics object through a Write() call. Each new surfacer should
also plug itself in through the New() method defined here.
*/
package surfacers

import (
	"context"
	"fmt"
	"strings"
	"sync"

	"github.com/google/cloudprober/logger"
	"github.com/google/cloudprober/metrics"
	"github.com/google/cloudprober/surfacers/file"
	"github.com/google/cloudprober/surfacers/postgres"
	"github.com/google/cloudprober/surfacers/prometheus"
	"github.com/google/cloudprober/surfacers/stackdriver"

	surfacerpb "github.com/google/cloudprober/surfacers/proto"
)

var (
	userDefinedSurfacers   = make(map[string]Surfacer)
	userDefinedSurfacersMu sync.Mutex
)

// Default surfacers. These surfacers are enabled if no surfacer is defined.
var defaultSurfacers = []*surfacerpb.SurfacerDef{
	&surfacerpb.SurfacerDef{
		Type: surfacerpb.Type_PROMETHEUS.Enum(),
	},
	&surfacerpb.SurfacerDef{
		Type: surfacerpb.Type_FILE.Enum(),
	},
}

// initSurfacer initializes and returns a new surfacer based on the config.
func initSurfacer(s *surfacerpb.SurfacerDef) (Surfacer, interface{}, error) {
	// Create a new logger
	logName := s.GetName()
	if logName == "" {
		logName = strings.ToLower(s.GetType().String())
	}

	// TODO(manugarg): Plumb context here too.
	l, err := logger.New(context.TODO(), logName)
	if err != nil {
		return nil, nil, fmt.Errorf("unable to create cloud logger: %v", err)
	}

	var conf interface{}
	var surfacer Surfacer

	switch s.GetType() {
	case surfacerpb.Type_PROMETHEUS:
		surfacer, err = prometheus.New(s.GetPrometheusSurfacer(), l)
		conf = s.GetPrometheusSurfacer()
	case surfacerpb.Type_STACKDRIVER:
		surfacer, err = stackdriver.New(s.GetStackdriverSurfacer(), l)
		conf = s.GetStackdriverSurfacer()
	case surfacerpb.Type_FILE:
		surfacer, err = file.New(s.GetFileSurfacer(), l)
		conf = s.GetFileSurfacer()
	case surfacerpb.Type_POSTGRES:
		surfacer, err = postgres.New(s.GetPostgresSurfacer(), l)
		conf = s.GetPostgresSurfacer()
	case surfacerpb.Type_USER_DEFINED:
		userDefinedSurfacersMu.Lock()
		defer userDefinedSurfacersMu.Unlock()
		surfacer := userDefinedSurfacers[s.GetName()]
		if surfacer == nil {
			return nil, nil, fmt.Errorf("unregistered user defined surfacer: %s", s.GetName())
		}
	default:
		return nil, nil, fmt.Errorf("unknown surfacer type: %s", s.GetType())
	}

	return surfacer, conf, err
}

// Surfacer is an interface for all metrics surfacing systems
type Surfacer interface {
	// Function for writing a piece of metric data to a specified metric
	// store (or other location).
	Write(ctx context.Context, em *metrics.EventMetrics)
}

// SurfacerInfo encapsulates a Surfacer and related info.
type SurfacerInfo struct {
	Surfacer
	Type string
	Name string
	Conf string
}

// Init initializes the surfacers from the config protobufs and returns them as
// a list.
func Init(sDefs []*surfacerpb.SurfacerDef) ([]*SurfacerInfo, error) {
	// If no surfacers are defined, return default surfacers. This behavior
	// can be disabled by explicitly specifying "surfacer {}" in the config.
	if len(sDefs) == 0 {
		sDefs = defaultSurfacers
	}

	var result []*SurfacerInfo
	for _, sDef := range sDefs {
		if sDef.GetType() == surfacerpb.Type_NONE {
			continue
		}
		s, conf, err := initSurfacer(sDef)
		if err != nil {
			return nil, err
		}

		confStr := ""
		if conf != nil {
			if stringer, ok := conf.(fmt.Stringer); ok {
				confStr = stringer.String()
			}
		}

		result = append(result, &SurfacerInfo{
			Surfacer: s,
			Type:     sDef.GetType().String(),
			Name:     sDef.GetName(),
			Conf:     confStr,
		})
	}
	return result, nil
}

// Register allows you to register a user defined surfacer with cloudprober.
// Example usage:
//	import (
//		"github.com/google/cloudprober"
//		"github.com/google/cloudprober/surfacers"
//	)
//
//	s := &FancySurfacer{}
//	surfacers.Register("fancy_surfacer", s)
//	pr, err := cloudprober.InitFromConfig(*configFile)
//	if err != nil {
//		log.Exitf("Error initializing cloudprober. Err: %v", err)
//	}
func Register(name string, s Surfacer) {
	userDefinedSurfacersMu.Lock()
	defer userDefinedSurfacersMu.Unlock()
	userDefinedSurfacers[name] = s
}
<commit_msg>Fix a bug in surfacers when using an user-defined surfacer.<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package surfacers is the base package for creating Surfacer objects that are
used for writing metics data to different monitoring services.

Any Surfacer that is created for writing metrics data to a monitor system
should implement the below Surfacer interface and should accept
metrics.EventMetrics object through a Write() call. Each new surfacer should
also plug itself in through the New() method defined here.
*/
package surfacers

import (
	"context"
	"fmt"
	"strings"
	"sync"

	"github.com/google/cloudprober/logger"
	"github.com/google/cloudprober/metrics"
	"github.com/google/cloudprober/surfacers/file"
	"github.com/google/cloudprober/surfacers/postgres"
	"github.com/google/cloudprober/surfacers/prometheus"
	"github.com/google/cloudprober/surfacers/stackdriver"

	surfacerpb "github.com/google/cloudprober/surfacers/proto"
)

var (
	userDefinedSurfacers   = make(map[string]Surfacer)
	userDefinedSurfacersMu sync.Mutex
)

// Default surfacers. These surfacers are enabled if no surfacer is defined.
var defaultSurfacers = []*surfacerpb.SurfacerDef{
	&surfacerpb.SurfacerDef{
		Type: surfacerpb.Type_PROMETHEUS.Enum(),
	},
	&surfacerpb.SurfacerDef{
		Type: surfacerpb.Type_FILE.Enum(),
	},
}

// initSurfacer initializes and returns a new surfacer based on the config.
func initSurfacer(s *surfacerpb.SurfacerDef) (Surfacer, interface{}, error) {
	// Create a new logger
	logName := s.GetName()
	if logName == "" {
		logName = strings.ToLower(s.GetType().String())
	}

	// TODO(manugarg): Plumb context here too.
	l, err := logger.New(context.TODO(), logName)
	if err != nil {
		return nil, nil, fmt.Errorf("unable to create cloud logger: %v", err)
	}

	var conf interface{}
	var surfacer Surfacer

	switch s.GetType() {
	case surfacerpb.Type_PROMETHEUS:
		surfacer, err = prometheus.New(s.GetPrometheusSurfacer(), l)
		conf = s.GetPrometheusSurfacer()
	case surfacerpb.Type_STACKDRIVER:
		surfacer, err = stackdriver.New(s.GetStackdriverSurfacer(), l)
		conf = s.GetStackdriverSurfacer()
	case surfacerpb.Type_FILE:
		surfacer, err = file.New(s.GetFileSurfacer(), l)
		conf = s.GetFileSurfacer()
	case surfacerpb.Type_POSTGRES:
		surfacer, err = postgres.New(s.GetPostgresSurfacer(), l)
		conf = s.GetPostgresSurfacer()
	case surfacerpb.Type_USER_DEFINED:
		userDefinedSurfacersMu.Lock()
		defer userDefinedSurfacersMu.Unlock()
		surfacer = userDefinedSurfacers[s.GetName()]
		if surfacer == nil {
			return nil, nil, fmt.Errorf("unregistered user defined surfacer: %s", s.GetName())
		}
	default:
		return nil, nil, fmt.Errorf("unknown surfacer type: %s", s.GetType())
	}

	return surfacer, conf, err
}

// Surfacer is an interface for all metrics surfacing systems
type Surfacer interface {
	// Function for writing a piece of metric data to a specified metric
	// store (or other location).
	Write(ctx context.Context, em *metrics.EventMetrics)
}

// SurfacerInfo encapsulates a Surfacer and related info.
type SurfacerInfo struct {
	Surfacer
	Type string
	Name string
	Conf string
}

// Init initializes the surfacers from the config protobufs and returns them as
// a list.
func Init(sDefs []*surfacerpb.SurfacerDef) ([]*SurfacerInfo, error) {
	// If no surfacers are defined, return default surfacers. This behavior
	// can be disabled by explicitly specifying "surfacer {}" in the config.
	if len(sDefs) == 0 {
		sDefs = defaultSurfacers
	}

	var result []*SurfacerInfo
	for _, sDef := range sDefs {
		if sDef.GetType() == surfacerpb.Type_NONE {
			continue
		}
		s, conf, err := initSurfacer(sDef)
		if err != nil {
			return nil, err
		}

		confStr := ""
		if conf != nil {
			if stringer, ok := conf.(fmt.Stringer); ok {
				confStr = stringer.String()
			}
		}

		result = append(result, &SurfacerInfo{
			Surfacer: s,
			Type:     sDef.GetType().String(),
			Name:     sDef.GetName(),
			Conf:     confStr,
		})
	}
	return result, nil
}

// Register allows you to register a user defined surfacer with cloudprober.
// Example usage:
//	import (
//		"github.com/google/cloudprober"
//		"github.com/google/cloudprober/surfacers"
//	)
//
//	s := &FancySurfacer{}
//	surfacers.Register("fancy_surfacer", s)
//	pr, err := cloudprober.InitFromConfig(*configFile)
//	if err != nil {
//		log.Exitf("Error initializing cloudprober. Err: %v", err)
//	}
func Register(name string, s Surfacer) {
	userDefinedSurfacersMu.Lock()
	defer userDefinedSurfacersMu.Unlock()
	userDefinedSurfacers[name] = s
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/slack-go/slack"
)

//Slackcat client
type Slackcat struct {
	queue       *StreamQ
	shutdown    chan os.Signal
	username    string
	iconEmoji   string
	channelID   string
	channelName string
}

func newSlackcat(username, iconEmoji, channelname string) *Slackcat {
	sc := &Slackcat{
		queue:       newStreamQ(),
		shutdown:    make(chan os.Signal, 1),
		username:    username,
		iconEmoji:   iconEmoji,
		channelName: channelname,
	}

	sc.channelID = lookupSlackID(sc.channelName)

	signal.Notify(sc.shutdown, os.Interrupt)
	return sc
}

func (sc *Slackcat) trap() {
	sigcount := 0
	for sig := range sc.shutdown {
		if sigcount > 0 {
			exitErr(fmt.Errorf("aborted"))
		}
		output(fmt.Sprintf("got signal: %s", sig.String()))
		output("press ctrl+c again to exit immediately")
		sigcount++
		go sc.exit()
	}
}

func (sc *Slackcat) exit() {
	for {
		if sc.queue.IsEmpty() {
			os.Exit(0)
		} else {
			output("flushing remaining messages to Slack...")
			time.Sleep(3 * time.Second)
		}
	}
}

func (sc *Slackcat) stream(lines chan string) {
	output("starting stream")

	go func() {
		for line := range lines {
			sc.queue.Add(line)
		}
		sc.exit()
	}()

	go sc.processStreamQ()
	go sc.trap()
	select {}
}

func (sc *Slackcat) processStreamQ() {
	if !(sc.queue.IsEmpty()) {
		msglines := sc.queue.Flush()
		if noop {
			output(fmt.Sprintf("skipped posting of %s message lines to %s", strconv.Itoa(len(msglines)), sc.channelName))
		} else {
			sc.postMsg(msglines)
		}
		sc.queue.Ack()
	}
	time.Sleep(3 * time.Second)
	sc.processStreamQ()
}

var CurMsgTS string

func (sc *Slackcat) postMsg(msglines []string) {
	msg := strings.Join(msglines, "\n")
	msg = strings.Replace(msg, "&", "%26amp%3B", -1)
	msg = strings.Replace(msg, "<", "%26lt%3B", -1)
	msg = strings.Replace(msg, ">", "%26gt%3B", -1)

	msgOpts := []slack.MsgOption{slack.MsgOptionText(msg, false)}
	if sc.username != "" {
		msgOpts = append(msgOpts, slack.MsgOptionAsUser(false))
		msgOpts = append(msgOpts, slack.MsgOptionUsername(sc.username))
	} else {
		msgOpts = append(msgOpts, slack.MsgOptionAsUser(true))
	}
	if sc.iconEmoji != "" {
		msgOpts = append(msgOpts, slack.MsgOptionIconEmoji(sc.iconEmoji))
	}

	if thread {
		if CurMsgTS != "" {
			msgOpts = append(msgOpts, slack.MsgOptionTS(CurMsgTS))
		}
	}

	var err error

	_, CurMsgTS, err = api.PostMessage(sc.channelID, msgOpts...)

	output(CurMsgTS)

	failOnError(err)
	count := strconv.Itoa(len(msglines))
	output(fmt.Sprintf("posted %s message lines to %s", count, sc.channelName))
}

func (sc *Slackcat) postFile(filePath, fileName, fileType, fileComment string) {
	//default to timestamp for filename
	if fileName == "" {
		fileName = strconv.FormatInt(time.Now().Unix(), 10)
	}

	if noop {
		output(fmt.Sprintf("skipping upload of file %s to %s", fileName, sc.channelName))
		return
	}

	start := time.Now()
	_, err := api.UploadFile(slack.FileUploadParameters{
		File:           filePath,
		Filename:       fileName,
		Filetype:       fileType,
		Title:          fileName,
		InitialComment: fileComment,
		Channels:       []string{sc.channelID},
	})
	failOnError(err, "error uploading file to Slack")
	duration := strconv.FormatFloat(time.Since(start).Seconds(), 'f', 3, 64)
	output(fmt.Sprintf("file %s uploaded to %s (%ss)", fileName, sc.channelName, duration))
	os.Exit(0)
}
<commit_msg>Remove debug output<commit_after>package main

import (
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/slack-go/slack"
)

//Slackcat client
type Slackcat struct {
	queue       *StreamQ
	shutdown    chan os.Signal
	username    string
	iconEmoji   string
	channelID   string
	channelName string
}

func newSlackcat(username, iconEmoji, channelname string) *Slackcat {
	sc := &Slackcat{
		queue:       newStreamQ(),
		shutdown:    make(chan os.Signal, 1),
		username:    username,
		iconEmoji:   iconEmoji,
		channelName: channelname,
	}

	sc.channelID = lookupSlackID(sc.channelName)

	signal.Notify(sc.shutdown, os.Interrupt)
	return sc
}

func (sc *Slackcat) trap() {
	sigcount := 0
	for sig := range sc.shutdown {
		if sigcount > 0 {
			exitErr(fmt.Errorf("aborted"))
		}
		output(fmt.Sprintf("got signal: %s", sig.String()))
		output("press ctrl+c again to exit immediately")
		sigcount++
		go sc.exit()
	}
}

func (sc *Slackcat) exit() {
	for {
		if sc.queue.IsEmpty() {
			os.Exit(0)
		} else {
			output("flushing remaining messages to Slack...")
			time.Sleep(3 * time.Second)
		}
	}
}

func (sc *Slackcat) stream(lines chan string) {
	output("starting stream")

	go func() {
		for line := range lines {
			sc.queue.Add(line)
		}
		sc.exit()
	}()

	go sc.processStreamQ()
	go sc.trap()
	select {}
}

func (sc *Slackcat) processStreamQ() {
	if !(sc.queue.IsEmpty()) {
		msglines := sc.queue.Flush()
		if noop {
			output(fmt.Sprintf("skipped posting of %s message lines to %s", strconv.Itoa(len(msglines)), sc.channelName))
		} else {
			sc.postMsg(msglines)
		}
		sc.queue.Ack()
	}
	time.Sleep(3 * time.Second)
	sc.processStreamQ()
}

var CurMsgTS string

func (sc *Slackcat) postMsg(msglines []string) {
	msg := strings.Join(msglines, "\n")
	msg = strings.Replace(msg, "&", "%26amp%3B", -1)
	msg = strings.Replace(msg, "<", "%26lt%3B", -1)
	msg = strings.Replace(msg, ">", "%26gt%3B", -1)

	msgOpts := []slack.MsgOption{slack.MsgOptionText(msg, false)}
	if sc.username != "" {
		msgOpts = append(msgOpts, slack.MsgOptionAsUser(false))
		msgOpts = append(msgOpts, slack.MsgOptionUsername(sc.username))
	} else {
		msgOpts = append(msgOpts, slack.MsgOptionAsUser(true))
	}
	if sc.iconEmoji != "" {
		msgOpts = append(msgOpts, slack.MsgOptionIconEmoji(sc.iconEmoji))
	}

	if thread {
		if CurMsgTS != "" {
			msgOpts = append(msgOpts, slack.MsgOptionTS(CurMsgTS))
		}
	}

	var err error

	_, CurMsgTS, err = api.PostMessage(sc.channelID, msgOpts...)

	failOnError(err)
	count := strconv.Itoa(len(msglines))
	output(fmt.Sprintf("posted %s message lines to %s", count, sc.channelName))
}

func (sc *Slackcat) postFile(filePath, fileName, fileType, fileComment string) {
	//default to timestamp for filename
	if fileName == "" {
		fileName = strconv.FormatInt(time.Now().Unix(), 10)
	}

	if noop {
		output(fmt.Sprintf("skipping upload of file %s to %s", fileName, sc.channelName))
		return
	}

	start := time.Now()
	_, err := api.UploadFile(slack.FileUploadParameters{
		File:           filePath,
		Filename:       fileName,
		Filetype:       fileType,
		Title:          fileName,
		InitialComment: fileComment,
		Channels:       []string{sc.channelID},
	})
	failOnError(err, "error uploading file to Slack")
	duration := strconv.FormatFloat(time.Since(start).Seconds(), 'f', 3, 64)
	output(fmt.Sprintf("file %s uploaded to %s (%ss)", fileName, sc.channelName, duration))
	os.Exit(0)
}
<|endoftext|>