<commit_before>package main

import (
	"io"
	"log"
	"net"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gholt/experimental-ring"
)

type msgMap struct {
	lock    sync.RWMutex
	mapping map[ring.MsgType]ring.MsgUnmarshaller
}

func newMsgMap() *msgMap {
	return &msgMap{mapping: make(map[ring.MsgType]ring.MsgUnmarshaller)}
}

func (mm *msgMap) set(t ring.MsgType, f ring.MsgUnmarshaller) ring.MsgUnmarshaller {
	mm.lock.Lock()
	p := mm.mapping[t]
	mm.mapping[t] = f
	mm.lock.Unlock()
	return p
}

func (mm *msgMap) get(t ring.MsgType) ring.MsgUnmarshaller {
	mm.lock.RLock()
	f := mm.mapping[t]
	mm.lock.RUnlock()
	return f
}

type ringPipe struct {
	nodeID          uint64
	conn            net.Conn
	lock            sync.RWMutex
	msgMap          *msgMap
	logError        *log.Logger
	logWarning      *log.Logger
	typeBytes       int
	lengthBytes     int
	writeChan       chan ring.Msg
	writingDoneChan chan struct{}
	sendDrops       uint32
}

func NewRingPipe(nodeID uint64, c net.Conn) *ringPipe {
	rp := &ringPipe{
		nodeID:          nodeID,
		conn:            c,
		msgMap:          newMsgMap(),
		logError:        log.New(os.Stderr, "", log.LstdFlags),
		logWarning:      log.New(os.Stderr, "", log.LstdFlags),
		typeBytes:       1,
		lengthBytes:     3,
		writeChan:       make(chan ring.Msg, 40),
		writingDoneChan: make(chan struct{}, 1),
	}
	return rp
}

func (rp *ringPipe) ID() uint64 {
	return 1
}

func (rp *ringPipe) PartitionPower() uint16 {
	return 8
}

func (rp *ringPipe) NodeID() uint64 {
	return rp.nodeID
}

func (rp *ringPipe) Responsible(partition uint32) bool {
	return partition&1 == uint32(rp.nodeID)&1
}

func (rp *ringPipe) Start() {
	go rp.reading()
	go rp.writing()
}

const _GLH_SEND_MSG_TIMEOUT = 1

func (rp *ringPipe) SetMsgHandler(t ring.MsgType, h ring.MsgUnmarshaller) {
	rp.msgMap.set(t, h)
}

func (rp *ringPipe) MsgToNode(nodeID uint64, m ring.Msg) bool {
	select {
	case rp.writeChan <- m:
		return true
	case <-time.After(_GLH_SEND_MSG_TIMEOUT * time.Second):
		atomic.AddUint32(&rp.sendDrops, 1)
		return false
	}
}

func (rp *ringPipe) MsgToOtherReplicas(ringID uint64, partition uint32, m ring.Msg) bool {
	// TODO: If ringID has changed, partition invalid, etc. return false
	select {
	case rp.writeChan <- m:
		return true
	case <-time.After(_GLH_SEND_MSG_TIMEOUT * time.Second):
		atomic.AddUint32(&rp.sendDrops, 1)
		return false
	}
}

func (rp *ringPipe) reading() {
	b := make([]byte, rp.typeBytes+rp.lengthBytes)
	d := make([]byte, 65536)
	for {
		var n int
		var sn int
		var err error
		for n != len(b) {
			if err != nil {
				if n != 0 || err != io.EOF {
					rp.logError.Print("error reading msg", err)
				}
				return
			}
			sn, err = rp.conn.Read(b[n:])
			n += sn
		}
		if err != nil {
			rp.logError.Print("error reading msg content", err)
			return
		}
		var t uint64
		for i := 0; i < rp.typeBytes; i++ {
			t = (t << 8) | uint64(b[i])
		}
		var l uint64
		for i := 0; i < rp.lengthBytes; i++ {
			l = (l << 8) | uint64(b[rp.typeBytes+i])
		}
		f := rp.msgMap.get(ring.MsgType(t))
		if f != nil {
			_, err = f(rp.conn, l)
			if err != nil {
				rp.logError.Print("error reading msg content", err)
				return
			}
		} else {
			rp.logWarning.Printf("unknown msg type %d", t)
			for l > 0 {
				if err != nil {
					rp.logError.Print("err reading msg content", err)
					return
				}
				if l >= uint64(len(d)) {
					sn, err = rp.conn.Read(d)
				} else {
					sn, err = rp.conn.Read(d[:l])
				}
				l -= uint64(sn)
			}
		}
	}
}

func (rp *ringPipe) writing() {
	b := make([]byte, rp.typeBytes+rp.lengthBytes)
	for {
		m := <-rp.writeChan
		if m == nil {
			break
		}
		t := m.MsgType()
		for i := rp.typeBytes - 1; i >= 0; i-- {
			b[i] = byte(t)
			t >>= 8
		}
		l := m.MsgLength()
		for i := rp.lengthBytes - 1; i >= 0; i-- {
			b[rp.typeBytes+i] = byte(l)
			l >>= 8
		}
		_, err := rp.conn.Write(b)
		if err != nil {
			rp.logError.Print("err writing msg", err)
			break
		}
		_, err = m.WriteContent(rp.conn)
		if err != nil {
			rp.logError.Print("err writing msg content", err)
			break
		}
		m.Done()
	}
	rp.writingDoneChan <- struct{}{}
}
<commit_msg>Push replication<commit_after>package main

import (
	"io"
	"log"
	"net"
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/gholt/experimental-ring"
)

type msgMap struct {
	lock    sync.RWMutex
	mapping map[ring.MsgType]ring.MsgUnmarshaller
}

func newMsgMap() *msgMap {
	return &msgMap{mapping: make(map[ring.MsgType]ring.MsgUnmarshaller)}
}

func (mm *msgMap) set(t ring.MsgType, f ring.MsgUnmarshaller) ring.MsgUnmarshaller {
	mm.lock.Lock()
	p := mm.mapping[t]
	mm.mapping[t] = f
	mm.lock.Unlock()
	return p
}

func (mm *msgMap) get(t ring.MsgType) ring.MsgUnmarshaller {
	mm.lock.RLock()
	f := mm.mapping[t]
	mm.lock.RUnlock()
	return f
}

type ringPipe struct {
	nodeID          uint64
	conn            net.Conn
	lock            sync.RWMutex
	msgMap          *msgMap
	logError        *log.Logger
	logWarning      *log.Logger
	typeBytes       int
	lengthBytes     int
	writeChan       chan ring.Msg
	writingDoneChan chan struct{}
	sendDrops       uint32
}

func NewRingPipe(nodeID uint64, c net.Conn) *ringPipe {
	rp := &ringPipe{
		nodeID:          nodeID,
		conn:            c,
		msgMap:          newMsgMap(),
		logError:        log.New(os.Stderr, "", log.LstdFlags),
		logWarning:      log.New(os.Stderr, "", log.LstdFlags),
		typeBytes:       1,
		lengthBytes:     3,
		writeChan:       make(chan ring.Msg, 40),
		writingDoneChan: make(chan struct{}, 1),
	}
	return rp
}

func (rp *ringPipe) ID() uint64 {
	return 1
}

func (rp *ringPipe) PartitionPower() uint16 {
	return 8
}

func (rp *ringPipe) NodeID() uint64 {
	return rp.nodeID
}

func (rp *ringPipe) Responsible(partition uint32) bool {
	// TODO: Testing push replication, so node 2 is responsible for everything
	// but we're putting everything into node 1.
	return rp.nodeID == 2
}

func (rp *ringPipe) Start() {
	go rp.reading()
	go rp.writing()
}

const _GLH_SEND_MSG_TIMEOUT = 1

func (rp *ringPipe) SetMsgHandler(t ring.MsgType, h ring.MsgUnmarshaller) {
	rp.msgMap.set(t, h)
}

func (rp *ringPipe) MsgToNode(nodeID uint64, m ring.Msg) bool {
	select {
	case rp.writeChan <- m:
		return true
	case <-time.After(_GLH_SEND_MSG_TIMEOUT * time.Second):
		atomic.AddUint32(&rp.sendDrops, 1)
		return false
	}
}

func (rp *ringPipe) MsgToOtherReplicas(ringID uint64, partition uint32, m ring.Msg) bool {
	// TODO: If ringID has changed, partition invalid, etc. return false
	select {
	case rp.writeChan <- m:
		return true
	case <-time.After(_GLH_SEND_MSG_TIMEOUT * time.Second):
		atomic.AddUint32(&rp.sendDrops, 1)
		return false
	}
}

func (rp *ringPipe) reading() {
	b := make([]byte, rp.typeBytes+rp.lengthBytes)
	d := make([]byte, 65536)
	for {
		var n int
		var sn int
		var err error
		for n != len(b) {
			if err != nil {
				if n != 0 || err != io.EOF {
					rp.logError.Print("error reading msg", err)
				}
				return
			}
			sn, err = rp.conn.Read(b[n:])
			n += sn
		}
		if err != nil {
			rp.logError.Print("error reading msg content", err)
			return
		}
		var t uint64
		for i := 0; i < rp.typeBytes; i++ {
			t = (t << 8) | uint64(b[i])
		}
		var l uint64
		for i := 0; i < rp.lengthBytes; i++ {
			l = (l << 8) | uint64(b[rp.typeBytes+i])
		}
		f := rp.msgMap.get(ring.MsgType(t))
		if f != nil {
			_, err = f(rp.conn, l)
			if err != nil {
				rp.logError.Print("error reading msg content", err)
				return
			}
		} else {
			rp.logWarning.Printf("unknown msg type %d", t)
			for l > 0 {
				if err != nil {
					rp.logError.Print("err reading msg content", err)
					return
				}
				if l >= uint64(len(d)) {
					sn, err = rp.conn.Read(d)
				} else {
					sn, err = rp.conn.Read(d[:l])
				}
				l -= uint64(sn)
			}
		}
	}
}

func (rp *ringPipe) writing() {
	b := make([]byte, rp.typeBytes+rp.lengthBytes)
	for {
		m := <-rp.writeChan
		if m == nil {
			break
		}
		t := m.MsgType()
		for i := rp.typeBytes - 1; i >= 0; i-- {
			b[i] = byte(t)
			t >>= 8
		}
		l := m.MsgLength()
		for i := rp.lengthBytes - 1; i >= 0; i-- {
			b[rp.typeBytes+i] = byte(l)
			l >>= 8
		}
		_, err := rp.conn.Write(b)
		if err != nil {
			rp.logError.Print("err writing msg", err)
			break
		}
		_, err = m.WriteContent(rp.conn)
		if err != nil {
			rp.logError.Print("err writing msg content", err)
			break
		}
		m.Done()
	}
	rp.writingDoneChan <- struct{}{}
}
<|endoftext|>


<commit_before>// Copyright 2017 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package riot

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"strings"

	"encoding/binary"
	"encoding/gob"

	"github.com/go-ego/murmur"
	"github.com/go-ego/riot/core"
	"github.com/go-ego/riot/types"
	toml "github.com/go-vgo/gt/conf"
)

// New create a new engine with mode
func New(conf ...interface{}) *Engine {
	// func (engine *Engine) New(conf com.Config) *Engine{
	var (
		config   types.EngineOpts
		searcher = &Engine{}
	)

	if len(conf) > 0 && strings.HasSuffix(conf[0].(string), ".toml") {
		fs := conf[0].(string)
		toml.Init(fs, &config)
		go toml.Watch(fs, &config)

		searcher.Init(config)
		return searcher
	}

	return NewEngine(conf...)
}

// NewEngine create a new engine
func NewEngine(conf ...interface{}) *Engine {
	var (
		searcher = &Engine{}

		path        = DefaultPath
		storeShards = 10
		numShards   = 10

		segmentDict string
	)

	if len(conf) > 0 {
		segmentDict = conf[0].(string)
	}

	if len(conf) > 1 {
		path = conf[1].(string)
	}

	if len(conf) > 2 {
		numShards = conf[2].(int)
		storeShards = conf[2].(int)
	}

	searcher.Init(types.EngineOpts{
		// Using:         using,
		StorageShards: storeShards,
		NumShards:     numShards,
		IndexerOpts: &types.IndexerOpts{
			IndexType: types.DocIdsIndex,
		},
		UseStorage:    true,
		StorageFolder: path,
		// StorageEngine: storeEngine,
		GseDict: segmentDict,
		// StopTokenFile: stopTokenFile,
	})

	// defer searcher.Close()
	os.MkdirAll(path, 0777)

	// 等待索引刷新完毕
	// searcher.Flush()
	// log.Println("recover index number: ", searcher.NumDocsIndexed())

	return searcher
}

// func (engine *Engine) IsDocExist(docId uint64) bool {
// 	return core.IsDocExist(docId)
// }

// HasDoc if the document is exist return true
func (engine *Engine) HasDoc(docId uint64) bool {
	for shard := 0; shard < engine.initOptions.NumShards; shard++ {
		engine.indexers = append(engine.indexers, core.Indexer{})

		has := engine.indexers[shard].HasDoc(docId)

		if has {
			return true
		}
	}

	return false
}

// HasDocDB if the document is exist in the database
// return true
func (engine *Engine) HasDocDB(docId uint64) bool {
	b := make([]byte, 10)
	length := binary.PutUvarint(b, docId)

	shard := murmur.Sum32(fmt.Sprintf("%d", docId)) %
		uint32(engine.initOptions.StorageShards)

	has, err := engine.dbs[shard].Has(b[0:length])
	if err != nil {
		log.Println("engine.dbs[shard].Has(b[0:length]): ", err)
	}

	return has
}

// GetDBAllIds get all the DocId from the storage database
// and return
// 从数据库遍历所有的 DocId, 并返回
func (engine *Engine) GetDBAllIds() []uint64 {
	docsId := make([]uint64, 0)
	for i := range engine.dbs {
		engine.dbs[i].ForEach(func(k, v []byte) error {
			// fmt.Println(k, v)
			docId, _ := binary.Uvarint(k)
			docsId = append(docsId, docId)
			return nil
		})
	}

	return docsId
}

// GetDBAllDocs get the db all docs
func (engine *Engine) GetDBAllDocs() (
	docsId []uint64, docsData []types.DocData) {
	for i := range engine.dbs {
		engine.dbs[i].ForEach(func(key, val []byte) error {
			// fmt.Println(k, v)
			docId, _ := binary.Uvarint(key)
			docsId = append(docsId, docId)

			buf := bytes.NewReader(val)
			dec := gob.NewDecoder(buf)

			var data types.DocData
			err := dec.Decode(&data)
			if err != nil {
				log.Println("dec.decode: ", err)
			}

			docsData = append(docsData, data)

			return nil
		})
	}

	return docsId, docsData
}

// GetAllDocIds get all the DocId from the storage database
// and return
// 从数据库遍历所有的 DocId, 并返回
func (engine *Engine) GetAllDocIds() []uint64 {
	return engine.GetDBAllIds()
}

// Try handler(err)
func Try(fun func(), handler func(interface{})) {
	defer func() {
		if err := recover(); err != nil {
			handler(err)
		}
	}()
	fun()
}
<commit_msg>optimize new code reduce life cycle<commit_after>// Copyright 2017 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package riot

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"strings"

	"encoding/binary"
	"encoding/gob"

	"github.com/go-ego/murmur"
	"github.com/go-ego/riot/core"
	"github.com/go-ego/riot/types"
	toml "github.com/go-vgo/gt/conf"
)

// New create a new engine with mode
func New(conf ...interface{}) *Engine {
	// func (engine *Engine) New(conf com.Config) *Engine{
	if len(conf) > 0 && strings.HasSuffix(conf[0].(string), ".toml") {
		var (
			config   types.EngineOpts
			searcher = &Engine{}
		)

		fs := conf[0].(string)
		toml.Init(fs, &config)
		go toml.Watch(fs, &config)

		searcher.Init(config)
		return searcher
	}

	return NewEngine(conf...)
}

// NewEngine create a new engine
func NewEngine(conf ...interface{}) *Engine {
	var (
		searcher = &Engine{}

		path        = DefaultPath
		storeShards = 10
		numShards   = 10

		segmentDict string
	)

	if len(conf) > 0 {
		segmentDict = conf[0].(string)
	}

	if len(conf) > 1 {
		path = conf[1].(string)
	}

	if len(conf) > 2 {
		numShards = conf[2].(int)
		storeShards = conf[2].(int)
	}

	searcher.Init(types.EngineOpts{
		// Using:         using,
		StorageShards: storeShards,
		NumShards:     numShards,
		IndexerOpts: &types.IndexerOpts{
			IndexType: types.DocIdsIndex,
		},
		UseStorage:    true,
		StorageFolder: path,
		// StorageEngine: storeEngine,
		GseDict: segmentDict,
		// StopTokenFile: stopTokenFile,
	})

	// defer searcher.Close()
	os.MkdirAll(path, 0777)

	// 等待索引刷新完毕
	// searcher.Flush()
	// log.Println("recover index number: ", searcher.NumDocsIndexed())

	return searcher
}

// func (engine *Engine) IsDocExist(docId uint64) bool {
// 	return core.IsDocExist(docId)
// }

// HasDoc if the document is exist return true
func (engine *Engine) HasDoc(docId uint64) bool {
	for shard := 0; shard < engine.initOptions.NumShards; shard++ {
		engine.indexers = append(engine.indexers, core.Indexer{})

		has := engine.indexers[shard].HasDoc(docId)

		if has {
			return true
		}
	}

	return false
}

// HasDocDB if the document is exist in the database
// return true
func (engine *Engine) HasDocDB(docId uint64) bool {
	b := make([]byte, 10)
	length := binary.PutUvarint(b, docId)

	shard := murmur.Sum32(fmt.Sprintf("%d", docId)) %
		uint32(engine.initOptions.StorageShards)

	has, err := engine.dbs[shard].Has(b[0:length])
	if err != nil {
		log.Println("engine.dbs[shard].Has(b[0:length]): ", err)
	}

	return has
}

// GetDBAllIds get all the DocId from the storage database
// and return
// 从数据库遍历所有的 DocId, 并返回
func (engine *Engine) GetDBAllIds() []uint64 {
	docsId := make([]uint64, 0)
	for i := range engine.dbs {
		engine.dbs[i].ForEach(func(k, v []byte) error {
			// fmt.Println(k, v)
			docId, _ := binary.Uvarint(k)
			docsId = append(docsId, docId)
			return nil
		})
	}

	return docsId
}

// GetDBAllDocs get the db all docs
func (engine *Engine) GetDBAllDocs() (
	docsId []uint64, docsData []types.DocData) {
	for i := range engine.dbs {
		engine.dbs[i].ForEach(func(key, val []byte) error {
			// fmt.Println(k, v)
			docId, _ := binary.Uvarint(key)
			docsId = append(docsId, docId)

			buf := bytes.NewReader(val)
			dec := gob.NewDecoder(buf)

			var data types.DocData
			err := dec.Decode(&data)
			if err != nil {
				log.Println("dec.decode: ", err)
			}

			docsData = append(docsData, data)

			return nil
		})
	}

	return docsId, docsData
}

// GetAllDocIds get all the DocId from the storage database
// and return
// 从数据库遍历所有的 DocId, 并返回
func (engine *Engine) GetAllDocIds() []uint64 {
	return engine.GetDBAllIds()
}

// Try handler(err)
func Try(fun func(), handler func(interface{})) {
	defer func() {
		if err := recover(); err != nil {
			handler(err)
		}
	}()
	fun()
}
<|endoftext|>


<commit_before>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package main

import (
	"context"
	"database/sql"
	"errors"
	"flag"
	"strconv"
	"time"

	"github.com/trackit/jsonlog"

	"github.com/trackit/trackit/aws"
	"github.com/trackit/trackit/aws/usageReports/ebs"
	"github.com/trackit/trackit/aws/usageReports/ec2"
	"github.com/trackit/trackit/aws/usageReports/elasticache"
	"github.com/trackit/trackit/aws/usageReports/es"
	"github.com/trackit/trackit/aws/usageReports/history"
	"github.com/trackit/trackit/aws/usageReports/lambda"
	"github.com/trackit/trackit/aws/usageReports/rds"
	"github.com/trackit/trackit/aws/usageReports/riEc2"
	"github.com/trackit/trackit/aws/usageReports/riRdS"
	"github.com/trackit/trackit/cache"
	"github.com/trackit/trackit/db"
	onDemandToRiEc2 "github.com/trackit/trackit/onDemandToRI/ec2"
)

// taskProcessAccount processes an AwsAccount to retrieve data from the AWS api.
func taskProcessAccount(ctx context.Context) error {
	args := flag.Args()
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger.Debug("Running task 'process-account'.", map[string]interface{}{
		"args": args,
	})
	if len(args) != 1 {
		return errors.New("taskProcessAccount requires an integer argument")
	} else if aaId, err := strconv.Atoi(args[0]); err != nil {
		return err
	} else {
		return ingestDataForAccount(ctx, aaId)
	}
}

// ingestDataForAccount ingests the AWS api data for an AwsAccount.
func ingestDataForAccount(ctx context.Context, aaId int) (err error) {
	var tx *sql.Tx
	var aa aws.AwsAccount
	var updateId int64
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	defer func() {
		if tx != nil {
			if err != nil {
				tx.Rollback()
			} else {
				tx.Commit()
			}
		}
	}()
	if tx, err = db.Db.BeginTx(ctx, nil); err != nil {
	} else if aa, err = aws.GetAwsAccountWithId(aaId, tx); err != nil {
	} else if updateId, err = registerAccountProcessing(db.Db, aa); err != nil {
	} else {
		ec2Err := processAccountEC2(ctx, aa)
		rdsErr := processAccountRDS(ctx, aa)
		esErr := processAccountES(ctx, aa)
		elastiCacheErr := processAccountElastiCache(ctx, aa)
		lambdaErr := processAccountLambda(ctx, aa)
		riEc2Err := riEc2.FetchDailyReservationsStats(ctx, aa)
		riRdsErr := riRdS.FetchDailyInstancesStats(ctx, aa)
		odToRiEc2Err := onDemandToRiEc2.RunOnDemandToRiEc2(ctx, aa)
		ebsErr := processAccountEbsSnapshot(ctx, aa)
		historyCreated, historyErr := processAccountHistory(ctx, aa)
		updateAccountProcessingCompletion(ctx, aaId, db.Db, updateId, nil, rdsErr, ec2Err, esErr, elastiCacheErr, lambdaErr, riEc2Err, riRdsErr, odToRiEc2Err, historyErr, ebsErr, historyCreated)
	}
	if err != nil {
		updateAccountProcessingCompletion(ctx, aaId, db.Db, updateId, err, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,false)
		logger.Error("Failed to process account data.", map[string]interface{}{
			"awsAccountId": aaId,
			"error":        err.Error(),
		})
	}
	var affectedRoutes = []string {
		"/ec2",
		"/ec2/coverage",
		"/ec2/unused",
		"/elasticache",
		"/elasticache/unused",
		"/es",
		"/es/unused",
		"/lambda",
		"/rds",
		"/rds/unused",
		"/ri/ec2",
		"/ri/rds",
	}
	_ = cache.RemoveMatchingCache(affectedRoutes, []string {aa.AwsIdentity}, logger)
	return
}

func registerAccountProcessing(db *sql.DB, aa aws.AwsAccount) (int64, error) {
	const sqlstr = `INSERT INTO aws_account_update_job(
		aws_account_id,
		worker_id
	) VALUES (?, ?)`
	res, err := db.Exec(sqlstr, aa.Id, backendId)
	if err != nil {
		return 0, err
	}
	return res.LastInsertId()
}

func updateAccountProcessingCompletion(ctx context.Context, aaId int, db *sql.DB, updateId int64, jobErr, rdsErr, ec2Err, esErr, elastiCacheErr, lambdaErr, riEc2Err, riRdsErr, odToRiEc2Err, historyErr, ebsErr error, historyCreated bool) {
	updateNextUpdateAccount(db, aaId)
	rErr := registerAccountProcessingCompletion(db, updateId, jobErr, rdsErr, ec2Err, esErr, elastiCacheErr, lambdaErr, riEc2Err, riRdsErr, odToRiEc2Err, historyErr, ebsErr, historyCreated)
	if rErr != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to register account processing completion.", map[string]interface{}{
			"awsAccountId": aaId,
			"error":        rErr.Error(),
			"updateId":     updateId,
		})
	}
}

func errToStr(err error) string {
	if err != nil {
		return err.Error()
	}
	return ""
}

func updateNextUpdateAccount(db *sql.DB, aaId int) error {
	const sqlstr = `UPDATE aws_account SET
		next_update=?
	WHERE id=?`
	_, err := db.Exec(sqlstr, time.Now().AddDate(0, 0, 1), aaId)
	return err
}

func registerAccountProcessingCompletion(db *sql.DB, updateId int64, jobErr, rdsErr, ec2Err, esErr, elastiCacheErr, lambdaErr, riEc2Err, riRdsErr, odToRiEc2Err, historyErr, ebsErr error, historyCreated bool) error {
	const sqlstr = `UPDATE aws_account_update_job SET
		completed=?,
		jobError=?,
		rdsError=?,
		ec2Error=?,
		esError=?,
		elastiCacheError=?,
		lambdaError=?,
		ebsError=?,
		riEc2Error=?,
		riRdsError=?,
		odToRiEc2Error=?,
		historyError=?,
		monthly_reports_generated=?
	WHERE id=?`
	_, err := db.Exec(sqlstr, time.Now(), errToStr(jobErr), errToStr(rdsErr), errToStr(ec2Err), errToStr(esErr), errToStr(elastiCacheErr), errToStr(lambdaErr), errToStr(ebsErr), errToStr(riEc2Err), errToStr(riRdsErr), errToStr(odToRiEc2Err), errToStr(historyErr), historyCreated, updateId)
	return err
}

// processAccountRDS processes all the RDS data for an AwsAccount
func processAccountRDS(ctx context.Context, aa aws.AwsAccount) error {
	err := rds.FetchDailyInstancesStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest RDS data.", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountEC2 processes all the EC2 data for an AwsAccount
func processAccountEC2(ctx context.Context, aa aws.AwsAccount) error {
	err := ec2.FetchDailyInstancesStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest EC2 data.", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountES processes all the ES data for an AwsAccount
func processAccountES(ctx context.Context, aa aws.AwsAccount) error {
	err := es.FetchDomainsStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest ES data", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountElastiCache processes all the ElastiCache data for an AwsAccount
func processAccountElastiCache(ctx context.Context, aa aws.AwsAccount) error {
	err := elasticache.FetchDailyInstancesStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest ElastiCache data", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountEbsSnapshot process all the EBS Snapshot data for an AwsAccount
func processAccountEbsSnapshot(ctx context.Context, aa aws.AwsAccount) error {
	err := ebs.FetchDailySnapshotsStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest EBS Snapshots data", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountLambda processes all the Lambda data for an AwsAccount
func processAccountLambda(ctx context.Context, aa aws.AwsAccount) error {
	err := lambda.FetchDailyFunctionsStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest Lambda data", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountHistory processes EC2, RDS, ES, ElastiCache, Lambda and EC2 Coverage data with billing data for an AwsAccount
func processAccountHistory(ctx context.Context, aa aws.AwsAccount) (bool, error) {
	status, err := history.FetchHistoryInfos(ctx, aa)
	if err != nil && err != history.ErrBillingDataIncomplete {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest History data.", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return status, err
}
<commit_msg>get data for ebs snapshot from aws, put data in ES and get data from ES to add a new sheet for EBS Snapshots in spreadsheets<commit_after>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package main

import (
	"context"
	"database/sql"
	"errors"
	"flag"
	"github.com/trackit/trackit-server/aws/usageReports/ebs"
	"strconv"
	"time"

	"github.com/trackit/jsonlog"

	"github.com/trackit/trackit/aws"
	"github.com/trackit/trackit/aws/usageReports/ebs"
	"github.com/trackit/trackit/aws/usageReports/ec2"
	"github.com/trackit/trackit/aws/usageReports/elasticache"
	"github.com/trackit/trackit/aws/usageReports/es"
	"github.com/trackit/trackit/aws/usageReports/history"
	"github.com/trackit/trackit/aws/usageReports/lambda"
	"github.com/trackit/trackit/aws/usageReports/rds"
	"github.com/trackit/trackit/aws/usageReports/riEc2"
	"github.com/trackit/trackit/aws/usageReports/riRdS"
	"github.com/trackit/trackit/cache"
	"github.com/trackit/trackit/db"
	onDemandToRiEc2 "github.com/trackit/trackit/onDemandToRI/ec2"
)

// taskProcessAccount processes an AwsAccount to retrieve data from the AWS api.
func taskProcessAccount(ctx context.Context) error {
	args := flag.Args()
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger.Debug("Running task 'process-account'.", map[string]interface{}{
		"args": args,
	})
	if len(args) != 1 {
		return errors.New("taskProcessAccount requires an integer argument")
	} else if aaId, err := strconv.Atoi(args[0]); err != nil {
		return err
	} else {
		return ingestDataForAccount(ctx, aaId)
	}
}

// ingestDataForAccount ingests the AWS api data for an AwsAccount.
func ingestDataForAccount(ctx context.Context, aaId int) (err error) {
	var tx *sql.Tx
	var aa aws.AwsAccount
	var updateId int64
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	defer func() {
		if tx != nil {
			if err != nil {
				tx.Rollback()
			} else {
				tx.Commit()
			}
		}
	}()
	if tx, err = db.Db.BeginTx(ctx, nil); err != nil {
	} else if aa, err = aws.GetAwsAccountWithId(aaId, tx); err != nil {
	} else if updateId, err = registerAccountProcessing(db.Db, aa); err != nil {
	} else {
		ec2Err := processAccountEC2(ctx, aa)
		rdsErr := processAccountRDS(ctx, aa)
		esErr := processAccountES(ctx, aa)
		elastiCacheErr := processAccountElastiCache(ctx, aa)
		lambdaErr := processAccountLambda(ctx, aa)
		riEc2Err := riEc2.FetchDailyReservationsStats(ctx, aa)
		riRdsErr := riRdS.FetchDailyInstancesStats(ctx, aa)
		odToRiEc2Err := onDemandToRiEc2.RunOnDemandToRiEc2(ctx, aa)
		ebsErr := processAccountEbsSnapshot(ctx, aa)
		historyCreated, historyErr := processAccountHistory(ctx, aa)
		updateAccountProcessingCompletion(ctx, aaId, db.Db, updateId, nil, rdsErr, ec2Err, esErr, elastiCacheErr, lambdaErr, riEc2Err, riRdsErr, odToRiEc2Err, historyErr, ebsErr, historyCreated)
	}
	if err != nil {
		updateAccountProcessingCompletion(ctx, aaId, db.Db, updateId, err, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil,false)
		logger.Error("Failed to process account data.", map[string]interface{}{
			"awsAccountId": aaId,
			"error":        err.Error(),
		})
	}
	var affectedRoutes = []string {
		"/ec2",
		"/ec2/coverage",
		"/ec2/unused",
		"/elasticache",
		"/elasticache/unused",
		"/es",
		"/es/unused",
		"/lambda",
		"/rds",
		"/rds/unused",
		"/ri/ec2",
		"/ri/rds",
	}
	_ = cache.RemoveMatchingCache(affectedRoutes, []string {aa.AwsIdentity}, logger)
	return
}

func registerAccountProcessing(db *sql.DB, aa aws.AwsAccount) (int64, error) {
	const sqlstr = `INSERT INTO aws_account_update_job(
		aws_account_id,
		worker_id
	) VALUES (?, ?)`
	res, err := db.Exec(sqlstr, aa.Id, backendId)
	if err != nil {
		return 0, err
	}
	return res.LastInsertId()
}

func updateAccountProcessingCompletion(ctx context.Context, aaId int, db *sql.DB, updateId int64, jobErr, rdsErr, ec2Err, esErr, elastiCacheErr, lambdaErr, riEc2Err, riRdsErr, odToRiEc2Err, historyErr, ebsErr error, historyCreated bool) {
	updateNextUpdateAccount(db, aaId)
	rErr := registerAccountProcessingCompletion(db, updateId, jobErr, rdsErr, ec2Err, esErr, elastiCacheErr, lambdaErr, riEc2Err, riRdsErr, odToRiEc2Err, historyErr, ebsErr, historyCreated)
	if rErr != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to register account processing completion.", map[string]interface{}{
			"awsAccountId": aaId,
			"error":        rErr.Error(),
			"updateId":     updateId,
		})
	}
}

func errToStr(err error) string {
	if err != nil {
		return err.Error()
	}
	return ""
}

func updateNextUpdateAccount(db *sql.DB, aaId int) error {
	const sqlstr = `UPDATE aws_account SET
		next_update=?
	WHERE id=?`
	_, err := db.Exec(sqlstr, time.Now().AddDate(0, 0, 1), aaId)
	return err
}

func registerAccountProcessingCompletion(db *sql.DB, updateId int64, jobErr, rdsErr, ec2Err, esErr, elastiCacheErr, lambdaErr, riEc2Err, riRdsErr, odToRiEc2Err, historyErr, ebsErr error, historyCreated bool) error {
	const sqlstr = `UPDATE aws_account_update_job SET
		completed=?,
		jobError=?,
		rdsError=?,
		ec2Error=?,
		esError=?,
		elastiCacheError=?,
		lambdaError=?,
		ebsError=?,
		riEc2Error=?,
		riRdsError=?,
		odToRiEc2Error=?,
		historyError=?,
		monthly_reports_generated=?
	WHERE id=?`
	_, err := db.Exec(sqlstr, time.Now(), errToStr(jobErr), errToStr(rdsErr), errToStr(ec2Err), errToStr(esErr), errToStr(elastiCacheErr), errToStr(lambdaErr), errToStr(ebsErr), errToStr(riEc2Err), errToStr(riRdsErr), errToStr(odToRiEc2Err), errToStr(historyErr), historyCreated, updateId)
	return err
}

// processAccountRDS processes all the RDS data for an AwsAccount
func processAccountRDS(ctx context.Context, aa aws.AwsAccount) error {
	err := rds.FetchDailyInstancesStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest RDS data.", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountEC2 processes all the EC2 data for an AwsAccount
func processAccountEC2(ctx context.Context, aa aws.AwsAccount) error {
	err := ec2.FetchDailyInstancesStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest EC2 data.", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountES processes all the ES data for an AwsAccount
func processAccountES(ctx context.Context, aa aws.AwsAccount) error {
	err := es.FetchDomainsStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest ES data", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountElastiCache processes all the ElastiCache data for an AwsAccount
func processAccountElastiCache(ctx context.Context, aa aws.AwsAccount) error {
	err := elasticache.FetchDailyInstancesStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest ElastiCache data", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountEbsSnapshot process all the EBS Snapshot data for an AwsAccount
func processAccountEbsSnapshot(ctx context.Context, aa aws.AwsAccount) error {
	err := ebs.FetchDailySnapshotsStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest EBS Snapshots data", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountLambda processes all the Lambda data for an AwsAccount
func processAccountLambda(ctx context.Context, aa aws.AwsAccount) error {
	err := lambda.FetchDailyFunctionsStats(ctx, aa)
	if err != nil {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest Lambda data", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return err
}

// processAccountHistory processes EC2, RDS, ES, ElastiCache, Lambda and EC2 Coverage data with billing data for an AwsAccount
func processAccountHistory(ctx context.Context, aa aws.AwsAccount) (bool, error) {
	status, err := history.FetchHistoryInfos(ctx, aa)
	if err != nil && err != history.ErrBillingDataIncomplete {
		logger := jsonlog.LoggerFromContextOrDefault(ctx)
		logger.Error("Failed to ingest History data.", map[string]interface{}{
			"awsAccountId": aa.Id,
			"error":        err.Error(),
		})
	}
	return status, err
}
<|endoftext|>


<commit_before>package sisyphus

import (
	"bufio"
	"errors"
	"log"
	"math"
	"mime/quotedprintable"
	"os"
	"regexp"
	"strings"

	"github.com/kennygrant/sanitize"
	"github.com/luksen/maildir"
)

// Maildir represents the address to a Maildir directory
type Maildir string

// Mail includes the key of a mail in Maildir
type Mail struct {
	Key           string
	Subject, Body *string
	Junk, New     bool
}

// CreateDirs creates all the required dirs -- if not already there.
func (d Maildir) CreateDirs() error {

	dir := string(d)

	log.Println("create missing directories for Maildir " + dir)

	err := os.MkdirAll(dir+"/.Junk/cur", 0700)
	if err != nil {
		return err
	}
	err = os.MkdirAll(dir+"/new", 0700)
	if err != nil {
		return err
	}
	err = os.MkdirAll(dir+"/cur", 0700)

	return err
}

// Index loads all mail keys from the Maildir directory for processing.
func (d Maildir) Index() (m []*Mail, err error) {

	dir := string(d)

	log.Println("start indexing mails in " + dir)
	dirs := []string{dir, dir + "/.Junk"}
	for _, val := range dirs {
		j, err := maildir.Dir(val).Keys()
		if err != nil {
			return m, err
		}
		for _, v := range j {
			var new Mail
			new.Key = v
			if val == dir+"/.Junk" {
				new.Junk = true
			}
			m = append(m, &new)
		}
	}

	log.Println("all mails in " + dir + " indexed")

	return m, nil
}

// Load reads a mail's subject and body
func (m *Mail) Load(d string) error {

	if m.Junk {
		d = d + "/.Junk"
	}
	message, err := maildir.Dir(d).Message(m.Key)
	if err != nil {
		return err
	}

	// get Subject
	if m.Subject != nil {
		return errors.New("there is already a subject")
	}
	subject := message.Header.Get("Subject")
	m.Subject = &subject

	// get Body
	bQ := quotedprintable.NewReader(message.Body)
	var b []string
	bScanner := bufio.NewScanner(bQ)
	for bScanner.Scan() {
		raw := bScanner.Text()
		b = append(b, raw)
	}

	body := strings.Join(b, " ")
	if m.Body != nil {
		return errors.New("there is already a body")
	}
	m.Body = &body

	return nil
}

func trimStringFromBase64(s string) string {
	if idx := strings.Index(s, "Content-Transfer-Encoding: base64"); idx != -1 {
		return s[:idx-1]
	}
	return s
}

func cleanString(i string) (s string) {

	s = sanitize.Accents(i)
	s = sanitize.HTML(s)
	s = strings.ToLower(s)

	bad := []string{
		"boundary=", "charset", "content-transfer-encoding",
		"content-type", "image/jpeg", "multipart/alternative",
		"multipart/related", "name=", "nextpart", "quoted-printable",
		"text/html", "text/plain", "this email must be viewed in html mode",
		"this is a multi-part message in mime format",
		"windows-1251", "windows-1252", "!", "#", "$", "%", "&", "'",
		"(", ")", "*", "+", ",", ". ", "<", "=", ">", "?", "@", "[",
		"\"", "\\", "\n", "\t", "]", "^", "_", "{", "|", "}",
	}
	for _, b := range bad {
		s = strings.Replace(s, b, " ", -1)
	}
	for i := 0; i < 10; i++ {
		s = strings.Replace(s, "  ", " ", -1)
	}

	return s
}

// Clean cleans the mail's subject and body
func (m *Mail) Clean() error {
	if m.Subject != nil {
		s := trimStringFromBase64(*m.Subject)
		s = cleanString(s)
		m.Subject = &s
	}

	if m.Body != nil {
		b := trimStringFromBase64(*m.Body)
		b = cleanString(b)
		m.Body = &b
	}
	return nil
}

// wordlist takes a string of space separated text and returns a list of unique
// words in a space separated string
func wordlist(s string) (l []string, err error) {
	list := make(map[string]int)

	raw := strings.Split(s, " ")
	var clean []string

	// use regexp compile for use in the loop that follows
	regexMatcher, err := regexp.Compile("(^[a-z]+$)")
	if err != nil {
		return l, err
	}

	for _, w := range raw {

		// no long or too short words
		length := len(w)
		if length < 4 || length > 10 {
			continue
		}

		// no numbers, special characters, etc. -- only words
		match := regexMatcher.MatchString(w)
		if !match {
			continue
		} else {
			clean = append(clean, w)
		}
	}

	// only the first 200 words count
	maxWords := int(math.Min(200, float64(len(clean))))
	for i := 0; i < maxWords; i++ {
		w := clean[i]
		list[w]++
	}

	for word, count := range list {
		if count > 10 {
			continue
		}

		l = append(l, word)
	}

	return l, nil
}

// Wordlist prepares the mail for training
func (m *Mail) Wordlist() (w []string, err error) {
	var s string

	if m.Subject != nil {
		s = s + " " + *m.Subject
	}

	if m.Body != nil {
		s = s + " " + *m.Body
	}

	w, err = wordlist(s)

	return w, err
}

// cleanWordlist combines Clean and Wordlist in one internal function
func (m *Mail) cleanWordlist() (w []string, err error) {
	err = m.Clean()
	if err != nil {
		return w, err
	}

	w, err = m.Wordlist()
	if err != nil {
		return w, err
	}

	return w, err
}

// LoadMails creates missing directories and then loads all mails from a given
// slice of Maildirs
func LoadMails(d []Maildir) (mails map[Maildir][]*Mail, err error) {
	mails = make(map[Maildir][]*Mail)

	// create missing directories and write index
	for _, val := range d {
		err := val.CreateDirs()
		if err != nil {
			return mails, err
		}

		var m []*Mail
		m, err = val.Index()
		if err != nil {
			return mails, err
		}

		mails[val] = m
	}

	return mails, nil
}
<commit_msg>Please gosimplify<commit_after>package sisyphus

import (
	"bufio"
	"errors"
	"log"
	"math"
	"mime/quotedprintable"
	"os"
	"regexp"
	"strings"

	"github.com/kennygrant/sanitize"
	"github.com/luksen/maildir"
)

// Maildir represents the address to a Maildir directory
type Maildir string

// Mail includes the key of a mail in Maildir
type Mail struct {
	Key           string
	Subject, Body *string
	Junk, New     bool
}

// CreateDirs creates all the required dirs -- if not already there.
func (d Maildir) CreateDirs() error {

	dir := string(d)

	log.Println("create missing directories for Maildir " + dir)

	err := os.MkdirAll(dir+"/.Junk/cur", 0700)
	if err != nil {
		return err
	}
	err = os.MkdirAll(dir+"/new", 0700)
	if err != nil {
		return err
	}
	err = os.MkdirAll(dir+"/cur", 0700)

	return err
}

// Index loads all mail keys from the Maildir directory for processing.
func (d Maildir) Index() (m []*Mail, err error) {

	dir := string(d)

	log.Println("start indexing mails in " + dir)
	dirs := []string{dir, dir + "/.Junk"}
	for _, val := range dirs {
		j, err := maildir.Dir(val).Keys()
		if err != nil {
			return m, err
		}
		for _, v := range j {
			var new Mail
			new.Key = v
			if val == dir+"/.Junk" {
				new.Junk = true
			}
			m = append(m, &new)
		}
	}

	log.Println("all mails in " + dir + " indexed")

	return m, nil
}

// Load reads a mail's subject and body
func (m *Mail) Load(d string) error {

	if m.Junk {
		d = d + "/.Junk"
	}
	message, err := maildir.Dir(d).Message(m.Key)
	if err != nil {
		return err
	}

	// get Subject
	if m.Subject != nil {
		return errors.New("there is already a subject")
	}
	subject := message.Header.Get("Subject")
	m.Subject = &subject

	// get Body
	bQ := quotedprintable.NewReader(message.Body)
	var b []string
	bScanner := bufio.NewScanner(bQ)
	for bScanner.Scan() {
		raw := bScanner.Text()
		b = append(b, raw)
	}

	body := strings.Join(b, " ")
	if m.Body != nil {
		return errors.New("there is already a body")
	}
	m.Body = &body

	return nil
}

func trimStringFromBase64(s string) string {
	if idx := strings.Index(s, "Content-Transfer-Encoding: base64"); idx != -1 {
		return s[:idx-1]
	}
	return s
}

func cleanString(i string) (s string) {

	s = sanitize.Accents(i)
	s = sanitize.HTML(s)
	s = strings.ToLower(s)

	bad := []string{
		"boundary=", "charset", "content-transfer-encoding",
		"content-type", "image/jpeg", "multipart/alternative",
		"multipart/related", "name=", "nextpart", "quoted-printable",
		"text/html", "text/plain", "this email must be viewed in html mode",
		"this is a multi-part message in mime format",
		"windows-1251", "windows-1252", "!", "#", "$", "%", "&", "'",
		"(", ")", "*", "+", ",", ". ", "<", "=", ">", "?", "@", "[",
		"\"", "\\", "\n", "\t", "]", "^", "_", "{", "|", "}",
	}
	for _, b := range bad {
		s = strings.Replace(s, b, " ", -1)
	}
	for i := 0; i < 10; i++ {
		s = strings.Replace(s, "  ", " ", -1)
	}

	return s
}

// Clean cleans the mail's subject and body
func (m *Mail) Clean() error {
	if m.Subject != nil {
		s := trimStringFromBase64(*m.Subject)
		s = cleanString(s)
		m.Subject = &s
	}

	if m.Body != nil {
		b := trimStringFromBase64(*m.Body)
		b = cleanString(b)
		m.Body = &b
	}
	return nil
}

// wordlist takes a string of space separated text and returns a list of unique
// words in a space separated string
func wordlist(s string) (l []string, err error) {
	list := make(map[string]int)

	raw := strings.Split(s, " ")
	var clean []string

	// use regexp compile for use in the loop that follows
	regexMatcher, err := regexp.Compile("(^[a-z]+$)")
	if err != nil {
		return l, err
	}

	for _, w := range raw {

		// no long or too short words
		length := len(w)
		if length < 4 || length > 10 {
			continue
		}

		// no numbers, special characters, etc. -- only words
		match := regexMatcher.MatchString(w)
		if !match {
			continue
		} else {
			clean = append(clean, w)
		}
	}

	// only the first 200 words count
	maxWords := int(math.Min(200, float64(len(clean))))
	for i := 0; i < maxWords; i++ {
		w := clean[i]
		list[w]++
	}

	for word, count := range list {
		if count > 10 {
			continue
		}

		l = append(l, word)
	}

	return l, nil
}

// Wordlist prepares the mail for training
func (m *Mail) Wordlist() (w []string, err error) {
	var s string

	if m.Subject != nil {
		s = s + " " + *m.Subject
	}

	if m.Body != nil {
		s = s + " " + *m.Body
	}

	w, err = wordlist(s)

	return w, err
}

// cleanWordlist combines Clean and Wordlist in one internal function
func (m *Mail) cleanWordlist() (w []string, err error) {
	err = m.Clean()
	if err != nil {
		return w, err
	}

	w, err = m.Wordlist()

	return w, err
}

// LoadMails creates missing directories and then loads all mails from a given
// slice of Maildirs
func LoadMails(d []Maildir) (mails map[Maildir][]*Mail, err error) {
	mails = make(map[Maildir][]*Mail)

	// create missing directories and write index
	for _, val := range d {
		err := val.CreateDirs()
		if err != nil {
			return mails, err
		}

		var m []*Mail
		m, err = val.Index()
		if err != nil {
			return mails, err
		}

		mails[val] = m
	}

	return mails, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/influxdb/influxdb/client/v2"
)

var (
	listURL               = flag.String("list-url", "", "URL to JSON-formatted list of URLs to check")
	pollInterval          = flag.Uint("poll-interval", 60, "Polling interval (in seconds)")
	requestTimeout        = flag.Uint("req-timeout", 20, "Timeout for each HTTP request, must be < than poll-interval")
	precision             = flag.String("precision", "ns", "Precision, valid values are ns and ms")
	debug                 = flag.Bool("debug", false, "To enable a more verbose mode for debugging")
	influxBatchSize       = flag.Int("influx-batch", 0, "Batch size to write to Influx, 0 means batch size = number of URLs")
	influxHost            = flag.String("influx-host", "http://localhost:8086", "InfluxDB host")
	influxDBName          = flag.String("influx-db", "uptime", "InfluxDB database name")
	influxMeasurementName = flag.String("influx-measurement", "page_load_time", "InfluxDB measurement to write to")

	// Global list of urls to fetch
	list *urlList

	errTimeout = errors.New("Request timed out")
)

type fetchableURL struct {
	URL         string
	Description string
	// More fields to come
}

type urlList struct {
	urls []fetchableURL
	lock sync.Mutex
}

type request struct {
	URL        string
	Duration   time.Duration
	StatusCode int
	done       chan struct{} // closes when request is done
	err        error
}

func main() {
	// Validate command line arguments
	if len(*listURL) == 0 {
		fmt.Println("Must pass list-url")
		os.Exit(1)
	}
	if *precision != "ns" && *precision != "ms" {
		fmt.Println("precision must be ms or ns")
		os.Exit(1)
	}

	if *debug {
		fmt.Printf("Connecting to DB '%s' at %s\n", *influxDBName, *influxHost)
	}
	influxClient, err := client.NewHTTPClient(client.HTTPConfig{
		Addr: *influxHost,
	})
	if err != nil {
		fmt.Println("Error creating InfluxDB client:", err)
	}

	// Initialize global list
	list = &urlList{
		lock: sync.Mutex{},
	}

	// Fetch the initial list of URLs
	if err := updateURLsToFecth(*listURL, list); err != nil {
		fmt.Println("Error getting list of URLs:", err)
		os.Exit(1)
	}

	// Update the list of URLs to fetch periodically in a separate goroutine
	go func() {
		c := time.Tick(1 * time.Minute)
		for range c {
			if err := updateURLsToFecth(*listURL, list); err != nil {
				fmt.Println("Error updating list of URLs:", err)
			}
		}
	}()

	// Channel to send concurrent responses on
	respCh := make(chan request, 10)

	// Read responses (concurrently) as they come in
	go handleReponses(influxClient, *influxBatchSize, respCh)

	// The actual fetching of URLs
	for {
		list.lock.Lock()
		for _, u := range list.urls {
			// Setup request object
			req := request{
				URL:  u.URL,
				done: make(chan struct{}),
			}
			// Send it in it's own goroutine
			go send(req, time.Second*time.Duration(int(*requestTimeout)), respCh)
		}
		list.lock.Unlock()
		time.Sleep(time.Second * time.Duration(int(*pollInterval)))
	}
}

func send(req request, timeout time.Duration, resp chan request) {
	if *debug {
		fmt.Println("Sending a request to", req.URL)
	}
	start := time.Now()
	// Send the HTTP request in it's own goroutine to be able to abort the request if reached timeout
	go func() {
		resp, err := http.Get(req.URL)
		if err != nil {
			req.err = err
		} else {
			req.StatusCode = resp.StatusCode
		}
		// Indicate the request is finished
		close(req.done)

	}()
	// Either the request finished, or the timeout triggers
	select {
	case <-req.done:
		// How long did the request take, only applies if not timed out
		req.Duration = time.Since(start)
	case <-time.After(timeout):
		req.Duration = timeout
		req.err = errTimeout
	}
	// Send back on the response channel
	resp <- req
}

// Handles responses sent on passed channel
func handleReponses(cl client.Client, batchSize int, ch chan request) {
	bpConfig := client.BatchPointsConfig{
		Database:  *influxDBName,
		Precision: *precision,
	}
	// Create initial batch
	bp, _ := client.NewBatchPoints(bpConfig)
	for {
		select {
		case req := <-ch:
			tags := map[string]string{"url": req.URL}
			dur := req.Duration.Nanoseconds()
			if *precision == "ms" {
				dur = dur / 1000000
			}
			fields := map[string]interface{}{
				"value": dur,
			}
			if req.err != nil {
				if req.err == errTimeout {
					fields["status_code"] = 408
				} else {
					// If error from the HTTP request, print to stdout and dont add new point
					fmt.Printf("Error for %s: %s\n", req.URL, req.err)
					continue
				}
			} else {
				fields["status_code"] = req.StatusCode
			}
			pt, err := client.NewPoint(*influxMeasurementName, tags, fields)
			if err != nil {
				fmt.Println("Error creating InfluxDB point: ", err.Error())
			}
			// Add point to current batch
			bp.AddPoint(pt)
			if *debug {
				fmt.Printf("%s\t%d\t%s\n", req.URL, req.StatusCode, req.Duration)
			}
			// If should send current batch
			if len(bp.Points()) == batchSize {
				if *debug {
					fmt.Println("Writing points to InfluxDB")
				}
				cl.Write(bp)
				// Create a new batch
				bp, _ = client.NewBatchPoints(bpConfig)
			}
		}
	}
}

// Update the list of urls to fecth
func updateURLsToFecth(url string, ul *urlList) error {
	if *debug {
		fmt.Println("Checking for URLs to fetch...")
	}
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	// Decode the JSON response
	decoder := json.NewDecoder(resp.Body)
	fetchableURLs := []fetchableURL{}
	if err := decoder.Decode(&fetchableURLs); err != nil {
		return err
	}
	// If new URLs were fetched, clear out the old ones and add the new ones
	list.lock.Lock()
	list.urls = fetchableURLs
	list.lock.Unlock()
	// Update InfluxDB batch size (if not manually set)
	if *influxBatchSize == 0 {
		*influxBatchSize = len(list.urls)
	}

	if *debug {
		fmt.Println("Current list of URLs is:")
		for _, u := range list.urls {
			fmt.Println(u.URL)
		}
	}
	return nil
}

func init() {
	flag.Parse()
}
<commit_msg>Fix some comments and use real http client for better control over sending the requests<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"os"
	"sync"
	"time"

	"github.com/influxdb/influxdb/client/v2"
)

var (
	listURL               = flag.String("list-url", "", "URL to JSON-formatted list of URLs to check")
	pollInterval          = flag.Uint("poll-interval", 60, "Polling interval (in seconds)")
	requestTimeout        = flag.Uint("req-timeout", 20, "Timeout for each HTTP request (in seconds)")
	precision             = flag.String("precision", "ns", "Precision, valid values are ns and ms")
	debug                 = flag.Bool("debug", false, "To enable a more verbose mode for debugging")
	influxBatchSize       = flag.Int("influx-batch", 0, "Batch size to write to Influx, 0 means batch size = number of URLs")
	influxHost            = flag.String("influx-host", "http://localhost:8086", "InfluxDB host")
	influxDBName          = flag.String("influx-db", "uptime", "InfluxDB database name")
	influxMeasurementName = flag.String("influx-measurement", "page_load_time", "InfluxDB measurement to write to")

	// Global list of urls to fetch
	list *urlList

	errTimeout = errors.New("Request timed out")
)

type fetchableURL struct {
	URL string
}

type urlList struct {
	urls []fetchableURL
	lock sync.Mutex
}

type request struct {
	httpReq    *http.Request
	Duration   time.Duration
	StatusCode int
	done       chan struct{} // closes when request is done
	err        error
}

func main() {
	// Validate command line arguments
	if len(*listURL) == 0 {
		fmt.Println("Must pass list-url")
		os.Exit(1)
	}
	if *precision != "ns" && *precision != "ms" {
		fmt.Println("precision must be ms or ns")
		os.Exit(1)
	}

	if *debug {
		fmt.Printf("Connecting to InfluxDB %s@%s\n", *influxDBName, *influxHost)
	}
	influxClient, err := client.NewHTTPClient(client.HTTPConfig{
		Addr: *influxHost,
	})
	if err != nil {
		fmt.Println("Error creating InfluxDB client:", err)
		os.Exit(1)
	}

	// Initialize global list
	list = &urlList{
		lock: sync.Mutex{},
	}

	// Fetch the initial list of URLs
	if err := updateURLsToFecth(*listURL, list); err != nil {
		fmt.Println("Error getting list of URLs:", err)
		os.Exit(1)
	}

	// Update the list of URLs to fetch periodically in a separate goroutine
	go func() {
		c := time.Tick(1 * time.Minute)
		for range c {
			if err := updateURLsToFecth(*listURL, list); err != nil {
				fmt.Println("Error updating list of URLs:", err)
			}
		}
	}()

	// Channel to send concurrent responses on
	respCh := make(chan request)

	// Read responses (concurrently) as they come in
	go handleReponses(influxClient, *influxBatchSize, respCh)

	// Setup HTTP client to use to send the requests
	tr := &http.Transport{}
	httpClient := &http.Client{Transport: tr}

	// The actual fetching of URLs
	for {
		// Make sure the list is not tampered with when iterating over it
		list.lock.Lock()
		for _, u := range list.urls {
			// Setup underlying http request
			httpReq, err := http.NewRequest("GET", u.URL, nil)
			if err != nil {
				fmt.Println(err)
				continue
			}

			// Setup request object
			req := request{
				done:    make(chan struct{}),
				httpReq: httpReq,
			}
			// Send it in it's own goroutine
			go send(httpClient, tr, req, time.Second*time.Duration(int(*requestTimeout)), respCh)
		}
		list.lock.Unlock()
		time.Sleep(time.Second * time.Duration(int(*pollInterval)))
	}
}

// send sends a request, kills it after passed timeout and writes the result to passed channel
func send(httpClient *http.Client, tr *http.Transport, req request, timeout time.Duration, resp chan request) {
	if *debug {
		fmt.Println("Sending a request to", req.httpReq.URL)
	}
	start := time.Now()
	// Send the HTTP request in it's own goroutine using a HTTP client to be able to abort the request if reached timeout
	go func() {
		// Setup real HTTP client to be able to cancel the request
		resp, err := httpClient.Do(req.httpReq)
		if err != nil {
			req.err = err
		} else {
			req.StatusCode = resp.StatusCode
		}
		// Indicate the request is finished
		close(req.done)
	}()
	// Either the request finished, or the timeout triggers
	select {
	case <-req.done:
		// How long did the request take, only applies if not timed out
		req.Duration = time.Since(start)
	case <-time.After(timeout):
		// Manually cancel the request in flight
		tr.CancelRequest(req.httpReq)
		req.Duration = timeout
		req.err = errTimeout
	}
	// Send back on the response channel
	resp <- req
}

// Handles responses sent on passed channel
func handleReponses(cl client.Client, batchSize int, ch chan request) {
	bpConfig := client.BatchPointsConfig{
		Database:  *influxDBName,
		Precision: *precision,
	}
	// Create initial batch
	bp, _ := client.NewBatchPoints(bpConfig)
	for {
		select {
		case req := <-ch:
			tags := map[string]string{"url": req.httpReq.URL.String()}
			dur := req.Duration.Nanoseconds()
			if *precision == "ms" {
				dur = dur / 1000000
			}
			fields := map[string]interface{}{
				"value": dur,
			}
			if req.err != nil {
				if req.err == errTimeout {
					fields["status_code"] = 408
				} else {
					// If error from the HTTP request, print to stdout and dont add new point
					fmt.Printf("Error for %s: %s\n", req.httpReq.URL, req.err)
					continue
				}
			} else {
				fields["status_code"] = req.StatusCode
			}
			pt, err := client.NewPoint(*influxMeasurementName, tags, fields)
			if err != nil {
				fmt.Println("Error creating InfluxDB point: ", err.Error())
			}
			// Add point to current batch
			bp.AddPoint(pt)
			if *debug {
				fmt.Printf("%s\t%d\t%s\n", req.httpReq.URL, req.StatusCode, req.Duration)
			}
			// If should send current batch
			if len(bp.Points()) == batchSize {
				if *debug {
					fmt.Println("Writing points to InfluxDB")
				}
				cl.Write(bp)
				// Create a new batch
				bp, _ = client.NewBatchPoints(bpConfig)
			}
		}
	}
}

// Update the list of urls to fecth
func updateURLsToFecth(url string, ul *urlList) error {
	if *debug {
		fmt.Println("Checking for URLs to fetch...")
	}
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	// Decode the JSON response
	decoder := json.NewDecoder(resp.Body)
	fetchableURLs := []fetchableURL{}
	if err := decoder.Decode(&fetchableURLs); err != nil {
		return err
	}
	// If new URLs were fetched, clear out the old ones and add the new ones
	list.lock.Lock()
	list.urls = fetchableURLs
	list.lock.Unlock()
	// Update InfluxDB batch size (if not manually set)
	if *influxBatchSize == 0 {
		*influxBatchSize = len(list.urls)
	}

	if *debug {
		fmt.Println("Current list of URLs is:")
		for _, u := range list.urls {
			fmt.Println(u.URL)
		}
	}
	return nil
}

func init() {
	flag.Parse()
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "bufio"
import "os"
import "sync/atomic"

func main() {
	myActor := ActorOf(new(MyActor))
	myActor.Tell(Hello{Name: "Roger"})
	myActor.Tell(Hello{Name: "Go"})
	bufio.NewReader(os.Stdin).ReadString('\n')
}

type ActorRef interface {
	Tell(message interface{})
	SendSystemMessage(message interface{})
}

type ChannelActorRef struct {
	actorCell *ActorCell
}

func (ref *ChannelActorRef) Tell(message interface{}) {
	ref.actorCell.mailbox.userMailbox <- message
	ref.actorCell.schedule()
}

func (ref *ChannelActorRef) SendSystemMessage(message interface{}) {
	ref.actorCell.mailbox.userMailbox <- message
	ref.actorCell.schedule()
}

type Actor interface {
	Receive(message interface{})
}

func ActorOf(actor Actor) ActorRef {
	userMailbox := make(chan interface{}, 100)
	systemMailbox := make(chan interface{}, 100)
	mailbox := Mailbox{
		userMailbox:   userMailbox,
		systemMailbox: systemMailbox,
	}
	cell := &ActorCell{
		mailbox:         &mailbox,
		actor:           actor,
		hasMoreMessages: int32(0),
		schedulerStatus: int32(0),
	}
	ref := ChannelActorRef{
		actorCell: cell,
	}

	return &ref
}

const MailboxIdle int32 = 0
const MailboxBussy int32 = 1
const MailboxHasMoreMessages int32 = 1
const MailboxHasNoMessages int32 = 0

func (cell *ActorCell) schedule() {
	swapped := atomic.CompareAndSwapInt32(&cell.schedulerStatus, MailboxIdle, MailboxBussy)
	atomic.StoreInt32(&cell.hasMoreMessages, MailboxHasMoreMessages) //we have more messages to process
	if swapped {
		go cell.processMessages()
	}
}

func (cell *ActorCell) processMessages() {
	atomic.StoreInt32(&cell.hasMoreMessages, MailboxHasNoMessages)
	for i := 0; i < 30; i++ {
		select {
		case sysMsg := <-cell.mailbox.systemMailbox:
			//prioritize system messages
			cell.invokeSystemMessage(sysMsg)
		default:
			//if no system message is present, try read user message
			select {
			case userMsg := <-cell.mailbox.userMailbox:
				cell.invokeUserMessage(userMsg)
			default:
			}
		}
	}
	atomic.StoreInt32(&cell.schedulerStatus, MailboxIdle)
	hasMore := atomic.LoadInt32(&cell.hasMoreMessages) //was there any messages scheduled since we began processing?
	status := atomic.LoadInt32(&cell.schedulerStatus)  //have there been any new scheduling of the mailbox? (e.g. race condition from the two above lines)
	if hasMore == MailboxHasMoreMessages && status == MailboxIdle {
		atomic.StoreInt32(&cell.schedulerStatus, MailboxBussy)
		go cell.processMessages()
	}
}

type ActorCell struct {
	mailbox         *Mailbox
	actor           Actor
	schedulerStatus int32
	hasMoreMessages int32
}

type Mailbox struct {
	userMailbox   chan interface{}
	systemMailbox chan interface{}
}

func (cell *ActorCell) invokeSystemMessage(message interface{}) {
	fmt.Printf("Received system message %v\n", message)
}

func (cell *ActorCell) invokeUserMessage(message interface{}) {
	cell.actor.Receive(message)
}

type MyActor struct{ messageCount int }
type Hello struct{ Name string }

func (state *MyActor) Receive(message interface{}) {
	switch msg := message.(type) {
	default:
		fmt.Printf("unexpected type %T\n", msg) // %T prints whatever type t has
	case Hello:
		fmt.Printf("Hello %v\n", msg.Name) // t has type bool
		state.messageCount++
	}
}
<commit_msg>async scheduler<commit_after>package main

import "fmt"
import "bufio"
import "os"
import "sync/atomic"

func main() {
	myActor := ActorOf(new(MyActor))
	myActor.Tell(Hello{Name: "Roger"})
	myActor.Tell(Hello{Name: "Go"})
	bufio.NewReader(os.Stdin).ReadString('\n')
}

type ActorRef interface {
	Tell(message interface{})
	SendSystemMessage(message interface{})
}

type ChannelActorRef struct {
	actorCell *ActorCell
}

func (ref *ChannelActorRef) Tell(message interface{}) {
	ref.actorCell.mailbox.userMailbox <- message
	ref.actorCell.schedule()
}

func (ref *ChannelActorRef) SendSystemMessage(message interface{}) {
	ref.actorCell.mailbox.userMailbox <- message
	ref.actorCell.schedule()
}

type Actor interface {
	Receive(message interface{})
}

func ActorOf(actor Actor) ActorRef {
	userMailbox := make(chan interface{}, 100)
	systemMailbox := make(chan interface{}, 100)
	mailbox := Mailbox{
		userMailbox:   userMailbox,
		systemMailbox: systemMailbox,
	}
	cell := &ActorCell{
		mailbox:         &mailbox,
		actor:           actor,
		hasMoreMessages: int32(0),
		schedulerStatus: int32(0),
	}
	ref := ChannelActorRef{
		actorCell: cell,
	}

	return &ref
}

const MailboxIdle int32 = 0
const MailboxBussy int32 = 1
const MailboxHasMoreMessages int32 = 1
const MailboxHasNoMessages int32 = 0

func (cell *ActorCell) schedule() {
	swapped := atomic.CompareAndSwapInt32(&cell.schedulerStatus, MailboxIdle, MailboxBussy)
	atomic.StoreInt32(&cell.hasMoreMessages, MailboxHasMoreMessages) //we have more messages to process
	if swapped {
		go cell.processMessages()
	}
}

func (cell *ActorCell) processMessages() {
	atomic.StoreInt32(&cell.hasMoreMessages, MailboxHasNoMessages)
	for i := 0; i < 30; i++ {
		select {
		case sysMsg := <-cell.mailbox.systemMailbox:
			//prioritize system messages
			cell.invokeSystemMessage(sysMsg)
		default:
			//if no system message is present, try read user message
			select {
			case userMsg := <-cell.mailbox.userMailbox:
				cell.invokeUserMessage(userMsg)
			default:
			}
		}
	}
	atomic.StoreInt32(&cell.schedulerStatus, MailboxIdle)
	hasMore := atomic.LoadInt32(&cell.hasMoreMessages) //was there any messages scheduled since we began processing?
	status := atomic.LoadInt32(&cell.schedulerStatus)  //have there been any new scheduling of the mailbox? (e.g. race condition from the two above lines)
	if hasMore == MailboxHasMoreMessages && status == MailboxIdle {
		swapped := atomic.CompareAndSwapInt32(&cell.schedulerStatus, MailboxIdle, MailboxBussy)
		if swapped {
			go cell.processMessages()
		}
	}
}

type ActorCell struct {
	mailbox         *Mailbox
	actor           Actor
	schedulerStatus int32
	hasMoreMessages int32
}

type Mailbox struct {
	userMailbox   chan interface{}
	systemMailbox chan interface{}
}

func (cell *ActorCell) invokeSystemMessage(message interface{}) {
	fmt.Printf("Received system message %v\n", message)
}

func (cell *ActorCell) invokeUserMessage(message interface{}) {
	cell.actor.Receive(message)
}

type MyActor struct{ messageCount int }
type Hello struct{ Name string }

func (state *MyActor) Receive(message interface{}) {
	switch msg := message.(type) {
	default:
		fmt.Printf("unexpected type %T\n", msg) // %T prints whatever type t has
	case Hello:
		fmt.Printf("Hello %v\n", msg.Name) // t has type bool
		state.messageCount++
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/matryer/moq/pkg/moq"
)

// Version is the command version, injected at build time.
var Version string = "dev"

type userFlags struct {
	outFile    string
	pkgName    string
	formatter  string
	stubImpl   bool
	skipEnsure bool
	args       []string
}

func main() {
	var flags userFlags
	flag.StringVar(&flags.outFile, "out", "", "output file (default stdout)")
	flag.StringVar(&flags.pkgName, "pkg", "", "package name (default will infer)")
	flag.StringVar(&flags.formatter, "fmt", "", "go pretty-printer: gofmt, goimports or noop (default gofmt)")
	flag.BoolVar(&flags.stubImpl, "stub", false,
		"return zero values when no mock implementation is provided, do not panic")
	printVersion := flag.Bool("version", false, "show the version for moq")
	flag.BoolVar(&flags.skipEnsure, "skip-ensure", false,
		"suppress mock implementation check, avoid import cycle if mocks generated outside of the tested package")

	flag.Usage = func() {
		fmt.Println(`moq [flags] source-dir interface [interface2 [interface3 [...]]]`)
		flag.PrintDefaults()
		fmt.Println(`Specifying an alias for the mock is also supported with the format 'interface:alias'`)
		fmt.Println(`Ex: moq -pkg different . MyInterface:MyMock`)
	}

	flag.Parse()
	flags.args = flag.Args()

	if *printVersion {
		fmt.Printf("moq version %s\n", version)
		os.Exit(0)
	}

	if err := run(flags); err != nil {
		fmt.Fprintln(os.Stderr, err)
		flag.Usage()
		os.Exit(1)
	}
}

func run(flags userFlags) error {
	if len(flags.args) < 2 {
		return errors.New("not enough arguments")
	}

	var buf bytes.Buffer
	var out io.Writer = os.Stdout
	if flags.outFile != "" {
		out = &buf
	}

	srcDir, args := flags.args[0], flags.args[1:]
	m, err := moq.New(moq.Config{
		SrcDir:     srcDir,
		PkgName:    flags.pkgName,
		Formatter:  flags.formatter,
		StubImpl:   flags.stubImpl,
		SkipEnsure: flags.skipEnsure,
	})
	if err != nil {
		return err
	}

	if err = m.Mock(out, args...); err != nil {
		return err
	}

	if flags.outFile == "" {
		return nil
	}

	// create the file
	err = os.MkdirAll(filepath.Dir(flags.outFile), 0750)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(flags.outFile, buf.Bytes(), 0600)
}
<commit_msg>fixed bug after changing version var name<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/matryer/moq/pkg/moq"
)

// Version is the command version, injected at build time.
var Version string = "dev"

type userFlags struct {
	outFile    string
	pkgName    string
	formatter  string
	stubImpl   bool
	skipEnsure bool
	args       []string
}

func main() {
	var flags userFlags
	flag.StringVar(&flags.outFile, "out", "", "output file (default stdout)")
	flag.StringVar(&flags.pkgName, "pkg", "", "package name (default will infer)")
	flag.StringVar(&flags.formatter, "fmt", "", "go pretty-printer: gofmt, goimports or noop (default gofmt)")
	flag.BoolVar(&flags.stubImpl, "stub", false,
		"return zero values when no mock implementation is provided, do not panic")
	printVersion := flag.Bool("version", false, "show the version for moq")
	flag.BoolVar(&flags.skipEnsure, "skip-ensure", false,
		"suppress mock implementation check, avoid import cycle if mocks generated outside of the tested package")

	flag.Usage = func() {
		fmt.Println(`moq [flags] source-dir interface [interface2 [interface3 [...]]]`)
		flag.PrintDefaults()
		fmt.Println(`Specifying an alias for the mock is also supported with the format 'interface:alias'`)
		fmt.Println(`Ex: moq -pkg different . MyInterface:MyMock`)
	}

	flag.Parse()
	flags.args = flag.Args()

	if *printVersion {
		fmt.Printf("moq version %s\n", Version)
		os.Exit(0)
	}

	if err := run(flags); err != nil {
		fmt.Fprintln(os.Stderr, err)
		flag.Usage()
		os.Exit(1)
	}
}

func run(flags userFlags) error {
	if len(flags.args) < 2 {
		return errors.New("not enough arguments")
	}

	var buf bytes.Buffer
	var out io.Writer = os.Stdout
	if flags.outFile != "" {
		out = &buf
	}

	srcDir, args := flags.args[0], flags.args[1:]
	m, err := moq.New(moq.Config{
		SrcDir:     srcDir,
		PkgName:    flags.pkgName,
		Formatter:  flags.formatter,
		StubImpl:   flags.stubImpl,
		SkipEnsure: flags.skipEnsure,
	})
	if err != nil {
		return err
	}

	if err = m.Mock(out, args...); err != nil {
		return err
	}

	if flags.outFile == "" {
		return nil
	}

	// create the file
	err = os.MkdirAll(filepath.Dir(flags.outFile), 0750)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(flags.outFile, buf.Bytes(), 0600)
}
<|endoftext|>


<commit_before>package hammer

import (
	"context"
	"log"

	"github.com/antihax/evedata/internal/datapackages"
	"github.com/antihax/goesi"
	"github.com/antihax/goesi/esi"
)

func init() {
	registerConsumer("structure", structureConsumer)
	registerConsumer("structureOrders", structureOrdersConsumer)
}

func structureConsumer(s *Hammer, parameter interface{}) {
	structureID := parameter.(int64)

	if s.inQueue.CheckWorkExpired("evedata_structure_failure", structureID) {
		return
	}

	ctx := context.WithValue(context.Background(), goesi.ContextOAuth2, *s.token)
	struc, _, err := s.esi.ESI.UniverseApi.GetUniverseStructuresStructureId(ctx, structureID, nil)
	if err != nil {
		log.Printf("Bad structure: %s %d\n", err, structureID)
		err := s.inQueue.SetWorkExpire("evedata_structure_failure", structureID, 86400)
		if err != nil {
			log.Printf("failed setting failure: %s %d\n", err, structureID)
		}
		return
	}
	log.Printf("Good structure: %s %d\n", err, structureID)
	// Send out the result
	err = s.QueueResult(&datapackages.Structure{Structure: struc, StructureID: structureID}, "structure")
	if err != nil {
		log.Println(err)
		return
	}
}

func structureOrdersConsumer(s *Hammer, parameter interface{}) {
	structureID := parameter.(int64)
	var page int32 = 1
	orders := []esi.GetMarketsStructuresStructureId200Ok{}

	if s.inQueue.CheckWorkExpired("evedata_structure_failure", structureID) {
		return
	}

	ctx := context.WithValue(context.Background(), goesi.ContextOAuth2, *s.token)

	for {
		o, _, err := s.esi.ESI.MarketApi.GetMarketsStructuresStructureId(ctx, structureID, map[string]interface{}{"page": page})
		if err != nil {
			log.Printf("Bad structure market: %s %d\n", err, structureID)
			err := s.inQueue.SetWorkExpire("evedata_structure_failure", structureID, 86400)
			if err != nil {
				log.Printf("failed setting failure: %s %d\n", err, structureID)
			}
			return
		} else if len(o) == 0 { // end of the pages
			break
		}

		orders = append(orders, o...)

		page++
	}
	log.Printf("Good structure market: %d\n", structureID)
	// early out if there are no orders
	if len(orders) == 0 {
		return
	}

	// Send out the result
	err := s.QueueResult(&datapackages.StructureOrders{Orders: orders, StructureID: structureID}, "structureOrders")
	if err != nil {
		log.Println(err)
		return
	}
}
<commit_msg>remove positive message<commit_after>package hammer

import (
	"context"
	"log"

	"github.com/antihax/evedata/internal/datapackages"
	"github.com/antihax/goesi"
	"github.com/antihax/goesi/esi"
)

func init() {
	registerConsumer("structure", structureConsumer)
	registerConsumer("structureOrders", structureOrdersConsumer)
}

func structureConsumer(s *Hammer, parameter interface{}) {
	structureID := parameter.(int64)

	if s.inQueue.CheckWorkExpired("evedata_structure_failure", structureID) {
		return
	}

	ctx := context.WithValue(context.Background(), goesi.ContextOAuth2, *s.token)
	struc, _, err := s.esi.ESI.UniverseApi.GetUniverseStructuresStructureId(ctx, structureID, nil)
	if err != nil {
		log.Printf("Bad structure: %s %d\n", err, structureID)
		err := s.inQueue.SetWorkExpire("evedata_structure_failure", structureID, 86400)
		if err != nil {
			log.Printf("failed setting failure: %s %d\n", err, structureID)
		}
		return
	}
	// Send out the result
	err = s.QueueResult(&datapackages.Structure{Structure: struc, StructureID: structureID}, "structure")
	if err != nil {
		log.Println(err)
		return
	}
}

func structureOrdersConsumer(s *Hammer, parameter interface{}) {
	structureID := parameter.(int64)
	var page int32 = 1
	orders := []esi.GetMarketsStructuresStructureId200Ok{}

	if s.inQueue.CheckWorkExpired("evedata_structure_failure", structureID) {
		return
	}

	ctx := context.WithValue(context.Background(), goesi.ContextOAuth2, *s.token)

	for {
		o, _, err := s.esi.ESI.MarketApi.GetMarketsStructuresStructureId(ctx, structureID, map[string]interface{}{"page": page})
		if err != nil {
			log.Printf("Bad structure market: %s %d\n", err, structureID)
			err := s.inQueue.SetWorkExpire("evedata_structure_failure", structureID, 86400)
			if err != nil {
				log.Printf("failed setting failure: %s %d\n", err, structureID)
			}
			return
		} else if len(o) == 0 { // end of the pages
			break
		}

		orders = append(orders, o...)

		page++
	}
	// early out if there are no orders
	if len(orders) == 0 {
		return
	}

	// Send out the result
	err := s.QueueResult(&datapackages.StructureOrders{Orders: orders, StructureID: structureID}, "structureOrders")
	if err != nil {
		log.Println(err)
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/clstokes/aero/commands"
	"github.com/clstokes/aero/providers"
	"github.com/clstokes/aero/structs"
	"github.com/mitchellh/cli"
	"os"
)

type Factory func() structs.Provider

var fingerprinters = map[string]Factory{
	"amazon": providers.InitAmazon,
	"google": providers.InitGoogle,
}

var currentProvider structs.Provider

func main() {
	os.Exit(Run(os.Args[1:]))
}

func Run(args []string) int {
	cli := &cli.CLI{
		Args:     os.Args[1:],
		Commands: commands.Commands(&currentProvider),
	}

	// TODO: This is passed above but set here - feel wonky.
	currentProvider = getCurrentProvider()
	if currentProvider == nil {
		fmt.Printf("Unknown provider.\n")
		return 1
	}

	exitCode, err := cli.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
		return 1
	}

	return exitCode
}

func getCurrentProvider() structs.Provider {
	for _, val := range fingerprinters {
		provider := val()
		if provider.IsCurrentProvider() {
			return provider
		}
	}
	return nil
}
<commit_msg>Add version - 0.1.0.<commit_after>package main

import (
	"fmt"
	"github.com/clstokes/aero/commands"
	"github.com/clstokes/aero/providers"
	"github.com/clstokes/aero/structs"
	"github.com/mitchellh/cli"
	"os"
)

const VERSION = "0.1.0"

type Factory func() structs.Provider

var fingerprinters = map[string]Factory{
	"amazon": providers.InitAmazon,
	"google": providers.InitGoogle,
}

var currentProvider structs.Provider

func main() {
	os.Exit(Run(os.Args[1:]))
}

func Run(args []string) int {
	cli := &cli.CLI{
		Args:     os.Args[1:],
		Commands: commands.Commands(&currentProvider),
		Version:  VERSION,
	}

	// TODO: This is passed above but set here - feels wonky.
	currentProvider = getCurrentProvider()
	if currentProvider == nil {
		fmt.Printf("Unknown provider.\n")
		return 1
	}

	exitCode, err := cli.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
		return 1
	}

	return exitCode
}

func getCurrentProvider() structs.Provider {
	for _, val := range fingerprinters {
		provider := val()
		if provider.IsCurrentProvider() {
			return provider
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Christian Funkhouser <christian.funkhouser@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/cfunkhouser/preppi/preppi"
)

var (
	version = "0.1.0"
	buildID = "dev" // Overriden at build time by build scripts.

	mapFile = flag.String("config", "/boot/preppi/preppi.conf", "Mappings file path")
	verFlag = flag.Bool("version", false, "If true, print version and exit.")
)

func main() {
	flag.Parse()

	if *verFlag {
		fmt.Printf("preppi v%v (%v)\n", version, buildID)
		os.Exit(0)
	}

	log.Printf("preppi v%v (%v) starting", version, buildID)
	start := time.Now()
	if *mapFile == "" {
		log.Fatal("No --config specified, nothing to do!")
	}
	mapper, err := preppi.MapperFromConfig(*mapFile)
	if err != nil {
		log.Fatal(err)
	}
	if err := mapper.Apply(); err != nil {
		log.Fatal(err)
	}
	log.Printf("preppi applied %v files in %v", len(mapper.Mappings), time.Since(start))
}
<commit_msg>Add a -dry_run flag<commit_after>// Copyright (c) 2017 Christian Funkhouser <christian.funkhouser@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/cfunkhouser/preppi/preppi"
)

var (
	version = "0.1.0"
	buildID = "dev" // Overriden at build time by build scripts.

	mapFile = flag.String("config", "/boot/preppi/preppi.conf", "Mappings file path")
	verFlag = flag.Bool("version", false, "If true, print version and exit.")
	dryRun  = flag.Bool("dry_run", false, "If true, parses the config but changes nothing.")
)

func main() {
	flag.Parse()

	if *verFlag {
		fmt.Printf("preppi v%v (%v)\n", version, buildID)
		os.Exit(0)
	}

	log.Printf("preppi v%v (%v) starting", version, buildID)
	start := time.Now()
	if *mapFile == "" {
		log.Fatal("No --config specified, nothing to do!")
	}
	mapper, err := preppi.MapperFromConfig(*mapFile)
	if err != nil {
		log.Fatal(err)
	}
	if !*dryRun {
		if err := mapper.Apply(); err != nil {
			log.Fatal(err)
		}
		log.Printf("preppi applied %v files in %v", len(mapper.Mappings), time.Since(start))
	} else {
		log.Printf("preppi would have applied %v files if not for --dry_run", len(mapper.Mappings))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/exec"
	"runtime"
	"syscall"

	"github.com/dotcloud/docker/pkg/user"
)

const VERSION = "1.0"

func init() {
	// make sure we only have one process and that it runs on the main thread (so that ideally, when we Exec, we keep our user switches and stuff)
	runtime.GOMAXPROCS(1)
	runtime.LockOSThread()
}

func main() {
	log.SetFlags(0) // no timestamps on our logs

	if len(os.Args) <= 2 {
		log.Printf("Usage: %s user-spec command [args]", os.Args[0])
		log.Printf("   ie: %s tianon bash", os.Args[0])
		log.Printf("       %s nobody:root bash -c 'whoami && id'", os.Args[0])
		log.Printf("       %s 1000:1 id", os.Args[0])
		log.Printf("%s version: %s", os.Args[0], VERSION)
		os.Exit(1)
	}

	uid, gid, suppGids, err := user.GetUserGroupSupplementary(os.Args[1], syscall.Getuid(), syscall.Getgid())
	if err != nil {
		log.Fatalf("error: failed parsing '%s': %v", os.Args[1], err)
	}

	if err := syscall.Setgroups(suppGids); err != nil {
		log.Fatalf("error: failed to setgroups: %v", err)
	}
	if err := syscall.Setgid(gid); err != nil {
		log.Fatalf("error: failed to setgid: %v", err)
	}
	if err := syscall.Setuid(uid); err != nil {
		log.Fatalf("error: failed to setuid: %v", err)
	}

	name, err := exec.LookPath(os.Args[2])
	if err != nil {
		log.Fatalf("error: %v", err)
	}

	err = syscall.Exec(name, os.Args[2:], os.Environ())
	if err != nil {
		log.Fatalf("error: exec failed: %v", err)
	}
}
<commit_msg>bump version<commit_after>package main

import (
	"log"
	"os"
	"os/exec"
	"runtime"
	"syscall"

	"github.com/dotcloud/docker/pkg/user"
)

const VERSION = "1.1"

func init() {
	// make sure we only have one process and that it runs on the main thread (so that ideally, when we Exec, we keep our user switches and stuff)
	runtime.GOMAXPROCS(1)
	runtime.LockOSThread()
}

func main() {
	log.SetFlags(0) // no timestamps on our logs

	if len(os.Args) <= 2 {
		log.Printf("Usage: %s user-spec command [args]", os.Args[0])
		log.Printf("   ie: %s tianon bash", os.Args[0])
		log.Printf("       %s nobody:root bash -c 'whoami && id'", os.Args[0])
		log.Printf("       %s 1000:1 id", os.Args[0])
		log.Printf("%s version: %s", os.Args[0], VERSION)
		os.Exit(1)
	}

	uid, gid, suppGids, err := user.GetUserGroupSupplementary(os.Args[1], syscall.Getuid(), syscall.Getgid())
	if err != nil {
		log.Fatalf("error: failed parsing '%s': %v", os.Args[1], err)
	}

	if err := syscall.Setgroups(suppGids); err != nil {
		log.Fatalf("error: failed to setgroups: %v", err)
	}
	if err := syscall.Setgid(gid); err != nil {
		log.Fatalf("error: failed to setgid: %v", err)
	}
	if err := syscall.Setuid(uid); err != nil {
		log.Fatalf("error: failed to setuid: %v", err)
	}

	name, err := exec.LookPath(os.Args[2])
	if err != nil {
		log.Fatalf("error: %v", err)
	}

	err = syscall.Exec(name, os.Args[2:], os.Environ())
	if err != nil {
		log.Fatalf("error: exec failed: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/nlopes/slack"
	"os"
)

var SLACK_TOKEN = os.Getenv("SLACK_TOKEN")
var SEND_AS_USER = os.Getenv("SEND_AS_USER")

func main() {
	api := slack.New(SLACK_TOKEN)
	rtm := api.NewRTM()

	go rtm.ManageConnection()

Loop:
	for {
		select {
		case msg := <-rtm.IncomingEvents:
			fmt.Print("Event Received: ")
			switch ev := msg.Data.(type) {
			case *slack.TeamJoinEvent:
				welcomeUser(ev.User.ID, api)
			case *slack.LatencyReport:
				fmt.Printf("Current latency: %v\n", ev.Value)

			case *slack.RTMError:
				fmt.Printf("Error: %s\n", ev.Error())

			case *slack.InvalidAuthEvent:
				fmt.Printf("Invalid credentials")
				break Loop

			default:

				// Ignore other events..
				// fmt.Printf("Unexpected: %v\n", msg.Data)
			}
		}
	}
}

func welcomeUser(user string, api *slack.Client) {
	fmt.Printf("New user joined: %s", user)
	_, _, channelID, err := api.OpenIMChannel(user)
	if err != nil {
		fmt.Printf("Error: %s\n", err)
		return
	}
	defer api.CloseIMChannel(channelID)
	text := `
Hi! Welcome to the Slack group! When you have a moment, head over to the #intros channel to introduce yourself to everyone here. You should mention your name, your org + position, and any expertise you have that might be helpful to the group.
`
	params := slack.PostMessageParameters{
		Username: SEND_AS_USER,
		AsUser:   true,
	}
	_, timestamp, err := api.PostMessage(channelID, text, params)
	if err != nil {
		fmt.Printf("Error: %s\n", err)
		return
	}
	fmt.Printf("Message successfully sent to channel %s at %s", channelID, timestamp)
}
<commit_msg>LinkNames=1<commit_after>package main

import (
	"fmt"
	"github.com/nlopes/slack"
	"os"
)

var SLACK_TOKEN = os.Getenv("SLACK_TOKEN")
var SEND_AS_USER = os.Getenv("SEND_AS_USER")

func main() {
	api := slack.New(SLACK_TOKEN)
	rtm := api.NewRTM()

	go rtm.ManageConnection()

Loop:
	for {
		select {
		case msg := <-rtm.IncomingEvents:
			fmt.Print("Event Received: ")
			switch ev := msg.Data.(type) {
			case *slack.TeamJoinEvent:
				welcomeUser(ev.User.ID, api)
			case *slack.LatencyReport:
				fmt.Printf("Current latency: %v\n", ev.Value)

			case *slack.RTMError:
				fmt.Printf("Error: %s\n", ev.Error())

			case *slack.InvalidAuthEvent:
				fmt.Printf("Invalid credentials")
				break Loop

			default:

				// Ignore other events..
				// fmt.Printf("Unexpected: %v\n", msg.Data)
			}
		}
	}
}

func welcomeUser(user string, api *slack.Client) {
	fmt.Printf("New user joined: %s", user)
	_, _, channelID, err := api.OpenIMChannel(user)
	if err != nil {
		fmt.Printf("Error: %s\n", err)
		return
	}
	defer api.CloseIMChannel(channelID)
	text := `
Hi! Welcome to the Slack group! When you have a moment, head over to the #intros channel to introduce yourself to everyone here. You should mention your name, your org + position, and any expertise you have that might be helpful to the group.
`
	params := slack.PostMessageParameters{
		Username:  SEND_AS_USER,
		AsUser:    true,
		LinkNames: 1,
	}
	_, timestamp, err := api.PostMessage(channelID, text, params)
	if err != nil {
		fmt.Printf("Error: %s\n", err)
		return
	}
	fmt.Printf("Message successfully sent to channel %s at %s", channelID, timestamp)
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/codegangsta/cli"
)

var Version string = "0.6.4"

func main() {
	newApp().Run(os.Args)
}

func newApp() *cli.App {
	app := cli.NewApp()
	app.Name = "ghq"
	app.Usage = "Manage GitHub repository clones"
	app.Version = Version
	app.Author = "motemen"
	app.Email = "motemen@gmail.com"
	app.Commands = Commands
	return app
}
<commit_msg>bump version to 0.6.5<commit_after>package main

import (
	"os"

	"github.com/codegangsta/cli"
)

var Version string = "0.6.5"

func main() {
	newApp().Run(os.Args)
}

func newApp() *cli.App {
	app := cli.NewApp()
	app.Name = "ghq"
	app.Usage = "Manage GitHub repository clones"
	app.Version = Version
	app.Author = "motemen"
	app.Email = "motemen@gmail.com"
	app.Commands = Commands
	return app
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"net/http"
	"os"
	"regexp"
	"sync"

	"github.com/fatih/color"
	"github.com/jmks/uristat/options"
)

func main() {
	opts := options.Parse()

	if !opts.IsValid() {
		opts.PrintError()
		os.Exit(1)
	}

	existingSrc := filepathProducer(opts.Filepaths)
	uriSrc := uriProducer(existingSrc)

	if opts.ListOnly() {
		for uri := range uriSrc {
			fmt.Println(uri)
		}
	} else {
		printStatuses(uriSrc)
	}
}

func filepathProducer(filepaths []string) <-chan string {
	dest := make(chan string, 100)

	go func() {
		for _, filepath := range filepaths {
			if len(filepath) == 0 {
				continue
			}

			if _, err := os.Stat(filepath); !os.IsNotExist(err) {
				dest <- filepath
			}
		}
		close(dest)
	}()

	return dest
}

func uriProducer(filepathSrc <-chan string) <-chan string {
	dest := make(chan string, 100)

	go func() {
		wg := sync.WaitGroup{}

		for filepath := range filepathSrc {
			wg.Add(1)

			go func(path string) {
				defer wg.Done()

				if uris := urisIn(path); len(uris) > 0 {
					for _, uri := range uris {
						dest <- uri
					}
				}
			}(filepath)
		}

		wg.Wait()
		close(dest)
	}()

	return dest
}

func urisIn(filepath string) (uris []string) {
	file, err := os.Open(filepath)
	if err != nil {
		fmt.Printf("Error '%v': %v\n", filepath, err)
		return uris
	}

	// https://github.com/matthewrudy/regexpert/blob/master/lib/regexpert.rb
	uriRe := regexp.MustCompile(`(^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(([0-9]{1,5})?\/.*)?$)`)

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		if matches := uriRe.FindAllString(line, -1); matches != nil {
			uris = append(uris, matches...)
		}
	}

	return uris
}

func printStatuses(uriSrc <-chan string) {
	wg := sync.WaitGroup{}

	for uri := range uriSrc {
		wg.Add(1)

		go func(uri string) {
			if resp, err := http.Head(uri); err != nil {
				redden := color.New(color.FgRed).SprintFunc()
				fmt.Printf("%v : %v\n", redden("HTTP ERROR"), uri)
			} else {
				defer resp.Body.Close()
				colorize := statusCodePrinterFunc(resp.StatusCode)
				fmt.Printf("%v : %v\n", colorize(resp.Status), uri)
			}

			wg.Done()
		}(uri)
	}

	wg.Wait()
}

func statusCodePrinterFunc(code int) func(...interface{}) string {
	switch code {
	case 200:
		return color.New(color.FgGreen).SprintFunc()
	case 404:
		return color.New(color.FgYellow).SprintFunc()
	default:
		return color.New(color.FgRed).SprintFunc()
	}
}
<commit_msg>Added URI uniqueness sink to minimize web requests<commit_after>package main

import (
	"bufio"
	"fmt"
	"net/http"
	"os"
	"regexp"
	"sync"

	"github.com/fatih/color"
	"github.com/jmks/uristat/options"
)

func main() {
	opts := options.Parse()

	if !opts.IsValid() {
		opts.PrintError()
		os.Exit(1)
	}

	filepathSrc := filepathProducer(opts.Filepaths)
	uriSrc := uriProducer(filepathSrc)
	uniqURIs := uniqAccumulator(uriSrc)

	if opts.ListOnly() {
		for _, uri := range uniqURIs {
			fmt.Println(uri)
		}
	} else {
		printStatuses(uniqURIs)
	}
}

func filepathProducer(filepaths []string) <-chan string {
	dest := make(chan string, 100)

	go func() {
		for _, filepath := range filepaths {
			if len(filepath) == 0 {
				continue
			}

			if _, err := os.Stat(filepath); !os.IsNotExist(err) {
				dest <- filepath
			}
		}
		close(dest)
	}()

	return dest
}

func uriProducer(filepathSrc <-chan string) <-chan string {
	dest := make(chan string, 100)

	go func() {
		wg := sync.WaitGroup{}

		for filepath := range filepathSrc {
			wg.Add(1)

			go func(path string) {
				defer wg.Done()

				if uris := urisIn(path); len(uris) > 0 {
					for _, uri := range uris {
						dest <- uri
					}
				}
			}(filepath)
		}

		wg.Wait()
		close(dest)
	}()

	return dest
}

func uniqAccumulator(src <-chan string) []string {
	uniq := make(map[string]bool)

	for str := range src {
		uniq[str] = true
	}

	uris := make([]string, len(uniq))
	i := 0
	for uri := range uniq {
		uris[i] = uri
		i++
	}

	return uris
}

func urisIn(filepath string) (uris []string) {
	file, err := os.Open(filepath)
	if err != nil {
		fmt.Printf("Error '%v': %v\n", filepath, err)
		return uris
	}

	// https://github.com/matthewrudy/regexpert/blob/master/lib/regexpert.rb
	uriRe := regexp.MustCompile(`(^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(([0-9]{1,5})?\/.*)?$)`)

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		if matches := uriRe.FindAllString(line, -1); matches != nil {
			uris = append(uris, matches...)
		}
	}

	return uris
}

func printStatuses(uris []string) {
	wg := sync.WaitGroup{}

	for _, uri := range uris {
		wg.Add(1)

		go func(uri string) {
			if resp, err := http.Head(uri); err != nil {
				redden := color.New(color.FgRed).SprintFunc()
				fmt.Printf("%v : %v\n", redden("HTTP ERROR"), uri)
			} else {
				defer resp.Body.Close()
				colorize := statusCodePrinterFunc(resp.StatusCode)
				fmt.Printf("%v : %v\n", colorize(resp.Status), uri)
			}

			wg.Done()
		}(uri)
	}

	wg.Wait()
}

func statusCodePrinterFunc(code int) func(...interface{}) string {
	switch code {
	case 200:
		return color.New(color.FgGreen).SprintFunc()
	case 404:
		return color.New(color.FgYellow).SprintFunc()
	default:
		return color.New(color.FgRed).SprintFunc()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/CiscoCloud/consul-cli/commands"
)

const Name = "consul-cli"
const Version = "0.2.0"

func main() {
	log.SetOutput(ioutil.Discard)

	root := commands.Init()
	if err := root.Execute(); err != nil {
		fmt.Fprintln(root.Err, err)
		os.Exit(1)
	}

	os.Exit(0)
	//	args := os.Args[1:]
	//	for _, arg := range args {
	//		if arg == "--" {
	//			break
	//		}
	//
	//		if arg == "-v" || arg == "--version" {
	//			fmt.Printf("%s v%s\n", Name, Version)
	//			os.Exit(0)
	//		}
	//	}
	//
	//	cli := &cli.CLI{
	//		Args:		args,
	//		Commands:	Commands,
	//		HelpFunc:	cli.BasicHelpFunc(Name),
	//	}
	//
	//	exitCode, err := cli.Run()
	//	if err != nil {
	//		fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err.Error())
	//		os.Exit(1)
	//	}
	//
	//	os.Exit(exitCode)
}
<commit_msg>Remove old code<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/CiscoCloud/consul-cli/commands"
)

const Name = "consul-cli"
const Version = "0.2.0"

func main() {
	log.SetOutput(ioutil.Discard)

	root := commands.Init()
	if err := root.Execute(); err != nil {
		fmt.Fprintln(root.Err, err)
		os.Exit(1)
	}

	os.Exit(0)
}
<|endoftext|>