<commit_before>package honoka

import (
    "crypto/sha256"
    "encoding/hex"
    "encoding/json"
    "errors"
    "io/ioutil"
    "os"
    "path/filepath"
    "strconv"
    "time"

    homedir "github.com/mitchellh/go-homedir"
    "github.com/mitchellh/mapstructure"

    // for Debug
    // "fmt"
    // "github.com/davecgh/go-spew/spew"
)

// Cache client
type Client struct {
    // Cache Index list
    Indexer IndexList
}

// Cache index list
type IndexList map[string]Index

// Cache index
type Index struct {
    // The index key.
    Key        string

    // The bucket name that saved cache data.
    Bucket     string

    // The maximum elapsed time since the last file update.
    Expiration int64
}

// The structure is used when use clean method.
type CleanResult struct {
    // The bucket name that saved cache data.
    Bucket string

    // Error when delete the specified bucket.
    Error  error
}

type UpdateFunc func() (interface{}, error)

var (
    Version = "0.0.1"
    BucketFileNotFound = errors.New("Not found specified bucket file")
    IndexFileNotFound  = errors.New("Not found specified index file")
    CacheIsExpired     = errors.New("specified cache is expired")
)

// New is a function for making a new cache
func New() (*Client, error) {
    idx, err := getIndexList()
    if err != nil {
        if err == IndexFileNotFound {
            idx = nil
        } else {
            return nil, err
        }
    }

    c := &Client{
        Indexer: idx,
    }
    return c, nil
}

// Get is used to retrieve a cache by specified key.
// Example:
//   cli, err := honoka.New()
//   var output interface{}
//   cli.Get("foobar", &output)
//   // OR
//   result, err := cli.Get("foobar", &output)
func (c *Client) Get(key string, output interface{}) (interface{}, error) {
    if c.Expire(key) {
        return nil, CacheIsExpired
    }
    cache, err := c.GetJson(key)
    if err != nil {
        return nil, err
    }
    var result interface{}
    err = json.Unmarshal(cache, &result)
    if err != nil {
        return nil, err
    }
    err = mapstructure.WeakDecode(result, &output);
    return &output, err
}

// Get is used to retrieve a cache by specified key.
// Return value is JSON string
// Example:
//   cli, err := honoka.New()
//   result, err := cli.GetJson("foobar")
func (c *Client) GetJson(key string) ([]byte, error) {
    if c.Expire(key) {
        return nil, CacheIsExpired
    }

    idx := c.Indexer[key]
    cache, err := getCacheFromBucket(idx.Bucket)
    if err != nil {
        return nil, err
    }
    return cache, nil
}

// Get is used to create a cache if specified key has not used yet.
// Example:
//   cli, err := honoka.New()
//   err := cli.Set("foobar", "fizzbizz", 100)
func (c *Client) Set(key string, val interface{}, expire int64) error {
    if ! c.Expire(key) {
        return nil
    }

    exp := createExpiration(expire)
    name := getBucketName(key, exp)
    _, err := createNewBucket(name, val)
    if err != nil {
        return err
    }
    var idx IndexList
    idx, err = getIndexList()
    if err != nil {
        if (err == IndexFileNotFound) {
            idx = map[string]Index{}
        } else {
            return err
        }
    }

    idx[key] = Index{
        Key:        key,
        Bucket:     name,
        Expiration: exp,
    }
    c.setIndexer(idx)

    return nil
}

// Update calls the cache update function on the cached data.
// Get is used to retrieve a cache by specified key.
// Example:
//   cli, err := honoka.New()
//   var output interface{}
//   f := func() { return "fizzbizz" }
//   cli.Update("foobar", f, 100, &output)
//   // OR
//   result, err := cli.Get("foobar", f, 100, &output)
func (c *Client) Update(key string, updater UpdateFunc, expire int64, output interface{}) (interface{}, error) {
    b, err := c.UpdateJson(key, updater, expire)
    if b != nil {
        var result interface{}
        e := json.Unmarshal(b, &result)
        if e != nil {
            return nil, e
        }

        e = mapstructure.WeakDecode(result, &output);
        if e != nil {
            return nil, e
        }
    }

    return output, err
}

// Update calls the cache update function on the cached data.
// Return value is JSON string.
// Example:
//   cli, err := honoka.New()
//   f := func() { return "fizzbizz" }
//   result, err := cli.UpdateJson("foobar", f, 100)
func (c *Client) UpdateJson(key string, updater UpdateFunc, expire int64) ([]byte, error) {
    if ! c.Expire(key) {
        return c.GetJson(key)
    }

    val, err := updater()
    if err != nil {
        return nil, err
    }

    exp := createExpiration(expire)
    name := getBucketName(key, exp)
    jval, err := createNewBucket(name, val)
    if err != nil {
        return jval, err
    }
    var idx IndexList
    idx, err = getIndexList()
    if err != nil {
        idx = c.Indexer
    }

    idx[key] = Index{
        Key:        key,
        Bucket:     name,
        Expiration: exp,
    }
    c.setIndexer(idx)

    return jval, nil
}

// Delete is used to delete a cache by specified key.
// Example:
//   cli, err := honoka.New()
//   err = cli.Delete("foobar")
func (c *Client) Delete(key string) error {
    idx := c.Indexer[key]
    path, err := getBucketPath(idx.Bucket)
    if err != nil {
        return err
    }
    if fileExists(path) {
        err = os.Remove(path)
        if err != nil {
            return err
        }
    }

    delete(c.Indexer, key)
    c.setIndexer(c.Indexer)
    return nil
}

// Expire is a predicate which determines if the cache should be updated.
// Example:
//   cli, err := honoka.New()
//   expired := cli.Expire("foobar")
func (c *Client) Expire(key string) bool {
    if nil == c.Indexer {
        return true
    }

    idx, exists := c.Indexer[key]
    if exists {
        if idx.Expiration <= time.Now().Unix() {
            c.Delete(key)
            return true
        } else {
            return false
        }
    }
    return true
}

// Outdated is used to retrive no-indexed bucket.
// Example:
//   cli, err := honoka.New()
//   list, err := cli.Outdated()
func (c *Client) Outdated() ([]string, error) {
    idx, err := c.getIndexer(true)
    if err != nil {
        return nil, err
    }
    currents := make(map[string]string)
    for _, i := range idx {
        currents[i.Bucket] = ""
    }

    var list []string
    buckets, err := getBucketList()
    if err != nil {
        return nil, err
    }
    for _, bucket := range buckets {
        if _, exists := currents[bucket]; !exists {
            list = append(list, bucket)
        }
    }
    return list, nil
}

// Clean is used to delete no-indexed bucket.
// Example:
//   cli, err := honoka.New()
//   result, err := cli.Clean()
func (c *Client) Clean() ([]CleanResult, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return nil, err
    }
    list, err := c.Outdated()
    if err != nil {
        return nil, err
    }

    var result []CleanResult
    for _, bucket := range list {
        e := os.Remove(filepath.Join(bucketsDir, bucket))
        r := CleanResult{
            Bucket: bucket,
            Error:  e,
        }
        result = append(result, r)
    }
    return result, nil
}

// List is used to retrive cache indexes.
// Example:
//   cli, err := honoka.New()
//   list, err := cli.List()
func (c *Client) List() ([]Index, error) {
    idx, err := c.getIndexer(true)
    if err != nil {
        return nil, err
    }
    var list []Index
    for _, i := range idx {
        list = append(list, i)
    }

    return list, nil
}

func (c *Client) getIndexer(replace bool) (IndexList, error) {
    if replace || c.Indexer == nil {
        idx, err := getIndexList()
        if err != nil {
            return nil, err
        }
        c.Indexer = idx
    }
    return c.Indexer, nil
}

func (c *Client) setIndexer(indexes IndexList) error {
    idx, err := json.Marshal(indexes)
    if err != nil {
        return err
    }

    if err = updateIndexFile(idx); err != nil {
        return err
    }
    c.Indexer = indexes
    return nil
}

func getBucketsDirPath() (string, error) {
    home, err := homedir.Dir()
    if err != nil {
        return "", err
    }
    bucketsDir := filepath.Join(home, ".honoka", "buckets")
    os.MkdirAll(bucketsDir, 0700)
    return bucketsDir, err
}

func getBucketPath(bucketName string) (string, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return "", err
    }
    return filepath.Join(bucketsDir, bucketName), nil
}

func getCacheFromBucket(bucketName string) ([]byte, error) {
    path, err := getBucketPath(bucketName)
    if err != nil {
        return nil, err
    }
    if !fileExists(path) {
        return nil, BucketFileNotFound
    }
    return ioutil.ReadFile(path);
}

func getBucketList() ([]string, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return nil, err
    }
    files, err := ioutil.ReadDir(bucketsDir)
    var list []string
    for _, fi := range files {
        if !fi.IsDir() {
            filename := fi.Name()
            list = append(list, filename)
        }
    }
    return list, nil
}

func createNewBucket(name string, val interface{}) ([]byte, error) {
    jval, err := json.Marshal(val)
    if err != nil {
        return nil, err
    }
    path, err := getBucketPath(name)
    if err != nil {
        return jval, err
    }
    err = ioutil.WriteFile(path, jval, 0644)
    return jval, err
}

func getBucketName(key string, expiration int64) string {
    k := key + "." + strconv.FormatInt(expiration, 10)
    bytes := sha256.Sum256([]byte(k))
    return hex.EncodeToString(bytes[:])
}

func getIndexPath() (string, error) {
    home, err := homedir.Dir()
    if err != nil {
        return "", err
    }
    indexDir := filepath.Join(home, ".honoka")
    os.MkdirAll(indexDir, 0700)
    return filepath.Join(indexDir, "index"), err
}

func getIndexList() (IndexList, error) {
    b, err := getIndexFromFile()
    if err != nil {
        return nil, err
    }
    var list IndexList
    err = json.Unmarshal(b, &list)
    if  err != nil {
        return nil, err
    }
    return list, nil
}

func getIndexFromFile() ([]byte, error) {
    path, err := getIndexPath()
    if err != nil {
        return nil, err
    }
    if !fileExists(path) {
        return nil, IndexFileNotFound
    }
    return ioutil.ReadFile(path);
}

func updateIndexFile(indexes []byte) error {
    path, err := getIndexPath()
    if err != nil {
        return err
    }
    return ioutil.WriteFile(path, indexes, 0644);
}

func fileExists(filename string) bool {
    _, err := os.Stat(filename)
    return err == nil
}

func createExpiration(expire int64) int64 {
    return time.Now().Unix() + expire
}
<commit_msg>fix godoc<commit_after>package honoka

import (
    "crypto/sha256"
    "encoding/hex"
    "encoding/json"
    "errors"
    "io/ioutil"
    "os"
    "path/filepath"
    "strconv"
    "time"

    homedir "github.com/mitchellh/go-homedir"
    "github.com/mitchellh/mapstructure"

    // for Debug
    // "fmt"
    // "github.com/davecgh/go-spew/spew"
)

// Cache client
type Client struct {
    // Cache Index list
    Indexer IndexList
}

// Cache index list
type IndexList map[string]Index

// Cache index
type Index struct {
    // The index key.
    Key        string

    // The bucket name that saved cache data.
    Bucket     string

    // The maximum elapsed time since the last file update.
    Expiration int64
}

// The structure is used when use clean method.
type CleanResult struct {
    // The bucket name that saved cache data.
    Bucket string

    // Error when delete the specified bucket.
    Error  error
}

type UpdateFunc func() (interface{}, error)

var (
    Version = "0.0.1"
    BucketFileNotFound = errors.New("Not found specified bucket file")
    IndexFileNotFound  = errors.New("Not found specified index file")
    CacheIsExpired     = errors.New("specified cache is expired")
)

// New is a function for making a new cache
func New() (*Client, error) {
    idx, err := getIndexList()
    if err != nil {
        if err == IndexFileNotFound {
            idx = nil
        } else {
            return nil, err
        }
    }

    c := &Client{
        Indexer: idx,
    }
    return c, nil
}

// Get is used to retrieve a cache by specified key.
//
// Example:
//   cli, err := honoka.New()
//   var output interface{}
//   cli.Get("foobar", &output)
//   // OR
//   result, err := cli.Get("foobar", &output)
func (c *Client) Get(key string, output interface{}) (interface{}, error) {
    if c.Expire(key) {
        return nil, CacheIsExpired
    }
    cache, err := c.GetJson(key)
    if err != nil {
        return nil, err
    }
    var result interface{}
    err = json.Unmarshal(cache, &result)
    if err != nil {
        return nil, err
    }
    err = mapstructure.WeakDecode(result, &output);
    return &output, err
}

// Get is used to retrieve a cache by specified key.
// Return value is JSON string
// Example:
//   cli, err := honoka.New()
//   result, err := cli.GetJson("foobar")
func (c *Client) GetJson(key string) ([]byte, error) {
    if c.Expire(key) {
        return nil, CacheIsExpired
    }

    idx := c.Indexer[key]
    cache, err := getCacheFromBucket(idx.Bucket)
    if err != nil {
        return nil, err
    }
    return cache, nil
}

// Get is used to create a cache if specified key has not used yet.
//
// Example:
//   cli, err := honoka.New()
//   err := cli.Set("foobar", "fizzbizz", 100)
func (c *Client) Set(key string, val interface{}, expire int64) error {
    if ! c.Expire(key) {
        return nil
    }

    exp := createExpiration(expire)
    name := getBucketName(key, exp)
    _, err := createNewBucket(name, val)
    if err != nil {
        return err
    }
    var idx IndexList
    idx, err = getIndexList()
    if err != nil {
        if (err == IndexFileNotFound) {
            idx = map[string]Index{}
        } else {
            return err
        }
    }

    idx[key] = Index{
        Key:        key,
        Bucket:     name,
        Expiration: exp,
    }
    c.setIndexer(idx)

    return nil
}

// Update calls the cache update function on the cached data.
// Get is used to retrieve a cache by specified key.
//
// Example:
//   cli, err := honoka.New()
//   var output interface{}
//   f := func() { return "fizzbizz" }
//   cli.Update("foobar", f, 100, &output)
//   // OR
//   result, err := cli.Get("foobar", f, 100, &output)
func (c *Client) Update(key string, updater UpdateFunc, expire int64, output interface{}) (interface{}, error) {
    b, err := c.UpdateJson(key, updater, expire)
    if b != nil {
        var result interface{}
        e := json.Unmarshal(b, &result)
        if e != nil {
            return nil, e
        }

        e = mapstructure.WeakDecode(result, &output);
        if e != nil {
            return nil, e
        }
    }

    return output, err
}

// Update calls the cache update function on the cached data.
// Return value is JSON string.
//
// Example:
//   cli, err := honoka.New()
//   f := func() { return "fizzbizz" }
//   result, err := cli.UpdateJson("foobar", f, 100)
func (c *Client) UpdateJson(key string, updater UpdateFunc, expire int64) ([]byte, error) {
    if ! c.Expire(key) {
        return c.GetJson(key)
    }

    val, err := updater()
    if err != nil {
        return nil, err
    }

    exp := createExpiration(expire)
    name := getBucketName(key, exp)
    jval, err := createNewBucket(name, val)
    if err != nil {
        return jval, err
    }
    var idx IndexList
    idx, err = getIndexList()
    if err != nil {
        idx = c.Indexer
    }

    idx[key] = Index{
        Key:        key,
        Bucket:     name,
        Expiration: exp,
    }
    c.setIndexer(idx)

    return jval, nil
}

// Delete is used to delete a cache by specified key.
//
// Example:
//   cli, err := honoka.New()
//   err = cli.Delete("foobar")
func (c *Client) Delete(key string) error {
    idx := c.Indexer[key]
    path, err := getBucketPath(idx.Bucket)
    if err != nil {
        return err
    }
    if fileExists(path) {
        err = os.Remove(path)
        if err != nil {
            return err
        }
    }

    delete(c.Indexer, key)
    c.setIndexer(c.Indexer)
    return nil
}

// Expire is a predicate which determines if the cache should be updated.
//
// Example:
//   cli, err := honoka.New()
//   expired := cli.Expire("foobar")
func (c *Client) Expire(key string) bool {
    if nil == c.Indexer {
        return true
    }

    idx, exists := c.Indexer[key]
    if exists {
        if idx.Expiration <= time.Now().Unix() {
            c.Delete(key)
            return true
        } else {
            return false
        }
    }
    return true
}

// Outdated is used to retrive no-indexed bucket.
//
// Example:
//   cli, err := honoka.New()
//   list, err := cli.Outdated()
func (c *Client) Outdated() ([]string, error) {
    idx, err := c.getIndexer(true)
    if err != nil {
        return nil, err
    }
    currents := make(map[string]string)
    for _, i := range idx {
        currents[i.Bucket] = ""
    }

    var list []string
    buckets, err := getBucketList()
    if err != nil {
        return nil, err
    }
    for _, bucket := range buckets {
        if _, exists := currents[bucket]; !exists {
            list = append(list, bucket)
        }
    }
    return list, nil
}

// Clean is used to delete no-indexed bucket.
// Example:
//   cli, err := honoka.New()
//   result, err := cli.Clean()
func (c *Client) Clean() ([]CleanResult, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return nil, err
    }
    list, err := c.Outdated()
    if err != nil {
        return nil, err
    }

    var result []CleanResult
    for _, bucket := range list {
        e := os.Remove(filepath.Join(bucketsDir, bucket))
        r := CleanResult{
            Bucket: bucket,
            Error:  e,
        }
        result = append(result, r)
    }
    return result, nil
}

// List is used to retrive cache indexes.
//
// Example:
//   cli, err := honoka.New()
//   list, err := cli.List()
func (c *Client) List() ([]Index, error) {
    idx, err := c.getIndexer(true)
    if err != nil {
        return nil, err
    }
    var list []Index
    for _, i := range idx {
        list = append(list, i)
    }

    return list, nil
}

func (c *Client) getIndexer(replace bool) (IndexList, error) {
    if replace || c.Indexer == nil {
        idx, err := getIndexList()
        if err != nil {
            return nil, err
        }
        c.Indexer = idx
    }
    return c.Indexer, nil
}

func (c *Client) setIndexer(indexes IndexList) error {
    idx, err := json.Marshal(indexes)
    if err != nil {
        return err
    }

    if err = updateIndexFile(idx); err != nil {
        return err
    }
    c.Indexer = indexes
    return nil
}

func getBucketsDirPath() (string, error) {
    home, err := homedir.Dir()
    if err != nil {
        return "", err
    }
    bucketsDir := filepath.Join(home, ".honoka", "buckets")
    os.MkdirAll(bucketsDir, 0700)
    return bucketsDir, err
}

func getBucketPath(bucketName string) (string, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return "", err
    }
    return filepath.Join(bucketsDir, bucketName), nil
}

func getCacheFromBucket(bucketName string) ([]byte, error) {
    path, err := getBucketPath(bucketName)
    if err != nil {
        return nil, err
    }
    if !fileExists(path) {
        return nil, BucketFileNotFound
    }
    return ioutil.ReadFile(path);
}

func getBucketList() ([]string, error) {
    bucketsDir, err := getBucketsDirPath()
    if err != nil {
        return nil, err
    }
    files, err := ioutil.ReadDir(bucketsDir)
    var list []string
    for _, fi := range files {
        if !fi.IsDir() {
            filename := fi.Name()
            list = append(list, filename)
        }
    }
    return list, nil
}

func createNewBucket(name string, val interface{}) ([]byte, error) {
    jval, err := json.Marshal(val)
    if err != nil {
        return nil, err
    }
    path, err := getBucketPath(name)
    if err != nil {
        return jval, err
    }
    err = ioutil.WriteFile(path, jval, 0644)
    return jval, err
}

func getBucketName(key string, expiration int64) string {
    k := key + "." + strconv.FormatInt(expiration, 10)
    bytes := sha256.Sum256([]byte(k))
    return hex.EncodeToString(bytes[:])
}

func getIndexPath() (string, error) {
    home, err := homedir.Dir()
    if err != nil {
        return "", err
    }
    indexDir := filepath.Join(home, ".honoka")
    os.MkdirAll(indexDir, 0700)
    return filepath.Join(indexDir, "index"), err
}

func getIndexList() (IndexList, error) {
    b, err := getIndexFromFile()
    if err != nil {
        return nil, err
    }
    var list IndexList
    err = json.Unmarshal(b, &list)
    if  err != nil {
        return nil, err
    }
    return list, nil
}

func getIndexFromFile() ([]byte, error) {
    path, err := getIndexPath()
    if err != nil {
        return nil, err
    }
    if !fileExists(path) {
        return nil, IndexFileNotFound
    }
    return ioutil.ReadFile(path);
}

func updateIndexFile(indexes []byte) error {
    path, err := getIndexPath()
    if err != nil {
        return err
    }
    return ioutil.WriteFile(path, indexes, 0644);
}

func fileExists(filename string) bool {
    _, err := os.Stat(filename)
    return err == nil
}

func createExpiration(expire int64) int64 {
    return time.Now().Unix() + expire
}
<|endoftext|>


<commit_before>package check

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	kafka "github.com/Shopify/sarama"
	log "github.com/Sirupsen/logrus"
	"github.com/pkg/errors"
	"github.com/samuel/go-zookeeper/zk"
)

func (check *HealthCheck) connect(firstConnection bool, stop <-chan struct{}) error {
	var createHealthTopicIfMissing = firstConnection
	var createReplicationTopicIfMissing = firstConnection
	ticker := time.NewTicker(check.config.retryInterval)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			if err := check.tryConnectOnce(&createHealthTopicIfMissing, &createReplicationTopicIfMissing); err == nil {
				return nil
			}
		case <-stop:
			return errors.New("connect was asked to stop")
		}
	}
}

func (check *HealthCheck) tryConnectOnce(createBrokerTopic, createReplicationTopic *bool) error {
	pauseTime := check.config.retryInterval
	// connect to kafka cluster
	connectString := fmt.Sprintf("localhost:%d", check.config.brokerPort)
	err := check.broker.Dial(connectString, check.brokerConfig())
	if err != nil {
		log.Printf("unable to connect to broker, retrying in %s (%s)", pauseTime.String(), err)
		return err
	}

	metadata, err := check.broker.Metadata()
	if err != nil {
		return errors.Wrap(err, "failure retrieving metadata")
	}

	check.partitionID, err = check.findPartitionID(check.config.topicName, true, createBrokerTopic, metadata)
	if err != nil {
		log.Printf("%s retrying in %s", err.Error(), pauseTime)
		check.broker.Close()
		return err
	}

	check.replicationPartitionID, err = check.findPartitionID(check.config.replicationTopicName, false, createReplicationTopic, metadata)
	if err != nil {
		log.Printf("%s retrying in %s", err.Error(), pauseTime)
		check.broker.Close()
		return err
	}

	consumer, err := check.broker.Consumer(check.consumerConfig())
	if err != nil {
		log.Printf("unable to create consumer, retrying in %s: %s", pauseTime.String(), err)
		check.broker.Close()
		return err
	}

	producer, err := check.broker.Producer(check.producerConfig())
	if err != nil {
		log.Printf("unable to create producer, retrying in %s: %s", pauseTime.String(), err)
		check.broker.Close()
		return err
	}

	check.consumer = consumer
	check.producer = producer
	return nil
}

func (check *HealthCheck) findPartitionID(topicName string, forHealthCheck bool, createIfMissing *bool, metadata *kafka.MetadataResponse) (int32, error) {
	brokerID := int32(check.config.brokerID)

	if !brokerExists(brokerID, metadata) {
		return 0, fmt.Errorf("unable to find broker %d in metadata", brokerID)
	}

	topic, ok := findTopic(topicName, metadata)

	if ok {
		for _, partition := range topic.Partitions {
			if forHealthCheck && partition.Leader != brokerID {
				continue
			}
			if !contains(partition.Replicas, brokerID) {
				continue
			}

			log.Printf(`found partition id %d for broker %d in topic "%s"`, partition.ID, brokerID, topicName)
			return partition.ID, nil
		}
	}

	if *createIfMissing {
		err := check.createTopic(topicName, forHealthCheck)
		if err != nil {
			return 0, errors.Wrapf(err, `unable to create topic "%s"`, topicName)
		}
		log.Printf(`topic "%s" created`, topicName)
		*createIfMissing = false
		return 0, errors.New("topic created, try again")
	}

	if ok {
		return 0, fmt.Errorf(`Unable to find broker's parition in topic "%s" in metadata`, topicName)
	}
	return 0, fmt.Errorf(`Unable to find broker's topic "%s" in metadata`, topicName)
}

func findTopic(name string, metadata *kafka.MetadataResponse) (*kafka.TopicMetadata, bool) {
	for _, topic := range metadata.Topics {
		if topic.Name == name {
			return topic, true
		}
	}

	return nil, false
}

func brokerExists(brokerID int32, metadata *kafka.MetadataResponse) bool {
	for _, broker := range metadata.Brokers {
		if broker.ID() == brokerID {
			return true
		}
	}
	return false
}

func zookeeperEnsembleAndChroot(connectString string) (ensemble []string, chroot string) {
	result := strings.Split(connectString, "/")
	switch len(result) {
	case 1:
		ensemble = strings.Split(result[0], ",")
		chroot = ""
	default:
		ensemble = strings.Split(result[0], ",")
		chroot = "/" + strings.Join(result[1:], "/")
		if strings.HasSuffix(chroot, "/") {
			chroot = chroot[:len(chroot)-1]
		}
	}
	return
}

func (check *HealthCheck) createTopic(name string, forHealthCheck bool) (err error) {
	log.Printf("connecting to ZooKeeper ensemble %s", check.config.zookeeperConnect)
	connectString, chroot := zookeeperEnsembleAndChroot(check.config.zookeeperConnect)
	zkConn := check.zookeeper

	if _, err = zkConn.Connect(connectString, 10*time.Second); err != nil {
		return
	}
	defer zkConn.Close()

	topicPath := chroot + "/config/topics/" + name

	exists := false
	if !forHealthCheck {
		exists, _, err = zkConn.Exists(topicPath)
		if err != nil {
			return
		}
	}

	brokerID := int32(check.config.brokerID)

	if !exists {
		topicConfig := `{"version":1,"config":{"delete.retention.ms":"10000",` +
			`"cleanup.policy":"delete","compression.type":"uncompressed"}}`
		log.Infof(`creating topic "%s" configuration node`, name)

		if err = createZkNode(zkConn, topicPath, topicConfig, forHealthCheck); err != nil {
			return
		}

		partitionAssignment := fmt.Sprintf(`{"version":1,"partitions":{"0":[%d]}}`, brokerID)
		log.Infof(`creating topic "%s" partition assignment node`, name)

		if err = createZkNode(zkConn, chroot+"/brokers/topics/"+name, partitionAssignment, forHealthCheck); err != nil {
			return
		}
	}

	if !forHealthCheck {
		err = maybeExpandReplicationTopic(zkConn, brokerID, check.replicationPartitionID, name, chroot)
	}

	return

}

func maybeExpandReplicationTopic(zk ZkConnection, brokerID, partitionID int32, topicName, chroot string) error {
	topic := ZkTopic{Name: topicName}
	err := zkPartitions(&topic, zk, topicName, chroot)
	if err != nil {
		return errors.Wrap(err, "Unable to determine if replication topic should be expanded")
	}

	replicas, ok := topic.Partitions[strconv.Itoa(int(partitionID))]
	if !ok {
		return fmt.Errorf(`Cannot find partition with ID %d in topic "%s"`, partitionID, topicName)
	}

	if !contains(replicas, brokerID) {
		log.Info("Expanding replication check topic to include broker ", brokerID)
		replicas = append(replicas, brokerID)

		return reassignPartition(zk, partitionID, replicas, topicName, chroot)
	}
	return nil
}

func reassignPartition(zk ZkConnection, partitionID int32, replicas []int32, topicName, chroot string) (err error) {

	repeat := true
	for repeat {
		time.Sleep(1 * time.Second)
		exists, _, rpErr := zk.Exists(chroot + "/admin/reassign_partitions")
		if rpErr != nil {
			log.Warn("Error while checking if reassign_partitions node exists", rpErr)
		}
		repeat = exists || rpErr != nil
	}

	var replicasStr []string
	for _, ID := range replicas {
		replicasStr = append(replicasStr, fmt.Sprintf("%d", ID))
	}

	reassign := fmt.Sprintf(`{"version":1,"partitions":[{"topic":"%s","partition":%d,"replicas":[%s]}]}`,
		topicName, partitionID, strings.Join(replicasStr, ","))

	repeat = true
	for repeat {
		log.Info("Creating reassign partition node")
		err = createZkNode(zk, chroot+"/admin/reassign_partitions", reassign, true)
		if err != nil {
			log.Warn("Error while creating reassignment node", err)
		}
		repeat = err != nil
	}

	return
}

func createZkNode(zookeeper ZkConnection, path string, content string, failIfExists bool) error {
	nodeExists, _, err := zookeeper.Exists(path)

	if err != nil {
		return err
	}

	if nodeExists {
		if failIfExists {
			return fmt.Errorf("node %s cannot be created, exists already", path)
		}
		return nil
	}

	log.Println("creating node", path)
	flags := int32(0) // permanent node.
	acl := zk.WorldACL(zk.PermAll)
	_, err = zookeeper.Create(path, []byte(content), flags, acl)
	return err
}

func (check *HealthCheck) closeConnection(deleteTopicIfPresent bool) {
	if deleteTopicIfPresent {
		log.Infof("connecting to ZooKeeper ensemble %s", check.config.zookeeperConnect)
		connectString, chroot := zookeeperEnsembleAndChroot(check.config.zookeeperConnect)

		zkConn := check.zookeeper
		_, err := zkConn.Connect(connectString, 10*time.Second)
		if err != nil {
			return
		}
		defer zkConn.Close()

		check.deleteTopic(zkConn, chroot, check.config.topicName, check.partitionID)
		check.deleteTopic(zkConn, chroot, check.config.replicationTopicName, check.replicationPartitionID)
	}
	check.broker.Close()
}

func (check *HealthCheck) deleteTopic(zkConn ZkConnection, chroot, name string, partitionID int32) error {
	topic := ZkTopic{Name: name}
	err := zkPartitions(&topic, zkConn, name, chroot)
	if err != nil {
		return err
	}

	replicas, ok := topic.Partitions[strconv.Itoa(int(partitionID))]
	if !ok {
		return fmt.Errorf(`Cannot find partition with ID %d in topic "%s"`, partitionID, name)
	}

	brokerID := int32(check.config.brokerID)
	if len(replicas) > 1 {
		log.Info("Shrinking replication check topic to exclude broker ", brokerID)
		replicas = delAll(replicas, brokerID)
		return reassignPartition(zkConn, partitionID, replicas, name, chroot)
	}

	delTopicPath := chroot + "/admin/delete_topics/" + name

	err = createZkNode(check.zookeeper, delTopicPath, "", true)
	if err != nil {
		return err
	}
	return check.waitForTopicDeletion(delTopicPath)
}

func (check *HealthCheck) waitForTopicDeletion(topicPath string) error {
	for {
		exists, _, err := check.zookeeper.Exists(topicPath)
		if err != nil {
			return err
		}
		if !exists {
			return nil
		}
		time.Sleep(check.config.retryInterval)
	}
}

func (check *HealthCheck) reconnect(stop <-chan struct{}) error {
	check.closeConnection(false)
	return check.connect(false, stop)
}
<commit_msg>Don't fail if topic exists<commit_after>package check

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	kafka "github.com/Shopify/sarama"
	log "github.com/Sirupsen/logrus"
	"github.com/pkg/errors"
	"github.com/samuel/go-zookeeper/zk"
)

func (check *HealthCheck) connect(firstConnection bool, stop <-chan struct{}) error {
	var createHealthTopicIfMissing = firstConnection
	var createReplicationTopicIfMissing = firstConnection
	ticker := time.NewTicker(check.config.retryInterval)
	defer ticker.Stop()
	for {
		select {
		case <-ticker.C:
			if err := check.tryConnectOnce(&createHealthTopicIfMissing, &createReplicationTopicIfMissing); err == nil {
				return nil
			}
		case <-stop:
			return errors.New("connect was asked to stop")
		}
	}
}

func (check *HealthCheck) tryConnectOnce(createBrokerTopic, createReplicationTopic *bool) error {
	pauseTime := check.config.retryInterval
	// connect to kafka cluster
	connectString := fmt.Sprintf("localhost:%d", check.config.brokerPort)
	err := check.broker.Dial(connectString, check.brokerConfig())
	if err != nil {
		log.Printf("unable to connect to broker, retrying in %s (%s)", pauseTime.String(), err)
		return err
	}

	metadata, err := check.broker.Metadata()
	if err != nil {
		return errors.Wrap(err, "failure retrieving metadata")
	}

	check.partitionID, err = check.findPartitionID(check.config.topicName, true, createBrokerTopic, metadata)
	if err != nil {
		log.Printf("%s retrying in %s", err.Error(), pauseTime)
		check.broker.Close()
		return err
	}

	check.replicationPartitionID, err = check.findPartitionID(check.config.replicationTopicName, false, createReplicationTopic, metadata)
	if err != nil {
		log.Printf("%s retrying in %s", err.Error(), pauseTime)
		check.broker.Close()
		return err
	}

	consumer, err := check.broker.Consumer(check.consumerConfig())
	if err != nil {
		log.Printf("unable to create consumer, retrying in %s: %s", pauseTime.String(), err)
		check.broker.Close()
		return err
	}

	producer, err := check.broker.Producer(check.producerConfig())
	if err != nil {
		log.Printf("unable to create producer, retrying in %s: %s", pauseTime.String(), err)
		check.broker.Close()
		return err
	}

	check.consumer = consumer
	check.producer = producer
	return nil
}

func (check *HealthCheck) findPartitionID(topicName string, forHealthCheck bool, createIfMissing *bool, metadata *kafka.MetadataResponse) (int32, error) {
	brokerID := int32(check.config.brokerID)

	if !brokerExists(brokerID, metadata) {
		return 0, fmt.Errorf("unable to find broker %d in metadata", brokerID)
	}

	topic, ok := findTopic(topicName, metadata)

	if ok {
		for _, partition := range topic.Partitions {
			if forHealthCheck && partition.Leader != brokerID {
				continue
			}
			if !contains(partition.Replicas, brokerID) {
				continue
			}

			log.Printf(`found partition id %d for broker %d in topic "%s"`, partition.ID, brokerID, topicName)
			return partition.ID, nil
		}
	}

	if *createIfMissing {
		err := check.createTopic(topicName, forHealthCheck)
		if err != nil {
			return 0, errors.Wrapf(err, `unable to create topic "%s"`, topicName)
		}
		log.Printf(`topic "%s" created`, topicName)
		*createIfMissing = false
		return 0, errors.New("topic created, try again")
	}

	if ok {
		return 0, fmt.Errorf(`Unable to find broker's parition in topic "%s" in metadata`, topicName)
	}
	return 0, fmt.Errorf(`Unable to find broker's topic "%s" in metadata`, topicName)
}

func findTopic(name string, metadata *kafka.MetadataResponse) (*kafka.TopicMetadata, bool) {
	for _, topic := range metadata.Topics {
		if topic.Name == name {
			return topic, true
		}
	}

	return nil, false
}

func brokerExists(brokerID int32, metadata *kafka.MetadataResponse) bool {
	for _, broker := range metadata.Brokers {
		if broker.ID() == brokerID {
			return true
		}
	}
	return false
}

func zookeeperEnsembleAndChroot(connectString string) (ensemble []string, chroot string) {
	result := strings.Split(connectString, "/")
	switch len(result) {
	case 1:
		ensemble = strings.Split(result[0], ",")
		chroot = ""
	default:
		ensemble = strings.Split(result[0], ",")
		chroot = "/" + strings.Join(result[1:], "/")
		if strings.HasSuffix(chroot, "/") {
			chroot = chroot[:len(chroot)-1]
		}
	}
	return
}

func (check *HealthCheck) createTopic(name string, forHealthCheck bool) (err error) {
	log.Printf("connecting to ZooKeeper ensemble %s", check.config.zookeeperConnect)
	connectString, chroot := zookeeperEnsembleAndChroot(check.config.zookeeperConnect)
	zkConn := check.zookeeper

	if _, err = zkConn.Connect(connectString, 10*time.Second); err != nil {
		return
	}
	defer zkConn.Close()

	topicPath := chroot + "/config/topics/" + name

	exists := false
	if !forHealthCheck {
		exists, _, err = zkConn.Exists(topicPath)
		if err != nil {
			return
		}
	}

	brokerID := int32(check.config.brokerID)

	if !exists {
		topicConfig := `{"version":1,"config":{"delete.retention.ms":"10000",` +
			`"cleanup.policy":"delete","compression.type":"uncompressed"}}`
		log.Infof(`creating topic "%s" configuration node`, name)

		if err = createZkNode(zkConn, topicPath, topicConfig, false); err != nil {
			return
		}

		partitionAssignment := fmt.Sprintf(`{"version":1,"partitions":{"0":[%d]}}`, brokerID)
		log.Infof(`creating topic "%s" partition assignment node`, name)

		if err = createZkNode(zkConn, chroot+"/brokers/topics/"+name, partitionAssignment, false); err != nil {
			return
		}
	}

	if !forHealthCheck {
		err = maybeExpandReplicationTopic(zkConn, brokerID, check.replicationPartitionID, name, chroot)
	}

	return

}

func maybeExpandReplicationTopic(zk ZkConnection, brokerID, partitionID int32, topicName, chroot string) error {
	topic := ZkTopic{Name: topicName}
	err := zkPartitions(&topic, zk, topicName, chroot)
	if err != nil {
		return errors.Wrap(err, "Unable to determine if replication topic should be expanded")
	}

	replicas, ok := topic.Partitions[strconv.Itoa(int(partitionID))]
	if !ok {
		return fmt.Errorf(`Cannot find partition with ID %d in topic "%s"`, partitionID, topicName)
	}

	if !contains(replicas, brokerID) {
		log.Info("Expanding replication check topic to include broker ", brokerID)
		replicas = append(replicas, brokerID)

		return reassignPartition(zk, partitionID, replicas, topicName, chroot)
	}
	return nil
}

func reassignPartition(zk ZkConnection, partitionID int32, replicas []int32, topicName, chroot string) (err error) {

	repeat := true
	for repeat {
		time.Sleep(1 * time.Second)
		exists, _, rpErr := zk.Exists(chroot + "/admin/reassign_partitions")
		if rpErr != nil {
			log.Warn("Error while checking if reassign_partitions node exists", rpErr)
		}
		repeat = exists || rpErr != nil
	}

	var replicasStr []string
	for _, ID := range replicas {
		replicasStr = append(replicasStr, fmt.Sprintf("%d", ID))
	}

	reassign := fmt.Sprintf(`{"version":1,"partitions":[{"topic":"%s","partition":%d,"replicas":[%s]}]}`,
		topicName, partitionID, strings.Join(replicasStr, ","))

	repeat = true
	for repeat {
		log.Info("Creating reassign partition node")
		err = createZkNode(zk, chroot+"/admin/reassign_partitions", reassign, true)
		if err != nil {
			log.Warn("Error while creating reassignment node", err)
		}
		repeat = err != nil
	}

	return
}

func createZkNode(zookeeper ZkConnection, path string, content string, failIfExists bool) error {
	nodeExists, _, err := zookeeper.Exists(path)

	if err != nil {
		return err
	}

	if nodeExists {
		if failIfExists {
			return fmt.Errorf("node %s cannot be created, exists already", path)
		}
		return nil
	}

	log.Println("creating node", path)
	flags := int32(0) // permanent node.
	acl := zk.WorldACL(zk.PermAll)
	_, err = zookeeper.Create(path, []byte(content), flags, acl)
	return err
}

func (check *HealthCheck) closeConnection(deleteTopicIfPresent bool) {
	if deleteTopicIfPresent {
		log.Infof("connecting to ZooKeeper ensemble %s", check.config.zookeeperConnect)
		connectString, chroot := zookeeperEnsembleAndChroot(check.config.zookeeperConnect)

		zkConn := check.zookeeper
		_, err := zkConn.Connect(connectString, 10*time.Second)
		if err != nil {
			return
		}
		defer zkConn.Close()

		check.deleteTopic(zkConn, chroot, check.config.topicName, check.partitionID)
		check.deleteTopic(zkConn, chroot, check.config.replicationTopicName, check.replicationPartitionID)
	}
	check.broker.Close()
}

func (check *HealthCheck) deleteTopic(zkConn ZkConnection, chroot, name string, partitionID int32) error {
	topic := ZkTopic{Name: name}
	err := zkPartitions(&topic, zkConn, name, chroot)
	if err != nil {
		return err
	}

	replicas, ok := topic.Partitions[strconv.Itoa(int(partitionID))]
	if !ok {
		return fmt.Errorf(`Cannot find partition with ID %d in topic "%s"`, partitionID, name)
	}

	brokerID := int32(check.config.brokerID)
	if len(replicas) > 1 {
		log.Info("Shrinking replication check topic to exclude broker ", brokerID)
		replicas = delAll(replicas, brokerID)
		return reassignPartition(zkConn, partitionID, replicas, name, chroot)
	}

	delTopicPath := chroot + "/admin/delete_topics/" + name

	err = createZkNode(check.zookeeper, delTopicPath, "", true)
	if err != nil {
		return err
	}
	return check.waitForTopicDeletion(delTopicPath)
}

func (check *HealthCheck) waitForTopicDeletion(topicPath string) error {
	for {
		exists, _, err := check.zookeeper.Exists(topicPath)
		if err != nil {
			return err
		}
		if !exists {
			return nil
		}
		time.Sleep(check.config.retryInterval)
	}
}

func (check *HealthCheck) reconnect(stop <-chan struct{}) error {
	check.closeConnection(false)
	return check.connect(false, stop)
}
<|endoftext|>


<commit_before>// +build linux

// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	httpaws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package config

func parseGMSACapability() bool {
	return false
}

func parseFSxWindowsFileServerCapability() bool {
	return false
}

// GetOperatingSystemFamily() returns "linux" as operating system family for linux based ecs instances
func GetOperatingSystemFamily() string {
	return OSType
}
<commit_msg>Revert "Changes to advertise OSType while registering the container instance with cluster"<commit_after>// +build linux

// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	httpaws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package config

func parseGMSACapability() bool {
	return false
}

func parseFSxWindowsFileServerCapability() bool {
	return false
}
// GetOperatingSystemFamily() returns "linux" as operating system family for linux based ecs instances
func GetOperatingSystemFamily() string {
	return OSType
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gochimp

// see http://apidocs.mailchimp.com/api/2.0/
const (
	lists_subscribe_endpoint         string = "/lists/subscribe.json"
	lists_unsubscribe_endpoint       string = "/lists/unsubscribe.json"
	lists_list_endpoint              string = "/lists/list.json"
	lists_update_member_endpoint     string = "/lists/update-member.json"
	lists_members_endpoint           string = "/lists/members.json"
	lists_member_info_endpoint       string = "/lists/member-info.json"
	lists_batch_unsubscribe_endpoint string = "/lists/batch-unsubscribe.json"
)

func (a *ChimpAPI) BatchUnsubscribe(req BatchUnsubscribe) (BatchResponse, error) {
	var response BatchResponse
	req.ApiKey = a.Key
	err := parseChimpJson(a, lists_batch_unsubscribe_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) ListsSubscribe(req ListsSubscribe) (Email, error) {
	var response Email
	req.ApiKey = a.Key
	err := parseChimpJson(a, lists_subscribe_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) ListsUnsubscribe(req ListsUnsubscribe) error {
	req.ApiKey = a.Key
	return parseChimpJson(a, lists_unsubscribe_endpoint, req, nil)
}

func (a *ChimpAPI) ListsList(req ListsList) (ListsListResponse, error) {
	req.ApiKey = a.Key
	var response ListsListResponse
	err := parseChimpJson(a, lists_list_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) UpdateMember(req UpdateMember) error {
	req.ApiKey = a.Key
	return parseChimpJson(a, lists_update_member_endpoint, req, nil)
}

func (a *ChimpAPI) Members(req ListsMembers) (ListsMembersResponse, error) {
	req.ApiKey = a.Key
	var response ListsMembersResponse
	err := parseChimpJson(a, lists_members_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) MemberInfo(req ListsMemberInfo) (ListsMemberInfoResponse, error) {
	req.ApiKey = a.Key
	var response ListsMemberInfoResponse
	err := parseChimpJson(a, lists_member_info_endpoint, req, &response)
	return response, err
}

type BatchUnsubscribe struct {
	ApiKey       string  `json:"apikey"`
	ListId       string  `json:"id"`
	Batch        []Email `json:"batch"`
	DeleteMember bool    `json:"delete_member"`
	SendGoodbye  bool    `json:"send_goodbye"`
	SendNotify   bool    `json:"send_notify"`
}

type ListsUnsubscribe struct {
	ApiKey       string `json:"apikey"`
	ListId       string `json:"id"`
	Email        Email  `json:"email"`
	DeleteMember bool   `json:"delete_member"`
	SendGoodbye  bool   `json:"send_goodbye"`
	SendNotify   bool   `json:"send_notify"`
}

type ListsSubscribe struct {
	ApiKey           string                 `json:"apikey"`
	ListId           string                 `json:"id"`
	Email            Email                  `json:"email"`
	MergeVars        map[string]interface{} `json:"merge_vars,omitempty"`
	EmailType        string                 `json:"email_type,omitempty"`
	DoubleOptIn      bool                   `json:"double_optin"`
	UpdateExisting   bool                   `json:"update_existing"`
	ReplaceInterests bool                   `json:"replace_interests"`
	SendWelcome      bool                   `json:"send_welcome"`
}

type ListFilter struct {
	ListId        string `json:"list_id"`
	ListName      string `json:"list_name"`
	FromName      string `json:"from_name"`
	FromEmail     string `json:"from_email"`
	FromSubject   string `json:"from_subject"`
	CreatedBefore string `json:"created_before"`
	CreatedAfter  string `json:"created_after"`
	Exact         bool   `json:"exact"`
}

type ListStat struct {
	MemberCount               float64 `json:"member_count"`
	UnsubscribeCount          float64 `json:"unsubscribe_count"`
	CleanedCount              float64 `json:"cleaned_count"`
	MemberCountSinceSend      float64 `json:"member_count_since_send"`
	UnsubscribeCountSinceSend float64 `json:"unsubscribe_count_since_send"`
	CleanedCountSinceSend     float64 `json:"cleaned_count_since_send"`
	CampaignCount             float64 `json:"campaign_count"`
	GroupingCount             float64 `json:"grouping_count"`
	GroupCount                float64 `json:"group_count"`
	MergeVarCount             float64 `json:"merge_var_count"`
	AvgSubRate                float64 `json:"avg_sub_rate"`
	AvgUnsubRate              float64 `json:"avg_unsub_rate"`
	TargetSubRate             float64 `json:"target_sub_rate "`
	OpenRate                  float64 `json:"open_rate "`
	ClickRate                 float64 `json:"click_rate "`
}

type ListData struct {
	Id                string   `json:"id"`
	WebId             int      `json:"web_id"`
	Name              string   `json:"name"`
	DateCreated       string   `json:"date_created"`
	EmailTypeOption   bool     `json:"email_type_option"`
	UseAwesomeBar     bool     `json:"use_awesomebar"`
	DefaultFromName   string   `json:"default_from_name"`
	DefaultFromEmail  string   `json:"default_from_email"`
	DefaultSubject    string   `json:"default_subject"`
	DefaultLanguage   string   `json:"default_language"`
	ListRating        float64  `json:"list_rating"`
	SubscribeShortUrl string   `json:"subscribe_url_short"`
	SubscribeLongUrl  string   `json:"subscribe_url_long"`
	BeamerAddress     string   `json:"beamer_address"`
	Visibility        string   `json:"visibility"`
	Stats             ListStat `json:"stats"`
	Modules           []string `json:"modules"`
}

type BatchResponse struct {
	Success     int          `json:"success_count"`
	ErrorCount  int          `json:"error_count"`
	BatchErrors []BatchError `json:"errors"`
}

type BatchError struct {
	Emails Email  `json:"email"`
	Code   int    `json:"code"`
	Error  string `json:"error"`
}

type ListError struct {
	Param string `json:"param"`
	Code  int    `json:"code"`
	Error string `json:"error"`
}

type ListsListResponse struct {
	Total  int         `json:"total"`
	Data   []ListData  `json:"data"`
	Errors []ListError `json:"errors"`
}

type ListsList struct {
	ApiKey        string     `json:"apikey"`
	Filters       ListFilter `json:"filters,omitempty"`
	Start         int        `json:"start,omitempty"`
	Limit         int        `json:"limit,omitempty"`
	SortField     string     `json:"sort_field,omitempty"`
	SortDirection string     `json:"sort_dir,omitempty"`
}

type UpdateMember struct {
	ApiKey           string                 `json:"apikey"`
	ListId           string                 `json:"id"`
	Email            Email                  `json:"email"`
	MergeVars        map[string]interface{} `json:"merge_vars,omitempty"`
	EmailType        string                 `json:"email_type,omitempty"`
	ReplaceInterests bool                   `json:"replace_interests"`
}

type Email struct {
	Email string `json:"email"`
	Euid  string `json:"euid"`
	Leid  string `json:"leid"`
}

type ListsMembers struct {
	ApiKey  string          `json:"apikey"`
	ListId  string          `json:"id"`
	Options ListsMembersOpt `json:"opts,omitempty"`
}

type ListsMembersOpt struct {
	Start         int    `json:"start,omitempty"`
	Limit         int    `json:"limit,omitempty"`
	SortField     string `json:"sort_field,omitempty"`
	SortDirection string `json:"sort_dir,omitempty"`
}

type ListsMembersResponse struct {
	Total int          `json:"total"`
	Data  []MemberInfo `json:"data"`
}

type ListsMemberInfo struct {
	ApiKey string  `json:"apikey"`
	ListId string  `json:"id"`
	Emails []Email `json:"emails"`
}

type ListsMemberInfoResponse struct {
	SuccessCount      int          `json:"success_count"`
	ErrorCount        int          `json:"error_count"`
	Errors            []ListError  `json:"errors"`
	MemberInfoRecords []MemberInfo `json:"data"`
}

type MemberInfo struct {
	Email           string                 `json:"email"`
	Euid            string                 `json:"euid"`
	EmailType       string                 `json:"email_type"`
	IpSignup        string                 `json:"ip_signup,omitempty"`
	TimestampSignup string                 `json:"timestamp_signup,omitempty"`
	IpOpt           string                 `json:"ip_opt"`
	TimestampOpt    string                 `json:"timestamp_opt"`
	MemberRating    int                    `json:"member_rating"`
	InfoChanged     string                 `json:"info_changed"`
	Leid            int                    `json:"leid"`
	Language        string                 `json:"language,omitempty"`
	ListId          string                 `json:"list_id"`
	ListName        string                 `json:"list_name"`
	Merges          map[string]interface{} `json:"merges"`
	Status          string                 `json:"status"`
	Timestamp       string                 `json:"timestamp"`
}
<commit_msg>ListsMembers Status was missing<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gochimp

// see http://apidocs.mailchimp.com/api/2.0/
const (
	lists_subscribe_endpoint         string = "/lists/subscribe.json"
	lists_unsubscribe_endpoint       string = "/lists/unsubscribe.json"
	lists_list_endpoint              string = "/lists/list.json"
	lists_update_member_endpoint     string = "/lists/update-member.json"
	lists_members_endpoint           string = "/lists/members.json"
	lists_member_info_endpoint       string = "/lists/member-info.json"
	lists_batch_unsubscribe_endpoint string = "/lists/batch-unsubscribe.json"
)

func (a *ChimpAPI) BatchUnsubscribe(req BatchUnsubscribe) (BatchResponse, error) {
	var response BatchResponse
	req.ApiKey = a.Key
	err := parseChimpJson(a, lists_batch_unsubscribe_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) ListsSubscribe(req ListsSubscribe) (Email, error) {
	var response Email
	req.ApiKey = a.Key
	err := parseChimpJson(a, lists_subscribe_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) ListsUnsubscribe(req ListsUnsubscribe) error {
	req.ApiKey = a.Key
	return parseChimpJson(a, lists_unsubscribe_endpoint, req, nil)
}

func (a *ChimpAPI) ListsList(req ListsList) (ListsListResponse, error) {
	req.ApiKey = a.Key
	var response ListsListResponse
	err := parseChimpJson(a, lists_list_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) UpdateMember(req UpdateMember) error {
	req.ApiKey = a.Key
	return parseChimpJson(a, lists_update_member_endpoint, req, nil)
}

func (a *ChimpAPI) Members(req ListsMembers) (ListsMembersResponse, error) {
	req.ApiKey = a.Key
	var response ListsMembersResponse
	err := parseChimpJson(a, lists_members_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) MemberInfo(req ListsMemberInfo) (ListsMemberInfoResponse, error) {
	req.ApiKey = a.Key
	var response ListsMemberInfoResponse
	err := parseChimpJson(a, lists_member_info_endpoint, req, &response)
	return response, err
}

type BatchUnsubscribe struct {
	ApiKey       string  `json:"apikey"`
	ListId       string  `json:"id"`
	Batch        []Email `json:"batch"`
	DeleteMember bool    `json:"delete_member"`
	SendGoodbye  bool    `json:"send_goodbye"`
	SendNotify   bool    `json:"send_notify"`
}

type ListsUnsubscribe struct {
	ApiKey       string `json:"apikey"`
	ListId       string `json:"id"`
	Email        Email  `json:"email"`
	DeleteMember bool   `json:"delete_member"`
	SendGoodbye  bool   `json:"send_goodbye"`
	SendNotify   bool   `json:"send_notify"`
}

type ListsSubscribe struct {
	ApiKey           string                 `json:"apikey"`
	ListId           string                 `json:"id"`
	Email            Email                  `json:"email"`
	MergeVars        map[string]interface{} `json:"merge_vars,omitempty"`
	EmailType        string                 `json:"email_type,omitempty"`
	DoubleOptIn      bool                   `json:"double_optin"`
	UpdateExisting   bool                   `json:"update_existing"`
	ReplaceInterests bool                   `json:"replace_interests"`
	SendWelcome      bool                   `json:"send_welcome"`
}

type ListFilter struct {
	ListId        string `json:"list_id"`
	ListName      string `json:"list_name"`
	FromName      string `json:"from_name"`
	FromEmail     string `json:"from_email"`
	FromSubject   string `json:"from_subject"`
	CreatedBefore string `json:"created_before"`
	CreatedAfter  string `json:"created_after"`
	Exact         bool   `json:"exact"`
}

type ListStat struct {
	MemberCount               float64 `json:"member_count"`
	UnsubscribeCount          float64 `json:"unsubscribe_count"`
	CleanedCount              float64 `json:"cleaned_count"`
	MemberCountSinceSend      float64 `json:"member_count_since_send"`
	UnsubscribeCountSinceSend float64 `json:"unsubscribe_count_since_send"`
	CleanedCountSinceSend     float64 `json:"cleaned_count_since_send"`
	CampaignCount             float64 `json:"campaign_count"`
	GroupingCount             float64 `json:"grouping_count"`
	GroupCount                float64 `json:"group_count"`
	MergeVarCount             float64 `json:"merge_var_count"`
	AvgSubRate                float64 `json:"avg_sub_rate"`
	AvgUnsubRate              float64 `json:"avg_unsub_rate"`
	TargetSubRate             float64 `json:"target_sub_rate "`
	OpenRate                  float64 `json:"open_rate "`
	ClickRate                 float64 `json:"click_rate "`
}

type ListData struct {
	Id                string   `json:"id"`
	WebId             int      `json:"web_id"`
	Name              string   `json:"name"`
	DateCreated       string   `json:"date_created"`
	EmailTypeOption   bool     `json:"email_type_option"`
	UseAwesomeBar     bool     `json:"use_awesomebar"`
	DefaultFromName   string   `json:"default_from_name"`
	DefaultFromEmail  string   `json:"default_from_email"`
	DefaultSubject    string   `json:"default_subject"`
	DefaultLanguage   string   `json:"default_language"`
	ListRating        float64  `json:"list_rating"`
	SubscribeShortUrl string   `json:"subscribe_url_short"`
	SubscribeLongUrl  string   `json:"subscribe_url_long"`
	BeamerAddress     string   `json:"beamer_address"`
	Visibility        string   `json:"visibility"`
	Stats             ListStat `json:"stats"`
	Modules           []string `json:"modules"`
}

type BatchResponse struct {
	Success     int          `json:"success_count"`
	ErrorCount  int          `json:"error_count"`
	BatchErrors []BatchError `json:"errors"`
}

type BatchError struct {
	Emails Email  `json:"email"`
	Code   int    `json:"code"`
	Error  string `json:"error"`
}

type ListError struct {
	Param string `json:"param"`
	Code  int    `json:"code"`
	Error string `json:"error"`
}

type ListsListResponse struct {
	Total  int         `json:"total"`
	Data   []ListData  `json:"data"`
	Errors []ListError `json:"errors"`
}

type ListsList struct {
	ApiKey        string     `json:"apikey"`
	Filters       ListFilter `json:"filters,omitempty"`
	Start         int        `json:"start,omitempty"`
	Limit         int        `json:"limit,omitempty"`
	SortField     string     `json:"sort_field,omitempty"`
	SortDirection string     `json:"sort_dir,omitempty"`
}

type UpdateMember struct {
	ApiKey           string                 `json:"apikey"`
	ListId           string                 `json:"id"`
	Email            Email                  `json:"email"`
	MergeVars        map[string]interface{} `json:"merge_vars,omitempty"`
	EmailType        string                 `json:"email_type,omitempty"`
	ReplaceInterests bool                   `json:"replace_interests"`
}

type Email struct {
	Email string `json:"email"`
	Euid  string `json:"euid"`
	Leid  string `json:"leid"`
}

type ListsMembers struct {
	ApiKey  string          `json:"apikey"`
	ListId  string          `json:"id"`
	Status  string          `json:"status"`
	Options ListsMembersOpt `json:"opts,omitempty"`
}

type ListsMembersOpt struct {
	Start         int    `json:"start,omitempty"`
	Limit         int    `json:"limit,omitempty"`
	SortField     string `json:"sort_field,omitempty"`
	SortDirection string `json:"sort_dir,omitempty"`
}

type ListsMembersResponse struct {
	Total int          `json:"total"`
	Data  []MemberInfo `json:"data"`
}

type ListsMemberInfo struct {
	ApiKey string  `json:"apikey"`
	ListId string  `json:"id"`
	Emails []Email `json:"emails"`
}

type ListsMemberInfoResponse struct {
	SuccessCount      int          `json:"success_count"`
	ErrorCount        int          `json:"error_count"`
	Errors            []ListError  `json:"errors"`
	MemberInfoRecords []MemberInfo `json:"data"`
}

type MemberInfo struct {
	Email           string                 `json:"email"`
	Euid            string                 `json:"euid"`
	EmailType       string                 `json:"email_type"`
	IpSignup        string                 `json:"ip_signup,omitempty"`
	TimestampSignup string                 `json:"timestamp_signup,omitempty"`
	IpOpt           string                 `json:"ip_opt"`
	TimestampOpt    string                 `json:"timestamp_opt"`
	MemberRating    int                    `json:"member_rating"`
	InfoChanged     string                 `json:"info_changed"`
	Leid            int                    `json:"leid"`
	Language        string                 `json:"language,omitempty"`
	ListId          string                 `json:"list_id"`
	ListName        string                 `json:"list_name"`
	Merges          map[string]interface{} `json:"merges"`
	Status          string                 `json:"status"`
	Timestamp       string                 `json:"timestamp"`
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
)

type handler struct {
	dest url.URL
}

var rewriter *strings.Replacer

func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	du := h.dest
	du.Path = req.URL.Path
	du.RawQuery = req.URL.RawQuery
	req.URL = &du
	req.RequestURI = ""

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	defer res.Body.Close()

	for k, v := range res.Header {
		res.Header[k] = v
	}
	w.WriteHeader(res.StatusCode)

	io.Copy(w, rewriteJson(res.Body, rewriter.Replace))
}

func initRewrite(conf string) {
	if conf == "" {
		rewriter = strings.NewReplacer()
		return
	}

	f, err := os.Open(conf)
	if err != nil {
		log.Fatalf("Error opening %v: %v", conf, err)
	}
	defer f.Close()
	d := json.NewDecoder(f)
	m := map[string]string{}
	err = d.Decode(&m)
	if err != nil {
		log.Fatalf("Error parsing %v: %v", conf, err)
	}

	params := []string{}
	for k, v := range m {
		params = append(params, k, v)
	}

	rewriter = strings.NewReplacer(params...)
}

func main() {
	bindAddr := flag.String("bind", ":7081", "Address to listen")
	rewriteConf := flag.String("rewriteconf", "",
		"Path to json rewrite rules")

	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatalf("Where to, sir?")
	}

	initRewrite(*rewriteConf)

	u, err := url.Parse(flag.Arg(0))
	if err != nil {
		log.Fatalf("Error parsing url: %v", err)
	}

	log.Fatal(http.ListenAndServe(*bindAddr, &handler{*u}))
}
<commit_msg>Only sed things claiming to be json<commit_after>package main

import (
	"encoding/json"
	"flag"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
)

type handler struct {
	dest url.URL
}

var rewriter *strings.Replacer

func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	du := h.dest
	du.Path = req.URL.Path
	du.RawQuery = req.URL.RawQuery
	req.URL = &du
	req.RequestURI = ""

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	defer res.Body.Close()

	for k, vs := range res.Header {
		h := w.Header()
		h.Del(k)
		for _, v := range vs {
			h.Add(k, v)
		}
	}
	w.WriteHeader(res.StatusCode)

	if strings.Contains(res.Header.Get("content-type"), "json") {
		io.Copy(w, rewriteJson(res.Body, rewriter.Replace))
	} else {
		io.Copy(w, res.Body)
	}
}

func initRewrite(conf string) {
	if conf == "" {
		rewriter = strings.NewReplacer()
		return
	}

	f, err := os.Open(conf)
	if err != nil {
		log.Fatalf("Error opening %v: %v", conf, err)
	}
	defer f.Close()
	d := json.NewDecoder(f)
	m := map[string]string{}
	err = d.Decode(&m)
	if err != nil {
		log.Fatalf("Error parsing %v: %v", conf, err)
	}

	params := []string{}
	for k, v := range m {
		params = append(params, k, v)
	}

	rewriter = strings.NewReplacer(params...)
}

func main() {
	bindAddr := flag.String("bind", ":7081", "Address to listen")
	rewriteConf := flag.String("rewriteconf", "",
		"Path to json rewrite rules")

	flag.Parse()

	if flag.NArg() != 1 {
		log.Fatalf("Where to, sir?")
	}

	initRewrite(*rewriteConf)

	u, err := url.Parse(flag.Arg(0))
	if err != nil {
		log.Fatalf("Error parsing url: %v", err)
	}

	log.Fatal(http.ListenAndServe(*bindAddr, &handler{*u}))
}
<|endoftext|>


<commit_before>// +build mage

package main

import (
	"encoding/json"
	"fmt"
	"os"
	"path"

	"github.com/magefile/mage/mg"
	"github.com/magefile/mage/sh"
)

// getBuildMatrix returns the build matrix from the current version of the go compiler
func getBuildMatrix() (map[string][]string, error) {
	jsonData, err := sh.Output("go", "tool", "dist", "list", "-json")
	if err != nil {
		return nil, err
	}
	var data []struct {
		Goos   string
		Goarch string
	}
	if err := json.Unmarshal([]byte(jsonData), &data); err != nil {
		return nil, err
	}

	matrix := map[string][]string{}
	for _, v := range data {
		if val, ok := matrix[v.Goos]; ok {
			matrix[v.Goos] = append(val, v.Goarch)
		} else {
			matrix[v.Goos] = []string{v.Goarch}
		}
	}

	return matrix, nil
}

func CrossBuild() error {
	matrix, err := getBuildMatrix()
	if err != nil {
		return err
	}

	for os, arches := range matrix {
		for _, arch := range arches {
			env := map[string]string{
				"GOOS":   os,
				"GOARCH": arch,
			}
			if mg.Verbose() {
				fmt.Printf("Building for GOOS=%s GOARCH=%s\n", os, arch)
			}
			if err := sh.RunWith(env, "go", "build", "./..."); err != nil {
				return err
			}
		}
	}
	return nil
}

func Lint() error {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		return fmt.Errorf("cannot retrieve GOPATH")
	}

	return sh.Run(path.Join(gopath, "bin", "golangci-lint"), "run", "./...")
}

// Run the test suite
func Test() error {
	return sh.RunWith(map[string]string{"GORACE": "halt_on_error=1"},
		"go", "test", "-race", "-v", "./...")
}
<commit_msg>reduce the list of cross build target<commit_after>//go:build mage
// +build mage

package main

import (
	"encoding/json"
	"fmt"
	"os"
	"path"
	"sort"

	"github.com/magefile/mage/mg"
	"github.com/magefile/mage/sh"
)

func intersect(a, b []string) []string {
	sort.Strings(a)
	sort.Strings(b)

	res := make([]string, 0, func() int {
		if len(a) < len(b) {
			return len(a)
		}
		return len(b)
	}())

	for _, v := range a {
		idx := sort.SearchStrings(b, v)
		if idx < len(b) && b[idx] == v {
			res = append(res, v)
		}
	}
	return res
}

// getBuildMatrix returns the build matrix from the current version of the go compiler
func getFullBuildMatrix() (map[string][]string, error) {
	jsonData, err := sh.Output("go", "tool", "dist", "list", "-json")
	if err != nil {
		return nil, err
	}
	var data []struct {
		Goos   string
		Goarch string
	}
	if err := json.Unmarshal([]byte(jsonData), &data); err != nil {
		return nil, err
	}

	matrix := map[string][]string{}
	for _, v := range data {
		if val, ok := matrix[v.Goos]; ok {
			matrix[v.Goos] = append(val, v.Goarch)
		} else {
			matrix[v.Goos] = []string{v.Goarch}
		}
	}

	return matrix, nil
}

func getBuildMatrix() (map[string][]string, error) {
	minimalMatrix := map[string][]string{
		"linux":   []string{"amd64"},
		"darwin":  []string{"amd64", "arm64"},
		"freebsd": []string{"amd64"},
		"js":      []string{"wasm"},
		"solaris": []string{"amd64"},
		"windows": []string{"amd64", "arm64"},
	}

	fullMatrix, err := getFullBuildMatrix()
	if err != nil {
		return nil, err
	}

	for os, arches := range minimalMatrix {
		if fullV, ok := fullMatrix[os]; !ok {
			delete(minimalMatrix, os)
		} else {
			minimalMatrix[os] = intersect(arches, fullV)
		}
	}
	return minimalMatrix, nil
}

func CrossBuild() error {
	matrix, err := getBuildMatrix()
	if err != nil {
		return err
	}

	for os, arches := range matrix {
		for _, arch := range arches {
			env := map[string]string{
				"GOOS":   os,
				"GOARCH": arch,
			}
			if mg.Verbose() {
				fmt.Printf("Building for GOOS=%s GOARCH=%s\n", os, arch)
			}
			if err := sh.RunWith(env, "go", "build", "./..."); err != nil {
				return err
			}
		}
	}
	return nil
}

func Lint() error {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		return fmt.Errorf("cannot retrieve GOPATH")
	}

	return sh.Run(path.Join(gopath, "bin", "golangci-lint"), "run", "./...")
}

// Run the test suite
func Test() error {
	return sh.RunWith(map[string]string{"GORACE": "halt_on_error=1"},
		"go", "test", "-race", "-v", "./...")
}
<|endoftext|>


<commit_before>package downloader

import (
    "bytes"

    "github.com/PuerkitoBio/goquery"
    "github.com/bitly/go-simplejson"
    //    iconv "github.com/djimenez/iconv-go"
    "github.com/hu17889/go_spider/core/common/mlog"
    "github.com/hu17889/go_spider/core/common/page"
    "github.com/hu17889/go_spider/core/common/request"
    "github.com/hu17889/go_spider/core/common/util"
    //    "golang.org/x/text/encoding/simplifiedchinese"
    //    "golang.org/x/text/transform"
    "io"
    "io/ioutil"
    "net/http"
    "net/url"
    //"fmt"
    "golang.org/x/net/html/charset"
    //    "regexp"
    //    "golang.org/x/net/html"
    "strings"
	"compress/gzip"
)

// The HttpDownloader download page by package net/http.
// The "html" content is contained in dom parser of package goquery.
// The "json" content is saved.
// The "jsonp" content is modified to json.
// The "text" content will save body plain text only.
// The page result is saved in Page.
type HttpDownloader struct {
}

func NewHttpDownloader() *HttpDownloader {
    return &HttpDownloader{}
}

func (this *HttpDownloader) Download(req *request.Request) *page.Page {
    var mtype string
    var p = page.NewPage(req)
    mtype = req.GetResponceType()
    switch mtype {
    case "html":
        return this.downloadHtml(p, req)
    case "json":
        fallthrough
    case "jsonp":
        return this.downloadJson(p, req)
    case "text":
        return this.downloadText(p, req)
    default:
        mlog.LogInst().LogError("error request type:" + mtype)
    }
    return p
}

/*
// The acceptableCharset is test for whether Content-Type is UTF-8 or not
func (this *HttpDownloader) acceptableCharset(contentTypes []string) bool {
    // each type is like [text/html; charset=UTF-8]
    // we want the UTF-8 only
    for _, cType := range contentTypes {
        if strings.Index(cType, "UTF-8") != -1 || strings.Index(cType, "utf-8") != -1 {
            return true
        }
    }
    return false
}

// The getCharset used for parsing the header["Content-Type"] string to get charset of the page.
func (this *HttpDownloader) getCharset(header http.Header) string {
    reg, err := regexp.Compile("charset=(.*)$")
    if err != nil {
        mlog.LogInst().LogError(err.Error())
        return ""
    }

    var charset string
    for _, cType := range header["Content-Type"] {
        substrings := reg.FindStringSubmatch(cType)
        if len(substrings) == 2 {
            charset = substrings[1]
        }
    }

    return charset
}

// Use golang.org/x/text/encoding. Get page body and change it to utf-8
func (this *HttpDownloader) changeCharsetEncoding(charset string, sor io.ReadCloser) string {
    ischange := true
    var tr transform.Transformer
    cs := strings.ToLower(charset)
    if cs == "gbk" {
        tr = simplifiedchinese.GBK.NewDecoder()
    } else if cs == "gb18030" {
        tr = simplifiedchinese.GB18030.NewDecoder()
    } else if cs == "hzgb2312" || cs == "gb2312" || cs == "hz-gb2312" {
        tr = simplifiedchinese.HZGB2312.NewDecoder()
    } else {
        ischange = false
    }

    var destReader io.Reader
    if ischange {
        transReader := transform.NewReader(sor, tr)
        destReader = transReader
    } else {
        destReader = sor
    }

    var sorbody []byte
    var err error
    if sorbody, err = ioutil.ReadAll(destReader); err != nil {
        mlog.LogInst().LogError(err.Error())
        return ""
    }
    bodystr := string(sorbody)

    return bodystr
}

// Use go-iconv. Get page body and change it to utf-8

func (this *HttpDownloader) changeCharsetGoIconv(charset string, sor io.ReadCloser) string {
    var err error
    var converter *iconv.Converter
    if charset != "" && strings.ToLower(charset) != "utf-8" && strings.ToLower(charset) != "utf8" {
        converter, err = iconv.NewConverter(charset, "utf-8")
        if err != nil {
            mlog.LogInst().LogError(err.Error())
            return ""
        }
        defer converter.Close()
    }

    var sorbody []byte
    if sorbody, err = ioutil.ReadAll(sor); err != nil {
        mlog.LogInst().LogError(err.Error())
        return ""
    }
    bodystr := string(sorbody)

    var destbody string
    if converter != nil {
        // convert to utf8
        destbody, err = converter.ConvertString(bodystr)
        if err != nil {
            mlog.LogInst().LogError(err.Error())
            return ""
        }
    } else {
        destbody = bodystr
    }
    return destbody
}
*/

// Charset auto determine. Use golang.org/x/net/html/charset. Get page body and change it to utf-8
func (this *HttpDownloader) changeCharsetEncodingAuto(contentTypeStr string, sor io.ReadCloser) string {
    var err error
    destReader, err := charset.NewReader(sor, contentTypeStr)

    if err != nil {
        mlog.LogInst().LogError(err.Error())
        destReader = sor
    }

    var sorbody []byte
    if sorbody, err = ioutil.ReadAll(destReader); err != nil {
        mlog.LogInst().LogError(err.Error())
        // For gb2312, an error will be returned.
        // Error like: simplifiedchinese: invalid GBK encoding
        // return ""
    }
    //e,name,certain := charset.DetermineEncoding(sorbody,contentTypeStr)
    bodystr := string(sorbody)

    return bodystr
}

func (this *HttpDownloader) changeCharsetEncodingAutoGzipSupport(contentTypeStr string, sor io.ReadCloser) string {
	var err error
	gzipReader, err := gzip.NewReader(sor)
	if err != nil {
		mlog.LogInst().LogError(err.Error())
		return ""
	}
	destReader, err := charset.NewReader(gzipReader, contentTypeStr)

	if err != nil {
		mlog.LogInst().LogError(err.Error())
		destReader = sor
	}

	var sorbody []byte
	if sorbody, err = ioutil.ReadAll(destReader); err != nil {
		mlog.LogInst().LogError(err.Error())
		// For gb2312, an error will be returned.
		// Error like: simplifiedchinese: invalid GBK encoding
		// return ""
	}
	//e,name,certain := charset.DetermineEncoding(sorbody,contentTypeStr)
	bodystr := string(sorbody)

	return bodystr
}

// choose http GET/method to download
func connectByHttp(p *page.Page, req *request.Request) (*http.Response, error) {
    client := &http.Client{
        CheckRedirect: req.GetRedirectFunc(),
    }

    httpreq, err := http.NewRequest(req.GetMethod(), req.GetUrl(), strings.NewReader(req.GetPostdata()))
    if header := req.GetHeader(); header != nil {
        httpreq.Header = req.GetHeader()
    }

    if cookies := req.GetCookies(); cookies != nil {
        for i := range cookies {
            httpreq.AddCookie(cookies[i])
        }
    }

    var resp *http.Response
    if resp, err = client.Do(httpreq); err != nil {
        if e, ok := err.(*url.Error); ok && e.Err != nil && e.Err.Error() == "normal" {
            //  normal
        } else {
            mlog.LogInst().LogError(err.Error())
            p.SetStatus(true, err.Error())
            //fmt.Printf("client do error %v \r\n", err)
            return nil, err
        }
    }

    return resp, nil
}

// choose a proxy server to excute http GET/method to download
func connectByHttpProxy(p *page.Page, in_req *request.Request) (*http.Response, error) {
    request, _ := http.NewRequest("GET", in_req.GetUrl(), nil)
    proxy, err := url.Parse(in_req.GetProxyHost())
    if err != nil {
        return nil, err
    }
    client := &http.Client{
        Transport: &http.Transport{
            Proxy: http.ProxyURL(proxy),
        },
    }
    resp, err := client.Do(request)
    if err != nil {
        return nil, err
    }
    return resp, nil

}

// Download file and change the charset of page charset.
func (this *HttpDownloader) downloadFile(p *page.Page, req *request.Request) (*page.Page, string) {
    var err error
    var urlstr string
    if urlstr = req.GetUrl(); len(urlstr) == 0 {
        mlog.LogInst().LogError("url is empty")
        p.SetStatus(true, "url is empty")
        return p, ""
    }

    var resp *http.Response

    if proxystr := req.GetProxyHost(); len(proxystr) != 0 {
        //using http proxy
        //fmt.Print("HttpProxy Enter ",proxystr,"\n")
        resp, err = connectByHttpProxy(p, req)
    } else {
        //normal http download
        //fmt.Print("Http Normal Enter \n",proxystr,"\n")
        resp, err = connectByHttp(p, req)
    }

    if err != nil {
        return p, ""
    }

    //b, _ := ioutil.ReadAll(resp.Body)
    //fmt.Printf("Resp body %v \r\n", string(b))

    p.SetHeader(resp.Header)
    p.SetCookies(resp.Cookies())

    // get converter to utf-8
	var bodyStr string
	if resp.Header.Get("Content-Encoding") == "gzip" {
		bodyStr = this.changeCharsetEncodingAutoGzipSupport(resp.Header.Get("Content-Type"), resp.Body)
	} else {
		bodyStr = this.changeCharsetEncodingAuto(resp.Header.Get("Content-Type"), resp.Body)
	}
    //fmt.Printf("utf-8 body %v \r\n", bodyStr)
    defer resp.Body.Close()
    return p, bodyStr
}

func (this *HttpDownloader) downloadHtml(p *page.Page, req *request.Request) *page.Page {
    var err error
    p, destbody := this.downloadFile(p, req)
    //fmt.Printf("Destbody %v \r\n", destbody)
    if !p.IsSucc() {
        //fmt.Print("Page error \r\n")
        return p
    }
    bodyReader := bytes.NewReader([]byte(destbody))

    var doc *goquery.Document
    if doc, err = goquery.NewDocumentFromReader(bodyReader); err != nil {
        mlog.LogInst().LogError(err.Error())
        p.SetStatus(true, err.Error())
        return p
    }

    var body string
    if body, err = doc.Html(); err != nil {
        mlog.LogInst().LogError(err.Error())
        p.SetStatus(true, err.Error())
        return p
    }

    p.SetBodyStr(body).SetHtmlParser(doc).SetStatus(false, "")

    return p
}

func (this *HttpDownloader) downloadJson(p *page.Page, req *request.Request) *page.Page {
    var err error
    p, destbody := this.downloadFile(p, req)
    if !p.IsSucc() {
        return p
    }

    var body []byte
    body = []byte(destbody)
    mtype := req.GetResponceType()
    if mtype == "jsonp" {
        tmpstr := util.JsonpToJson(destbody)
        body = []byte(tmpstr)
    }

    var r *simplejson.Json
    if r, err = simplejson.NewJson(body); err != nil {
        mlog.LogInst().LogError(string(body) + "\t" + err.Error())
        p.SetStatus(true, err.Error())
        return p
    }

    // json result
    p.SetBodyStr(string(body)).SetJson(r).SetStatus(false, "")

    return p
}

func (this *HttpDownloader) downloadText(p *page.Page, req *request.Request) *page.Page {
    p, destbody := this.downloadFile(p, req)
    if !p.IsSucc() {
        return p
    }

    p.SetBodyStr(destbody).SetStatus(false, "")
    return p
}
<commit_msg>#修复了GzipReader没有Close内存泄露的可能<commit_after>package downloader

import (
    "bytes"

    "github.com/PuerkitoBio/goquery"
    "github.com/bitly/go-simplejson"
    //    iconv "github.com/djimenez/iconv-go"
    "github.com/hu17889/go_spider/core/common/mlog"
    "github.com/hu17889/go_spider/core/common/page"
    "github.com/hu17889/go_spider/core/common/request"
    "github.com/hu17889/go_spider/core/common/util"
    //    "golang.org/x/text/encoding/simplifiedchinese"
    //    "golang.org/x/text/transform"
    "io"
    "io/ioutil"
    "net/http"
    "net/url"
    //"fmt"
    "golang.org/x/net/html/charset"
    //    "regexp"
    //    "golang.org/x/net/html"
    "strings"
	"compress/gzip"
)

// The HttpDownloader download page by package net/http.
// The "html" content is contained in dom parser of package goquery.
// The "json" content is saved.
// The "jsonp" content is modified to json.
// The "text" content will save body plain text only.
// The page result is saved in Page.
type HttpDownloader struct {
}

func NewHttpDownloader() *HttpDownloader {
    return &HttpDownloader{}
}

func (this *HttpDownloader) Download(req *request.Request) *page.Page {
    var mtype string
    var p = page.NewPage(req)
    mtype = req.GetResponceType()
    switch mtype {
    case "html":
        return this.downloadHtml(p, req)
    case "json":
        fallthrough
    case "jsonp":
        return this.downloadJson(p, req)
    case "text":
        return this.downloadText(p, req)
    default:
        mlog.LogInst().LogError("error request type:" + mtype)
    }
    return p
}

/*
// The acceptableCharset is test for whether Content-Type is UTF-8 or not
func (this *HttpDownloader) acceptableCharset(contentTypes []string) bool {
    // each type is like [text/html; charset=UTF-8]
    // we want the UTF-8 only
    for _, cType := range contentTypes {
        if strings.Index(cType, "UTF-8") != -1 || strings.Index(cType, "utf-8") != -1 {
            return true
        }
    }
    return false
}

// The getCharset used for parsing the header["Content-Type"] string to get charset of the page.
func (this *HttpDownloader) getCharset(header http.Header) string {
    reg, err := regexp.Compile("charset=(.*)$")
    if err != nil {
        mlog.LogInst().LogError(err.Error())
        return ""
    }

    var charset string
    for _, cType := range header["Content-Type"] {
        substrings := reg.FindStringSubmatch(cType)
        if len(substrings) == 2 {
            charset = substrings[1]
        }
    }

    return charset
}

// Use golang.org/x/text/encoding. Get page body and change it to utf-8
func (this *HttpDownloader) changeCharsetEncoding(charset string, sor io.ReadCloser) string {
    ischange := true
    var tr transform.Transformer
    cs := strings.ToLower(charset)
    if cs == "gbk" {
        tr = simplifiedchinese.GBK.NewDecoder()
    } else if cs == "gb18030" {
        tr = simplifiedchinese.GB18030.NewDecoder()
    } else if cs == "hzgb2312" || cs == "gb2312" || cs == "hz-gb2312" {
        tr = simplifiedchinese.HZGB2312.NewDecoder()
    } else {
        ischange = false
    }

    var destReader io.Reader
    if ischange {
        transReader := transform.NewReader(sor, tr)
        destReader = transReader
    } else {
        destReader = sor
    }

    var sorbody []byte
    var err error
    if sorbody, err = ioutil.ReadAll(destReader); err != nil {
        mlog.LogInst().LogError(err.Error())
        return ""
    }
    bodystr := string(sorbody)

    return bodystr
}

// Use go-iconv. Get page body and change it to utf-8

func (this *HttpDownloader) changeCharsetGoIconv(charset string, sor io.ReadCloser) string {
    var err error
    var converter *iconv.Converter
    if charset != "" && strings.ToLower(charset) != "utf-8" && strings.ToLower(charset) != "utf8" {
        converter, err = iconv.NewConverter(charset, "utf-8")
        if err != nil {
            mlog.LogInst().LogError(err.Error())
            return ""
        }
        defer converter.Close()
    }

    var sorbody []byte
    if sorbody, err = ioutil.ReadAll(sor); err != nil {
        mlog.LogInst().LogError(err.Error())
        return ""
    }
    bodystr := string(sorbody)

    var destbody string
    if converter != nil {
        // convert to utf8
        destbody, err = converter.ConvertString(bodystr)
        if err != nil {
            mlog.LogInst().LogError(err.Error())
            return ""
        }
    } else {
        destbody = bodystr
    }
    return destbody
}
*/

// Charset auto determine. Use golang.org/x/net/html/charset. Get page body and change it to utf-8
func (this *HttpDownloader) changeCharsetEncodingAuto(contentTypeStr string, sor io.ReadCloser) string {
    var err error
    destReader, err := charset.NewReader(sor, contentTypeStr)

    if err != nil {
        mlog.LogInst().LogError(err.Error())
        destReader = sor
    }

    var sorbody []byte
    if sorbody, err = ioutil.ReadAll(destReader); err != nil {
        mlog.LogInst().LogError(err.Error())
        // For gb2312, an error will be returned.
        // Error like: simplifiedchinese: invalid GBK encoding
        // return ""
    }
    //e,name,certain := charset.DetermineEncoding(sorbody,contentTypeStr)
    bodystr := string(sorbody)

    return bodystr
}

func (this *HttpDownloader) changeCharsetEncodingAutoGzipSupport(contentTypeStr string, sor io.ReadCloser) string {
	var err error
	gzipReader, err := gzip.NewReader(sor)
	defer gzipReader.Close()
	if err != nil {
		mlog.LogInst().LogError(err.Error())
		return ""
	}
	destReader, err := charset.NewReader(gzipReader, contentTypeStr)

	if err != nil {
		mlog.LogInst().LogError(err.Error())
		destReader = sor
	}

	var sorbody []byte
	if sorbody, err = ioutil.ReadAll(destReader); err != nil {
		mlog.LogInst().LogError(err.Error())
		// For gb2312, an error will be returned.
		// Error like: simplifiedchinese: invalid GBK encoding
		// return ""
	}
	//e,name,certain := charset.DetermineEncoding(sorbody,contentTypeStr)
	bodystr := string(sorbody)

	return bodystr
}

// choose http GET/method to download
func connectByHttp(p *page.Page, req *request.Request) (*http.Response, error) {
    client := &http.Client{
        CheckRedirect: req.GetRedirectFunc(),
    }

    httpreq, err := http.NewRequest(req.GetMethod(), req.GetUrl(), strings.NewReader(req.GetPostdata()))
    if header := req.GetHeader(); header != nil {
        httpreq.Header = req.GetHeader()
    }

    if cookies := req.GetCookies(); cookies != nil {
        for i := range cookies {
            httpreq.AddCookie(cookies[i])
        }
    }

    var resp *http.Response
    if resp, err = client.Do(httpreq); err != nil {
        if e, ok := err.(*url.Error); ok && e.Err != nil && e.Err.Error() == "normal" {
            //  normal
        } else {
            mlog.LogInst().LogError(err.Error())
            p.SetStatus(true, err.Error())
            //fmt.Printf("client do error %v \r\n", err)
            return nil, err
        }
    }

    return resp, nil
}

// choose a proxy server to excute http GET/method to download
func connectByHttpProxy(p *page.Page, in_req *request.Request) (*http.Response, error) {
    request, _ := http.NewRequest("GET", in_req.GetUrl(), nil)
    proxy, err := url.Parse(in_req.GetProxyHost())
    if err != nil {
        return nil, err
    }
    client := &http.Client{
        Transport: &http.Transport{
            Proxy: http.ProxyURL(proxy),
        },
    }
    resp, err := client.Do(request)
    if err != nil {
        return nil, err
    }
    return resp, nil

}

// Download file and change the charset of page charset.
func (this *HttpDownloader) downloadFile(p *page.Page, req *request.Request) (*page.Page, string) {
    var err error
    var urlstr string
    if urlstr = req.GetUrl(); len(urlstr) == 0 {
        mlog.LogInst().LogError("url is empty")
        p.SetStatus(true, "url is empty")
        return p, ""
    }

    var resp *http.Response

    if proxystr := req.GetProxyHost(); len(proxystr) != 0 {
        //using http proxy
        //fmt.Print("HttpProxy Enter ",proxystr,"\n")
        resp, err = connectByHttpProxy(p, req)
    } else {
        //normal http download
        //fmt.Print("Http Normal Enter \n",proxystr,"\n")
        resp, err = connectByHttp(p, req)
    }

    if err != nil {
        return p, ""
    }

    //b, _ := ioutil.ReadAll(resp.Body)
    //fmt.Printf("Resp body %v \r\n", string(b))

    p.SetHeader(resp.Header)
    p.SetCookies(resp.Cookies())

    // get converter to utf-8
	var bodyStr string
	if resp.Header.Get("Content-Encoding") == "gzip" {
		bodyStr = this.changeCharsetEncodingAutoGzipSupport(resp.Header.Get("Content-Type"), resp.Body)
	} else {
		bodyStr = this.changeCharsetEncodingAuto(resp.Header.Get("Content-Type"), resp.Body)
	}
    //fmt.Printf("utf-8 body %v \r\n", bodyStr)
    defer resp.Body.Close()
    return p, bodyStr
}

func (this *HttpDownloader) downloadHtml(p *page.Page, req *request.Request) *page.Page {
    var err error
    p, destbody := this.downloadFile(p, req)
    //fmt.Printf("Destbody %v \r\n", destbody)
    if !p.IsSucc() {
        //fmt.Print("Page error \r\n")
        return p
    }
    bodyReader := bytes.NewReader([]byte(destbody))

    var doc *goquery.Document
    if doc, err = goquery.NewDocumentFromReader(bodyReader); err != nil {
        mlog.LogInst().LogError(err.Error())
        p.SetStatus(true, err.Error())
        return p
    }

    var body string
    if body, err = doc.Html(); err != nil {
        mlog.LogInst().LogError(err.Error())
        p.SetStatus(true, err.Error())
        return p
    }

    p.SetBodyStr(body).SetHtmlParser(doc).SetStatus(false, "")

    return p
}

func (this *HttpDownloader) downloadJson(p *page.Page, req *request.Request) *page.Page {
    var err error
    p, destbody := this.downloadFile(p, req)
    if !p.IsSucc() {
        return p
    }

    var body []byte
    body = []byte(destbody)
    mtype := req.GetResponceType()
    if mtype == "jsonp" {
        tmpstr := util.JsonpToJson(destbody)
        body = []byte(tmpstr)
    }

    var r *simplejson.Json
    if r, err = simplejson.NewJson(body); err != nil {
        mlog.LogInst().LogError(string(body) + "\t" + err.Error())
        p.SetStatus(true, err.Error())
        return p
    }

    // json result
    p.SetBodyStr(string(body)).SetJson(r).SetStatus(false, "")

    return p
}

func (this *HttpDownloader) downloadText(p *page.Page, req *request.Request) *page.Page {
    p, destbody := this.downloadFile(p, req)
    if !p.IsSucc() {
        return p
    }

    p.SetBodyStr(destbody).SetStatus(false, "")
    return p
}
<|endoftext|>