<commit_before>package siesta

import (
	"fmt"
	"log"
	"sync"
	"time"
)

type ProducerRecord struct {
	Topic string
	Key   interface{}
	Value interface{}

	metadataChan chan *RecordMetadata
	partition    int32
	encodedKey   []byte
	encodedValue []byte
}

type RecordMetadata struct {
	Offset    int64
	Topic     string
	Partition int32
	Error     error
}

type PartitionInfo struct{}
type Metric struct{}
type ProducerConfig struct {
	MetadataFetchTimeout int64
	MaxRequestSize       int
	TotalMemorySize      int
	CompressionType      string
	BatchSize            int
	LingerMs             int64
	RetryBackoffMs       int64
	BlockOnBufferFull    bool

	ClientID        string
	MaxRequests     int
	SendRoutines    int
	ReceiveRoutines int
	ReadTimeout     time.Duration
	WriteTimeout    time.Duration
	RequiredAcks    int
}

type Serializer func(interface{}) ([]byte, error)

func ByteSerializer(value interface{}) ([]byte, error) {
	if value == nil {
		return nil, nil
	}

	if array, ok := value.([]byte); ok {
		return array, nil
	}

	return nil, fmt.Errorf("Can't serialize %v", value)
}

func StringSerializer(value interface{}) ([]byte, error) {
	if str, ok := value.(string); ok {
		return []byte(str), nil
	}

	return nil, fmt.Errorf("Can't serialize %v to string", value)
}

type Producer interface {
	// Send the given record asynchronously and return a channel which will eventually contain the response information.
	Send(*ProducerRecord) <-chan *RecordMetadata

	// Flush any accumulated records from the producer. Blocks until all sends are complete.
	Flush()

	// Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change
	// over time so this list should not be cached.
	PartitionsFor(topic string) []PartitionInfo

	// Return a map of metrics maintained by the producer
	Metrics() map[string]Metric

	// Tries to close the producer cleanly within the specified timeout. If the close does not complete within the
	// timeout, fail any pending send requests and force close the producer.
	Close(timeout int)
}

type KafkaProducer struct {
	config                 *ProducerConfig
	time                   time.Time
	partitioner            Partitioner
	keySerializer          Serializer
	valueSerializer        Serializer
	metadataFetchTimeoutMs int64
	metadata               *Metadata
	maxRequestSize         int
	totalMemorySize        int
	metrics                map[string]Metric
	compressionType        string
	accumulator            *RecordAccumulator
	metricTags             map[string]string
	connector              Connector
	topicMetadataLock      sync.Mutex
}

func NewKafkaProducer(config *ProducerConfig, keySerializer Serializer, valueSerializer Serializer, connector Connector) *KafkaProducer {
	log.Println("Starting the Kafka producer")
	producer := &KafkaProducer{}
	producer.config = config
	producer.time = time.Now()
	producer.metrics = make(map[string]Metric)
	producer.partitioner = NewHashPartitioner()
	producer.keySerializer = keySerializer
	producer.valueSerializer = valueSerializer
	producer.metadataFetchTimeoutMs = config.MetadataFetchTimeout
	producer.metadata = NewMetadata()
	producer.maxRequestSize = config.MaxRequestSize
	producer.totalMemorySize = config.TotalMemorySize
	producer.compressionType = config.CompressionType
	producer.connector = connector
	metricTags := make(map[string]string)

	networkClientConfig := NetworkClientConfig{}
	client := NewNetworkClient(networkClientConfig, connector, config)

	accumulatorConfig := &RecordAccumulatorConfig{
		batchSize:         config.BatchSize,
		totalMemorySize:   producer.totalMemorySize,
		compressionType:   producer.compressionType,
		lingerMs:          config.LingerMs,
		blockOnBufferFull: config.BlockOnBufferFull,
		metrics:           producer.metrics,
		time:              producer.time,
		metricTags:        metricTags,
		networkClient:     client,
	}
	producer.accumulator = NewRecordAccumulator(accumulatorConfig)

	log.Println("Kafka producer started")

	return producer
}

func (kp *KafkaProducer) Send(record *ProducerRecord) <-chan *RecordMetadata {
	metadata := make(chan *RecordMetadata, 1)
	kp.send(record, metadata)
	return metadata
}

func (kp *KafkaProducer) send(record *ProducerRecord, metadataChan chan *RecordMetadata) {
	metadata := new(RecordMetadata)

	serializedKey, err := kp.keySerializer(record.Key)
	if err != nil {
		metadata.Error = err
		metadataChan <- metadata
		return
	}

	serializedValue, err := kp.valueSerializer(record.Value)
	if err != nil {
		metadata.Error = err
		metadataChan <- metadata
		return
	}

	record.encodedKey = serializedKey
	record.encodedValue = serializedValue

	partitions, err := kp.partitionsForTopic(record.Topic)
	if err != nil {
		metadata.Error = err
		metadataChan <- metadata
		return
	}

	partition, err := kp.partitioner.Partition(record, partitions)
	if err != nil {
		metadata.Error = err
		metadataChan <- metadata
		return
	}
	record.partition = partition
	record.metadataChan = metadataChan

	kp.accumulator.addChan <- record
}

//TODO cache
func (kp *KafkaProducer) partitionsForTopic(topic string) ([]int32, error) {
	kp.topicMetadataLock.Lock()
	defer kp.topicMetadataLock.Unlock()

	topicMetadataResponse, err := kp.connector.GetTopicMetadata([]string{topic})
	if err != nil {
		return nil, err
	}

	for _, topicMetadata := range topicMetadataResponse.TopicsMetadata {
		if topic == topicMetadata.Topic {
			partitions := make([]int32, 0)
			for _, partitionMetadata := range topicMetadata.PartitionsMetadata {
				partitions = append(partitions, partitionMetadata.PartitionID)
			}
			return partitions, nil
		}
	}

	return nil, fmt.Errorf("Topic Metadata response did not contain metadata for topic %s", topic)
}

//func (kp *KafkaProducer) SendCallback(ProducerRecord, Callback) <-chan RecordMetadata {
//	return make(chan RecordMetadata)
//}

func (kp *KafkaProducer) Flush() {}

func (kp *KafkaProducer) PartitionsFor(topic string) []PartitionInfo {
	return []PartitionInfo{}
}

func (kp *KafkaProducer) Metrics() map[string]Metric {
	return make(map[string]Metric)
}

func (kp *KafkaProducer) Close(timeout int) {

	kp.accumulator.close()
}
<commit_msg>Added topic metadata cache<commit_after>package siesta

import (
	"fmt"
	"log"
	"sync"
	"time"
)

type ProducerRecord struct {
	Topic string
	Key   interface{}
	Value interface{}

	metadataChan chan *RecordMetadata
	partition    int32
	encodedKey   []byte
	encodedValue []byte
}

type RecordMetadata struct {
	Offset    int64
	Topic     string
	Partition int32
	Error     error
}

type PartitionInfo struct{}
type Metric struct{}
type ProducerConfig struct {
	MetadataFetchTimeout int64
	MetadataExpireMs     int64
	MaxRequestSize       int
	TotalMemorySize      int
	CompressionType      string
	BatchSize            int
	LingerMs             int64
	RetryBackoffMs       int64
	BlockOnBufferFull    bool

	ClientID        string
	MaxRequests     int
	SendRoutines    int
	ReceiveRoutines int
	ReadTimeout     time.Duration
	WriteTimeout    time.Duration
	RequiredAcks    int
}

type Serializer func(interface{}) ([]byte, error)

func ByteSerializer(value interface{}) ([]byte, error) {
	if value == nil {
		return nil, nil
	}

	if array, ok := value.([]byte); ok {
		return array, nil
	}

	return nil, fmt.Errorf("Can't serialize %v", value)
}

func StringSerializer(value interface{}) ([]byte, error) {
	if str, ok := value.(string); ok {
		return []byte(str), nil
	}

	return nil, fmt.Errorf("Can't serialize %v to string", value)
}

type Producer interface {
	// Send the given record asynchronously and return a channel which will eventually contain the response information.
	Send(*ProducerRecord) <-chan *RecordMetadata

	// Flush any accumulated records from the producer. Blocks until all sends are complete.
	Flush()

	// Get a list of partitions for the given topic for custom partition assignment. The partition metadata will change
	// over time so this list should not be cached.
	PartitionsFor(topic string) []PartitionInfo

	// Return a map of metrics maintained by the producer
	Metrics() map[string]Metric

	// Tries to close the producer cleanly within the specified timeout. If the close does not complete within the
	// timeout, fail any pending send requests and force close the producer.
	Close(timeout int)
}

type KafkaProducer struct {
	config                 *ProducerConfig
	time                   time.Time
	partitioner            Partitioner
	keySerializer          Serializer
	valueSerializer        Serializer
	metadataFetchTimeoutMs int64
	metadata               *Metadata
	maxRequestSize         int
	totalMemorySize        int
	metrics                map[string]Metric
	compressionType        string
	accumulator            *RecordAccumulator
	metricTags             map[string]string
	connector              Connector
	topicMetadataLock      sync.Mutex
	metadataCache          *topicMetadataCache
}

func NewKafkaProducer(config *ProducerConfig, keySerializer Serializer, valueSerializer Serializer, connector Connector) *KafkaProducer {
	log.Println("Starting the Kafka producer")
	producer := &KafkaProducer{}
	producer.config = config
	producer.time = time.Now()
	producer.metrics = make(map[string]Metric)
	producer.partitioner = NewHashPartitioner()
	producer.keySerializer = keySerializer
	producer.valueSerializer = valueSerializer
	producer.metadataFetchTimeoutMs = config.MetadataFetchTimeout
	producer.metadata = NewMetadata()
	producer.maxRequestSize = config.MaxRequestSize
	producer.totalMemorySize = config.TotalMemorySize
	producer.compressionType = config.CompressionType
	producer.connector = connector
	producer.metadataCache = newTopicMetadataCache(connector, time.Duration(config.MetadataExpireMs)*time.Millisecond) //TODO we should probably accept configs in time.Duration and not like BlaBlaMs
	metricTags := make(map[string]string)

	networkClientConfig := NetworkClientConfig{}
	client := NewNetworkClient(networkClientConfig, connector, config)

	accumulatorConfig := &RecordAccumulatorConfig{
		batchSize:         config.BatchSize,
		totalMemorySize:   producer.totalMemorySize,
		compressionType:   producer.compressionType,
		lingerMs:          config.LingerMs,
		blockOnBufferFull: config.BlockOnBufferFull,
		metrics:           producer.metrics,
		time:              producer.time,
		metricTags:        metricTags,
		networkClient:     client,
	}
	producer.accumulator = NewRecordAccumulator(accumulatorConfig)

	log.Println("Kafka producer started")

	return producer
}

func (kp *KafkaProducer) Send(record *ProducerRecord) <-chan *RecordMetadata {
	metadata := make(chan *RecordMetadata, 1)
	kp.send(record, metadata)
	return metadata
}

func (kp *KafkaProducer) send(record *ProducerRecord, metadataChan chan *RecordMetadata) {
	metadata := new(RecordMetadata)

	serializedKey, err := kp.keySerializer(record.Key)
	if err != nil {
		metadata.Error = err
		metadataChan <- metadata
		return
	}

	serializedValue, err := kp.valueSerializer(record.Value)
	if err != nil {
		metadata.Error = err
		metadataChan <- metadata
		return
	}

	record.encodedKey = serializedKey
	record.encodedValue = serializedValue

	partitions, err := kp.metadataCache.Get(record.Topic)
	if err != nil {
		metadata.Error = err
		metadataChan <- metadata
		return
	}

	partition, err := kp.partitioner.Partition(record, partitions)
	if err != nil {
		metadata.Error = err
		metadataChan <- metadata
		return
	}
	record.partition = partition
	record.metadataChan = metadataChan

	kp.accumulator.addChan <- record
}

//func (kp *KafkaProducer) SendCallback(ProducerRecord, Callback) <-chan RecordMetadata {
//	return make(chan RecordMetadata)
//}

func (kp *KafkaProducer) Flush() {}

func (kp *KafkaProducer) PartitionsFor(topic string) []PartitionInfo {
	return []PartitionInfo{}
}

func (kp *KafkaProducer) Metrics() map[string]Metric {
	return make(map[string]Metric)
}

func (kp *KafkaProducer) Close(timeout int) {

	kp.accumulator.close()
}

type topicMetadataCache struct {
	connector   Connector
	ttl         time.Duration
	cache       map[string]*topicMetadataCacheEntry
	refreshLock sync.Mutex
}

func newTopicMetadataCache(connector Connector, ttl time.Duration) *topicMetadataCache {
	return &topicMetadataCache{
		connector: connector,
		ttl:       ttl,
		cache:     make(map[string]*topicMetadataCacheEntry),
	}
}

func (tmc *topicMetadataCache) Get(topic string) ([]int32, error) {
	cache := tmc.cache[topic]
	if cache == nil {
		err := tmc.Refresh([]string{topic})
		if err != nil {
			return nil, err
		}
	}

	cache = tmc.cache[topic]
	if cache != nil {
		if cache.timestamp.Add(tmc.ttl).Before(time.Now()) {
			err := tmc.Refresh([]string{topic})
			if err != nil {
				return nil, err
			}
		}

		cache = tmc.cache[topic]
		if cache != nil {
			return cache.partitions, nil
		}
	}

	return nil, fmt.Errorf("Could not get topic metadata for topic %s", topic)
}

func (tmc *topicMetadataCache) Refresh(topics []string) error {
	tmc.refreshLock.Lock()
	defer tmc.refreshLock.Unlock()

	topicMetadataResponse, err := tmc.connector.GetTopicMetadata(topics)
	if err != nil {
		return err
	}

	for _, topicMetadata := range topicMetadataResponse.TopicsMetadata {
		partitions := make([]int32, 0)
		for _, partitionMetadata := range topicMetadata.PartitionsMetadata {
			partitions = append(partitions, partitionMetadata.PartitionID)
		}
		tmc.cache[topicMetadata.Topic] = newTopicMetadataCacheEntry(partitions)
	}

	return nil
}

type topicMetadataCacheEntry struct {
	partitions []int32
	timestamp  time.Time
}

func newTopicMetadataCacheEntry(partitions []int32) *topicMetadataCacheEntry {
	return &topicMetadataCacheEntry{
		partitions: partitions,
		timestamp:  time.Now(),
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 TiKV Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License

package cluster

import (
	"sync"

	"github.com/pingcap/kvproto/pkg/pdpb"
	"github.com/pingcap/log"
	"github.com/tikv/pd/server/schedule"
	"github.com/tikv/pd/server/schedule/opt"
	"github.com/tikv/pd/server/schedule/storelimit"
	"go.uber.org/zap"
)

// StoreLimiter adjust the store limit dynamically
type StoreLimiter struct {
	m       sync.RWMutex
	opt     opt.Options
	scene   map[storelimit.Type]*storelimit.Scene
	state   *State
	current LoadState
}

// NewStoreLimiter builds a store limiter object using the operator controller
func NewStoreLimiter(opt opt.Options) *StoreLimiter {
	defaultScene := map[storelimit.Type]*storelimit.Scene{
		storelimit.AddPeer:    storelimit.DefaultScene(storelimit.AddPeer),
		storelimit.RemovePeer: storelimit.DefaultScene(storelimit.RemovePeer),
	}

	return &StoreLimiter{
		opt:     opt,
		state:   NewState(),
		scene:   defaultScene,
		current: LoadStateNone,
	}
}

// Collect the store statistics and update the cluster state
func (s *StoreLimiter) Collect(stats *pdpb.StoreStats) {
	s.m.Lock()
	defer s.m.Unlock()

	log.Debug("collected statistics", zap.Reflect("stats", stats))
	s.state.Collect((*StatEntry)(stats))

	state := s.state.State()
	ratePeerAdd := s.calculateRate(storelimit.AddPeer, state)
	ratePeerRemove := s.calculateRate(storelimit.RemovePeer, state)

	if ratePeerAdd > 0 || ratePeerRemove > 0 {
		if ratePeerAdd > 0 {
			s.opt.SetAllStoresLimit(storelimit.AddPeer, ratePeerAdd)
			log.Info("change store region add limit for cluster", zap.Stringer("state", state), zap.Float64("rate", ratePeerAdd))
		}
		if ratePeerRemove > 0 {
			s.opt.SetAllStoresLimit(storelimit.RemovePeer, ratePeerRemove)
			log.Info("change store region remove limit for cluster", zap.Stringer("state", state), zap.Float64("rate", ratePeerRemove))
		}
		s.current = state
		collectClusterStateCurrent(state)
	}
}

func collectClusterStateCurrent(state LoadState) {
	for i := LoadStateNone; i <= LoadStateHigh; i++ {
		if i == state {
			clusterStateCurrent.WithLabelValues(state.String()).Set(1)
			continue
		}
		clusterStateCurrent.WithLabelValues(i.String()).Set(0)
	}
}

func (s *StoreLimiter) calculateRate(limitType storelimit.Type, state LoadState) float64 {
	rate := float64(0)
	switch state {
	case LoadStateIdle:
		rate = float64(s.scene[limitType].Idle) / schedule.StoreBalanceBaseTime
	case LoadStateLow:
		rate = float64(s.scene[limitType].Low) / schedule.StoreBalanceBaseTime
	case LoadStateNormal:
		rate = float64(s.scene[limitType].Normal) / schedule.StoreBalanceBaseTime
	case LoadStateHigh:
		rate = float64(s.scene[limitType].High) / schedule.StoreBalanceBaseTime
	}
	return rate
}

// ReplaceStoreLimitScene replaces the store limit values for different scenes
func (s *StoreLimiter) ReplaceStoreLimitScene(scene *storelimit.Scene, limitType storelimit.Type) {
	s.m.Lock()
	defer s.m.Unlock()
	if s.scene == nil {
		s.scene = make(map[storelimit.Type]*storelimit.Scene)
	}
	s.scene[limitType] = scene
}

// StoreLimitScene returns the current limit for different scenes
func (s *StoreLimiter) StoreLimitScene(limitType storelimit.Type) *storelimit.Scene {
	s.m.RLock()
	defer s.m.RUnlock()
	return s.scene[limitType]
}
<commit_msg>bug fix: calculate limit rate bases on minute when mode is auto (#2822)<commit_after>// Copyright 2019 TiKV Project Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License

package cluster

import (
	"sync"

	"github.com/pingcap/kvproto/pkg/pdpb"
	"github.com/pingcap/log"
	"github.com/tikv/pd/server/schedule/opt"
	"github.com/tikv/pd/server/schedule/storelimit"
	"go.uber.org/zap"
)

// StoreLimiter adjust the store limit dynamically
type StoreLimiter struct {
	m       sync.RWMutex
	opt     opt.Options
	scene   map[storelimit.Type]*storelimit.Scene
	state   *State
	current LoadState
}

// NewStoreLimiter builds a store limiter object using the operator controller
func NewStoreLimiter(opt opt.Options) *StoreLimiter {
	defaultScene := map[storelimit.Type]*storelimit.Scene{
		storelimit.AddPeer:    storelimit.DefaultScene(storelimit.AddPeer),
		storelimit.RemovePeer: storelimit.DefaultScene(storelimit.RemovePeer),
	}

	return &StoreLimiter{
		opt:     opt,
		state:   NewState(),
		scene:   defaultScene,
		current: LoadStateNone,
	}
}

// Collect the store statistics and update the cluster state
func (s *StoreLimiter) Collect(stats *pdpb.StoreStats) {
	s.m.Lock()
	defer s.m.Unlock()

	log.Debug("collected statistics", zap.Reflect("stats", stats))
	s.state.Collect((*StatEntry)(stats))

	state := s.state.State()
	ratePeerAdd := s.calculateRate(storelimit.AddPeer, state)
	ratePeerRemove := s.calculateRate(storelimit.RemovePeer, state)

	if ratePeerAdd > 0 || ratePeerRemove > 0 {
		if ratePeerAdd > 0 {
			s.opt.SetAllStoresLimit(storelimit.AddPeer, ratePeerAdd)
			log.Info("change store region add limit for cluster", zap.Stringer("state", state), zap.Float64("rate", ratePeerAdd))
		}
		if ratePeerRemove > 0 {
			s.opt.SetAllStoresLimit(storelimit.RemovePeer, ratePeerRemove)
			log.Info("change store region remove limit for cluster", zap.Stringer("state", state), zap.Float64("rate", ratePeerRemove))
		}
		s.current = state
		collectClusterStateCurrent(state)
	}
}

func collectClusterStateCurrent(state LoadState) {
	for i := LoadStateNone; i <= LoadStateHigh; i++ {
		if i == state {
			clusterStateCurrent.WithLabelValues(state.String()).Set(1)
			continue
		}
		clusterStateCurrent.WithLabelValues(i.String()).Set(0)
	}
}

func (s *StoreLimiter) calculateRate(limitType storelimit.Type, state LoadState) float64 {
	rate := float64(0)
	switch state {
	case LoadStateIdle:
		rate = float64(s.scene[limitType].Idle)
	case LoadStateLow:
		rate = float64(s.scene[limitType].Low)
	case LoadStateNormal:
		rate = float64(s.scene[limitType].Normal)
	case LoadStateHigh:
		rate = float64(s.scene[limitType].High)
	}
	return rate
}

// ReplaceStoreLimitScene replaces the store limit values for different scenes
func (s *StoreLimiter) ReplaceStoreLimitScene(scene *storelimit.Scene, limitType storelimit.Type) {
	s.m.Lock()
	defer s.m.Unlock()
	if s.scene == nil {
		s.scene = make(map[storelimit.Type]*storelimit.Scene)
	}
	s.scene[limitType] = scene
}

// StoreLimitScene returns the current limit for different scenes
func (s *StoreLimiter) StoreLimitScene(limitType storelimit.Type) *storelimit.Scene {
	s.m.RLock()
	defer s.m.RUnlock()
	return s.scene[limitType]
}
<|endoftext|>


<commit_before>//  Description = coldhak website
//  Authors = coldhak (C. // J. // R. // T.)
//  Version = 1.0

package main

import (
    "io/ioutil"
    "net/http"
	"github.com/pilu/traffic"
    "github.com/shurcooL/github_flavored_markdown"
)

func build(page string, location string) {
    resp, err := http.Get(page)
    if err != nil {}

    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {}

    output := github_flavored_markdown.Markdown(body)

    err = ioutil.WriteFile(location, output, 0644)
    if err != nil {}
}

func buildColdkernel(w traffic.ResponseWriter, r *traffic.Request) {
    build("https://raw.githubusercontent.com/coldhakca/coldkernel/master/README.md", "views/coldkernelreadme.tpl")
}

func buildSyncFamily(w traffic.ResponseWriter, r *traffic.Request) {
    build("https://raw.githubusercontent.com/coldhakca/sync_family/master/README.md", "views/sync_familyreadme.tpl")
}

func buildAtlasTools(w traffic.ResponseWriter, r *traffic.Request) {
	build("https://raw.githubusercontent.com/coldhakca/atlas_tools/master/README.md", "views/atlas_toolsreadme.tpl")
}

func indexHandler(w traffic.ResponseWriter, r *traffic.Request) {
	w.Render("index")
}

func relaysHandler(w traffic.ResponseWriter, r *traffic.Request) {
	w.Render("relays")
}

func sync_familyHandler(w traffic.ResponseWriter, r *traffic.Request) {
	w.Render("sync_family")
}

func aboutHandler(w traffic.ResponseWriter, r *traffic.Request) {
    w.Render("about")
}

func atlas_toolsHandler(w traffic.ResponseWriter, r *traffic.Request) {
	w.Render("atlas_tools")
}

func main() {
	router := traffic.New()

	// add a route for each page you add to the site
	// make sure you create a route handler for it
	router.Get("/", indexHandler)
	router.Get("/coldkernel", indexHandler)
	router.Get("/relays", relaysHandler)
	router.Get("/sync_family", sync_familyHandler)
	router.Get("/about", aboutHandler)
	router.Get("/atlas_tools", atlas_toolsHandler)
//	router.Get("/build/coldkernel", buildColdkernel)
//	router.Get("/build/sync_family", buildSyncFamily)
//	router.Get("/build/atlas_tools", buildAtlasTools)
	router.Run()
}
<commit_msg>Removing sync_family<commit_after>//  Description = coldhak website
//  Authors = coldhak (C. // J. // R. // T.)
//  Version = 1.0

package main

import (
    "io/ioutil"
    "net/http"
	"github.com/pilu/traffic"
    "github.com/shurcooL/github_flavored_markdown"
)

func build(page string, location string) {
    resp, err := http.Get(page)
    if err != nil {}

    defer resp.Body.Close()
    body, err := ioutil.ReadAll(resp.Body)
    if err != nil {}

    output := github_flavored_markdown.Markdown(body)

    err = ioutil.WriteFile(location, output, 0644)
    if err != nil {}
}

func buildColdkernel(w traffic.ResponseWriter, r *traffic.Request) {
    build("https://raw.githubusercontent.com/coldhakca/coldkernel/master/README.md", "views/coldkernelreadme.tpl")
}

func buildSyncFamily(w traffic.ResponseWriter, r *traffic.Request) {
    build("https://raw.githubusercontent.com/coldhakca/sync_family/master/README.md", "views/sync_familyreadme.tpl")
}

func buildAtlasTools(w traffic.ResponseWriter, r *traffic.Request) {
	build("https://raw.githubusercontent.com/coldhakca/atlas_tools/master/README.md", "views/atlas_toolsreadme.tpl")
}

func indexHandler(w traffic.ResponseWriter, r *traffic.Request) {
	w.Render("index")
}

func relaysHandler(w traffic.ResponseWriter, r *traffic.Request) {
	w.Render("relays")
}

func aboutHandler(w traffic.ResponseWriter, r *traffic.Request) {
    w.Render("about")
}

func atlas_toolsHandler(w traffic.ResponseWriter, r *traffic.Request) {
	w.Render("atlas_tools")
}

func main() {
	router := traffic.New()

	// add a route for each page you add to the site
	// make sure you create a route handler for it
	router.Get("/", indexHandler)
	router.Get("/coldkernel", indexHandler)
	router.Get("/relays", relaysHandler)
	router.Get("/about", aboutHandler)
	router.Get("/atlas_tools", atlas_toolsHandler)
//	router.Get("/build/coldkernel", buildColdkernel)
//	router.Get("/build/sync_family", buildSyncFamily)
//	router.Get("/build/atlas_tools", buildAtlasTools)
	router.Run()
}
<|endoftext|>


<commit_before>package commands

import(
	"errors"
	"fmt"
	"os"
	fp "path/filepath"
	"syscall"

	"github.com/codegangsta/cli"

	"github.com/yuuki1/dochroot/log"
	"github.com/yuuki1/dochroot/osutil"
)

var CommandArgRun = "--root ROOT_DIR [--user USER] [--group GROUP] [--bind SRC-PATH[:DEST-PATH]] COMMAND"
var CommandRun = cli.Command{
	Name:  "run",
	Usage: "Run an extracted docker image from s3",
	Action: fatalOnError(doRun),
	Flags: []cli.Flag{
		cli.StringFlag{Name: "root, r", Usage: "Root directory path for chrooting"},
		cli.StringFlag{Name: "user, u", Usage: "User (ID or name) to switch before running the program"},
		cli.StringFlag{Name: "group, g", Usage: "Group (ID or name) to switch to"},
		cli.StringSliceFlag{
			Name: "bind, b",
			Value: &cli.StringSlice{},
			Usage: "Bind mount directory (can be specifies multiple times)",
		},
		cli.BoolFlag{
			Name: "copy-files, cp",
			Usage: "Copy host from containersuch as /etc/hosts, /etc/group, /etc/passwd, /etc/hosts",
		},
	},
}

var copyFiles = []string{
	"etc/group",
	"etc/passwd",
	"etc/resolv.conf",
	"etc/hosts",
}

var keepCaps = map[uint]bool{
	2:	true,	// CAP_DAC_READ_SEARCH
	6:	true,	// CAP_SETGID
	7:	true,	// CAP_SETUID
	10:	true,	// CAP_NET_BIND_SERVICE
}

func doRun(c *cli.Context) error {
	command := c.Args()
	if len(command) < 1 {
		cli.ShowCommandHelp(c, "run")
		return errors.New("command required")
	}

	rootDir := c.String("root")
	bindDirs := c.StringSlice("bind")

	if rootDir == "" {
		cli.ShowCommandHelp(c, "run")
		return errors.New("--root option required")
	}

	if !osutil.ExistsDir(rootDir) {
		return fmt.Errorf("No such directory %s", rootDir)
	}

	// copy files
	if c.Bool("copy-files") {
		for _, f := range copyFiles {
			if err := osutil.Cp(fp.Join("/", f), fp.Join(rootDir, f)); err != nil {
				return err
			}
		}
	}

	// bind the directories
	for _, dir := range bindDirs {
		if err := bindMount(dir, rootDir); err != nil {
			return err
		}
	}

	// create symlinks
	if err := osutil.Symlink("../run/lock", fp.Join(rootDir, "/var/lock")); err != nil {
		return err
	}

	if err := createDevices(rootDir); err != nil {
		return err
	}

	log.Debug("chroot", rootDir, command)

	if err := syscall.Chroot(rootDir); err != nil {
		return err
	}
	if err := syscall.Chdir("/"); err != nil {
		return err
	}

	log.Debug("drop capabilities")
	if err := osutil.DropCapabilities(keepCaps); err != nil {
		return err
	}

	if group := c.String("group"); group != "" {
		log.Debug("setgid", group)
		if err := osutil.SetGroup(group); err != nil {
			return err
		}
	}
	if user := c.String("user"); user != "" {
		log.Debug("setuid", user)
		if err := osutil.SetUser(user); err != nil {
			return err
		}
	}

	return osutil.Execv(command[0], command[0:], os.Environ())
}

func bindMount(bindDir string, rootDir string) error {
	ok, err := osutil.IsDirEmpty(bindDir)
	if err != nil {
		return err
	}
	if ok {
		if _, err := os.Create(fp.Join(dir, ".dochroot.keep")); err != nil {
			return err
		}
	}

	containerDir := fp.Join(rootDir, bindDir)

	if err := os.MkdirAll(containerDir, os.FileMode(0755)); err != nil {
		return err
	}

	ok, err = osutil.IsDirEmpty(containerDir)
	if err != nil {
		return err
	}
	if ok {
		if err := osutil.BindMount(bindDir, containerDir); err != nil {
			return err
		}
		log.Debug("bind mount", bindDir, "to", containerDir)
	}

	return nil
}

func createDevices(rootDir string) error {
	if err := osutil.Mknod(fp.Join(rootDir, os.DevNull), syscall.S_IFCHR | uint32(os.FileMode(0666)), 1*256+3); err != nil {
		return err
	}

	if err := osutil.Mknod(fp.Join(rootDir, "/dev/zero"), syscall.S_IFCHR | uint32(os.FileMode(0666)), 1*256+3); err != nil {
		return err
	}

	for _, f := range []string{"/dev/random", "/dev/urandom"} {
		if err := osutil.Mknod(fp.Join(rootDir, f), syscall.S_IFCHR | uint32(os.FileMode(0666)), 1*256+9); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Enable to handle '--bind src-path:dest-path'<commit_after>package commands

import(
	"errors"
	"fmt"
	"os"
	fp "path/filepath"
	"strings"
	"syscall"

	"github.com/codegangsta/cli"

	"github.com/yuuki1/dochroot/log"
	"github.com/yuuki1/dochroot/osutil"
)

var CommandArgRun = "--root ROOT_DIR [--user USER] [--group GROUP] [--bind SRC-PATH[:DEST-PATH]] COMMAND"
var CommandRun = cli.Command{
	Name:  "run",
	Usage: "Run an extracted docker image from s3",
	Action: fatalOnError(doRun),
	Flags: []cli.Flag{
		cli.StringFlag{Name: "root, r", Usage: "Root directory path for chrooting"},
		cli.StringFlag{Name: "user, u", Usage: "User (ID or name) to switch before running the program"},
		cli.StringFlag{Name: "group, g", Usage: "Group (ID or name) to switch to"},
		cli.StringSliceFlag{
			Name: "bind, b",
			Value: &cli.StringSlice{},
			Usage: "Bind mount directory (can be specifies multiple times)",
		},
		cli.BoolFlag{
			Name: "copy-files, cp",
			Usage: "Copy host from containersuch as /etc/hosts, /etc/group, /etc/passwd, /etc/hosts",
		},
	},
}

var copyFiles = []string{
	"etc/group",
	"etc/passwd",
	"etc/resolv.conf",
	"etc/hosts",
}

var keepCaps = map[uint]bool{
	2:	true,	// CAP_DAC_READ_SEARCH
	6:	true,	// CAP_SETGID
	7:	true,	// CAP_SETUID
	10:	true,	// CAP_NET_BIND_SERVICE
}

func doRun(c *cli.Context) error {
	command := c.Args()
	if len(command) < 1 {
		cli.ShowCommandHelp(c, "run")
		return errors.New("command required")
	}

	rootDir := c.String("root")
	if rootDir == "" {
		cli.ShowCommandHelp(c, "run")
		return errors.New("--root option required")
	}

	if !osutil.ExistsDir(rootDir) {
		return fmt.Errorf("No such directory %s", rootDir)
	}

	// copy files
	if c.Bool("copy-files") {
		for _, f := range copyFiles {
			if err := osutil.Cp(fp.Join("/", f), fp.Join(rootDir, f)); err != nil {
				return err
			}
		}
	}

	// bind the directories
	for _, dir := range c.StringSlice("bind") {
		if err := bindMount(dir, rootDir); err != nil {
			return err
		}
	}

	// create symlinks
	if err := osutil.Symlink("../run/lock", fp.Join(rootDir, "/var/lock")); err != nil {
		return err
	}

	if err := createDevices(rootDir); err != nil {
		return err
	}

	log.Debug("chroot", rootDir, command)

	if err := syscall.Chroot(rootDir); err != nil {
		return err
	}
	if err := syscall.Chdir("/"); err != nil {
		return err
	}

	log.Debug("drop capabilities")
	if err := osutil.DropCapabilities(keepCaps); err != nil {
		return err
	}

	if group := c.String("group"); group != "" {
		log.Debug("setgid", group)
		if err := osutil.SetGroup(group); err != nil {
			return err
		}
	}
	if user := c.String("user"); user != "" {
		log.Debug("setuid", user)
		if err := osutil.SetUser(user); err != nil {
			return err
		}
	}

	return osutil.Execv(command[0], command[0:], os.Environ())
}

func bindMount(bindDir string, rootDir string) error {
	var srcDir, destDir string

	d := strings.SplitN(bindDir, ":", 2)
	if len(d) < 2 {
		srcDir = d[0]
	} else {
		srcDir, destDir = d[0], d[1]
	}
	if destDir == "" {
		destDir = srcDir
	}

	ok, err := osutil.IsDirEmpty(srcDir)
	if err != nil {
		return err
	}
	if ok {
		if _, err := os.Create(fp.Join(srcDir, ".dochroot.keep")); err != nil {
			return err
		}
	}

	containerDir := fp.Join(rootDir, destDir)

	if err := os.MkdirAll(containerDir, os.FileMode(0755)); err != nil {
		return err
	}

	ok, err = osutil.IsDirEmpty(containerDir)
	if err != nil {
		return err
	}
	if ok {
		if err := osutil.BindMount(srcDir, containerDir); err != nil {
			return err
		}
		log.Debug("bind mount", bindDir, "to", containerDir)
	}

	return nil
}

func createDevices(rootDir string) error {
	if err := osutil.Mknod(fp.Join(rootDir, os.DevNull), syscall.S_IFCHR | uint32(os.FileMode(0666)), 1*256+3); err != nil {
		return err
	}

	if err := osutil.Mknod(fp.Join(rootDir, "/dev/zero"), syscall.S_IFCHR | uint32(os.FileMode(0666)), 1*256+3); err != nil {
		return err
	}

	for _, f := range []string{"/dev/random", "/dev/urandom"} {
		if err := osutil.Mknod(fp.Join(rootDir, f), syscall.S_IFCHR | uint32(os.FileMode(0666)), 1*256+9); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/persist"
)

var (
	errWrongNumberArguments = errors.New("Improper number of arguments")

	// TODO: possibly move this to ssh package
	baseSSHArgs = []string{
		"-o", "IdentitiesOnly=yes",
		"-o", "StrictHostKeyChecking=no",
		"-o", "UserKnownHostsFile=/dev/null",
		"-o", "LogLevel=quiet", // suppress "Warning: Permanently added '[localhost]:2022' (ECDSA) to the list of known hosts."
	}
)

// HostInfo gives the mandatory information to connect to a host.
type HostInfo interface {
	GetMachineName() string

	GetIP() (string, error)

	GetSSHUsername() string

	GetSSHKeyPath() string
}

// HostInfoLoader loads host information.
type HostInfoLoader interface {
	load(name string) (HostInfo, error)
}

type storeHostInfoLoader struct {
	store persist.Store
}

func (s *storeHostInfoLoader) load(name string) (HostInfo, error) {
	host, err := s.store.Load(name)
	if err != nil {
		return nil, fmt.Errorf("Error loading host: %s", err)
	}

	return host.Driver, nil
}

func cmdScp(c CommandLine, api libmachine.API) error {
	args := c.Args()
	if len(args) != 2 {
		c.ShowHelp()
		return errWrongNumberArguments
	}

	src := args[0]
	dest := args[1]

	hostInfoLoader := &storeHostInfoLoader{api}

	cmd, err := getScpCmd(src, dest, c.Bool("recursive"), hostInfoLoader)
	if err != nil {
		return err
	}

	if err := runCmdWithStdIo(*cmd); err != nil {
		return err
	}

	return runCmdWithStdIo(*cmd)
}

func getScpCmd(src, dest string, recursive bool, hostInfoLoader HostInfoLoader) (*exec.Cmd, error) {
	cmdPath, err := exec.LookPath("scp")
	if err != nil {
		return nil, errors.New("Error: You must have a copy of the scp binary locally to use the scp feature.")
	}

	srcHost, srcPath, srcOpts, err := getInfoForScpArg(src, hostInfoLoader)
	if err != nil {
		return nil, err
	}

	destHost, destPath, destOpts, err := getInfoForScpArg(dest, hostInfoLoader)
	if err != nil {
		return nil, err
	}

	// TODO: Check that "-3" flag is available in user's version of scp.
	// It is on every system I've checked, but the manual mentioned it's "newer"
	sshArgs := baseSSHArgs
	sshArgs = append(sshArgs, "-3")
	if recursive {
		sshArgs = append(sshArgs, "-r")
	}

	// Append needed -i / private key flags to command.
	sshArgs = append(sshArgs, srcOpts...)
	sshArgs = append(sshArgs, destOpts...)

	// Append actual arguments for the scp command (i.e. docker@<ip>:/path)
	locationArg, err := generateLocationArg(srcHost, srcPath)
	if err != nil {
		return nil, err
	}

	sshArgs = append(sshArgs, locationArg)
	locationArg, err = generateLocationArg(destHost, destPath)
	if err != nil {
		return nil, err
	}
	sshArgs = append(sshArgs, locationArg)

	cmd := exec.Command(cmdPath, sshArgs...)
	log.Debug(*cmd)
	return cmd, nil
}

func getInfoForScpArg(hostAndPath string, hostInfoLoader HostInfoLoader) (HostInfo, string, []string, error) {
	// Local path.  e.g. "/tmp/foo"
	if !strings.Contains(hostAndPath, ":") {
		return nil, hostAndPath, nil, nil
	}

	// Path with hostname.  e.g. "hostname:/usr/bin/cmatrix"
	parts := strings.SplitN(hostAndPath, ":", 2)
	hostName := parts[0]
	path := parts[1]
	if hostName == "localhost" {
		return nil, path, nil, nil
	}

	// Remote path
	hostInfo, err := hostInfoLoader.load(hostName)
	if err != nil {
		return nil, "", nil, fmt.Errorf("Error loading host: %s", err)
	}

	args := []string{
		"-i",
		hostInfo.GetSSHKeyPath(),
	}
	return hostInfo, path, args, nil
}

func generateLocationArg(hostInfo HostInfo, path string) (string, error) {
	if hostInfo == nil {
		return path, nil
	}

	ip, err := hostInfo.GetIP()
	if err != nil {
		return "", err
	}

	location := fmt.Sprintf("%s@%s:%s", hostInfo.GetSSHUsername(), ip, path)
	return location, nil
}

func runCmdWithStdIo(cmd exec.Cmd) error {
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}
<commit_msg>FIX 2444 Command scp being called twice<commit_after>package commands

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/log"
	"github.com/docker/machine/libmachine/persist"
)

var (
	errWrongNumberArguments = errors.New("Improper number of arguments")

	// TODO: possibly move this to ssh package
	baseSSHArgs = []string{
		"-o", "IdentitiesOnly=yes",
		"-o", "StrictHostKeyChecking=no",
		"-o", "UserKnownHostsFile=/dev/null",
		"-o", "LogLevel=quiet", // suppress "Warning: Permanently added '[localhost]:2022' (ECDSA) to the list of known hosts."
	}
)

// HostInfo gives the mandatory information to connect to a host.
type HostInfo interface {
	GetMachineName() string

	GetIP() (string, error)

	GetSSHUsername() string

	GetSSHKeyPath() string
}

// HostInfoLoader loads host information.
type HostInfoLoader interface {
	load(name string) (HostInfo, error)
}

type storeHostInfoLoader struct {
	store persist.Store
}

func (s *storeHostInfoLoader) load(name string) (HostInfo, error) {
	host, err := s.store.Load(name)
	if err != nil {
		return nil, fmt.Errorf("Error loading host: %s", err)
	}

	return host.Driver, nil
}

func cmdScp(c CommandLine, api libmachine.API) error {
	args := c.Args()
	if len(args) != 2 {
		c.ShowHelp()
		return errWrongNumberArguments
	}

	src := args[0]
	dest := args[1]

	hostInfoLoader := &storeHostInfoLoader{api}

	cmd, err := getScpCmd(src, dest, c.Bool("recursive"), hostInfoLoader)
	if err != nil {
		return err
	}

	return runCmdWithStdIo(*cmd)
}

func getScpCmd(src, dest string, recursive bool, hostInfoLoader HostInfoLoader) (*exec.Cmd, error) {
	cmdPath, err := exec.LookPath("scp")
	if err != nil {
		return nil, errors.New("Error: You must have a copy of the scp binary locally to use the scp feature.")
	}

	srcHost, srcPath, srcOpts, err := getInfoForScpArg(src, hostInfoLoader)
	if err != nil {
		return nil, err
	}

	destHost, destPath, destOpts, err := getInfoForScpArg(dest, hostInfoLoader)
	if err != nil {
		return nil, err
	}

	// TODO: Check that "-3" flag is available in user's version of scp.
	// It is on every system I've checked, but the manual mentioned it's "newer"
	sshArgs := baseSSHArgs
	sshArgs = append(sshArgs, "-3")
	if recursive {
		sshArgs = append(sshArgs, "-r")
	}

	// Append needed -i / private key flags to command.
	sshArgs = append(sshArgs, srcOpts...)
	sshArgs = append(sshArgs, destOpts...)

	// Append actual arguments for the scp command (i.e. docker@<ip>:/path)
	locationArg, err := generateLocationArg(srcHost, srcPath)
	if err != nil {
		return nil, err
	}

	sshArgs = append(sshArgs, locationArg)
	locationArg, err = generateLocationArg(destHost, destPath)
	if err != nil {
		return nil, err
	}
	sshArgs = append(sshArgs, locationArg)

	cmd := exec.Command(cmdPath, sshArgs...)
	log.Debug(*cmd)
	return cmd, nil
}

func getInfoForScpArg(hostAndPath string, hostInfoLoader HostInfoLoader) (HostInfo, string, []string, error) {
	// Local path.  e.g. "/tmp/foo"
	if !strings.Contains(hostAndPath, ":") {
		return nil, hostAndPath, nil, nil
	}

	// Path with hostname.  e.g. "hostname:/usr/bin/cmatrix"
	parts := strings.SplitN(hostAndPath, ":", 2)
	hostName := parts[0]
	path := parts[1]
	if hostName == "localhost" {
		return nil, path, nil, nil
	}

	// Remote path
	hostInfo, err := hostInfoLoader.load(hostName)
	if err != nil {
		return nil, "", nil, fmt.Errorf("Error loading host: %s", err)
	}

	args := []string{
		"-i",
		hostInfo.GetSSHKeyPath(),
	}
	return hostInfo, path, args, nil
}

func generateLocationArg(hostInfo HostInfo, path string) (string, error) {
	if hostInfo == nil {
		return path, nil
	}

	ip, err := hostInfo.GetIP()
	if err != nil {
		return "", err
	}

	location := fmt.Sprintf("%s@%s:%s", hostInfo.GetSSHUsername(), ip, path)
	return location, nil
}

func runCmdWithStdIo(cmd exec.Cmd) error {
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}
<|endoftext|>


<commit_before>package mirror

import (
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/golib/gofmt"
	"github.com/funkygao/golib/ratelimiter"
	"github.com/funkygao/golib/signal"
	log "github.com/funkygao/log4go"
)

// Mirror maintains a replica of an existing kafka cluster.
//
//               mirror
// kafka(source) ------------------> kafka(target)
//               topics discover
//               consumer balancing
//
// target kafka auto.create.topics.enable=true
//
// TODO
// * sync pub to assure no message lost
// * pub pool
// * we might add a data channel between pub and sub
type Mirror struct {
	Config

	startedAt time.Time
	quit      chan struct{}
	once      sync.Once

	transferN     int64
	transferBytes int64

	bandwidthRateLimiter *ratelimiter.LeakyBucket
}

func New(cf *Config) *Mirror {
	return &Mirror{Config: *cf}
}

func (this *Mirror) Main() (exitCode int) {
	this.quit = make(chan struct{})
	signal.RegisterHandler(func(sig os.Signal) {
		log.Info("received signal: %s", strings.ToUpper(sig.String()))
		log.Info("quiting...")

		this.once.Do(func() {
			close(this.quit)
		})
	}, syscall.SIGINT, syscall.SIGTERM)

	limit := (1 << 20) * this.BandwidthLimit / 8
	if this.BandwidthLimit > 0 {
		this.bandwidthRateLimiter = ratelimiter.NewLeakyBucket(limit*10, time.Second*10)
	}

	log.Info("starting mirror@%s", gafka.BuildId)

	// pprof
	debugAddr := ":10009"
	go http.ListenAndServe(debugAddr, nil)
	log.Info("pprof ready on %s", debugAddr)

	z1 := zk.NewZkZone(zk.DefaultConfig(this.Z1, ctx.ZoneZkAddrs(this.Z1)))
	z2 := zk.NewZkZone(zk.DefaultConfig(this.Z2, ctx.ZoneZkAddrs(this.Z2)))
	c1 := z1.NewCluster(this.C1)
	c2 := z2.NewCluster(this.C2)

	this.runMirror(c1, c2, limit)

	log.Info("bye mirror@%s, %s", gafka.BuildId, time.Since(this.startedAt))
	log.Close()

	return
}

func (this *Mirror) runMirror(c1, c2 *zk.ZkCluster, limit int64) {
	this.startedAt = time.Now()

	log.Info("start [%s/%s] -> [%s/%s] with bandwidth %sbps",
		c1.ZkZone().Name(), c1.Name(),
		c2.ZkZone().Name(), c2.Name(),
		gofmt.Comma(limit*8))

	pub, err := this.makePub(c2)
	if err != nil {
		panic(err)
	}
	log.Trace("pub[%s/%s] created", c2.ZkZone().Name(), c2.Name())

	go func(pub sarama.AsyncProducer, c *zk.ZkCluster) {
		for {
			select {
			case <-this.quit:
				return

			case err := <-pub.Errors():
				// TODO
				log.Error("pub[%s/%s] %v", c.ZkZone().Name(), c.Name(), err)
			}
		}
	}(pub, c2)

	group := this.groupName(c1, c2)
	ever := true
	round := 0
	for ever {
		round++

		topics, topicsChanges, err := c1.WatchTopics()
		if err != nil {
			log.Error("#%d [%s/%s]watch topics: %v", round, c1.ZkZone().Name(), c1.Name(), err)
			time.Sleep(time.Second * 10)
			continue
		}

		topics = this.realTopics(topics)
		sub, err := this.makeSub(c1, group, topics)
		if err != nil {
			log.Error("#%d [%s/%s] %v", round, c1.ZkZone().Name(), c1.Name(), err)
			time.Sleep(time.Second * 10)
			continue
		}

		log.Info("#%d starting pump [%s/%s] -> [%s/%s] %d topics with group %s for %+v", round,
			c1.ZkZone().Name(), c1.Name(),
			c2.ZkZone().Name(), c2.Name(), len(topics), group, topics)

		pumpStopper := make(chan struct{})
		pumpStopped := make(chan struct{})
		go this.pump(sub, pub, pumpStopper, pumpStopped)

		select {
		case <-topicsChanges:
			// TODO log the diff the topics
			log.Warn("#%d [%s/%s] topics changed, stopping pump...", round, c1.Name(), c2.Name())
			pumpStopper <- struct{}{} // stop pump
			<-pumpStopped             // await pump cleanup

		case <-this.quit:
			log.Info("#%d awaiting pump cleanup...", round)
			<-pumpStopped

			ever = false

		case <-pumpStopped:
			// pump encounters problems, just retry
		}
	}

	log.Info("total transferred: %s %smsgs",
		gofmt.ByteSize(this.transferBytes),
		gofmt.Comma(this.transferN))

	log.Info("closing pub...")
	pub.Close()
}

func (this *Mirror) groupName(c1, c2 *zk.ZkCluster) string {
	return fmt.Sprintf("_mirror_.%s.%s.%s.%s", c1.ZkZone().Name(), c1.Name(), c2.ZkZone().Name(), c2.Name())
}
<commit_msg>error examples given<commit_after>package mirror

import (
	"fmt"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/golib/gofmt"
	"github.com/funkygao/golib/ratelimiter"
	"github.com/funkygao/golib/signal"
	log "github.com/funkygao/log4go"
)

// Mirror maintains a replica of an existing kafka cluster.
//
//               mirror
// kafka(source) ------------------> kafka(target)
//               topics discover
//               consumer balancing
//
// target kafka auto.create.topics.enable=true
//
// TODO
// * sync pub to assure no message lost
// * pub pool
// * we might add a data channel between pub and sub
type Mirror struct {
	Config

	startedAt time.Time
	quit      chan struct{}
	once      sync.Once

	transferN     int64
	transferBytes int64

	bandwidthRateLimiter *ratelimiter.LeakyBucket
}

func New(cf *Config) *Mirror {
	return &Mirror{Config: *cf}
}

func (this *Mirror) Main() (exitCode int) {
	this.quit = make(chan struct{})
	signal.RegisterHandler(func(sig os.Signal) {
		log.Info("received signal: %s", strings.ToUpper(sig.String()))
		log.Info("quiting...")

		this.once.Do(func() {
			close(this.quit)
		})
	}, syscall.SIGINT, syscall.SIGTERM)

	limit := (1 << 20) * this.BandwidthLimit / 8
	if this.BandwidthLimit > 0 {
		this.bandwidthRateLimiter = ratelimiter.NewLeakyBucket(limit*10, time.Second*10)
	}

	log.Info("starting mirror@%s", gafka.BuildId)

	// pprof
	debugAddr := ":10009"
	go http.ListenAndServe(debugAddr, nil)
	log.Info("pprof ready on %s", debugAddr)

	z1 := zk.NewZkZone(zk.DefaultConfig(this.Z1, ctx.ZoneZkAddrs(this.Z1)))
	z2 := zk.NewZkZone(zk.DefaultConfig(this.Z2, ctx.ZoneZkAddrs(this.Z2)))
	c1 := z1.NewCluster(this.C1)
	c2 := z2.NewCluster(this.C2)

	this.runMirror(c1, c2, limit)

	log.Info("bye mirror@%s, %s", gafka.BuildId, time.Since(this.startedAt))
	log.Close()

	return
}

func (this *Mirror) runMirror(c1, c2 *zk.ZkCluster, limit int64) {
	this.startedAt = time.Now()

	log.Info("start [%s/%s] -> [%s/%s] with bandwidth %sbps",
		c1.ZkZone().Name(), c1.Name(),
		c2.ZkZone().Name(), c2.Name(),
		gofmt.Comma(limit*8))

	pub, err := this.makePub(c2)
	if err != nil {
		panic(err)
	}
	log.Trace("pub[%s/%s] created", c2.ZkZone().Name(), c2.Name())

	go func(pub sarama.AsyncProducer, c *zk.ZkCluster) {
		for {
			select {
			case <-this.quit:
				return

			case err := <-pub.Errors():
				// e,g
				// Failed to produce message to topic xx: write tcp src->kfk: i/o timeout
				// kafka: broker not connected
				log.Error("pub[%s/%s] %v", c.ZkZone().Name(), c.Name(), err)
			}
		}
	}(pub, c2)

	group := this.groupName(c1, c2)
	ever := true
	round := 0
	for ever {
		round++

		topics, topicsChanges, err := c1.WatchTopics()
		if err != nil {
			log.Error("#%d [%s/%s]watch topics: %v", round, c1.ZkZone().Name(), c1.Name(), err)
			time.Sleep(time.Second * 10)
			continue
		}

		topics = this.realTopics(topics)
		sub, err := this.makeSub(c1, group, topics)
		if err != nil {
			log.Error("#%d [%s/%s] %v", round, c1.ZkZone().Name(), c1.Name(), err)
			time.Sleep(time.Second * 10)
			continue
		}

		log.Info("#%d starting pump [%s/%s] -> [%s/%s] %d topics with group %s for %+v", round,
			c1.ZkZone().Name(), c1.Name(),
			c2.ZkZone().Name(), c2.Name(), len(topics), group, topics)

		pumpStopper := make(chan struct{})
		pumpStopped := make(chan struct{})
		go this.pump(sub, pub, pumpStopper, pumpStopped)

		select {
		case <-topicsChanges:
			// TODO log the diff the topics
			log.Warn("#%d [%s/%s] topics changed, stopping pump...", round, c1.Name(), c2.Name())
			pumpStopper <- struct{}{} // stop pump
			<-pumpStopped             // await pump cleanup

		case <-this.quit:
			log.Info("#%d awaiting pump cleanup...", round)
			<-pumpStopped

			ever = false

		case <-pumpStopped:
			// pump encounters problems, just retry
		}
	}

	log.Info("total transferred: %s %smsgs",
		gofmt.ByteSize(this.transferBytes),
		gofmt.Comma(this.transferN))

	log.Info("closing pub...")
	pub.Close()
}

func (this *Mirror) groupName(c1, c2 *zk.ZkCluster) string {
	return fmt.Sprintf("_mirror_.%s.%s.%s.%s", c1.ZkZone().Name(), c1.Name(), c2.ZkZone().Name(), c2.Name())
}
<|endoftext|>


<commit_before>package common

import "math"

// Color is used to represent the color and color temperature of a light.
// The color is represented as a 48-bit HSB (Hue, Saturation, Brightness) value.
// The color temperature is represented in K (Kelvin) and is used to adjust the
// warmness / coolness of a white light, which is most obvious when saturation
// is close zero.
type Color struct {
	Hue        uint16 // range 0 to 65535
	Saturation uint16 // range 0 to 65535
	Brightness uint16 // range 0 to 65535
	Kelvin     uint16 // range 2500° (warm) to 9000° (cool)
}

// AverageColor returns the average of the provided colors
func AverageColor(colors ...Color) (color Color) {
	var (
		x, y               float64
		hue, sat, bri, kel int
	)

	// Sum sind/cosd for hues
	for _, c := range colors {
		// Convert hue to degrees
		h := float64(c.Hue) / float64(math.MaxUint16) * 360.0

		x += math.Cos(h / 180.0 * math.Pi)
		y += math.Sin(h / 180.0 * math.Pi)
		sat += int(c.Saturation)
		bri += int(c.Brightness)
		kel += int(c.Kelvin)
	}

	// Average sind/cosd
	x /= float64(len(colors))
	y /= float64(len(colors))

	// Take atan2 of averaged hue and convert to uint16 scale
	hue = int((math.Atan2(y, x) * 180.0 / math.Pi) / 360.0 * float64(math.MaxUint16))
	sat /= len(colors)
	bri /= len(colors)
	kel /= len(colors)

	color.Hue = uint16(hue)
	color.Saturation = uint16(sat)
	color.Brightness = uint16(bri)
	color.Kelvin = uint16(kel)

	return color
}
<commit_msg>Add json keys to `common.Color`<commit_after>package common

import "math"

// Color is used to represent the color and color temperature of a light.
// The color is represented as a 48-bit HSB (Hue, Saturation, Brightness) value.
// The color temperature is represented in K (Kelvin) and is used to adjust the
// warmness / coolness of a white light, which is most obvious when saturation
// is close zero.
type Color struct {
	Hue        uint16 `json:"hue"`        // range 0 to 65535
	Saturation uint16 `json:"saturation"` // range 0 to 65535
	Brightness uint16 `json:"brightness"` // range 0 to 65535
	Kelvin     uint16 `json:"kelvin"`     // range 2500° (warm) to 9000° (cool)
}

// AverageColor returns the average of the provided colors
func AverageColor(colors ...Color) (color Color) {
	var (
		x, y               float64
		hue, sat, bri, kel int
	)

	// Sum sind/cosd for hues
	for _, c := range colors {
		// Convert hue to degrees
		h := float64(c.Hue) / float64(math.MaxUint16) * 360.0

		x += math.Cos(h / 180.0 * math.Pi)
		y += math.Sin(h / 180.0 * math.Pi)
		sat += int(c.Saturation)
		bri += int(c.Brightness)
		kel += int(c.Kelvin)
	}

	// Average sind/cosd
	x /= float64(len(colors))
	y /= float64(len(colors))

	// Take atan2 of averaged hue and convert to uint16 scale
	hue = int((math.Atan2(y, x) * 180.0 / math.Pi) / 360.0 * float64(math.MaxUint16))
	sat /= len(colors)
	bri /= len(colors)
	kel /= len(colors)

	color.Hue = uint16(hue)
	color.Saturation = uint16(sat)
	color.Brightness = uint16(bri)
	color.Kelvin = uint16(kel)

	return color
}
<|endoftext|>


<commit_before>package libkb

import (
	"github.com/keybase/go-jsonw"
)

type ProofEngine struct {
	Force              bool
	Service, Username  string
	me                 *User
	st                 ServiceType
	usernameNormalized string
	supersede          bool
	proof              *jsonw.Wrapper
	sig                string
	sigId              *SigId
	postRes            *PostProofRes
	ProveUI            ProveUI
	LoginUI            LoginUI
	SecretUI           SecretUI
	LogUI              LogUI
}

func (v *ProofEngine) Init() error {
	if v.ProveUI == nil {
		v.ProveUI = G.UI.GetProveUI()
	}
	if v.LoginUI == nil {
		v.LoginUI = G.UI.GetLoginUI()
	}
	if v.SecretUI == nil {
		v.SecretUI = G.UI.GetSecretUI()
	}
	return nil
}

func (v *ProofEngine) Login() (err error) {
	return G.LoginState.Login(LoginArg{Ui: v.LoginUI})
}

func (v *ProofEngine) LoadMe() (err error) {
	v.me, err = LoadMe(LoadUserArg{LoadSecrets: true, AllKeys: false, ForceReload: true})
	return
}

func (v *ProofEngine) CheckExists1() (err error) {
	proofs := v.me.IdTable.GetActiveProofsFor(v.st)
	if len(proofs) != 0 && !v.Force && v.st.LastWriterWins() {
		lst := proofs[len(proofs)-1]
		var redo bool
		redo, err = v.ProveUI.PromptOverwrite1(lst.ToDisplayString())
		if err != nil {
		} else if !redo {
			err = NotConfirmedError{}
		} else {
			v.supersede = true
		}
	}
	return
}

func (v *ProofEngine) PromptRemoteName() (err error) {
	if len(v.Username) == 0 {
		var prevErr error
		for len(v.Username) == 0 && err == nil {
			var un string
			un, err = v.ProveUI.PromptUsername(v.st.GetPrompt(), prevErr)
			if err == nil {
				prevErr = v.st.CheckUsername(un)
				if prevErr == nil {
					v.Username = un
				}
			}
		}
	} else {
		err = v.st.CheckUsername(v.Username)
	}
	return
}

func (v *ProofEngine) NormalizeRemoteName() (err error) {
	v.usernameNormalized, err = v.st.NormalizeUsername(v.Username)
	return
}

func (v *ProofEngine) CheckExists2() (err error) {
	G.Log.Debug("+ CheckExists2")
	defer func() { G.Log.Debug("- CheckExists2 -> %s", ErrToOk(err)) }()
	if !v.st.LastWriterWins() {
		var found RemoteProofChainLink
		for _, p := range v.me.IdTable.GetActiveProofsFor(v.st) {
			_, name := p.ToKeyValuePair()
			if Cicmp(name, v.usernameNormalized) {
				found = p
				break
			}
		}
		if found != nil {
			var redo bool
			redo, err = v.ProveUI.PromptOverwrite2(found.ToDisplayString())
			if err != nil {
			} else if !redo {
				err = NotConfirmedError{}
			} else {
				v.supersede = true
			}
		}
	}
	return
}

func (v *ProofEngine) DoPrechecks() (err error) {
	var w *Markup
	w, err = v.st.PreProofCheck(v.usernameNormalized)
	if w != nil {
		v.ProveUI.OutputPrechecks(w.Export())
	}
	return
}

func (v *ProofEngine) DoWarnings() (err error) {
	if mu := v.st.PreProofWarning(v.usernameNormalized); mu != nil {
		var ok bool
		if ok, err = v.ProveUI.PreProofWarning(mu.Export()); err == nil && !ok {
			err = NotConfirmedError{}
		}
	}
	return
}
func (v *ProofEngine) GenerateProof() (err error) {
	var key GenericKey
	if v.proof, err = v.me.ServiceProof(v.st, v.usernameNormalized); err != nil {
		return
	}
	if key, err = G.Keyrings.GetSecretKey("proof signature", v.SecretUI); err != nil {
		return
	}
	if v.sig, v.sigId, _, err = SignJson(v.proof, key); err != nil {
		return
	}
	return
}

func (v *ProofEngine) PostProofToServer() (err error) {
	arg := PostProofArg{
		Sig:            v.sig,
		ProofType:      v.st.GetProofType(),
		Id:             *v.sigId,
		Supersede:      v.supersede,
		RemoteUsername: v.usernameNormalized,
		RemoteKey:      v.st.GetApiArgKey(),
	}
	v.postRes, err = PostProof(arg)
	return
}

func (v *ProofEngine) InstructAction() (err error) {
	mkp := v.st.PostInstructions(v.usernameNormalized)
	var txt string
	if txt, err = v.st.FormatProofText(v.postRes); err != nil {
		return
	}
	err = v.ProveUI.OutputInstructions(mkp.Export(), txt)
	return
}

func (v *ProofEngine) PromptPostedLoop() (err error) {
	found := false
	for i := 0; ; i++ {
		var retry bool
		var status int
		var warn *Markup
		retry, err = v.ProveUI.OkToCheck(v.st.DisplayName(v.usernameNormalized), i)
		if !retry || err != nil {
			break
		}
		found, status, err = CheckPosted(v.postRes.Id)
		if found || err != nil {
			break
		}
		warn, err = v.st.RecheckProofPosting(status, i)
		if warn != nil {
			v.ProveUI.DisplayRecheckWarning(warn.Export())
		}
		if err != nil {
			break
		}
	}
	if !found && err == nil {
		err = ProofNotYetAvailableError{}
	}

	return
}

func (v *ProofEngine) CheckProofText() error {
	return v.st.CheckProofText(v.postRes.Text, *v.sigId, v.sig)
}

func (v *ProofEngine) GetServiceType() (err error) {
	if v.st = GetServiceType(v.Service); v.st == nil {
		err = BadServiceError{v.Service}
	}
	return
}

func (v *ProofEngine) Run() (err error) {

	if err = v.Init(); err != nil {
		return
	}
	if err = v.GetServiceType(); err != nil {
		return
	}
	if err = v.Login(); err != nil {
		return
	}
	if err = v.LoadMe(); err != nil {
		return
	}
	if err = v.CheckExists1(); err != nil {
		return
	}
	if err = v.PromptRemoteName(); err != nil {
		return
	}
	if err = v.NormalizeRemoteName(); err != nil {
		return
	}
	if err = v.CheckExists2(); err != nil {
		return
	}
	if err = v.DoPrechecks(); err != nil {
		return
	}
	if err = v.DoWarnings(); err != nil {
		return
	}
	if err = v.GenerateProof(); err != nil {
		return
	}
	if err = v.PostProofToServer(); err != nil {
		return
	}
	if err = v.CheckProofText(); err != nil {
		return
	}
	if err = v.InstructAction(); err != nil {
		return
	}
	if err = v.PromptPostedLoop(); err != nil {
		return
	}
	v.LogUI.Notice("Success!")
	return nil
}
<commit_msg>login<commit_after>package libkb

import (
	"github.com/keybase/go-jsonw"
)

type ProofEngine struct {
	Force              bool
	Service, Username  string
	me                 *User
	st                 ServiceType
	usernameNormalized string
	supersede          bool
	proof              *jsonw.Wrapper
	sig                string
	sigId              *SigId
	postRes            *PostProofRes
	ProveUI            ProveUI
	LoginUI            LoginUI
	SecretUI           SecretUI
	LogUI              LogUI
}

func (v *ProofEngine) Init() error {
	if v.ProveUI == nil {
		v.ProveUI = G.UI.GetProveUI()
	}
	if v.LoginUI == nil {
		v.LoginUI = G.UI.GetLoginUI()
	}
	if v.SecretUI == nil {
		v.SecretUI = G.UI.GetSecretUI()
	}
	return nil
}

func (v *ProofEngine) Login() (err error) {
	return G.LoginState.Login(LoginArg{Ui: v.LoginUI, SecretUI: v.SecretUI})
}

func (v *ProofEngine) LoadMe() (err error) {
	v.me, err = LoadMe(LoadUserArg{LoadSecrets: true, AllKeys: false, ForceReload: true})
	return
}

func (v *ProofEngine) CheckExists1() (err error) {
	proofs := v.me.IdTable.GetActiveProofsFor(v.st)
	if len(proofs) != 0 && !v.Force && v.st.LastWriterWins() {
		lst := proofs[len(proofs)-1]
		var redo bool
		redo, err = v.ProveUI.PromptOverwrite1(lst.ToDisplayString())
		if err != nil {
		} else if !redo {
			err = NotConfirmedError{}
		} else {
			v.supersede = true
		}
	}
	return
}

func (v *ProofEngine) PromptRemoteName() (err error) {
	if len(v.Username) == 0 {
		var prevErr error
		for len(v.Username) == 0 && err == nil {
			var un string
			un, err = v.ProveUI.PromptUsername(v.st.GetPrompt(), prevErr)
			if err == nil {
				prevErr = v.st.CheckUsername(un)
				if prevErr == nil {
					v.Username = un
				}
			}
		}
	} else {
		err = v.st.CheckUsername(v.Username)
	}
	return
}

func (v *ProofEngine) NormalizeRemoteName() (err error) {
	v.usernameNormalized, err = v.st.NormalizeUsername(v.Username)
	return
}

func (v *ProofEngine) CheckExists2() (err error) {
	G.Log.Debug("+ CheckExists2")
	defer func() { G.Log.Debug("- CheckExists2 -> %s", ErrToOk(err)) }()
	if !v.st.LastWriterWins() {
		var found RemoteProofChainLink
		for _, p := range v.me.IdTable.GetActiveProofsFor(v.st) {
			_, name := p.ToKeyValuePair()
			if Cicmp(name, v.usernameNormalized) {
				found = p
				break
			}
		}
		if found != nil {
			var redo bool
			redo, err = v.ProveUI.PromptOverwrite2(found.ToDisplayString())
			if err != nil {
			} else if !redo {
				err = NotConfirmedError{}
			} else {
				v.supersede = true
			}
		}
	}
	return
}

func (v *ProofEngine) DoPrechecks() (err error) {
	var w *Markup
	w, err = v.st.PreProofCheck(v.usernameNormalized)
	if w != nil {
		v.ProveUI.OutputPrechecks(w.Export())
	}
	return
}

func (v *ProofEngine) DoWarnings() (err error) {
	if mu := v.st.PreProofWarning(v.usernameNormalized); mu != nil {
		var ok bool
		if ok, err = v.ProveUI.PreProofWarning(mu.Export()); err == nil && !ok {
			err = NotConfirmedError{}
		}
	}
	return
}
func (v *ProofEngine) GenerateProof() (err error) {
	var key GenericKey
	if v.proof, err = v.me.ServiceProof(v.st, v.usernameNormalized); err != nil {
		return
	}
	if key, err = G.Keyrings.GetSecretKey("proof signature", v.SecretUI); err != nil {
		return
	}
	if v.sig, v.sigId, _, err = SignJson(v.proof, key); err != nil {
		return
	}
	return
}

func (v *ProofEngine) PostProofToServer() (err error) {
	arg := PostProofArg{
		Sig:            v.sig,
		ProofType:      v.st.GetProofType(),
		Id:             *v.sigId,
		Supersede:      v.supersede,
		RemoteUsername: v.usernameNormalized,
		RemoteKey:      v.st.GetApiArgKey(),
	}
	v.postRes, err = PostProof(arg)
	return
}

func (v *ProofEngine) InstructAction() (err error) {
	mkp := v.st.PostInstructions(v.usernameNormalized)
	var txt string
	if txt, err = v.st.FormatProofText(v.postRes); err != nil {
		return
	}
	err = v.ProveUI.OutputInstructions(mkp.Export(), txt)
	return
}

func (v *ProofEngine) PromptPostedLoop() (err error) {
	found := false
	for i := 0; ; i++ {
		var retry bool
		var status int
		var warn *Markup
		retry, err = v.ProveUI.OkToCheck(v.st.DisplayName(v.usernameNormalized), i)
		if !retry || err != nil {
			break
		}
		found, status, err = CheckPosted(v.postRes.Id)
		if found || err != nil {
			break
		}
		warn, err = v.st.RecheckProofPosting(status, i)
		if warn != nil {
			v.ProveUI.DisplayRecheckWarning(warn.Export())
		}
		if err != nil {
			break
		}
	}
	if !found && err == nil {
		err = ProofNotYetAvailableError{}
	}

	return
}

func (v *ProofEngine) CheckProofText() error {
	return v.st.CheckProofText(v.postRes.Text, *v.sigId, v.sig)
}

func (v *ProofEngine) GetServiceType() (err error) {
	if v.st = GetServiceType(v.Service); v.st == nil {
		err = BadServiceError{v.Service}
	}
	return
}

func (v *ProofEngine) Run() (err error) {

	if err = v.Init(); err != nil {
		return
	}
	if err = v.GetServiceType(); err != nil {
		return
	}
	if err = v.Login(); err != nil {
		return
	}
	if err = v.LoadMe(); err != nil {
		return
	}
	if err = v.CheckExists1(); err != nil {
		return
	}
	if err = v.PromptRemoteName(); err != nil {
		return
	}
	if err = v.NormalizeRemoteName(); err != nil {
		return
	}
	if err = v.CheckExists2(); err != nil {
		return
	}
	if err = v.DoPrechecks(); err != nil {
		return
	}
	if err = v.DoWarnings(); err != nil {
		return
	}
	if err = v.GenerateProof(); err != nil {
		return
	}
	if err = v.PostProofToServer(); err != nil {
		return
	}
	if err = v.CheckProofText(); err != nil {
		return
	}
	if err = v.InstructAction(); err != nil {
		return
	}
	if err = v.PromptPostedLoop(); err != nil {
		return
	}
	v.LogUI.Notice("Success!")
	return nil
}
<|endoftext|>


<commit_before>package jit

/*
#cgo CFLAGS: -I./include
#cgo linux,amd64 LDFLAGS: -L./lib -lm -ljit

#include <stdio.h>
#include <jit/jit.h>
#include <jit/jit-dump.h>

extern int on_demand_compile(jit_function_t);

static void SetOnDemandCompileFunction(jit_function_t f) {
    jit_function_set_on_demand_compiler(f, on_demand_compile);
}

static FILE *getStdout(void) { return stdout; }
*/
import "C"
import "unsafe"
import "reflect"

const CDECL = C.jit_abi_cdecl

type Context struct {
	C C.jit_context_t
}

type Type struct {
	C C.jit_type_t
}

type Signature struct {
	C C.jit_type_t
}

type Function struct {
	C         C.jit_function_t
	retType   Type
	paramType []Type
}

type Value struct {
	C C.jit_value_t
}

type Label struct {
	C C.jit_label_t
}

func Int() Type {
	return Type{C.jit_type_int}
}

func UInt() Type {
	return Type{C.jit_type_uint}
}

func Void() Type {
	return Type{C.jit_type_void}
}

func VoidPtr() Type {
	return Type{C.jit_type_void_ptr}
}

func NewContext() *Context {
	return &Context{C.jit_context_create()}
}

func NewSignature(ret Type, params []Type) *Signature {
	if len(params) == 0 {
		signature := C.jit_type_create_signature(C.jit_abi_cdecl,
			ret.C,
			(*C.jit_type_t)(nil),
			C.uint(0), 1)
		return &Signature{signature}
	} else {
		signature := C.jit_type_create_signature(C.jit_abi_cdecl,
			ret.C,
			(*C.jit_type_t)(unsafe.Pointer(&params[0])),
			C.uint(len(params)), 1)
		return &Signature{signature}
	}
}

func NewLabel() *Label {
	return &Label{C.jit_label_undefined}
}

// ========== Context =============

func (c *Context) NewFunction(ret Type, params []Type) *Function {
	signature := NewSignature(ret, params)
	function := C.jit_function_create(c.C, signature.C)
	C.jit_type_free(signature.C)
	return &Function{C: function, retType: ret, paramType: params}
}

func (c *Context) BuildStart() {
	C.jit_context_build_start(c.C)
}

func (c *Context) BuildEnd() {
	C.jit_context_build_end(c.C)
}

func (c *Context) Destroy() {
	C.jit_context_destroy(c.C)
}

// ========== Context =============

// ========== Function =============

func (f *Function) Param(i int) *Value {
	return &Value{C.jit_value_get_param(f.C, C.uint(i))}
}

func (f *Function) Param2() (*Value, *Value) {
	return &Value{C.jit_value_get_param(f.C, C.uint(0))},
		&Value{C.jit_value_get_param(f.C, C.uint(1))}
}

func (f *Function) Param3() (*Value, *Value, *Value) {
	return &Value{C.jit_value_get_param(f.C, C.uint(0))},
		&Value{C.jit_value_get_param(f.C, C.uint(1))},
		&Value{C.jit_value_get_param(f.C, C.uint(2))}
}

func (f *Function) Mul(a, b *Value) *Value {
	return &Value{C.jit_insn_mul(f.C, a.C, b.C)}
}

func (f *Function) Add(a, b *Value) *Value {
	return &Value{C.jit_insn_add(f.C, a.C, b.C)}
}

func (f *Function) Sub(a, b *Value) *Value {
	return &Value{C.jit_insn_sub(f.C, a.C, b.C)}
}

func (f *Function) Return(ret *Value) {
	C.jit_insn_return(f.C, ret.C)
}

func (f *Function) Store(x, y *Value) {
	C.jit_insn_store(f.C, x.C, y.C)
}

func (f *Function) StoreRelative(x *Value, offset int, y *Value) {
	C.jit_insn_store_relative(f.C, x.C, C.jit_nint(offset), y.C)
}

func (f *Function) Eq(a, b *Value) *Value {
	return &Value{C.jit_insn_eq(f.C, a.C, b.C)}
}

func (f *Function) BranchIfNot(v *Value, label *Label) {
	C.jit_insn_branch_if_not(f.C, v.C, (*C.jit_label_t)(unsafe.Pointer(label)))
}

func (f *Function) Label(label *Label) {
	C.jit_insn_label(f.C, (*C.jit_label_t)(unsafe.Pointer(label)))
}

func (f *Function) LessThan(a, b *Value) *Value {
	return &Value{C.jit_insn_lt(f.C, a.C, b.C)}
}

func unwrap(values []*Value) *C.jit_value_t {
    args := make([]C.jit_value_t, len(values))
    for i := range values {
        args[i] = values[i].C
    }
    return (*C.jit_value_t)(&args[0])
}

func (f *Function) TailCall(target *Function, values ...*Value) *Value {
	if len(values) == 0 {
		return &Value{C.jit_insn_call(f.C,
			C.CString("noname"),
			target.C, nil,
			(*C.jit_value_t)(nil),
			C.uint(0), C.JIT_CALL_TAIL)}
	} else {
		return &Value{C.jit_insn_call(f.C,
			C.CString("noname"),
			target.C, nil,
			unwrap(values),
			C.uint(len(values)), C.JIT_CALL_TAIL)}
	}
}

func (f *Function) Call(target *Function, values ...*Value) *Value {
	if len(values) == 0 {
		return &Value{C.jit_insn_call(f.C,
			C.CString("noname"),
			target.C, nil,
			(*C.jit_value_t)(nil),
			C.uint(0), C.int(0))}
	} else {
		return &Value{C.jit_insn_call(f.C,
			C.CString("noname"),
			target.C, nil,
            unwrap(values),
			C.uint(len(values)), C.int(0))}
	}
}

func (f *Function) Call0(name string, target *Function, values ...*Value) *Value {
	if len(values) == 0 {
		return &Value{C.jit_insn_call(f.C,
			C.CString(name),
			target.C, nil,
			(*C.jit_value_t)(nil),
			C.uint(0), C.int(0))}
	} else {
		return &Value{C.jit_insn_call(f.C,
			C.CString(name),
			target.C, nil,
            unwrap(values),
			C.uint(len(values)), C.int(0))}
	}
}

func (f *Function) CallNative(name string, target unsafe.Pointer, sig *Signature, values ...*Value) *Value {
	if len(values) == 0 {
		return &Value{C.jit_insn_call_native(f.C,
			C.CString(name),
			target,
			sig.C, (*C.jit_value_t)(nil),
			C.uint(0), C.JIT_CALL_NOTHROW)}
	} else {
		return &Value{C.jit_insn_call_native(f.C,
			C.CString(name),
			target,
            sig.C, unwrap(values),
			C.uint(len(values)), C.JIT_CALL_NOTHROW)}
	}
}

func (f *Function) Compile() {
	C.jit_function_compile(f.C)
}

func (f *Function) Run(values ...interface{}) interface{} {
	args := make([]unsafe.Pointer, len(values))
	for i := range values {
		switch f.paramType[i].C {
		case C.jit_type_int:
			x := C.jit_int(values[i].(int))
			args[i] = (unsafe.Pointer)(&x)

		case C.jit_type_uint:
			x := C.jit_uint(values[i].(uint))
			args[i] = (unsafe.Pointer)(&x)

		case C.jit_type_void_ptr:
			x := (unsafe.Pointer)(reflect.ValueOf(values[i]).Pointer())
			args[i] = (unsafe.Pointer)(&x)
		}
	}

	switch f.retType.C {
	case C.jit_type_int:
		result := C.jit_int(0)
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(&result))
		return int(result)

	case C.jit_type_uint:
		result := C.jit_uint(0)
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(&result))
		return uint(result)

	case C.jit_type_void:
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(nil))
		return nil

	}

	return nil
}

func (f *Function) Dump(name string) {
	C.jit_dump_function(C.getStdout(), f.C, C.CString(name))
}

func (f *Function) SetRecompilable() {
	C.jit_function_set_recompilable(f.C)
}

type compileFunction struct {
	F           *Function
	compileFunc func(*Function) bool
}

var registry = make(map[C.jit_function_t]*compileFunction)

func (f *Function) SetOnDemandCompiler(function func(f *Function) bool) {
	registry[f.C] = &compileFunction{f, function}
	C.SetOnDemandCompileFunction(f.C)
}

func (f *Function) GetOnDemandCompiler() func(*Function) bool {
	return registry[f.C].compileFunc
}

// ========== Function =============

//export on_demand_compile
func on_demand_compile(f C.jit_function_t) C.int {
	cf := registry[f]
	result := cf.compileFunc(cf.F)
	if result {
		return C.int(1)
	}
	return C.int(0)
}
<commit_msg>gofmt<commit_after>package jit

/*
#cgo CFLAGS: -I./include
#cgo linux,amd64 LDFLAGS: -L./lib -lm -ljit

#include <stdio.h>
#include <jit/jit.h>
#include <jit/jit-dump.h>

extern int on_demand_compile(jit_function_t);

static void SetOnDemandCompileFunction(jit_function_t f) {
    jit_function_set_on_demand_compiler(f, on_demand_compile);
}

static FILE *getStdout(void) { return stdout; }
*/
import "C"
import "unsafe"
import "reflect"

const CDECL = C.jit_abi_cdecl

type Context struct {
	C C.jit_context_t
}

type Type struct {
	C C.jit_type_t
}

type Signature struct {
	C C.jit_type_t
}

type Function struct {
	C         C.jit_function_t
	retType   Type
	paramType []Type
}

type Value struct {
	C C.jit_value_t
}

type Label struct {
	C C.jit_label_t
}

func Int() Type {
	return Type{C.jit_type_int}
}

func UInt() Type {
	return Type{C.jit_type_uint}
}

func Void() Type {
	return Type{C.jit_type_void}
}

func VoidPtr() Type {
	return Type{C.jit_type_void_ptr}
}

func NewContext() *Context {
	return &Context{C.jit_context_create()}
}

func NewSignature(ret Type, params []Type) *Signature {
	if len(params) == 0 {
		signature := C.jit_type_create_signature(C.jit_abi_cdecl,
			ret.C,
			(*C.jit_type_t)(nil),
			C.uint(0), 1)
		return &Signature{signature}
	} else {
		signature := C.jit_type_create_signature(C.jit_abi_cdecl,
			ret.C,
			(*C.jit_type_t)(unsafe.Pointer(&params[0])),
			C.uint(len(params)), 1)
		return &Signature{signature}
	}
}

func NewLabel() *Label {
	return &Label{C.jit_label_undefined}
}

// ========== Context =============

func (c *Context) NewFunction(ret Type, params []Type) *Function {
	signature := NewSignature(ret, params)
	function := C.jit_function_create(c.C, signature.C)
	C.jit_type_free(signature.C)
	return &Function{C: function, retType: ret, paramType: params}
}

func (c *Context) BuildStart() {
	C.jit_context_build_start(c.C)
}

func (c *Context) BuildEnd() {
	C.jit_context_build_end(c.C)
}

func (c *Context) Destroy() {
	C.jit_context_destroy(c.C)
}

// ========== Context =============

// ========== Function =============

func (f *Function) Param(i int) *Value {
	return &Value{C.jit_value_get_param(f.C, C.uint(i))}
}

func (f *Function) Param2() (*Value, *Value) {
	return &Value{C.jit_value_get_param(f.C, C.uint(0))},
		&Value{C.jit_value_get_param(f.C, C.uint(1))}
}

func (f *Function) Param3() (*Value, *Value, *Value) {
	return &Value{C.jit_value_get_param(f.C, C.uint(0))},
		&Value{C.jit_value_get_param(f.C, C.uint(1))},
		&Value{C.jit_value_get_param(f.C, C.uint(2))}
}

func (f *Function) Mul(a, b *Value) *Value {
	return &Value{C.jit_insn_mul(f.C, a.C, b.C)}
}

func (f *Function) Add(a, b *Value) *Value {
	return &Value{C.jit_insn_add(f.C, a.C, b.C)}
}

func (f *Function) Sub(a, b *Value) *Value {
	return &Value{C.jit_insn_sub(f.C, a.C, b.C)}
}

func (f *Function) Return(ret *Value) {
	C.jit_insn_return(f.C, ret.C)
}

func (f *Function) Store(x, y *Value) {
	C.jit_insn_store(f.C, x.C, y.C)
}

func (f *Function) StoreRelative(x *Value, offset int, y *Value) {
	C.jit_insn_store_relative(f.C, x.C, C.jit_nint(offset), y.C)
}

func (f *Function) Eq(a, b *Value) *Value {
	return &Value{C.jit_insn_eq(f.C, a.C, b.C)}
}

func (f *Function) BranchIfNot(v *Value, label *Label) {
	C.jit_insn_branch_if_not(f.C, v.C, (*C.jit_label_t)(unsafe.Pointer(label)))
}

func (f *Function) Label(label *Label) {
	C.jit_insn_label(f.C, (*C.jit_label_t)(unsafe.Pointer(label)))
}

func (f *Function) LessThan(a, b *Value) *Value {
	return &Value{C.jit_insn_lt(f.C, a.C, b.C)}
}

func unwrap(values []*Value) *C.jit_value_t {
	args := make([]C.jit_value_t, len(values))
	for i := range values {
		args[i] = values[i].C
	}
	return (*C.jit_value_t)(&args[0])
}

func (f *Function) TailCall(target *Function, values ...*Value) *Value {
	if len(values) == 0 {
		return &Value{C.jit_insn_call(f.C,
			C.CString("noname"),
			target.C, nil,
			(*C.jit_value_t)(nil),
			C.uint(0), C.JIT_CALL_TAIL)}
	} else {
		return &Value{C.jit_insn_call(f.C,
			C.CString("noname"),
			target.C, nil,
			unwrap(values),
			C.uint(len(values)), C.JIT_CALL_TAIL)}
	}
}

func (f *Function) Call(target *Function, values ...*Value) *Value {
	if len(values) == 0 {
		return &Value{C.jit_insn_call(f.C,
			C.CString("noname"),
			target.C, nil,
			(*C.jit_value_t)(nil),
			C.uint(0), C.int(0))}
	} else {
		return &Value{C.jit_insn_call(f.C,
			C.CString("noname"),
			target.C, nil,
			unwrap(values),
			C.uint(len(values)), C.int(0))}
	}
}

func (f *Function) Call0(name string, target *Function, values ...*Value) *Value {
	if len(values) == 0 {
		return &Value{C.jit_insn_call(f.C,
			C.CString(name),
			target.C, nil,
			(*C.jit_value_t)(nil),
			C.uint(0), C.int(0))}
	} else {
		return &Value{C.jit_insn_call(f.C,
			C.CString(name),
			target.C, nil,
			unwrap(values),
			C.uint(len(values)), C.int(0))}
	}
}

func (f *Function) CallNative(name string, target unsafe.Pointer, sig *Signature, values ...*Value) *Value {
	if len(values) == 0 {
		return &Value{C.jit_insn_call_native(f.C,
			C.CString(name),
			target,
			sig.C, (*C.jit_value_t)(nil),
			C.uint(0), C.JIT_CALL_NOTHROW)}
	} else {
		return &Value{C.jit_insn_call_native(f.C,
			C.CString(name),
			target,
			sig.C, unwrap(values),
			C.uint(len(values)), C.JIT_CALL_NOTHROW)}
	}
}

func (f *Function) Compile() {
	C.jit_function_compile(f.C)
}

func (f *Function) Run(values ...interface{}) interface{} {
	args := make([]unsafe.Pointer, len(values))
	for i := range values {
		switch f.paramType[i].C {
		case C.jit_type_int:
			x := C.jit_int(values[i].(int))
			args[i] = (unsafe.Pointer)(&x)

		case C.jit_type_uint:
			x := C.jit_uint(values[i].(uint))
			args[i] = (unsafe.Pointer)(&x)

		case C.jit_type_void_ptr:
			x := (unsafe.Pointer)(reflect.ValueOf(values[i]).Pointer())
			args[i] = (unsafe.Pointer)(&x)
		}
	}

	switch f.retType.C {
	case C.jit_type_int:
		result := C.jit_int(0)
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(&result))
		return int(result)

	case C.jit_type_uint:
		result := C.jit_uint(0)
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(&result))
		return uint(result)

	case C.jit_type_void:
		C.jit_function_apply(f.C, (*unsafe.Pointer)(&args[0]), unsafe.Pointer(nil))
		return nil

	}

	return nil
}

func (f *Function) Dump(name string) {
	C.jit_dump_function(C.getStdout(), f.C, C.CString(name))
}

func (f *Function) SetRecompilable() {
	C.jit_function_set_recompilable(f.C)
}

type compileFunction struct {
	F           *Function
	compileFunc func(*Function) bool
}

var registry = make(map[C.jit_function_t]*compileFunction)

func (f *Function) SetOnDemandCompiler(function func(f *Function) bool) {
	registry[f.C] = &compileFunction{f, function}
	C.SetOnDemandCompileFunction(f.C)
}

func (f *Function) GetOnDemandCompiler() func(*Function) bool {
	return registry[f.C].compileFunc
}

// ========== Function =============

//export on_demand_compile
func on_demand_compile(f C.jit_function_t) C.int {
	cf := registry[f]
	result := cf.compileFunc(cf.F)
	if result {
		return C.int(1)
	}
	return C.int(0)
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package agent // not agent_test for no good reason

import (
	stdtesting "testing"

	coretesting "github.com/juju/juju/testing"
)

func TestPackage(t *stdtesting.T) {
	// TODO(waigani) 2014-03-19 bug 1294458
	// Refactor to use base suites
	coretesting.MgoTestPackage(t)
}
<commit_msg>cmd/jujud/agent: Registers components in tests<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package agent // not agent_test for no good reason

import (
	stdtesting "testing"

	"github.com/juju/juju/component/all"
	coretesting "github.com/juju/juju/testing"
)

func init() {
	// Required for resources support.
	if err := all.RegisterForServer(); err != nil {
		panic(err)
	}
}

func TestPackage(t *stdtesting.T) {
	// TODO(waigani) 2014-03-19 bug 1294458
	// Refactor to use base suites
	coretesting.MgoTestPackage(t)
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"

	"fmt"

	"os"

	"github.com/griffithsh/sqlite-squish/database"
)

func outputText(d *database.Database, dir string) error {
	// If dir does not exist, create it.
	info, err := os.Lstat(dir)
	if err != nil {
		if os.IsNotExist(err) {
			err = os.MkdirAll(dir, 0755)
			if err != nil {
				return err
			}
			info, err = os.Lstat(dir)
			if err != nil {
				return err
			}
		} else {
			return err
		}
	}

	// Else if dir is something that isnt a directory, error out.
	if !info.IsDir() {
		return fmt.Errorf("cannot output to non-directory %s", dir)
	}

	tableNames, err := d.SortedTables()
	if err != nil {
		return err
	}
	for _, name := range tableNames {
		file := d.Tables[name].String()
		filepath := fmt.Sprintf("%s/%s.sql", info.Name(), name)
		err = ioutil.WriteFile(filepath, []byte(file), 0644)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Cause output files to end with a newline<commit_after>package main

import (
	"io/ioutil"

	"fmt"

	"os"

	"github.com/griffithsh/sqlite-squish/database"
)

func outputText(d *database.Database, dir string) error {
	// If dir does not exist, create it.
	info, err := os.Lstat(dir)
	if err != nil {
		if os.IsNotExist(err) {
			err = os.MkdirAll(dir, 0755)
			if err != nil {
				return err
			}
			info, err = os.Lstat(dir)
			if err != nil {
				return err
			}
		} else {
			return err
		}
	}

	// Else if dir is something that isnt a directory, error out.
	if !info.IsDir() {
		return fmt.Errorf("cannot output to non-directory %s", dir)
	}

	tableNames, err := d.SortedTables()
	if err != nil {
		return err
	}
	for _, name := range tableNames {
		file := fmt.Sprintf("%s\n", d.Tables[name].String())
		filepath := fmt.Sprintf("%s/%s.sql", info.Name(), name)
		err = ioutil.WriteFile(filepath, []byte(file), 0644)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This binary fetches all repos of a user or organization and clones
// them.  It is strongly recommended to get a personal API token from
// https://github.com/settings/tokens, save the token in a file, and
// point the --token option to it.
package main

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"

	"github.com/google/zoekt/gitindex"
)

func main() {
	dest := flag.String("dest", "", "destination directory")
	githubURL := flag.String("url", "", "GitHub Enterprise url. If not set github.com will be used as the host.")
	org := flag.String("org", "", "organization to mirror")
	user := flag.String("user", "", "user to mirror")
	token := flag.String("token",
		filepath.Join(os.Getenv("HOME"), ".github-token"),
		"file holding API token.")
	forks := flag.Bool("forks", false, "also mirror forks.")
	deleteRepos := flag.Bool("delete", false, "delete missing repos")
	namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
	excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
	flag.Parse()

	if *dest == "" {
		log.Fatal("must set --dest")
	}
	if (*org == "") == (*user == "") {
		log.Fatal("must set either --org or --user")
	}

	var host string
	var apiBaseURL string
	var client *github.Client
	if *githubURL != "" {
		rootURL, err := url.Parse(*githubURL)
		if err != nil {
			log.Fatal(err)
		}
		host = rootURL.Host
		apiPath, err := url.Parse("/api/v3/")
		if err != nil {
			log.Fatal(err)
		}
		apiBaseURL = rootURL.ResolveReference(apiPath).String()
		client, err = github.NewEnterpriseClient(apiBaseURL, apiBaseURL, nil)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		host = "github.com"
		apiBaseURL = "https://github.com/"
		client = github.NewClient(nil)
	}
	destDir := filepath.Join(*dest, host)
	if err := os.MkdirAll(destDir, 0755); err != nil {
		log.Fatal(err)
	}

	if *token != "" {
		content, err := ioutil.ReadFile(*token)
		if err != nil {
			log.Fatal(err)
		}

		ts := oauth2.StaticTokenSource(
			&oauth2.Token{
				AccessToken: strings.TrimSpace(string(content)),
			})
		tc := oauth2.NewClient(context.Background(), ts)
		if *githubURL != "" {
			client, err = github.NewEnterpriseClient(apiBaseURL, apiBaseURL, tc)
			if err != nil {
				log.Fatal(err)
			}
		} else {
			client = github.NewClient(tc)
		}
	}

	var repos []*github.Repository
	var err error
	if *org != "" {
		repos, err = getOrgRepos(client, *org)
	} else if *user != "" {
		repos, err = getUserRepos(client, *user)
	}

	if err != nil {
		log.Fatal(err)
	}

	if !*forks {
		trimmed := repos[:0]
		for _, r := range repos {
			if r.Fork == nil || !*r.Fork {
				trimmed = append(trimmed, r)
			}
		}
		repos = trimmed
	}

	filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
	if err != nil {
		log.Fatal(err)
	}

	{
		trimmed := repos[:0]
		for _, r := range repos {
			if filter.Include(*r.Name) {
				trimmed = append(trimmed, r)
			}
		}
		repos = trimmed
	}

	if err := cloneRepos(destDir, repos); err != nil {
		log.Fatalf("cloneRepos: %v", err)
	}

	if *deleteRepos {
		if err := deleteStaleRepos(*dest, filter, repos, *org+*user); err != nil {
			log.Fatalf("deleteStaleRepos: %v", err)
		}
	}
}

func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []*github.Repository, user string) error {
	var baseURL string
	if len(repos) > 0 {
		baseURL = *repos[0].HTMLURL
	} else {
		return nil
	}
	u, err := url.Parse(baseURL + user)
	if err != nil {
		return err
	}

	paths, err := gitindex.ListRepos(destDir, u)
	if err != nil {
		return err
	}

	names := map[string]bool{}
	for _, r := range repos {
		u, err := url.Parse(*r.HTMLURL)
		if err != nil {
			return err
		}

		names[filepath.Join(u.Host, u.Path+".git")] = true
	}

	var toDelete []string
	for _, p := range paths {
		if filter.Include(p) && !names[p] {
			toDelete = append(toDelete, p)
		}
	}

	if len(toDelete) > 0 {
		log.Printf("deleting repos %v", toDelete)
	}

	var errs []string
	for _, d := range toDelete {
		if err := os.RemoveAll(filepath.Join(destDir, d)); err != nil {
			errs = append(errs, err.Error())
		}
	}
	if len(errs) > 0 {
		return fmt.Errorf("errors: %v", errs)
	}
	return nil
}

func getOrgRepos(client *github.Client, org string) ([]*github.Repository, error) {
	var allRepos []*github.Repository
	opt := &github.RepositoryListByOrgOptions{}
	for {
		repos, resp, err := client.Repositories.ListByOrg(context.Background(), org, opt)
		if err != nil {
			return nil, err
		}
		if len(repos) == 0 {
			break
		}

		opt.Page = resp.NextPage
		allRepos = append(allRepos, repos...)
		if resp.NextPage == 0 {
			break
		}
	}
	return allRepos, nil
}

func getUserRepos(client *github.Client, user string) ([]*github.Repository, error) {
	var allRepos []*github.Repository
	opt := &github.RepositoryListOptions{}
	for {
		repos, resp, err := client.Repositories.List(context.Background(), user, opt)
		if err != nil {
			return nil, err
		}
		if len(repos) == 0 {
			break
		}

		opt.Page = resp.NextPage
		allRepos = append(allRepos, repos...)
		if resp.NextPage == 0 {
			break
		}
	}
	return allRepos, nil
}

func itoa(p *int) string {
	if p != nil {
		return strconv.Itoa(*p)
	}
	return ""
}

func cloneRepos(destDir string, repos []*github.Repository) error {
	for _, r := range repos {
		host, err := url.Parse(*r.HTMLURL)
		if err != nil {
			return err
		}
		config := map[string]string{
			"zoekt.web-url-type": "github",
			"zoekt.web-url":      *r.HTMLURL,
			"zoekt.name":         filepath.Join(host.Hostname(), *r.FullName),

			"zoekt.github-stars":       itoa(r.StargazersCount),
			"zoekt.github-watchers":    itoa(r.WatchersCount),
			"zoekt.github-subscribers": itoa(r.SubscribersCount),
			"zoekt.github-forks":       itoa(r.ForksCount),
		}
		dest, err := gitindex.CloneRepo(destDir, *r.FullName, *r.CloneURL, config)
		if err != nil {
			return err
		}
		if dest != "" {
			fmt.Println(dest)
		}

	}

	return nil
}
<commit_msg>Fix deleteRepos in mirror-github<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This binary fetches all repos of a user or organization and clones
// them.  It is strongly recommended to get a personal API token from
// https://github.com/settings/tokens, save the token in a file, and
// point the --token option to it.
package main

import (
	"context"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/google/go-github/github"
	"golang.org/x/oauth2"

	"github.com/google/zoekt/gitindex"
)

func main() {
	dest := flag.String("dest", "", "destination directory")
	githubURL := flag.String("url", "", "GitHub Enterprise url. If not set github.com will be used as the host.")
	org := flag.String("org", "", "organization to mirror")
	user := flag.String("user", "", "user to mirror")
	token := flag.String("token",
		filepath.Join(os.Getenv("HOME"), ".github-token"),
		"file holding API token.")
	forks := flag.Bool("forks", false, "also mirror forks.")
	deleteRepos := flag.Bool("delete", false, "delete missing repos")
	namePattern := flag.String("name", "", "only clone repos whose name matches the given regexp.")
	excludePattern := flag.String("exclude", "", "don't mirror repos whose names match this regexp.")
	flag.Parse()

	if *dest == "" {
		log.Fatal("must set --dest")
	}
	if (*org == "") == (*user == "") {
		log.Fatal("must set either --org or --user")
	}

	var host string
	var apiBaseURL string
	var client *github.Client
	if *githubURL != "" {
		rootURL, err := url.Parse(*githubURL)
		if err != nil {
			log.Fatal(err)
		}
		host = rootURL.Host
		apiPath, err := url.Parse("/api/v3/")
		if err != nil {
			log.Fatal(err)
		}
		apiBaseURL = rootURL.ResolveReference(apiPath).String()
		client, err = github.NewEnterpriseClient(apiBaseURL, apiBaseURL, nil)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		host = "github.com"
		apiBaseURL = "https://github.com/"
		client = github.NewClient(nil)
	}
	destDir := filepath.Join(*dest, host)
	if err := os.MkdirAll(destDir, 0755); err != nil {
		log.Fatal(err)
	}

	if *token != "" {
		content, err := ioutil.ReadFile(*token)
		if err != nil {
			log.Fatal(err)
		}

		ts := oauth2.StaticTokenSource(
			&oauth2.Token{
				AccessToken: strings.TrimSpace(string(content)),
			})
		tc := oauth2.NewClient(context.Background(), ts)
		if *githubURL != "" {
			client, err = github.NewEnterpriseClient(apiBaseURL, apiBaseURL, tc)
			if err != nil {
				log.Fatal(err)
			}
		} else {
			client = github.NewClient(tc)
		}
	}

	var repos []*github.Repository
	var err error
	if *org != "" {
		repos, err = getOrgRepos(client, *org)
	} else if *user != "" {
		repos, err = getUserRepos(client, *user)
	}

	if err != nil {
		log.Fatal(err)
	}

	if !*forks {
		trimmed := repos[:0]
		for _, r := range repos {
			if r.Fork == nil || !*r.Fork {
				trimmed = append(trimmed, r)
			}
		}
		repos = trimmed
	}

	filter, err := gitindex.NewFilter(*namePattern, *excludePattern)
	if err != nil {
		log.Fatal(err)
	}

	{
		trimmed := repos[:0]
		for _, r := range repos {
			if filter.Include(*r.Name) {
				trimmed = append(trimmed, r)
			}
		}
		repos = trimmed
	}

	if err := cloneRepos(destDir, repos); err != nil {
		log.Fatalf("cloneRepos: %v", err)
	}

	if *deleteRepos {
		if err := deleteStaleRepos(*dest, filter, repos, *org+*user); err != nil {
			log.Fatalf("deleteStaleRepos: %v", err)
		}
	}
}

func deleteStaleRepos(destDir string, filter *gitindex.Filter, repos []*github.Repository, user string) error {
	var baseURL string
	if len(repos) > 0 {
		baseURL = *repos[0].HTMLURL
	} else {
		return nil
	}
	u, err := url.Parse(baseURL)
	if err != nil {
		return err
	}
	u.Path = user

	paths, err := gitindex.ListRepos(destDir, u)
	if err != nil {
		return err
	}

	names := map[string]bool{}
	for _, r := range repos {
		u, err := url.Parse(*r.HTMLURL)
		if err != nil {
			return err
		}

		names[filepath.Join(u.Host, u.Path+".git")] = true
	}

	var toDelete []string
	for _, p := range paths {
		if filter.Include(p) && !names[p] {
			toDelete = append(toDelete, p)
		}
	}

	if len(toDelete) > 0 {
		log.Printf("deleting repos %v", toDelete)
	}

	var errs []string
	for _, d := range toDelete {
		if err := os.RemoveAll(filepath.Join(destDir, d)); err != nil {
			errs = append(errs, err.Error())
		}
	}
	if len(errs) > 0 {
		return fmt.Errorf("errors: %v", errs)
	}
	return nil
}

func getOrgRepos(client *github.Client, org string) ([]*github.Repository, error) {
	var allRepos []*github.Repository
	opt := &github.RepositoryListByOrgOptions{}
	for {
		repos, resp, err := client.Repositories.ListByOrg(context.Background(), org, opt)
		if err != nil {
			return nil, err
		}
		if len(repos) == 0 {
			break
		}

		opt.Page = resp.NextPage
		allRepos = append(allRepos, repos...)
		if resp.NextPage == 0 {
			break
		}
	}
	return allRepos, nil
}

func getUserRepos(client *github.Client, user string) ([]*github.Repository, error) {
	var allRepos []*github.Repository
	opt := &github.RepositoryListOptions{}
	for {
		repos, resp, err := client.Repositories.List(context.Background(), user, opt)
		if err != nil {
			return nil, err
		}
		if len(repos) == 0 {
			break
		}

		opt.Page = resp.NextPage
		allRepos = append(allRepos, repos...)
		if resp.NextPage == 0 {
			break
		}
	}
	return allRepos, nil
}

func itoa(p *int) string {
	if p != nil {
		return strconv.Itoa(*p)
	}
	return ""
}

func cloneRepos(destDir string, repos []*github.Repository) error {
	for _, r := range repos {
		host, err := url.Parse(*r.HTMLURL)
		if err != nil {
			return err
		}
		config := map[string]string{
			"zoekt.web-url-type": "github",
			"zoekt.web-url":      *r.HTMLURL,
			"zoekt.name":         filepath.Join(host.Hostname(), *r.FullName),

			"zoekt.github-stars":       itoa(r.StargazersCount),
			"zoekt.github-watchers":    itoa(r.WatchersCount),
			"zoekt.github-subscribers": itoa(r.SubscribersCount),
			"zoekt.github-forks":       itoa(r.ForksCount),
		}
		dest, err := gitindex.CloneRepo(destDir, *r.FullName, *r.CloneURL, config)
		if err != nil {
			return err
		}
		if dest != "" {
			fmt.Println(dest)
		}

	}

	return nil
}
<|endoftext|>