<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
	"time"

	"go.etcd.io/etcd/raft/v3/raftpb"
)

func getSnapshotFn() (func() ([]byte, error), <-chan struct{}) {
	snapshotTriggeredC := make(chan struct{})
	return func() ([]byte, error) {
		snapshotTriggeredC <- struct{}{}
		return nil, nil
	}, snapshotTriggeredC
}

type cluster struct {
	peers              []string
	commitC            []<-chan *commit
	errorC             []<-chan error
	proposeC           []chan string
	confChangeC        []chan raftpb.ConfChange
	snapshotTriggeredC []<-chan struct{}
}

// newCluster creates a cluster of n nodes
func newCluster(n int) *cluster {
	peers := make([]string, n)
	for i := range peers {
		peers[i] = fmt.Sprintf("http://127.0.0.1:%d", 10000+i)
	}

	clus := &cluster{
		peers:              peers,
		commitC:            make([]<-chan *commit, len(peers)),
		errorC:             make([]<-chan error, len(peers)),
		proposeC:           make([]chan string, len(peers)),
		confChangeC:        make([]chan raftpb.ConfChange, len(peers)),
		snapshotTriggeredC: make([]<-chan struct{}, len(peers)),
	}

	for i := range clus.peers {
		os.RemoveAll(fmt.Sprintf("raftexample-%d", i+1))
		os.RemoveAll(fmt.Sprintf("raftexample-%d-snap", i+1))
		clus.proposeC[i] = make(chan string, 1)
		clus.confChangeC[i] = make(chan raftpb.ConfChange, 1)
		fn, snapshotTriggeredC := getSnapshotFn()
		clus.snapshotTriggeredC[i] = snapshotTriggeredC
		clus.commitC[i], clus.errorC[i], _ = newRaftNode(i+1, clus.peers, false, fn, clus.proposeC[i], clus.confChangeC[i])
	}

	return clus
}

// Close closes all cluster nodes and returns an error if any failed.
func (clus *cluster) Close() (err error) {
	for i := range clus.peers {
		go func(i int) {
			for range clus.commitC[i] {
				// drain pending commits
			}
		}(i)
		close(clus.proposeC[i])
		// wait for channel to close
		if erri := <-clus.errorC[i]; erri != nil {
			err = erri
		}
		// clean intermediates
		os.RemoveAll(fmt.Sprintf("raftexample-%d", i+1))
		os.RemoveAll(fmt.Sprintf("raftexample-%d-snap", i+1))
	}
	return err
}

func (clus *cluster) closeNoErrors(t *testing.T) {
	t.Log("closing cluster...")
	if err := clus.Close(); err != nil {
		t.Fatal(err)
	}
	t.Log("closing cluster [done]")
}

// TestProposeOnCommit starts three nodes and feeds commits back into the proposal
// channel. The intent is to ensure blocking on a proposal won't block raft progress.
func TestProposeOnCommit(t *testing.T) {
	clus := newCluster(3)
	defer clus.closeNoErrors(t)

	donec := make(chan struct{})
	for i := range clus.peers {
		// feedback for "n" committed entries, then update donec
		go func(pC chan<- string, cC <-chan *commit, eC <-chan error) {
			for n := 0; n < 100; n++ {
				c, ok := <-cC
				if !ok {
					pC = nil
				}
				select {
				case pC <- c.data[0]:
					continue
				case err := <-eC:
					t.Errorf("eC message (%v)", err)
				}
			}
			donec <- struct{}{}
			for range cC {
				// acknowledge the commits from other nodes so
				// raft continues to make progress
			}
		}(clus.proposeC[i], clus.commitC[i], clus.errorC[i])

		// one message feedback per node
		go func(i int) { clus.proposeC[i] <- "foo" }(i)
	}

	for range clus.peers {
		<-donec
	}
}

// TestCloseProposerBeforeReplay tests closing the producer before raft starts.
func TestCloseProposerBeforeReplay(t *testing.T) {
	clus := newCluster(1)
	// close before replay so raft never starts
	defer clus.closeNoErrors(t)
}

// TestCloseProposerInflight tests closing the producer while
// committed messages are being published to the client.
func TestCloseProposerInflight(t *testing.T) {
	clus := newCluster(1)
	defer clus.closeNoErrors(t)

	// some inflight ops
	go func() {
		clus.proposeC[0] <- "foo"
		clus.proposeC[0] <- "bar"
	}()

	// wait for one message
	if c, ok := <-clus.commitC[0]; !ok || c.data[0] != "foo" {
		t.Fatalf("Commit failed")
	}
}

func TestPutAndGetKeyValue(t *testing.T) {
	clusters := []string{"http://127.0.0.1:9021"}

	proposeC := make(chan string)
	defer close(proposeC)

	confChangeC := make(chan raftpb.ConfChange)
	defer close(confChangeC)

	var kvs *kvstore
	getSnapshot := func() ([]byte, error) { return kvs.getSnapshot() }
	commitC, errorC, snapshotterReady := newRaftNode(1, clusters, false, getSnapshot, proposeC, confChangeC)

	kvs = newKVStore(<-snapshotterReady, proposeC, commitC, errorC)

	srv := httptest.NewServer(&httpKVAPI{
		store:       kvs,
		confChangeC: confChangeC,
	})
	defer srv.Close()

	// wait server started
	<-time.After(time.Second * 3)

	wantKey, wantValue := "test-key", "test-value"
	url := fmt.Sprintf("%s/%s", srv.URL, wantKey)
	body := bytes.NewBufferString(wantValue)
	cli := srv.Client()

	req, err := http.NewRequest("PUT", url, body)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set("Content-Type", "text/html; charset=utf-8")
	_, err = cli.Do(req)
	if err != nil {
		t.Fatal(err)
	}

	// wait for a moment for processing message, otherwise get would be failed.
	<-time.After(time.Second)

	resp, err := cli.Get(url)
	if err != nil {
		t.Fatal(err)
	}

	data, err := io.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Body.Close()

	if gotValue := string(data); wantValue != gotValue {
		t.Fatalf("expect %s, got %s", wantValue, gotValue)
	}
}

// TestAddNewNode tests adding new node to the existing cluster.
func TestAddNewNode(t *testing.T) {
	clus := newCluster(3)
	defer clus.closeNoErrors(t)

	os.RemoveAll("raftexample-4")
	os.RemoveAll("raftexample-4-snap")
	defer func() {
		os.RemoveAll("raftexample-4")
		os.RemoveAll("raftexample-4-snap")
	}()

	newNodeURL := "http://127.0.0.1:10004"
	clus.confChangeC[0] <- raftpb.ConfChange{
		Type:    raftpb.ConfChangeAddNode,
		NodeID:  4,
		Context: []byte(newNodeURL),
	}

	proposeC := make(chan string)
	defer close(proposeC)

	confChangeC := make(chan raftpb.ConfChange)
	defer close(confChangeC)

	newRaftNode(4, append(clus.peers, newNodeURL), true, nil, proposeC, confChangeC)

	go func() {
		proposeC <- "foo"
	}()

	if c, ok := <-clus.commitC[0]; !ok || c.data[0] != "foo" {
		t.Fatalf("Commit failed")
	}
}

func TestSnapshot(t *testing.T) {
	prevDefaultSnapshotCount := defaultSnapshotCount
	prevSnapshotCatchUpEntriesN := snapshotCatchUpEntriesN
	defaultSnapshotCount = 4
	snapshotCatchUpEntriesN = 4
	defer func() {
		defaultSnapshotCount = prevDefaultSnapshotCount
		snapshotCatchUpEntriesN = prevSnapshotCatchUpEntriesN
	}()

	clus := newCluster(3)
	defer clus.closeNoErrors(t)

	go func() {
		clus.proposeC[0] <- "foo"
	}()

	c := <-clus.commitC[0]

	select {
	case <-clus.snapshotTriggeredC[0]:
		t.Fatalf("snapshot triggered before applying done")
	default:
	}
	close(c.applyDoneC)
	<-clus.snapshotTriggeredC[0]
}
<commit_msg>fix the DATA RACE issue of TestCloseProposerInflight<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"sync"
	"testing"
	"time"

	"go.etcd.io/etcd/raft/v3/raftpb"
)

func getSnapshotFn() (func() ([]byte, error), <-chan struct{}) {
	snapshotTriggeredC := make(chan struct{})
	return func() ([]byte, error) {
		snapshotTriggeredC <- struct{}{}
		return nil, nil
	}, snapshotTriggeredC
}

type cluster struct {
	peers              []string
	commitC            []<-chan *commit
	errorC             []<-chan error
	proposeC           []chan string
	confChangeC        []chan raftpb.ConfChange
	snapshotTriggeredC []<-chan struct{}
}

// newCluster creates a cluster of n nodes
func newCluster(n int) *cluster {
	peers := make([]string, n)
	for i := range peers {
		peers[i] = fmt.Sprintf("http://127.0.0.1:%d", 10000+i)
	}

	clus := &cluster{
		peers:              peers,
		commitC:            make([]<-chan *commit, len(peers)),
		errorC:             make([]<-chan error, len(peers)),
		proposeC:           make([]chan string, len(peers)),
		confChangeC:        make([]chan raftpb.ConfChange, len(peers)),
		snapshotTriggeredC: make([]<-chan struct{}, len(peers)),
	}

	for i := range clus.peers {
		os.RemoveAll(fmt.Sprintf("raftexample-%d", i+1))
		os.RemoveAll(fmt.Sprintf("raftexample-%d-snap", i+1))
		clus.proposeC[i] = make(chan string, 1)
		clus.confChangeC[i] = make(chan raftpb.ConfChange, 1)
		fn, snapshotTriggeredC := getSnapshotFn()
		clus.snapshotTriggeredC[i] = snapshotTriggeredC
		clus.commitC[i], clus.errorC[i], _ = newRaftNode(i+1, clus.peers, false, fn, clus.proposeC[i], clus.confChangeC[i])
	}

	return clus
}

// Close closes all cluster nodes and returns an error if any failed.
func (clus *cluster) Close() (err error) {
	for i := range clus.peers {
		go func(i int) {
			for range clus.commitC[i] {
				// drain pending commits
			}
		}(i)
		close(clus.proposeC[i])
		// wait for channel to close
		if erri := <-clus.errorC[i]; erri != nil {
			err = erri
		}
		// clean intermediates
		os.RemoveAll(fmt.Sprintf("raftexample-%d", i+1))
		os.RemoveAll(fmt.Sprintf("raftexample-%d-snap", i+1))
	}
	return err
}

func (clus *cluster) closeNoErrors(t *testing.T) {
	t.Log("closing cluster...")
	if err := clus.Close(); err != nil {
		t.Fatal(err)
	}
	t.Log("closing cluster [done]")
}

// TestProposeOnCommit starts three nodes and feeds commits back into the proposal
// channel. The intent is to ensure blocking on a proposal won't block raft progress.
func TestProposeOnCommit(t *testing.T) {
	clus := newCluster(3)
	defer clus.closeNoErrors(t)

	donec := make(chan struct{})
	for i := range clus.peers {
		// feedback for "n" committed entries, then update donec
		go func(pC chan<- string, cC <-chan *commit, eC <-chan error) {
			for n := 0; n < 100; n++ {
				c, ok := <-cC
				if !ok {
					pC = nil
				}
				select {
				case pC <- c.data[0]:
					continue
				case err := <-eC:
					t.Errorf("eC message (%v)", err)
				}
			}
			donec <- struct{}{}
			for range cC {
				// acknowledge the commits from other nodes so
				// raft continues to make progress
			}
		}(clus.proposeC[i], clus.commitC[i], clus.errorC[i])

		// one message feedback per node
		go func(i int) { clus.proposeC[i] <- "foo" }(i)
	}

	for range clus.peers {
		<-donec
	}
}

// TestCloseProposerBeforeReplay tests closing the producer before raft starts.
func TestCloseProposerBeforeReplay(t *testing.T) {
	clus := newCluster(1)
	// close before replay so raft never starts
	defer clus.closeNoErrors(t)
}

// TestCloseProposerInflight tests closing the producer while
// committed messages are being published to the client.
func TestCloseProposerInflight(t *testing.T) {
	clus := newCluster(1)
	defer clus.closeNoErrors(t)

	var wg sync.WaitGroup
	wg.Add(1)

	// some inflight ops
	go func() {
		defer wg.Done()
		clus.proposeC[0] <- "foo"
		clus.proposeC[0] <- "bar"
	}()

	// wait for one message
	if c, ok := <-clus.commitC[0]; !ok || c.data[0] != "foo" {
		t.Fatalf("Commit failed")
	}

	wg.Wait()
}

func TestPutAndGetKeyValue(t *testing.T) {
	clusters := []string{"http://127.0.0.1:9021"}

	proposeC := make(chan string)
	defer close(proposeC)

	confChangeC := make(chan raftpb.ConfChange)
	defer close(confChangeC)

	var kvs *kvstore
	getSnapshot := func() ([]byte, error) { return kvs.getSnapshot() }
	commitC, errorC, snapshotterReady := newRaftNode(1, clusters, false, getSnapshot, proposeC, confChangeC)

	kvs = newKVStore(<-snapshotterReady, proposeC, commitC, errorC)

	srv := httptest.NewServer(&httpKVAPI{
		store:       kvs,
		confChangeC: confChangeC,
	})
	defer srv.Close()

	// wait server started
	<-time.After(time.Second * 3)

	wantKey, wantValue := "test-key", "test-value"
	url := fmt.Sprintf("%s/%s", srv.URL, wantKey)
	body := bytes.NewBufferString(wantValue)
	cli := srv.Client()

	req, err := http.NewRequest("PUT", url, body)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set("Content-Type", "text/html; charset=utf-8")
	_, err = cli.Do(req)
	if err != nil {
		t.Fatal(err)
	}

	// wait for a moment for processing message, otherwise get would be failed.
	<-time.After(time.Second)

	resp, err := cli.Get(url)
	if err != nil {
		t.Fatal(err)
	}

	data, err := io.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Body.Close()

	if gotValue := string(data); wantValue != gotValue {
		t.Fatalf("expect %s, got %s", wantValue, gotValue)
	}
}

// TestAddNewNode tests adding new node to the existing cluster.
func TestAddNewNode(t *testing.T) {
	clus := newCluster(3)
	defer clus.closeNoErrors(t)

	os.RemoveAll("raftexample-4")
	os.RemoveAll("raftexample-4-snap")
	defer func() {
		os.RemoveAll("raftexample-4")
		os.RemoveAll("raftexample-4-snap")
	}()

	newNodeURL := "http://127.0.0.1:10004"
	clus.confChangeC[0] <- raftpb.ConfChange{
		Type:    raftpb.ConfChangeAddNode,
		NodeID:  4,
		Context: []byte(newNodeURL),
	}

	proposeC := make(chan string)
	defer close(proposeC)

	confChangeC := make(chan raftpb.ConfChange)
	defer close(confChangeC)

	newRaftNode(4, append(clus.peers, newNodeURL), true, nil, proposeC, confChangeC)

	go func() {
		proposeC <- "foo"
	}()

	if c, ok := <-clus.commitC[0]; !ok || c.data[0] != "foo" {
		t.Fatalf("Commit failed")
	}
}

func TestSnapshot(t *testing.T) {
	prevDefaultSnapshotCount := defaultSnapshotCount
	prevSnapshotCatchUpEntriesN := snapshotCatchUpEntriesN
	defaultSnapshotCount = 4
	snapshotCatchUpEntriesN = 4
	defer func() {
		defaultSnapshotCount = prevDefaultSnapshotCount
		snapshotCatchUpEntriesN = prevSnapshotCatchUpEntriesN
	}()

	clus := newCluster(3)
	defer clus.closeNoErrors(t)

	go func() {
		clus.proposeC[0] <- "foo"
	}()

	c := <-clus.commitC[0]

	select {
	case <-clus.snapshotTriggeredC[0]:
		t.Fatalf("snapshot triggered before applying done")
	default:
	}
	close(c.applyDoneC)
	<-clus.snapshotTriggeredC[0]
}
<|endoftext|>


<commit_before>package consistence

func (self *NsqdCoordinator) requestJoinCatchup(topic string, partition int) *CoordErr {
	coordLog.Infof("try to join catchup for topic: %v-%v", topic, partition)
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		coordLog.Infof("get lookup failed: %v", err)
		return err
	}
	//defer self.putLookupRemoteProxy(c)
	err = c.RequestJoinCatchup(topic, partition, self.myNode.GetID())
	if err != nil {
		coordLog.Infof("request join catchup failed: %v", err)
	}
	return err
}

func (self *NsqdCoordinator) requestCheckTopicConsistence(topic string, partition int) {
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		coordLog.Infof("get lookup failed: %v", err)
		return
	}
	//defer self.putLookupRemoteProxy(c)
	c.RequestCheckTopicConsistence(topic, partition)
}

func (self *NsqdCoordinator) requestNotifyNewTopicInfo(topic string, partition int) {
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		coordLog.Infof("get lookup failed: %v", err)
		return
	}
	//defer self.putLookupRemoteProxy(c)
	c.RequestNotifyNewTopicInfo(topic, partition, self.myNode.GetID())
}

func (self *NsqdCoordinator) requestJoinTopicISR(topicInfo *TopicPartitionMetaInfo) *CoordErr {
	// request change catchup to isr list and wait for nsqlookupd response to temp disable all new write.
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		return err
	}
	//defer self.putLookupRemoteProxy(c)
	err = c.RequestJoinTopicISR(topicInfo.Name, topicInfo.Partition, self.myNode.GetID())
	return err
}

func (self *NsqdCoordinator) notifyReadyForTopicISR(topicInfo *TopicPartitionMetaInfo, leaderSession *TopicLeaderSession, joinSession string) *CoordErr {
	// notify myself is ready for isr list for current session and can accept new write.
	// leader session should contain the (isr list, current leader session, leader epoch), to identify the
	// the different session stage.
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		return err
	}
	//defer self.putLookupRemoteProxy(c)
	return c.ReadyForTopicISR(topicInfo.Name, topicInfo.Partition, self.myNode.GetID(), leaderSession, joinSession)
}

// only move from isr to catchup, if restart, we can catchup directly.
func (self *NsqdCoordinator) requestLeaveFromISR(topic string, partition int) *CoordErr {
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		return err
	}
	//defer self.putLookupRemoteProxy(c)
	return c.RequestLeaveFromISR(topic, partition, self.myNode.GetID())
}

// this should only be called by leader to remove slow node in isr.
// Be careful to avoid removing most of the isr nodes, should only remove while
// only small part of isr is slow.
// TODO: If most of nodes is slow, the leader should check the leader itself and
// maybe giveup the leadership.
func (self *NsqdCoordinator) requestLeaveFromISRByLeader(topic string, partition int, nid string) *CoordErr {
	topicCoord, err := self.getTopicCoordData(topic, partition)
	if err != nil {
		return err
	}
	if topicCoord.GetLeaderSessionID() != self.myNode.GetID() || topicCoord.GetLeader() != self.myNode.GetID() {
		return ErrNotTopicLeader
	}

	// send request with leader session, so lookup can check the valid of session.
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		return err
	}
	//defer self.putLookupRemoteProxy(c)
	return c.RequestLeaveFromISRByLeader(topic, partition, self.myNode.GetID(), &topicCoord.topicLeaderSession)
}
<commit_msg>fix request isr leave by leader<commit_after>package consistence

func (self *NsqdCoordinator) requestJoinCatchup(topic string, partition int) *CoordErr {
	coordLog.Infof("try to join catchup for topic: %v-%v", topic, partition)
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		coordLog.Infof("get lookup failed: %v", err)
		return err
	}
	//defer self.putLookupRemoteProxy(c)
	err = c.RequestJoinCatchup(topic, partition, self.myNode.GetID())
	if err != nil {
		coordLog.Infof("request join catchup failed: %v", err)
	}
	return err
}

func (self *NsqdCoordinator) requestCheckTopicConsistence(topic string, partition int) {
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		coordLog.Infof("get lookup failed: %v", err)
		return
	}
	//defer self.putLookupRemoteProxy(c)
	c.RequestCheckTopicConsistence(topic, partition)
}

func (self *NsqdCoordinator) requestNotifyNewTopicInfo(topic string, partition int) {
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		coordLog.Infof("get lookup failed: %v", err)
		return
	}
	//defer self.putLookupRemoteProxy(c)
	c.RequestNotifyNewTopicInfo(topic, partition, self.myNode.GetID())
}

func (self *NsqdCoordinator) requestJoinTopicISR(topicInfo *TopicPartitionMetaInfo) *CoordErr {
	// request change catchup to isr list and wait for nsqlookupd response to temp disable all new write.
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		return err
	}
	//defer self.putLookupRemoteProxy(c)
	err = c.RequestJoinTopicISR(topicInfo.Name, topicInfo.Partition, self.myNode.GetID())
	return err
}

func (self *NsqdCoordinator) notifyReadyForTopicISR(topicInfo *TopicPartitionMetaInfo, leaderSession *TopicLeaderSession, joinSession string) *CoordErr {
	// notify myself is ready for isr list for current session and can accept new write.
	// leader session should contain the (isr list, current leader session, leader epoch), to identify the
	// the different session stage.
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		return err
	}
	//defer self.putLookupRemoteProxy(c)
	return c.ReadyForTopicISR(topicInfo.Name, topicInfo.Partition, self.myNode.GetID(), leaderSession, joinSession)
}

// only move from isr to catchup, if restart, we can catchup directly.
func (self *NsqdCoordinator) requestLeaveFromISR(topic string, partition int) *CoordErr {
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		return err
	}
	//defer self.putLookupRemoteProxy(c)
	return c.RequestLeaveFromISR(topic, partition, self.myNode.GetID())
}

// this should only be called by leader to remove slow node in isr.
// Be careful to avoid removing most of the isr nodes, should only remove while
// only small part of isr is slow.
// TODO: If most of nodes is slow, the leader should check the leader itself and
// maybe giveup the leadership.
func (self *NsqdCoordinator) requestLeaveFromISRByLeader(topic string, partition int, nid string) *CoordErr {
	topicCoord, err := self.getTopicCoordData(topic, partition)
	if err != nil {
		return err
	}
	if topicCoord.GetLeaderSessionID() != self.myNode.GetID() || topicCoord.GetLeader() != self.myNode.GetID() {
		return ErrNotTopicLeader
	}

	// send request with leader session, so lookup can check the valid of session.
	c, err := self.getLookupRemoteProxy()
	if err != nil {
		return err
	}
	//defer self.putLookupRemoteProxy(c)
	return c.RequestLeaveFromISRByLeader(topic, partition, nid, &topicCoord.topicLeaderSession)
}
<|endoftext|>


<commit_before>package controller

import (
	"fmt"
	"time"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	kv1core "k8s.io/client-go/kubernetes/typed/core/v1"
	kapi "k8s.io/kubernetes/pkg/api"
	kapiv1 "k8s.io/kubernetes/pkg/api/v1"
	kappsv1beta1 "k8s.io/kubernetes/pkg/apis/apps/v1beta1"
	kextensionsv1beta1 "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
	kclientsetexternal "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"

	buildclient "github.com/openshift/origin/pkg/build/client"
	"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
	imagecontroller "github.com/openshift/origin/pkg/image/controller"
	imagetriggercontroller "github.com/openshift/origin/pkg/image/controller/trigger"
	triggerannotations "github.com/openshift/origin/pkg/image/trigger/annotations"
	triggerbuildconfigs "github.com/openshift/origin/pkg/image/trigger/buildconfigs"
	triggerdeploymentconfigs "github.com/openshift/origin/pkg/image/trigger/deploymentconfigs"
)

type ImageTriggerControllerConfig struct {
	HasBuilderEnabled      bool
	HasDeploymentsEnabled  bool
	HasDaemonSetsEnabled   bool
	HasStatefulSetsEnabled bool
	HasCronJobsEnabled     bool
}

func (c *ImageTriggerControllerConfig) RunController(ctx ControllerContext) (bool, error) {
	//streamInformer := ctx.ImageInformers.Image().InternalVersion().ImageStreams().Informer()
	informer := ctx.ImageInformers.Image().InternalVersion().ImageStreams()

	oclient, err := ctx.ClientBuilder.DeprecatedOpenshiftClient(bootstrappolicy.InfraImageTriggerControllerServiceAccountName)
	if err != nil {
		return true, err
	}
	kclient := ctx.ClientBuilder.ClientOrDie(bootstrappolicy.InfraImageTriggerControllerServiceAccountName)

	updater := podSpecUpdater{kclient}
	bcInstantiator := buildclient.NewOSClientBuildConfigInstantiatorClient(oclient)
	broadcaster := imagetriggercontroller.NewTriggerEventBroadcaster(kv1core.New(kclient.CoreV1().RESTClient()))

	sources := []imagetriggercontroller.TriggerSource{
		{
			Resource:  schema.GroupResource{Group: "apps.openshift.io", Resource: "deploymentconfigs"},
			Informer:  ctx.AppInformers.Apps().InternalVersion().DeploymentConfigs().Informer(),
			Store:     ctx.AppInformers.Apps().InternalVersion().DeploymentConfigs().Informer().GetIndexer(),
			TriggerFn: triggerdeploymentconfigs.NewDeploymentConfigTriggerIndexer,
			Reactor:   &triggerdeploymentconfigs.DeploymentConfigReactor{Client: oclient},
		},
	}
	if !c.HasBuilderEnabled {
		sources = append(sources, imagetriggercontroller.TriggerSource{
			Resource:  schema.GroupResource{Group: "build.openshift.io", Resource: "buildconfigs"},
			Informer:  ctx.BuildInformers.Build().InternalVersion().BuildConfigs().Informer(),
			Store:     ctx.BuildInformers.Build().InternalVersion().BuildConfigs().Informer().GetIndexer(),
			TriggerFn: triggerbuildconfigs.NewBuildConfigTriggerIndexer,
			Reactor:   &triggerbuildconfigs.BuildConfigReactor{Instantiator: bcInstantiator},
		})
	}
	if !c.HasDeploymentsEnabled {
		sources = append(sources, imagetriggercontroller.TriggerSource{
			Resource:  schema.GroupResource{Group: "extensions", Resource: "deployments"},
			Informer:  ctx.ExternalKubeInformers.Extensions().V1beta1().Deployments().Informer(),
			Store:     ctx.ExternalKubeInformers.Extensions().V1beta1().Deployments().Informer().GetIndexer(),
			TriggerFn: triggerannotations.NewAnnotationTriggerIndexer,
			Reactor:   &triggerannotations.AnnotationReactor{Updater: updater, Copier: kapi.Scheme},
		})
	}
	if !c.HasDaemonSetsEnabled {
		sources = append(sources, imagetriggercontroller.TriggerSource{
			Resource:  schema.GroupResource{Group: "extensions", Resource: "daemonsets"},
			Informer:  ctx.ExternalKubeInformers.Extensions().V1beta1().DaemonSets().Informer(),
			Store:     ctx.ExternalKubeInformers.Extensions().V1beta1().DaemonSets().Informer().GetIndexer(),
			TriggerFn: triggerannotations.NewAnnotationTriggerIndexer,
			Reactor:   &triggerannotations.AnnotationReactor{Updater: updater, Copier: kapi.Scheme},
		})
	}
	if !c.HasStatefulSetsEnabled {
		sources = append(sources, imagetriggercontroller.TriggerSource{
			Resource:  schema.GroupResource{Group: "apps", Resource: "statefulsets"},
			Informer:  ctx.ExternalKubeInformers.Apps().V1beta1().StatefulSets().Informer(),
			Store:     ctx.ExternalKubeInformers.Apps().V1beta1().StatefulSets().Informer().GetIndexer(),
			TriggerFn: triggerannotations.NewAnnotationTriggerIndexer,
			Reactor:   &triggerannotations.AnnotationReactor{Updater: updater, Copier: kapi.Scheme},
		})
	}
	if !c.HasCronJobsEnabled {
		sources = append(sources, imagetriggercontroller.TriggerSource{
			Resource:  schema.GroupResource{Group: "batch", Resource: "cronjobs"},
			Informer:  ctx.ExternalKubeInformers.Batch().V2alpha1().CronJobs().Informer(),
			Store:     ctx.ExternalKubeInformers.Batch().V2alpha1().CronJobs().Informer().GetIndexer(),
			TriggerFn: triggerannotations.NewAnnotationTriggerIndexer,
			Reactor:   &triggerannotations.AnnotationReactor{Updater: updater, Copier: kapi.Scheme},
		})
	}

	go imagetriggercontroller.NewTriggerController(
		broadcaster,
		informer,
		sources...,
	).Run(5, ctx.Stop)

	return true, nil
}

type podSpecUpdater struct {
	kclient kclientsetexternal.Interface
}

func (u podSpecUpdater) Update(obj runtime.Object) error {
	switch t := obj.(type) {
	case *kextensionsv1beta1.DaemonSet:
		_, err := u.kclient.Extensions().DaemonSets(t.Namespace).Update(t)
		return err
	case *kappsv1beta1.Deployment:
		_, err := u.kclient.Apps().Deployments(t.Namespace).Update(t)
		return err
	case *kappsv1beta1.StatefulSet:
		_, err := u.kclient.Apps().StatefulSets(t.Namespace).Update(t)
		return err
	case *kapiv1.Pod:
		_, err := u.kclient.Core().Pods(t.Namespace).Update(t)
		return err
	default:
		return fmt.Errorf("unrecognized object - no trigger update possible for %T", obj)
	}
}

type ImageImportControllerConfig struct {
	MaxScheduledImageImportsPerMinute          int
	ScheduledImageImportMinimumIntervalSeconds int
	DisableScheduledImport                     bool
	ResyncPeriod                               time.Duration
}

func (c *ImageImportControllerConfig) RunController(ctx ControllerContext) (bool, error) {
	informer := ctx.ImageInformers.Image().InternalVersion().ImageStreams()

	controller := imagecontroller.NewImageStreamController(
		ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(bootstrappolicy.InfraImageImportControllerServiceAccountName),
		informer,
	)
	go controller.Run(5, ctx.Stop)

	if c.DisableScheduledImport {
		return true, nil
	}

	scheduledController := imagecontroller.NewScheduledImageStreamController(
		ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(bootstrappolicy.InfraImageImportControllerServiceAccountName),
		informer,
		imagecontroller.ScheduledImageStreamControllerOptions{
			Resync: time.Duration(c.ScheduledImageImportMinimumIntervalSeconds) * time.Second,

			Enabled:                  !c.DisableScheduledImport,
			DefaultBucketSize:        4,
			MaxImageImportsPerMinute: c.MaxScheduledImageImportsPerMinute,
		},
	)

	controller.SetNotifier(scheduledController)
	go scheduledController.Run(ctx.Stop)

	return true, nil
}
<commit_msg>Image trigger controller should handle extensions v1beta1<commit_after>package controller

import (
	"fmt"
	"time"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	kv1core "k8s.io/client-go/kubernetes/typed/core/v1"
	kapi "k8s.io/kubernetes/pkg/api"
	kapiv1 "k8s.io/kubernetes/pkg/api/v1"
	kappsv1beta1 "k8s.io/kubernetes/pkg/apis/apps/v1beta1"
	kbatchv1 "k8s.io/kubernetes/pkg/apis/batch/v1"
	kbatchv2alpha1 "k8s.io/kubernetes/pkg/apis/batch/v2alpha1"
	kextensionsv1beta1 "k8s.io/kubernetes/pkg/apis/extensions/v1beta1"
	kclientsetexternal "k8s.io/kubernetes/pkg/client/clientset_generated/clientset"

	buildclient "github.com/openshift/origin/pkg/build/client"
	"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
	imagecontroller "github.com/openshift/origin/pkg/image/controller"
	imagetriggercontroller "github.com/openshift/origin/pkg/image/controller/trigger"
	triggerannotations "github.com/openshift/origin/pkg/image/trigger/annotations"
	triggerbuildconfigs "github.com/openshift/origin/pkg/image/trigger/buildconfigs"
	triggerdeploymentconfigs "github.com/openshift/origin/pkg/image/trigger/deploymentconfigs"
)

type ImageTriggerControllerConfig struct {
	HasBuilderEnabled      bool
	HasDeploymentsEnabled  bool
	HasDaemonSetsEnabled   bool
	HasStatefulSetsEnabled bool
	HasCronJobsEnabled     bool
}

func (c *ImageTriggerControllerConfig) RunController(ctx ControllerContext) (bool, error) {
	//streamInformer := ctx.ImageInformers.Image().InternalVersion().ImageStreams().Informer()
	informer := ctx.ImageInformers.Image().InternalVersion().ImageStreams()

	oclient, err := ctx.ClientBuilder.DeprecatedOpenshiftClient(bootstrappolicy.InfraImageTriggerControllerServiceAccountName)
	if err != nil {
		return true, err
	}
	kclient := ctx.ClientBuilder.ClientOrDie(bootstrappolicy.InfraImageTriggerControllerServiceAccountName)

	updater := podSpecUpdater{kclient}
	bcInstantiator := buildclient.NewOSClientBuildConfigInstantiatorClient(oclient)
	broadcaster := imagetriggercontroller.NewTriggerEventBroadcaster(kv1core.New(kclient.CoreV1().RESTClient()))

	sources := []imagetriggercontroller.TriggerSource{
		{
			Resource:  schema.GroupResource{Group: "apps.openshift.io", Resource: "deploymentconfigs"},
			Informer:  ctx.AppInformers.Apps().InternalVersion().DeploymentConfigs().Informer(),
			Store:     ctx.AppInformers.Apps().InternalVersion().DeploymentConfigs().Informer().GetIndexer(),
			TriggerFn: triggerdeploymentconfigs.NewDeploymentConfigTriggerIndexer,
			Reactor:   &triggerdeploymentconfigs.DeploymentConfigReactor{Client: oclient},
		},
	}
	if !c.HasBuilderEnabled {
		sources = append(sources, imagetriggercontroller.TriggerSource{
			Resource:  schema.GroupResource{Group: "build.openshift.io", Resource: "buildconfigs"},
			Informer:  ctx.BuildInformers.Build().InternalVersion().BuildConfigs().Informer(),
			Store:     ctx.BuildInformers.Build().InternalVersion().BuildConfigs().Informer().GetIndexer(),
			TriggerFn: triggerbuildconfigs.NewBuildConfigTriggerIndexer,
			Reactor:   &triggerbuildconfigs.BuildConfigReactor{Instantiator: bcInstantiator},
		})
	}
	if !c.HasDeploymentsEnabled {
		sources = append(sources, imagetriggercontroller.TriggerSource{
			Resource:  schema.GroupResource{Group: "extensions", Resource: "deployments"},
			Informer:  ctx.ExternalKubeInformers.Extensions().V1beta1().Deployments().Informer(),
			Store:     ctx.ExternalKubeInformers.Extensions().V1beta1().Deployments().Informer().GetIndexer(),
			TriggerFn: triggerannotations.NewAnnotationTriggerIndexer,
			Reactor:   &triggerannotations.AnnotationReactor{Updater: updater, Copier: kapi.Scheme},
		})
	}
	if !c.HasDaemonSetsEnabled {
		sources = append(sources, imagetriggercontroller.TriggerSource{
			Resource:  schema.GroupResource{Group: "extensions", Resource: "daemonsets"},
			Informer:  ctx.ExternalKubeInformers.Extensions().V1beta1().DaemonSets().Informer(),
			Store:     ctx.ExternalKubeInformers.Extensions().V1beta1().DaemonSets().Informer().GetIndexer(),
			TriggerFn: triggerannotations.NewAnnotationTriggerIndexer,
			Reactor:   &triggerannotations.AnnotationReactor{Updater: updater, Copier: kapi.Scheme},
		})
	}
	if !c.HasStatefulSetsEnabled {
		sources = append(sources, imagetriggercontroller.TriggerSource{
			Resource:  schema.GroupResource{Group: "apps", Resource: "statefulsets"},
			Informer:  ctx.ExternalKubeInformers.Apps().V1beta1().StatefulSets().Informer(),
			Store:     ctx.ExternalKubeInformers.Apps().V1beta1().StatefulSets().Informer().GetIndexer(),
			TriggerFn: triggerannotations.NewAnnotationTriggerIndexer,
			Reactor:   &triggerannotations.AnnotationReactor{Updater: updater, Copier: kapi.Scheme},
		})
	}
	if !c.HasCronJobsEnabled {
		sources = append(sources, imagetriggercontroller.TriggerSource{
			Resource:  schema.GroupResource{Group: "batch", Resource: "cronjobs"},
			Informer:  ctx.ExternalKubeInformers.Batch().V2alpha1().CronJobs().Informer(),
			Store:     ctx.ExternalKubeInformers.Batch().V2alpha1().CronJobs().Informer().GetIndexer(),
			TriggerFn: triggerannotations.NewAnnotationTriggerIndexer,
			Reactor:   &triggerannotations.AnnotationReactor{Updater: updater, Copier: kapi.Scheme},
		})
	}

	go imagetriggercontroller.NewTriggerController(
		broadcaster,
		informer,
		sources...,
	).Run(5, ctx.Stop)

	return true, nil
}

type podSpecUpdater struct {
	kclient kclientsetexternal.Interface
}

func (u podSpecUpdater) Update(obj runtime.Object) error {
	switch t := obj.(type) {
	case *kextensionsv1beta1.DaemonSet:
		_, err := u.kclient.Extensions().DaemonSets(t.Namespace).Update(t)
		return err
	case *kappsv1beta1.Deployment:
		_, err := u.kclient.Apps().Deployments(t.Namespace).Update(t)
		return err
	case *kextensionsv1beta1.Deployment:
		_, err := u.kclient.Extensions().Deployments(t.Namespace).Update(t)
		return err
	case *kappsv1beta1.StatefulSet:
		_, err := u.kclient.Apps().StatefulSets(t.Namespace).Update(t)
		return err
	case *kbatchv1.Job:
		_, err := u.kclient.Batch().Jobs(t.Namespace).Update(t)
		return err
	case *kbatchv2alpha1.CronJob:
		_, err := u.kclient.BatchV2alpha1().CronJobs(t.Namespace).Update(t)
		return err
	case *kapiv1.Pod:
		_, err := u.kclient.Core().Pods(t.Namespace).Update(t)
		return err
	default:
		return fmt.Errorf("unrecognized object - no trigger update possible for %T", obj)
	}
}

type ImageImportControllerConfig struct {
	MaxScheduledImageImportsPerMinute          int
	ScheduledImageImportMinimumIntervalSeconds int
	DisableScheduledImport                     bool
	ResyncPeriod                               time.Duration
}

func (c *ImageImportControllerConfig) RunController(ctx ControllerContext) (bool, error) {
	informer := ctx.ImageInformers.Image().InternalVersion().ImageStreams()

	controller := imagecontroller.NewImageStreamController(
		ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(bootstrappolicy.InfraImageImportControllerServiceAccountName),
		informer,
	)
	go controller.Run(5, ctx.Stop)

	if c.DisableScheduledImport {
		return true, nil
	}

	scheduledController := imagecontroller.NewScheduledImageStreamController(
		ctx.ClientBuilder.DeprecatedOpenshiftClientOrDie(bootstrappolicy.InfraImageImportControllerServiceAccountName),
		informer,
		imagecontroller.ScheduledImageStreamControllerOptions{
			Resync: time.Duration(c.ScheduledImageImportMinimumIntervalSeconds) * time.Second,

			Enabled:                  !c.DisableScheduledImport,
			DefaultBucketSize:        4,
			MaxImageImportsPerMinute: c.MaxScheduledImageImportsPerMinute,
		},
	)

	controller.SetNotifier(scheduledController)
	go scheduledController.Run(ctx.Stop)

	return true, nil
}
<|endoftext|>


<commit_before>package certificates

import (
	"context"
	"fmt"
	"sync"
	"time"

	"github.com/golang/glog"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	coreinformers "k8s.io/client-go/informers/core/v1"
	extinformers "k8s.io/client-go/informers/extensions/v1beta1"
	"k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	extlisters "k8s.io/client-go/listers/extensions/v1beta1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"

	clientset "github.com/jetstack/cert-manager/pkg/client/clientset/versioned"
	cminformers "github.com/jetstack/cert-manager/pkg/client/informers/externalversions/certmanager/v1alpha1"
	cmlisters "github.com/jetstack/cert-manager/pkg/client/listers/certmanager/v1alpha1"
	controllerpkg "github.com/jetstack/cert-manager/pkg/controller"
	"github.com/jetstack/cert-manager/pkg/issuer"
	"github.com/jetstack/cert-manager/pkg/scheduler"
	"github.com/jetstack/cert-manager/pkg/util"
)

type Controller struct {
	client        kubernetes.Interface
	cmClient      clientset.Interface
	issuerFactory issuer.Factory
	recorder      record.EventRecorder

	// To allow injection for testing.
	syncHandler func(ctx context.Context, key string) error

	issuerLister        cmlisters.IssuerLister
	clusterIssuerLister cmlisters.ClusterIssuerLister
	certificateLister   cmlisters.CertificateLister
	secretLister        corelisters.SecretLister
	ingressLister       extlisters.IngressLister

	queue              workqueue.RateLimitingInterface
	scheduledWorkQueue scheduler.ScheduledWorkQueue
	workerWg           sync.WaitGroup
	syncedFuncs        []cache.InformerSynced
}

// New returns a new Certificates controller. It sets up the informer handler
// functions for all the types it watches.
func New(
	certificatesInformer cminformers.CertificateInformer,
	issuersInformer cminformers.IssuerInformer,
	clusterIssuersInformer cminformers.ClusterIssuerInformer,
	secretsInformer coreinformers.SecretInformer,
	ingressInformer extinformers.IngressInformer,
	podsInformer coreinformers.PodInformer,
	serviceInformer coreinformers.ServiceInformer,
	client kubernetes.Interface,
	cmClient clientset.Interface,
	issuerFactory issuer.Factory,
	recorder record.EventRecorder,
) *Controller {
	ctrl := &Controller{client: client, cmClient: cmClient, issuerFactory: issuerFactory, recorder: recorder}
	ctrl.syncHandler = ctrl.processNextWorkItem
	ctrl.queue = workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(time.Second*5, time.Minute*2), "certificates")
	// Create a scheduled work queue that calls the ctrl.queue.Add method for
	// each object in the queue. This is used to schedule re-checks of
	// Certificate resources when they get near to expiry
	ctrl.scheduledWorkQueue = scheduler.NewScheduledWorkQueue(ctrl.queue.AddRateLimited)

	certificatesInformer.Informer().AddEventHandler(&controllerpkg.QueuingEventHandler{Queue: ctrl.queue})
	ctrl.certificateLister = certificatesInformer.Lister()
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, certificatesInformer.Informer().HasSynced)

	ctrl.issuerLister = issuersInformer.Lister()
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, issuersInformer.Informer().HasSynced)

	// clusterIssuersInformer may be nil if cert-manager is scoped to a single
	// namespace
	if clusterIssuersInformer != nil {
		ctrl.clusterIssuerLister = clusterIssuersInformer.Lister()
		ctrl.syncedFuncs = append(ctrl.syncedFuncs, clusterIssuersInformer.Informer().HasSynced)
	}

	ctrl.secretLister = secretsInformer.Lister()
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, secretsInformer.Informer().HasSynced)

	ctrl.ingressLister = ingressInformer.Lister()
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, ingressInformer.Informer().HasSynced)

	// We also add pod and service informers to the list of informers to sync.
	// They are not actually used directly by the Certificates controller,
	// however the ACME HTTP challenge solver *does* require a Pod and Secret
	// lister, and due to the way the instantiation of issuers is performed it
	// is far more performant to perform the sync here.
	// We should consider moving this into pkg/issuer/acme at some point, some how.
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, podsInformer.Informer().HasSynced)
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, serviceInformer.Informer().HasSynced)

	return ctrl
}

func (c *Controller) Run(workers int, stopCh <-chan struct{}) error {
	glog.V(4).Infof("Starting %s control loop", ControllerName)
	// wait for all the informer caches we depend to sync
	if !cache.WaitForCacheSync(stopCh, c.syncedFuncs...) {
		return fmt.Errorf("error waiting for informer caches to sync")
	}

	glog.V(4).Infof("Synced all caches for %s control loop", ControllerName)

	for i := 0; i < workers; i++ {
		c.workerWg.Add(1)
		// TODO (@munnerz): make time.Second duration configurable
		go wait.Until(func() { c.worker(stopCh) }, time.Second, stopCh)
	}
	<-stopCh
	glog.V(4).Infof("Shutting down queue as workqueue signaled shutdown")
	c.queue.ShutDown()
	glog.V(4).Infof("Waiting for workers to exit...")
	c.workerWg.Wait()
	glog.V(4).Infof("Workers exited.")
	return nil
}

func (c *Controller) worker(stopCh <-chan struct{}) {
	defer c.workerWg.Done()
	glog.V(4).Infof("Starting %q worker", ControllerName)
	for {
		obj, shutdown := c.queue.Get()
		if shutdown {
			break
		}

		var key string
		err := func(obj interface{}) error {
			defer c.queue.Done(obj)
			var ok bool
			if key, ok = obj.(string); !ok {
				return nil
			}
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			ctx = util.ContextWithStopCh(ctx, stopCh)
			glog.Infof("%s controller: syncing item '%s'", ControllerName, key)
			if err := c.syncHandler(ctx, key); err != nil {
				return err
			}
			c.queue.Forget(obj)
			return nil
		}(obj)

		if err != nil {
			glog.Errorf("%s controller: Re-queuing item %q due to error processing: %s", ControllerName, key, err.Error())
			c.queue.AddRateLimited(obj)
			continue
		}

		glog.Infof("%s controller: Finished processing work item %q", ControllerName, key)
	}
	glog.V(4).Infof("Exiting %q worker loop", ControllerName)
}

func (c *Controller) processNextWorkItem(ctx context.Context, key string) error {
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		runtime.HandleError(fmt.Errorf("invalid resource key: %s", key))
		return nil
	}

	crt, err := c.certificateLister.Certificates(namespace).Get(name)

	if err != nil {
		if k8sErrors.IsNotFound(err) {
			c.scheduledWorkQueue.Forget(key)
			runtime.HandleError(fmt.Errorf("certificate '%s' in work queue no longer exists", key))
			return nil
		}

		return err
	}

	return c.Sync(ctx, crt)
}

var keyFunc = controllerpkg.KeyFunc

const (
	ControllerName = "certificates"
)

func init() {
	controllerpkg.Register(ControllerName, func(ctx *controllerpkg.Context) controllerpkg.Interface {
		return New(
			ctx.SharedInformerFactory.Certmanager().V1alpha1().Certificates(),
			ctx.SharedInformerFactory.Certmanager().V1alpha1().Issuers(),
			ctx.SharedInformerFactory.Certmanager().V1alpha1().ClusterIssuers(),
			ctx.KubeSharedInformerFactory.Core().V1().Secrets(),
			ctx.KubeSharedInformerFactory.Extensions().V1beta1().Ingresses(),
			ctx.KubeSharedInformerFactory.Core().V1().Pods(),
			ctx.KubeSharedInformerFactory.Core().V1().Services(),
			ctx.Client,
			ctx.CMClient,
			ctx.IssuerFactory,
			ctx.Recorder,
		).Run
	})
}
<commit_msg>Requeue Certificate if target secret is deleted<commit_after>package certificates

import (
	"context"
	"fmt"
	"sync"
	"time"

	"github.com/golang/glog"
	corev1 "k8s.io/api/core/v1"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	coreinformers "k8s.io/client-go/informers/core/v1"
	extinformers "k8s.io/client-go/informers/extensions/v1beta1"
	"k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	extlisters "k8s.io/client-go/listers/extensions/v1beta1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"

	clientset "github.com/jetstack/cert-manager/pkg/client/clientset/versioned"
	cminformers "github.com/jetstack/cert-manager/pkg/client/informers/externalversions/certmanager/v1alpha1"
	cmlisters "github.com/jetstack/cert-manager/pkg/client/listers/certmanager/v1alpha1"
	controllerpkg "github.com/jetstack/cert-manager/pkg/controller"
	"github.com/jetstack/cert-manager/pkg/issuer"
	"github.com/jetstack/cert-manager/pkg/scheduler"
	"github.com/jetstack/cert-manager/pkg/util"
)

type Controller struct {
	client        kubernetes.Interface
	cmClient      clientset.Interface
	issuerFactory issuer.Factory
	recorder      record.EventRecorder

	// To allow injection for testing.
	syncHandler func(ctx context.Context, key string) error

	issuerLister        cmlisters.IssuerLister
	clusterIssuerLister cmlisters.ClusterIssuerLister
	certificateLister   cmlisters.CertificateLister
	secretLister        corelisters.SecretLister
	ingressLister       extlisters.IngressLister

	queue              workqueue.RateLimitingInterface
	scheduledWorkQueue scheduler.ScheduledWorkQueue
	workerWg           sync.WaitGroup
	syncedFuncs        []cache.InformerSynced
}

// New returns a new Certificates controller. It sets up the informer handler
// functions for all the types it watches.
func New(
	certificatesInformer cminformers.CertificateInformer,
	issuersInformer cminformers.IssuerInformer,
	clusterIssuersInformer cminformers.ClusterIssuerInformer,
	secretsInformer coreinformers.SecretInformer,
	ingressInformer extinformers.IngressInformer,
	podsInformer coreinformers.PodInformer,
	serviceInformer coreinformers.ServiceInformer,
	client kubernetes.Interface,
	cmClient clientset.Interface,
	issuerFactory issuer.Factory,
	recorder record.EventRecorder,
) *Controller {
	ctrl := &Controller{client: client, cmClient: cmClient, issuerFactory: issuerFactory, recorder: recorder}
	ctrl.syncHandler = ctrl.processNextWorkItem
	ctrl.queue = workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(time.Second*5, time.Minute*2), "certificates")
	// Create a scheduled work queue that calls the ctrl.queue.Add method for
	// each object in the queue. This is used to schedule re-checks of
	// Certificate resources when they get near to expiry
	ctrl.scheduledWorkQueue = scheduler.NewScheduledWorkQueue(ctrl.queue.AddRateLimited)

	certificatesInformer.Informer().AddEventHandler(&controllerpkg.QueuingEventHandler{Queue: ctrl.queue})
	ctrl.certificateLister = certificatesInformer.Lister()
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, certificatesInformer.Informer().HasSynced)

	ctrl.issuerLister = issuersInformer.Lister()
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, issuersInformer.Informer().HasSynced)

	// clusterIssuersInformer may be nil if cert-manager is scoped to a single
	// namespace
	if clusterIssuersInformer != nil {
		ctrl.clusterIssuerLister = clusterIssuersInformer.Lister()
		ctrl.syncedFuncs = append(ctrl.syncedFuncs, clusterIssuersInformer.Informer().HasSynced)
	}

	secretsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		DeleteFunc: ctrl.secretDeleted,
	})
	ctrl.secretLister = secretsInformer.Lister()
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, secretsInformer.Informer().HasSynced)

	ctrl.ingressLister = ingressInformer.Lister()
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, ingressInformer.Informer().HasSynced)

	// We also add pod and service informers to the list of informers to sync.
	// They are not actually used directly by the Certificates controller,
	// however the ACME HTTP challenge solver *does* require a Pod and Secret
	// lister, and due to the way the instantiation of issuers is performed it
	// is far more performant to perform the sync here.
	// We should consider moving this into pkg/issuer/acme at some point, some how.
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, podsInformer.Informer().HasSynced)
	ctrl.syncedFuncs = append(ctrl.syncedFuncs, serviceInformer.Informer().HasSynced)

	return ctrl
}

// TODO: replace with generic handleObjet function (like Navigator)
func (c *Controller) secretDeleted(obj interface{}) {
	var secret *corev1.Secret
	var ok bool
	secret, ok = obj.(*corev1.Secret)
	if !ok {
		runtime.HandleError(fmt.Errorf("Object is not a Secret object %#v", obj))
		return
	}
	crts, err := c.certificatesForSecret(secret)
	if err != nil {
		runtime.HandleError(fmt.Errorf("Error looking up Certificates observing Secret: %s/%s", secret.Namespace, secret.Name))
		return
	}
	for _, crt := range crts {
		key, err := keyFunc(crt)
		if err != nil {
			runtime.HandleError(err)
			continue
		}
		c.queue.AddRateLimited(key)
	}
}

func (c *Controller) Run(workers int, stopCh <-chan struct{}) error {
	glog.V(4).Infof("Starting %s control loop", ControllerName)
	// wait for all the informer caches we depend to sync
	if !cache.WaitForCacheSync(stopCh, c.syncedFuncs...) {
		return fmt.Errorf("error waiting for informer caches to sync")
	}

	glog.V(4).Infof("Synced all caches for %s control loop", ControllerName)

	for i := 0; i < workers; i++ {
		c.workerWg.Add(1)
		// TODO (@munnerz): make time.Second duration configurable
		go wait.Until(func() { c.worker(stopCh) }, time.Second, stopCh)
	}
	<-stopCh
	glog.V(4).Infof("Shutting down queue as workqueue signaled shutdown")
	c.queue.ShutDown()
	glog.V(4).Infof("Waiting for workers to exit...")
	c.workerWg.Wait()
	glog.V(4).Infof("Workers exited.")
	return nil
}

func (c *Controller) worker(stopCh <-chan struct{}) {
	defer c.workerWg.Done()
	glog.V(4).Infof("Starting %q worker", ControllerName)
	for {
		obj, shutdown := c.queue.Get()
		if shutdown {
			break
		}

		var key string
		err := func(obj interface{}) error {
			defer c.queue.Done(obj)
			var ok bool
			if key, ok = obj.(string); !ok {
				return nil
			}
			ctx, cancel := context.WithCancel(context.Background())
			defer cancel()
			ctx = util.ContextWithStopCh(ctx, stopCh)
			glog.Infof("%s controller: syncing item '%s'", ControllerName, key)
			if err := c.syncHandler(ctx, key); err != nil {
				return err
			}
			c.queue.Forget(obj)
			return nil
		}(obj)

		if err != nil {
			glog.Errorf("%s controller: Re-queuing item %q due to error processing: %s", ControllerName, key, err.Error())
			c.queue.AddRateLimited(obj)
			continue
		}

		glog.Infof("%s controller: Finished processing work item %q", ControllerName, key)
	}
	glog.V(4).Infof("Exiting %q worker loop", ControllerName)
}

func (c *Controller) processNextWorkItem(ctx context.Context, key string) error {
	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		runtime.HandleError(fmt.Errorf("invalid resource key: %s", key))
		return nil
	}

	crt, err := c.certificateLister.Certificates(namespace).Get(name)

	if err != nil {
		if k8sErrors.IsNotFound(err) {
			c.scheduledWorkQueue.Forget(key)
			runtime.HandleError(fmt.Errorf("certificate '%s' in work queue no longer exists", key))
			return nil
		}

		return err
	}

	return c.Sync(ctx, crt)
}

var keyFunc = controllerpkg.KeyFunc

const (
	ControllerName = "certificates"
)

func init() {
	controllerpkg.Register(ControllerName, func(ctx *controllerpkg.Context) controllerpkg.Interface {
		return New(
			ctx.SharedInformerFactory.Certmanager().V1alpha1().Certificates(),
			ctx.SharedInformerFactory.Certmanager().V1alpha1().Issuers(),
			ctx.SharedInformerFactory.Certmanager().V1alpha1().ClusterIssuers(),
			ctx.KubeSharedInformerFactory.Core().V1().Secrets(),
			ctx.KubeSharedInformerFactory.Extensions().V1beta1().Ingresses(),
			ctx.KubeSharedInformerFactory.Core().V1().Pods(),
			ctx.KubeSharedInformerFactory.Core().V1().Services(),
			ctx.Client,
			ctx.CMClient,
			ctx.IssuerFactory,
			ctx.Recorder,
		).Run
	})
}
<|endoftext|>


<commit_before>//
// policy.go
//
package srnd

import (
	"regexp"
	"strings"
)

type FeedPolicy struct {
	rules map[string]string
}

// do we allow this newsgroup?
func (self *FeedPolicy) AllowsNewsgroup(newsgroup string) (result bool) {
	var k, v string
	var allows, disallows int
	for k, v = range self.rules {
		v = strings.Trim(v, " ")
		match, err := regexp.MatchString(k, newsgroup)
		if err == nil {
			if match {
				if v == "1" {
					allows++
				} else if v == "0" {
					disallows++
				}
			}
		}
	}

	result = allows > 0 && disallows == 0

	return
}
<commit_msg>fix ?<commit_after>//
// policy.go
//
package srnd

import (
	"regexp"
	"strings"
)

type FeedPolicy struct {
	rules map[string]string
}

// do we allow this newsgroup?
func (self *FeedPolicy) AllowsNewsgroup(newsgroup string) (result bool) {
	var k, v string
	var allows, disallows int
	for k, v = range self.rules {
		v = strings.Trim(v, " ")
		match, err := regexp.MatchString(k, newsgroup)
		if err == nil {
			if match {
				if v == "1" {
					allows++
				} else if v == "0" {
					return false
				}
			}
		}
	}

	result = allows > 0

	return
}
<|endoftext|>


<commit_before>package locations

import (
	"appengine"
	"net/http"

	"lib/constants"
	"lib/position"
	"lib/spreadsheet"
	"lib/subdomain"
)

// Parse the location data
func All(c appengine.Context) (cities []Location) {
	sheetId := "0"
	headerRow := 0
	citiesData := spreadsheet.Get(c, constants.LocationSpreadsheetId, sheetId, headerRow)

	for _, locationData := range citiesData {
		if locationData["visible"] != "y" {
			continue
		}

		location := Location{
			ID:            locationData["ID"],
			Name:          locationData["City"],
			Position:      position.Create(c, locationData["Lat"], locationData["Lng"]),
			SpreadsheetId: locationData["Spreadsheet ID"],
			SheetId:       locationData["Sheet ID"],
		}

		cities = append(cities, location)
	}

	return
}

// Get the location according to subdomain or position from the request
func Get(r *http.Request) (location Location) {
	c := appengine.NewContext(r)

	cities := All(c)
	calledSubdomain := subdomain.GetFromRequest(r)
	userPosition := position.GetFromRequest(r)

	if calledSubdomain != "" {
		location = selectByID(cities, calledSubdomain)
	}

	if location.ID == "" && userPosition.Lat != 0 && userPosition.Lng != 0 {
		location = selectClosest(c, cities, userPosition)
	}

	if location.ID == "" {
		location = cities[0]
	}

	return
}

// Get a location by it’s id
func GetById(r *http.Request, locationId string) (location Location, exists bool) {
	c := appengine.NewContext(r)

	cities := All(c)
	location = selectByID(cities, locationId)
	exists = location.ID != ""

	return
}
<commit_msg>refactor(locations): rename cities to locations<commit_after>package locations

import (
	"appengine"
	"net/http"

	"lib/constants"
	"lib/position"
	"lib/spreadsheet"
	"lib/subdomain"
)

// Parse the location data
func All(c appengine.Context) (locations []Location) {
	sheetId := "0"
	headerRow := 0
	locationsData := spreadsheet.Get(c, constants.LocationSpreadsheetId, sheetId, headerRow)

	for _, locationData := range locationsData {
		if locationData["visible"] != "y" {
			continue
		}

		location := Location{
			ID:            locationData["ID"],
			Name:          locationData["City"],
			Position:      position.Create(c, locationData["Lat"], locationData["Lng"]),
			SpreadsheetId: locationData["Spreadsheet ID"],
			SheetId:       locationData["Sheet ID"],
		}

		locations = append(locations, location)
	}

	return
}

// Get the location according to subdomain or position from the request
func Get(r *http.Request) (location Location) {
	c := appengine.NewContext(r)

	locations := All(c)
	calledSubdomain := subdomain.GetFromRequest(r)
	userPosition := position.GetFromRequest(r)

	if calledSubdomain != "" {
		location = selectByID(locations, calledSubdomain)
	}

	if location.ID == "" && userPosition.Lat != 0 && userPosition.Lng != 0 {
		location = selectClosest(c, locations, userPosition)
	}

	if location.ID == "" {
		location = locations[0]
	}

	return
}

// Get a location by it’s id
func GetById(r *http.Request, locationId string) (location Location, exists bool) {
	c := appengine.NewContext(r)

	locations := All(c)
	location = selectByID(locations, locationId)
	exists = location.ID != ""

	return
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package configcron

import (
	"testing"
	"time"

	"google.golang.org/protobuf/encoding/prototext"
	"google.golang.org/protobuf/proto"

	"go.chromium.org/luci/common/clock/testclock"
	"go.chromium.org/luci/config"
	"go.chromium.org/luci/config/cfgclient"
	cfgmemory "go.chromium.org/luci/config/impl/memory"
	"go.chromium.org/luci/gae/service/datastore"
	"go.chromium.org/luci/server/tq/tqtesting"

	cvconfig "go.chromium.org/luci/cv/internal/config"
	"go.chromium.org/luci/cv/internal/cvtesting"
	"go.chromium.org/luci/cv/internal/prjmanager/pmtest"

	. "github.com/smartystreets/goconvey/convey"
	. "go.chromium.org/luci/common/testing/assertions"
)

var testNow = testclock.TestTimeLocal.Round(1 * time.Millisecond)

func TestConfigRefreshCron(t *testing.T) {
	t.Parallel()

	Convey("Config refresh cron works", t, func() {
		ct := cvtesting.Test{}
		ctx, cancel := ct.SetUp()
		defer cancel()

		Convey("for a new project", func() {
			ctx = cfgclient.Use(ctx, cfgmemory.New(map[config.Set]cfgmemory.Files{
				config.ProjectSet("chromium"): {cvconfig.ConfigFileName: ""},
			}))
			// Project chromium doesn't exist in datastore.
			err := SubmitRefreshTasks(ctx)
			So(err, ShouldBeNil)
			So(ct.TQ.Tasks().Payloads(), ShouldResembleProto, []*RefreshProjectConfigTask{
				{Project: "chromium"},
			})
			ct.TQ.Run(ctx, tqtesting.StopAfterTask("refresh-project-config"))
			So(pmtest.Projects(ct.TQ.Tasks()), ShouldResemble, []string{"chromium"})
		})

		Convey("for an existing project", func() {
			ctx = cfgclient.Use(ctx, cfgmemory.New(map[config.Set]cfgmemory.Files{
				config.ProjectSet("chromium"): {cvconfig.ConfigFileName: ""},
			}))
			So(datastore.Put(ctx, &cvconfig.ProjectConfig{
				Project: "chromium",
				Enabled: true,
			}), ShouldBeNil)
			err := SubmitRefreshTasks(ctx)
			So(err, ShouldBeNil)
			So(ct.TQ.Tasks().Payloads(), ShouldResembleProto, []*RefreshProjectConfigTask{
				{Project: "chromium"},
			})
			ct.TQ.Run(ctx, tqtesting.StopAfterTask("refresh-project-config"))
			So(pmtest.Projects(ct.TQ.Tasks()), ShouldResemble, []string{"chromium"})
		})

		Convey("Disable project", func() {
			Convey("that doesn't have CV config", func() {
				ctx = cfgclient.Use(ctx, cfgmemory.New(map[config.Set]cfgmemory.Files{
					config.ProjectSet("chromium"): {"other.cfg": ""},
				}))
				So(datastore.Put(ctx, &cvconfig.ProjectConfig{
					Project: "chromium",
					Enabled: true,
				}), ShouldBeNil)
				err := SubmitRefreshTasks(ctx)
				So(err, ShouldBeNil)
				So(ct.TQ.Tasks().Payloads(), ShouldResembleProto, []*RefreshProjectConfigTask{
					{Project: "chromium", Disable: true},
				})
				ct.TQ.Run(ctx, tqtesting.StopAfterTask("refresh-project-config"))
				So(pmtest.Projects(ct.TQ.Tasks()), ShouldResemble, []string{"chromium"})
			})
			Convey("that doesn't exist in LUCI Config", func() {
				ctx = cfgclient.Use(ctx, cfgmemory.New(map[config.Set]cfgmemory.Files{}))
				So(datastore.Put(ctx, &cvconfig.ProjectConfig{
					Project: "chromium",
					Enabled: true,
				}), ShouldBeNil)
				err := SubmitRefreshTasks(ctx)
				So(err, ShouldBeNil)
				So(ct.TQ.Tasks().Payloads(), ShouldResembleProto, []*RefreshProjectConfigTask{
					{Project: "chromium", Disable: true},
				})
				ct.TQ.Run(ctx, tqtesting.StopAfterTask("refresh-project-config"))
				So(pmtest.Projects(ct.TQ.Tasks()), ShouldResemble, []string{"chromium"})
			})

			Convey("Skip already disabled Project", func() {
				ctx = cfgclient.Use(ctx, cfgmemory.New(map[config.Set]cfgmemory.Files{}))
				So(datastore.Put(ctx, &cvconfig.ProjectConfig{
					Project: "foo",
					Enabled: false,
				}), ShouldBeNil)
				err := SubmitRefreshTasks(ctx)
				So(err, ShouldBeNil)
				So(ct.TQ.Tasks(), ShouldBeEmpty)
			})
		})
	})
}

func toProtoText(msg proto.Message) string {
	bs, err := prototext.Marshal(msg)
	So(err, ShouldBeNil)
	return string(bs)
}
<commit_msg>[cv] mock PM dispatch in configcron tests.<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package configcron

import (
	"testing"
	"time"

	"google.golang.org/protobuf/encoding/prototext"
	"google.golang.org/protobuf/proto"

	"go.chromium.org/luci/common/clock/testclock"
	"go.chromium.org/luci/config"
	"go.chromium.org/luci/config/cfgclient"
	cfgmemory "go.chromium.org/luci/config/impl/memory"
	"go.chromium.org/luci/gae/service/datastore"
	"go.chromium.org/luci/server/tq/tqtesting"

	cvconfig "go.chromium.org/luci/cv/internal/config"
	"go.chromium.org/luci/cv/internal/cvtesting"
	"go.chromium.org/luci/cv/internal/prjmanager/pmtest"

	. "github.com/smartystreets/goconvey/convey"
	. "go.chromium.org/luci/common/testing/assertions"
)

var testNow = testclock.TestTimeLocal.Round(1 * time.Millisecond)

func TestConfigRefreshCron(t *testing.T) {
	t.Parallel()

	Convey("Config refresh cron works", t, func() {
		ct := cvtesting.Test{}
		ctx, cancel := ct.SetUp()
		defer cancel()
		ctx, pmDispatcher := pmtest.MockDispatch(ctx)

		Convey("for a new project", func() {
			ctx = cfgclient.Use(ctx, cfgmemory.New(map[config.Set]cfgmemory.Files{
				config.ProjectSet("chromium"): {cvconfig.ConfigFileName: ""},
			}))
			// Project chromium doesn't exist in datastore.
			err := SubmitRefreshTasks(ctx)
			So(err, ShouldBeNil)
			So(ct.TQ.Tasks().Payloads(), ShouldResembleProto, []*RefreshProjectConfigTask{
				{Project: "chromium"},
			})
			ct.TQ.Run(ctx, tqtesting.StopAfterTask("refresh-project-config"))
			So(pmDispatcher.PopProjects(), ShouldResemble, []string{"chromium"})
		})

		Convey("for an existing project", func() {
			ctx = cfgclient.Use(ctx, cfgmemory.New(map[config.Set]cfgmemory.Files{
				config.ProjectSet("chromium"): {cvconfig.ConfigFileName: ""},
			}))
			So(datastore.Put(ctx, &cvconfig.ProjectConfig{
				Project: "chromium",
				Enabled: true,
			}), ShouldBeNil)
			err := SubmitRefreshTasks(ctx)
			So(err, ShouldBeNil)
			So(ct.TQ.Tasks().Payloads(), ShouldResembleProto, []*RefreshProjectConfigTask{
				{Project: "chromium"},
			})
			ct.TQ.Run(ctx, tqtesting.StopAfterTask("refresh-project-config"))
			So(pmDispatcher.PopProjects(), ShouldResemble, []string{"chromium"})
		})

		Convey("Disable project", func() {
			Convey("that doesn't have CV config", func() {
				ctx = cfgclient.Use(ctx, cfgmemory.New(map[config.Set]cfgmemory.Files{
					config.ProjectSet("chromium"): {"other.cfg": ""},
				}))
				So(datastore.Put(ctx, &cvconfig.ProjectConfig{
					Project: "chromium",
					Enabled: true,
				}), ShouldBeNil)
				err := SubmitRefreshTasks(ctx)
				So(err, ShouldBeNil)
				So(ct.TQ.Tasks().Payloads(), ShouldResembleProto, []*RefreshProjectConfigTask{
					{Project: "chromium", Disable: true},
				})
				ct.TQ.Run(ctx, tqtesting.StopAfterTask("refresh-project-config"))
				So(pmDispatcher.PopProjects(), ShouldResemble, []string{"chromium"})
			})
			Convey("that doesn't exist in LUCI Config", func() {
				ctx = cfgclient.Use(ctx, cfgmemory.New(map[config.Set]cfgmemory.Files{}))
				So(datastore.Put(ctx, &cvconfig.ProjectConfig{
					Project: "chromium",
					Enabled: true,
				}), ShouldBeNil)
				err := SubmitRefreshTasks(ctx)
				So(err, ShouldBeNil)
				So(ct.TQ.Tasks().Payloads(), ShouldResembleProto, []*RefreshProjectConfigTask{
					{Project: "chromium", Disable: true},
				})
				ct.TQ.Run(ctx, tqtesting.StopAfterTask("refresh-project-config"))
				So(pmDispatcher.PopProjects(), ShouldResemble, []string{"chromium"})
			})

			Convey("Skip already disabled Project", func() {
				ctx = cfgclient.Use(ctx, cfgmemory.New(map[config.Set]cfgmemory.Files{}))
				So(datastore.Put(ctx, &cvconfig.ProjectConfig{
					Project: "foo",
					Enabled: false,
				}), ShouldBeNil)
				err := SubmitRefreshTasks(ctx)
				So(err, ShouldBeNil)
				So(ct.TQ.Tasks(), ShouldBeEmpty)
			})
		})
	})
}

func toProtoText(msg proto.Message) string {
	bs, err := prototext.Marshal(msg)
	So(err, ShouldBeNil)
	return string(bs)
}
<|endoftext|>


<commit_before>//go:build linux || freebsd || openbsd || netbsd || darwin || solaris || illumos || dragonfly
// +build linux freebsd openbsd netbsd darwin solaris illumos dragonfly

package client // import "github.com/docker/docker/client"

// DefaultDockerHost defines OS-specific default host if the DOCKER_HOST
// (EnvOverrideHost) environment variable is unset or empty.
const DefaultDockerHost = "unix:///var/run/docker.sock"

const defaultProto = "unix"
const defaultAddr = "/var/run/docker.sock"
<commit_msg>client: remove solaris build-tag, simplify and gofumpt<commit_after>//go:build !windows
// +build !windows

package client // import "github.com/docker/docker/client"

// DefaultDockerHost defines OS-specific default host if the DOCKER_HOST
// (EnvOverrideHost) environment variable is unset or empty.
const DefaultDockerHost = "unix:///var/run/docker.sock"

const defaultProto = "unix"
const defaultAddr = "/var/run/docker.sock"
<|endoftext|>


<commit_before>package store

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"jiacrontab/libs"
	"jiacrontab/libs/proto"
	"jiacrontab/model"
	"log"
	"os"
	"path/filepath"
	"time"
)

const (
	Update = "update"
	Sync   = "sync"
	Select = "select"
	Load   = "load"
	Search = "Search"
)

type result struct {
	value interface{}
}
type request struct {
	key      string
	state    string
	handler  handle
	body     string
	response chan<- result
}

type handle func(s *Store)

func NewStore(path string) *Store {
	s := &Store{
		dataFile: path,
		swapFile: filepath.Join(filepath.Dir(path), ".swap"),
		requests: make(chan request),
	}
	go s.Server()
	return s
}

type Store struct {
	Mail     proto.MailArgs
	TaskList map[string]model.CrontabTask

	swapFile string
	dataFile string
	requests chan request
}

func (s *Store) Server() {
	t := time.Tick(time.Duration(10 * time.Minute))
	for {
		select {
		case req := <-s.requests:
			s.requestHandle(req)
		case <-t:
			s.sync(s.swapFile)
		}

	}
}

func (s *Store) requestHandle(req request) {
	if req.handler != nil {
		req.handler(s)
	}
	if req.state == Update {
		// s.sync()
	}

	if req.state == Sync {
		s.sync(s.dataFile)
	}

	if req.state == Load {
		if err := s.load(s.dataFile); err != nil {
			log.Printf("failed recover %s", err)
			if err = s.load(s.swapFile); err != nil {
				log.Printf("failed recover %s", err)
			}
		}
	}
	switch req.key {
	case "Mail":
		req.response <- result{value: s.Mail}
	// case "TaskList":
	// 	if req.state == Search && req.body != "" {
	// 		if v, ok := s.TaskList[req.body]; ok {
	// 			req.response <- result{value: v}
	// 		} else {
	// 			req.response <- result{value: nil}
	// 		}
	// 	} else {
	// 		var taskList proto.Mdata
	// 		if b, err := json.Marshal(s.TaskList); err == nil {
	// 			json.Unmarshal(b, &taskList)
	// 		}
	// 		req.response <- result{value: taskList}
	// 	}

	case "dataFile":
		req.response <- result{value: s.dataFile}
	default:
		req.response <- result{value: nil}
	}
}

func (s *Store) Get(key string) result {
	return s.Query(key, Select, nil, "")
}

func (s *Store) Search(key, args string) result {
	return s.Query(key, Search, nil, args)
}

// func (s *Store) SearchTaskList(args string) (*proto.TaskArgs, bool) {
// 	ret, ok := s.Search("TaskList", args).value.(*proto.TaskArgs)
// 	return ret, ok
// }

func (s *Store) Update(fn handle) *Store {
	s.Query("", Update, fn, "")
	return s
}

func (s *Store) Sync() result {

	return s.Query("", Sync, nil, "")
}

func (s *Store) Load() result {
	return s.Query("", Load, nil, "")
}

func (s *Store) Query(key string, state string, fn handle, body string) result {
	response := make(chan result)
	s.requests <- request{key, state, fn, body, response}
	return <-response
}

func (s *Store) GetMail() (proto.MailArgs, bool) {
	ret, ok := (s.Get("Mail")).value.(proto.MailArgs)
	return ret, ok
}

// func (s *Store) GetRpcClient() (proto.Mdata, bool) {
// 	ret, ok := (s.Get("RpcClient")).value.(proto.Mdata)
// 	return ret, ok
// }

func (s *Store) GetDataFile() (string, bool) {
	ret, ok := (s.Get("dataFile")).value.(string)
	return ret, ok
}

// func (s *Store) GetTaskList() (proto.Mdata, bool) {
// 	ret, ok := (s.Get("TaskList")).value.(proto.Mdata)
// 	return ret, ok
// }

func (s *Store) Export2DB() {
	for _, v := range s.TaskList {
		ret := model.DB().Create(&v)
		if ret.Error == nil {
			log.Printf("import crontab %+v \n", v)
		} else {
			log.Printf("failed import crontab %+v \n", v, ret.Error)
		}

	}
}
func (s *Store) sync(fpath string) error {

	f, err := libs.TryOpen(fpath, os.O_CREATE|os.O_RDWR|os.O_TRUNC)
	defer func() {
		f.Close()
	}()
	if err != nil {
		return err
	}

	b, err := json.MarshalIndent(s, "", "  ")

	if err != nil {
		return err
	}
	_, err = f.Write(b)
	return err

}

func (s *Store) load(fpath string) error {

	f, err := libs.TryOpen(fpath, os.O_CREATE|os.O_RDWR)
	defer func() {
		f.Close()
	}()

	if err != nil {

		return err
	}
	b, err := ioutil.ReadAll(f)
	if err != nil {

		return err
	}

	if len(b) == 0 {
		err = errors.New("nothing to read from " + fpath)

		return err
	}

	err = json.Unmarshal(b, s)
	if err == nil {
		for _, v := range s.TaskList {
			if v.MaxConcurrent == 0 {
				v.MaxConcurrent = 1
			}
			v.NumberProcess = 0
			v.TimerCounter = 0
		}
	}
	return err
}
<commit_msg>update<commit_after>package store

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"jiacrontab/libs"
	"jiacrontab/libs/proto"
	"jiacrontab/model"
	"log"
	"os"
	"path/filepath"
	"time"
)

const (
	Update = "update"
	Sync   = "sync"
	Select = "select"
	Load   = "load"
	Search = "Search"
)

type result struct {
	value interface{}
}
type request struct {
	key      string
	state    string
	handler  handle
	body     string
	response chan<- result
}

type handle func(s *Store)

func NewStore(path string) *Store {
	s := &Store{
		dataFile: path,
		swapFile: filepath.Join(filepath.Dir(path), ".swap"),
		requests: make(chan request),
	}
	go s.Server()
	return s
}

type Store struct {
	Mail     proto.MailArgs
	TaskList map[string]model.CrontabTask

	swapFile string
	dataFile string
	requests chan request
}

func (s *Store) Server() {
	t := time.Tick(time.Duration(10 * time.Minute))
	for {
		select {
		case req := <-s.requests:
			s.requestHandle(req)
		case <-t:
			s.sync(s.swapFile)
		}

	}
}

func (s *Store) requestHandle(req request) {
	if req.handler != nil {
		req.handler(s)
	}
	if req.state == Update {
		// s.sync()
	}

	if req.state == Sync {
		s.sync(s.dataFile)
	}

	if req.state == Load {
		if err := s.load(s.dataFile); err != nil {
			log.Printf("failed recover %s", err)
			if err = s.load(s.swapFile); err != nil {
				log.Printf("failed recover %s", err)
			}
		}
	}
	switch req.key {
	case "Mail":
		req.response <- result{value: s.Mail}
	// case "TaskList":
	// 	if req.state == Search && req.body != "" {
	// 		if v, ok := s.TaskList[req.body]; ok {
	// 			req.response <- result{value: v}
	// 		} else {
	// 			req.response <- result{value: nil}
	// 		}
	// 	} else {
	// 		var taskList proto.Mdata
	// 		if b, err := json.Marshal(s.TaskList); err == nil {
	// 			json.Unmarshal(b, &taskList)
	// 		}
	// 		req.response <- result{value: taskList}
	// 	}

	case "dataFile":
		req.response <- result{value: s.dataFile}
	default:
		req.response <- result{value: nil}
	}
}

func (s *Store) Get(key string) result {
	return s.Query(key, Select, nil, "")
}

func (s *Store) Search(key, args string) result {
	return s.Query(key, Search, nil, args)
}

// func (s *Store) SearchTaskList(args string) (*proto.TaskArgs, bool) {
// 	ret, ok := s.Search("TaskList", args).value.(*proto.TaskArgs)
// 	return ret, ok
// }

func (s *Store) Update(fn handle) *Store {
	s.Query("", Update, fn, "")
	return s
}

func (s *Store) Sync() result {

	return s.Query("", Sync, nil, "")
}

func (s *Store) Load() result {
	return s.Query("", Load, nil, "")
}

func (s *Store) Query(key string, state string, fn handle, body string) result {
	response := make(chan result)
	s.requests <- request{key, state, fn, body, response}
	return <-response
}

func (s *Store) GetMail() (proto.MailArgs, bool) {
	ret, ok := (s.Get("Mail")).value.(proto.MailArgs)
	return ret, ok
}

// func (s *Store) GetRpcClient() (proto.Mdata, bool) {
// 	ret, ok := (s.Get("RpcClient")).value.(proto.Mdata)
// 	return ret, ok
// }

func (s *Store) GetDataFile() (string, bool) {
	ret, ok := (s.Get("dataFile")).value.(string)
	return ret, ok
}

// func (s *Store) GetTaskList() (proto.Mdata, bool) {
// 	ret, ok := (s.Get("TaskList")).value.(proto.Mdata)
// 	return ret, ok
// }

func (s *Store) Export2DB() {
	for _, v := range s.TaskList {
		ret := model.DB().Create(&v)
		if ret.Error == nil {
			log.Printf("import crontab %+v \n", v)
		} else {
			log.Printf("failed import crontab %+v \n", v, ret.Error)
		}

	}
}
func (s *Store) sync(fpath string) error {
	f, err := libs.TryOpen(fpath, os.O_CREATE|os.O_RDWR|os.O_TRUNC)
	defer func() {
		f.Close()
	}()
	if err != nil {
		return err
	}

	b, err := json.MarshalIndent(s, "", "  ")

	if err != nil {
		return err
	}
	_, err = f.Write(b)
	return err

}

func (s *Store) load(fpath string) error {

	f, err := libs.TryOpen(fpath, os.O_CREATE|os.O_RDWR)
	defer func() {
		f.Close()
	}()

	if err != nil {

		return err
	}
	b, err := ioutil.ReadAll(f)
	if err != nil {

		return err
	}

	if len(b) == 0 {
		err = errors.New("nothing to read from " + fpath)

		return err
	}

	err = json.Unmarshal(b, s)
	if err == nil {
		for _, v := range s.TaskList {
			if v.MaxConcurrent == 0 {
				v.MaxConcurrent = 1
			}
			v.NumberProcess = 0
			v.TimerCounter = 0
		}
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"fmt"
	"io/ioutil"
	"strings"

	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	madmin "github.com/minio/madmin-go"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/pkg/console"
)

var adminTierAddFlags = []cli.Flag{
	cli.StringFlag{
		Name:  "endpoint",
		Value: "",
		Usage: "remote tier endpoint. e.g https://s3.amazonaws.com",
	},
	cli.StringFlag{
		Name:  "region",
		Value: "",
		Usage: "remote tier region. e.g us-west-2",
	},
	cli.StringFlag{
		Name:  "access-key",
		Value: "",
		Usage: "AWS S3 or compatible object storage access-key",
	},
	cli.StringFlag{
		Name:  "secret-key",
		Value: "",
		Usage: "AWS S3 or compatible object storage secret-key",
	},
	cli.BoolFlag{
		Name:  "use-aws-role",
		Usage: "use AWS S3 role",
	},
	cli.StringFlag{
		Name:  "account-name",
		Value: "",
		Usage: "Azure Blob Storage account name",
	},
	cli.StringFlag{
		Name:  "account-key",
		Value: "",
		Usage: "Azure Blob Storage account key",
	},
	cli.StringFlag{
		Name:  "credentials-file",
		Value: "",
		Usage: "path to Google Cloud Storage credentials file",
	},
	cli.StringFlag{
		Name:  "bucket",
		Value: "",
		Usage: "remote tier bucket",
	},
	cli.StringFlag{
		Name:  "prefix",
		Value: "",
		Usage: "remote tier prefix",
	},
	cli.StringFlag{
		Name:  "storage-class",
		Value: "",
		Usage: "remote tier storage-class",
	},
}

var adminTierAddCmd = cli.Command{
	Name:         "add",
	Usage:        "add a new remote tier target",
	Action:       mainAdminTierAdd,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        append(globalFlags, adminTierAddFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TYPE ALIAS NAME [FLAGS]

TYPE:
  Transition objects to supported cloud storage backend tier. Supported values are minio, s3, azure and gcs.

NAME:
  Name of the remote tier target. e.g WARM-TIER

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Configure a new remote tier which transitions objects to a bucket in AWS S3 with STANDARD storage class:
     {{.Prompt}} {{.HelpName}} minio myminio WARM-MINIO-TIER --endpoint https://warm-minio.com \
        --access-key ACCESSKEY --secret-key SECRETKEY --bucket mybucket --prefix myprefix/

  2. Configure a new remote tier which transitions objects to a bucket in Azure Blob Storage:
     {{.Prompt}} {{.HelpName}} azure myminio AZTIER --account-name ACCOUNT-NAME --account-key ACCOUNT-KEY \
        --bucket myazurebucket --prefix myazureprefix/

  3. Configure a new remote tier which transitions objects to a bucket in AWS S3 with STANDARD storage class:
     {{.Prompt}} {{.HelpName}} s3 myminio S3TIER --endpoint https://s3.amazonaws.com \
        --access-key ACCESSKEY --secret-key SECRETKEY --bucket mys3bucket --prefix mys3prefix/ \
        --storage-class "STANDARD" --region us-west-2

  4. Configure a new remote tier which transitions objects to a bucket in Google Cloud Storage:
     {{.Prompt}} {{.HelpName}} gcs myminio GCSTIER --credentials-file /path/to/credentials.json \
        --bucket mygcsbucket  --prefix mygcsprefix/
`,
}

// checkAdminTierAddSyntax validates all the positional arguments
func checkAdminTierAddSyntax(ctx *cli.Context) {
	argsNr := len(ctx.Args())
	if argsNr < 3 {
		cli.ShowCommandHelpAndExit(ctx, ctx.Command.Name, 1) // last argument is exit code
	}
	if argsNr > 3 {
		fatalIf(errInvalidArgument().Trace(ctx.Args().Tail()...),
			"Incorrect number of arguments for tier add command.")
	}
}

const (
	s3Standard          = "STANDARD"
	s3ReducedRedundancy = "REDUCED_REDUNDANCY"
)

// fetchTierConfig returns a TierConfig given a tierName, a tierType and ctx to
// lookup command-line flags from. It exits with non-zero error code if any of
// the flags contain invalid values.
func fetchTierConfig(ctx *cli.Context, tierName string, tierType madmin.TierType) *madmin.TierConfig {
	switch tierType {
	case madmin.MinIO:
		accessKey := ctx.String("access-key")
		secretKey := ctx.String("secret-key")
		if accessKey == "" || secretKey == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires access credentials", tierType))
		}
		bucket := ctx.String("bucket")
		if bucket == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires target bucket", tierType))
		}

		endpoint := ctx.String("endpoint")
		if endpoint == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires target endpoint", tierType))
		}

		minioOpts := []madmin.MinIOOptions{}
		prefix := ctx.String("prefix")
		if prefix != "" {
			minioOpts = append(minioOpts, madmin.MinIOPrefix(prefix))
		}

		region := ctx.String("region")
		if region != "" {
			minioOpts = append(minioOpts, madmin.MinIORegion(region))
		}

		minioCfg, err := madmin.NewTierMinIO(tierName, endpoint, accessKey, secretKey, bucket, minioOpts...)
		if err != nil {
			fatalIf(probe.NewError(err), "Invalid configuration for MinIO tier")
		}

		return minioCfg

	case madmin.S3:
		accessKey := ctx.String("access-key")
		secretKey := ctx.String("secret-key")
		useAwsRole := ctx.IsSet("use-aws-role")
		if accessKey == "" && secretKey == "" && !useAwsRole {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires access credentials or AWS role", tierType))
		}
		if (accessKey != "" || secretKey != "") && useAwsRole {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires access credentials", tierType))
		}

		bucket := ctx.String("bucket")
		if bucket == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires target bucket", tierType))
		}

		s3Opts := []madmin.S3Options{}
		prefix := ctx.String("prefix")
		if prefix != "" {
			s3Opts = append(s3Opts, madmin.S3Prefix(prefix))
		}

		endpoint := ctx.String("endpoint")
		if endpoint != "" {
			s3Opts = append(s3Opts, madmin.S3Endpoint(endpoint))
		}

		region := ctx.String("region")
		if region != "" {
			s3Opts = append(s3Opts, madmin.S3Region(region))
		}

		s3SC := ctx.String("storage-class")
		if s3SC != "" {
			if s3SC != s3Standard && s3SC != s3ReducedRedundancy {
				fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("unsupported storage-class type %s", s3SC))
			}
			s3Opts = append(s3Opts, madmin.S3StorageClass(s3SC))
		}
		if ctx.IsSet("use-aws-role") {
			s3Opts = append(s3Opts, madmin.S3AWSRole())
		}
		s3Cfg, err := madmin.NewTierS3(tierName, accessKey, secretKey, bucket, s3Opts...)
		if err != nil {
			fatalIf(probe.NewError(err), "Invalid configuration for AWS S3 compatible remote tier")
		}

		return s3Cfg
	case madmin.Azure:
		accountName := ctx.String("account-name")
		accountKey := ctx.String("account-key")
		if accountName == "" || accountKey == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires access credentials", tierType))
		}

		bucket := ctx.String("bucket")
		if bucket == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires target bucket", tierType))
		}

		azOpts := []madmin.AzureOptions{}
		endpoint := ctx.String("endpoint")
		if endpoint != "" {
			azOpts = append(azOpts, madmin.AzureEndpoint(endpoint))
		}

		region := ctx.String("region")
		if region != "" {
			azOpts = append(azOpts, madmin.AzureRegion(region))
		}

		prefix := ctx.String("prefix")
		if prefix != "" {
			azOpts = append(azOpts, madmin.AzurePrefix(prefix))
		}

		azCfg, err := madmin.NewTierAzure(tierName, accountName, accountKey, bucket, azOpts...)
		if err != nil {
			fatalIf(probe.NewError(err), "Invalid configuration for Azure Blob Storage remote tier")
		}

		return azCfg
	case madmin.GCS:
		bucket := ctx.String("bucket")
		if bucket == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote requires target bucket", tierType))
		}

		gcsOpts := []madmin.GCSOptions{}
		prefix := ctx.String("prefix")
		if prefix != "" {
			gcsOpts = append(gcsOpts, madmin.GCSPrefix(prefix))
		}

		region := ctx.String("region")
		if region != "" {
			gcsOpts = append(gcsOpts, madmin.GCSRegion(region))
		}

		credsPath := ctx.String("credentials-file")
		credsBytes, err := ioutil.ReadFile(credsPath)
		if err != nil {
			fatalIf(probe.NewError(err), "Failed to read credentials file")
		}

		gcsCfg, err := madmin.NewTierGCS(tierName, credsBytes, bucket, gcsOpts...)
		if err != nil {
			fatalIf(probe.NewError(err), "Invalid configuration for Google Cloud Storage remote tier")
		}

		return gcsCfg
	}
	fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("Invalid remote tier type %s", tierType))
	return nil
}

type tierMessage struct {
	op         string
	Status     string            `json:"status"`
	TierName   string            `json:"tierName"`
	TierType   string            `json:"tierType"`
	Endpoint   string            `json:"tierEndpoint"`
	Bucket     string            `json:"bucket"`
	Prefix     string            `json:"prefix,omitempty"`
	Region     string            `json:"region,omitempty"`
	TierParams map[string]string `json:"tierParams,omitempty"`
}

// String returns string representation of msg
func (msg *tierMessage) String() string {
	switch msg.op {
	case "add":
		addMsg := fmt.Sprintf("Added remote tier %s of type %s", msg.TierName, msg.TierType)
		return console.Colorize("TierMessage", addMsg)
	case "rm":
		rmMsg := fmt.Sprintf("Removed remote tier %s", msg.TierName)
		return console.Colorize("TierMessage", rmMsg)
	case "verify":
		verifyMsg := fmt.Sprintf("Verified remote tier %s", msg.TierName)
		return console.Colorize("TierMessage", verifyMsg)
	case "edit":
		editMsg := fmt.Sprintf("Updated remote tier %s", msg.TierName)
		return console.Colorize("TierMessage", editMsg)
	}
	return ""
}

// JSON returns json encoded msg
func (msg *tierMessage) JSON() string {
	jsonMessageBytes, e := json.MarshalIndent(msg, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(jsonMessageBytes)
}

// SetTierConfig sets TierConfig related fields
func (msg *tierMessage) SetTierConfig(sCfg *madmin.TierConfig) {
	msg.TierName = sCfg.Name
	msg.TierType = sCfg.Type.String()
	msg.Endpoint = sCfg.Endpoint()
	msg.Bucket = sCfg.Bucket()
	msg.Prefix = sCfg.Prefix()
	msg.Region = sCfg.Region()
	switch sCfg.Type {
	case madmin.S3:
		msg.TierParams = map[string]string{
			"storageClass": sCfg.S3.StorageClass,
		}
	}
}

func mainAdminTierAdd(ctx *cli.Context) error {
	checkAdminTierAddSyntax(ctx)

	console.SetColor("TierMessage", color.New(color.FgGreen))

	args := ctx.Args()
	tierTypeStr := args.Get(0)
	tierType, err := madmin.NewTierType(tierTypeStr)
	fatalIf(probe.NewError(err), "Unsupported tier type")

	aliasedURL := args.Get(1)
	tierName := args.Get(2)
	if tierName == "" {
		fatalIf(errInvalidArgument(), "Tier name can't be empty")
	}

	// Create a new MinIO Admin Client
	client, cerr := newAdminClient(aliasedURL)
	fatalIf(cerr, "Unable to initialize admin connection.")

	tCfg := fetchTierConfig(ctx, strings.ToUpper(tierName), tierType)
	if err = client.AddTier(globalContext, tCfg); err != nil {
		fatalIf(probe.NewError(err).Trace(args...), "Unable to configure remote tier target")
	}

	msg := &tierMessage{
		op:     "add",
		Status: "success",
	}
	msg.SetTierConfig(tCfg)
	printMsg(msg)
	return nil
}
<commit_msg>fix: tier-add help (#4148)<commit_after>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"fmt"
	"io/ioutil"
	"strings"

	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	madmin "github.com/minio/madmin-go"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/pkg/console"
)

var adminTierAddFlags = []cli.Flag{
	cli.StringFlag{
		Name:  "endpoint",
		Value: "",
		Usage: "remote tier endpoint. e.g https://s3.amazonaws.com",
	},
	cli.StringFlag{
		Name:  "region",
		Value: "",
		Usage: "remote tier region. e.g us-west-2",
	},
	cli.StringFlag{
		Name:  "access-key",
		Value: "",
		Usage: "AWS S3 or compatible object storage access-key",
	},
	cli.StringFlag{
		Name:  "secret-key",
		Value: "",
		Usage: "AWS S3 or compatible object storage secret-key",
	},
	cli.BoolFlag{
		Name:  "use-aws-role",
		Usage: "use AWS S3 role",
	},
	cli.StringFlag{
		Name:  "account-name",
		Value: "",
		Usage: "Azure Blob Storage account name",
	},
	cli.StringFlag{
		Name:  "account-key",
		Value: "",
		Usage: "Azure Blob Storage account key",
	},
	cli.StringFlag{
		Name:  "credentials-file",
		Value: "",
		Usage: "path to Google Cloud Storage credentials file",
	},
	cli.StringFlag{
		Name:  "bucket",
		Value: "",
		Usage: "remote tier bucket",
	},
	cli.StringFlag{
		Name:  "prefix",
		Value: "",
		Usage: "remote tier prefix",
	},
	cli.StringFlag{
		Name:  "storage-class",
		Value: "",
		Usage: "remote tier storage-class",
	},
}

var adminTierAddCmd = cli.Command{
	Name:         "add",
	Usage:        "add a new remote tier target",
	Action:       mainAdminTierAdd,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        append(globalFlags, adminTierAddFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TYPE ALIAS NAME [FLAGS]

TYPE:
  Transition objects to supported cloud storage backend tier. Supported values are minio, s3, azure and gcs.

NAME:
  Name of the remote tier target. e.g WARM-TIER

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Configure a new remote tier which transitions objects to a bucket in a MinIO deployment:
     {{.Prompt}} {{.HelpName}} minio myminio WARM-MINIO-TIER --endpoint https://warm-minio.com \
        --access-key ACCESSKEY --secret-key SECRETKEY --bucket mybucket --prefix myprefix/

  2. Configure a new remote tier which transitions objects to a bucket in Azure Blob Storage:
     {{.Prompt}} {{.HelpName}} azure myminio AZTIER --account-name ACCOUNT-NAME --account-key ACCOUNT-KEY \
        --bucket myazurebucket --prefix myazureprefix/

  3. Configure a new remote tier which transitions objects to a bucket in AWS S3 with STANDARD storage class:
     {{.Prompt}} {{.HelpName}} s3 myminio S3TIER --endpoint https://s3.amazonaws.com \
        --access-key ACCESSKEY --secret-key SECRETKEY --bucket mys3bucket --prefix mys3prefix/ \
        --storage-class "STANDARD" --region us-west-2

  4. Configure a new remote tier which transitions objects to a bucket in Google Cloud Storage:
     {{.Prompt}} {{.HelpName}} gcs myminio GCSTIER --credentials-file /path/to/credentials.json \
        --bucket mygcsbucket  --prefix mygcsprefix/
`,
}

// checkAdminTierAddSyntax validates all the positional arguments
func checkAdminTierAddSyntax(ctx *cli.Context) {
	argsNr := len(ctx.Args())
	if argsNr < 3 {
		cli.ShowCommandHelpAndExit(ctx, ctx.Command.Name, 1) // last argument is exit code
	}
	if argsNr > 3 {
		fatalIf(errInvalidArgument().Trace(ctx.Args().Tail()...),
			"Incorrect number of arguments for tier add command.")
	}
}

const (
	s3Standard          = "STANDARD"
	s3ReducedRedundancy = "REDUCED_REDUNDANCY"
)

// fetchTierConfig returns a TierConfig given a tierName, a tierType and ctx to
// lookup command-line flags from. It exits with non-zero error code if any of
// the flags contain invalid values.
func fetchTierConfig(ctx *cli.Context, tierName string, tierType madmin.TierType) *madmin.TierConfig {
	switch tierType {
	case madmin.MinIO:
		accessKey := ctx.String("access-key")
		secretKey := ctx.String("secret-key")
		if accessKey == "" || secretKey == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires access credentials", tierType))
		}
		bucket := ctx.String("bucket")
		if bucket == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires target bucket", tierType))
		}

		endpoint := ctx.String("endpoint")
		if endpoint == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires target endpoint", tierType))
		}

		minioOpts := []madmin.MinIOOptions{}
		prefix := ctx.String("prefix")
		if prefix != "" {
			minioOpts = append(minioOpts, madmin.MinIOPrefix(prefix))
		}

		region := ctx.String("region")
		if region != "" {
			minioOpts = append(minioOpts, madmin.MinIORegion(region))
		}

		minioCfg, err := madmin.NewTierMinIO(tierName, endpoint, accessKey, secretKey, bucket, minioOpts...)
		if err != nil {
			fatalIf(probe.NewError(err), "Invalid configuration for MinIO tier")
		}

		return minioCfg

	case madmin.S3:
		accessKey := ctx.String("access-key")
		secretKey := ctx.String("secret-key")
		useAwsRole := ctx.IsSet("use-aws-role")
		if accessKey == "" && secretKey == "" && !useAwsRole {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires access credentials or AWS role", tierType))
		}
		if (accessKey != "" || secretKey != "") && useAwsRole {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires access credentials", tierType))
		}

		bucket := ctx.String("bucket")
		if bucket == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires target bucket", tierType))
		}

		s3Opts := []madmin.S3Options{}
		prefix := ctx.String("prefix")
		if prefix != "" {
			s3Opts = append(s3Opts, madmin.S3Prefix(prefix))
		}

		endpoint := ctx.String("endpoint")
		if endpoint != "" {
			s3Opts = append(s3Opts, madmin.S3Endpoint(endpoint))
		}

		region := ctx.String("region")
		if region != "" {
			s3Opts = append(s3Opts, madmin.S3Region(region))
		}

		s3SC := ctx.String("storage-class")
		if s3SC != "" {
			if s3SC != s3Standard && s3SC != s3ReducedRedundancy {
				fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("unsupported storage-class type %s", s3SC))
			}
			s3Opts = append(s3Opts, madmin.S3StorageClass(s3SC))
		}
		if ctx.IsSet("use-aws-role") {
			s3Opts = append(s3Opts, madmin.S3AWSRole())
		}
		s3Cfg, err := madmin.NewTierS3(tierName, accessKey, secretKey, bucket, s3Opts...)
		if err != nil {
			fatalIf(probe.NewError(err), "Invalid configuration for AWS S3 compatible remote tier")
		}

		return s3Cfg
	case madmin.Azure:
		accountName := ctx.String("account-name")
		accountKey := ctx.String("account-key")
		if accountName == "" || accountKey == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires access credentials", tierType))
		}

		bucket := ctx.String("bucket")
		if bucket == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote tier requires target bucket", tierType))
		}

		azOpts := []madmin.AzureOptions{}
		endpoint := ctx.String("endpoint")
		if endpoint != "" {
			azOpts = append(azOpts, madmin.AzureEndpoint(endpoint))
		}

		region := ctx.String("region")
		if region != "" {
			azOpts = append(azOpts, madmin.AzureRegion(region))
		}

		prefix := ctx.String("prefix")
		if prefix != "" {
			azOpts = append(azOpts, madmin.AzurePrefix(prefix))
		}

		azCfg, err := madmin.NewTierAzure(tierName, accountName, accountKey, bucket, azOpts...)
		if err != nil {
			fatalIf(probe.NewError(err), "Invalid configuration for Azure Blob Storage remote tier")
		}

		return azCfg
	case madmin.GCS:
		bucket := ctx.String("bucket")
		if bucket == "" {
			fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("%s remote requires target bucket", tierType))
		}

		gcsOpts := []madmin.GCSOptions{}
		prefix := ctx.String("prefix")
		if prefix != "" {
			gcsOpts = append(gcsOpts, madmin.GCSPrefix(prefix))
		}

		region := ctx.String("region")
		if region != "" {
			gcsOpts = append(gcsOpts, madmin.GCSRegion(region))
		}

		credsPath := ctx.String("credentials-file")
		credsBytes, err := ioutil.ReadFile(credsPath)
		if err != nil {
			fatalIf(probe.NewError(err), "Failed to read credentials file")
		}

		gcsCfg, err := madmin.NewTierGCS(tierName, credsBytes, bucket, gcsOpts...)
		if err != nil {
			fatalIf(probe.NewError(err), "Invalid configuration for Google Cloud Storage remote tier")
		}

		return gcsCfg
	}
	fatalIf(errInvalidArgument().Trace(), fmt.Sprintf("Invalid remote tier type %s", tierType))
	return nil
}

type tierMessage struct {
	op         string
	Status     string            `json:"status"`
	TierName   string            `json:"tierName"`
	TierType   string            `json:"tierType"`
	Endpoint   string            `json:"tierEndpoint"`
	Bucket     string            `json:"bucket"`
	Prefix     string            `json:"prefix,omitempty"`
	Region     string            `json:"region,omitempty"`
	TierParams map[string]string `json:"tierParams,omitempty"`
}

// String returns string representation of msg
func (msg *tierMessage) String() string {
	switch msg.op {
	case "add":
		addMsg := fmt.Sprintf("Added remote tier %s of type %s", msg.TierName, msg.TierType)
		return console.Colorize("TierMessage", addMsg)
	case "rm":
		rmMsg := fmt.Sprintf("Removed remote tier %s", msg.TierName)
		return console.Colorize("TierMessage", rmMsg)
	case "verify":
		verifyMsg := fmt.Sprintf("Verified remote tier %s", msg.TierName)
		return console.Colorize("TierMessage", verifyMsg)
	case "edit":
		editMsg := fmt.Sprintf("Updated remote tier %s", msg.TierName)
		return console.Colorize("TierMessage", editMsg)
	}
	return ""
}

// JSON returns json encoded msg
func (msg *tierMessage) JSON() string {
	jsonMessageBytes, e := json.MarshalIndent(msg, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(jsonMessageBytes)
}

// SetTierConfig sets TierConfig related fields
func (msg *tierMessage) SetTierConfig(sCfg *madmin.TierConfig) {
	msg.TierName = sCfg.Name
	msg.TierType = sCfg.Type.String()
	msg.Endpoint = sCfg.Endpoint()
	msg.Bucket = sCfg.Bucket()
	msg.Prefix = sCfg.Prefix()
	msg.Region = sCfg.Region()
	switch sCfg.Type {
	case madmin.S3:
		msg.TierParams = map[string]string{
			"storageClass": sCfg.S3.StorageClass,
		}
	}
}

func mainAdminTierAdd(ctx *cli.Context) error {
	checkAdminTierAddSyntax(ctx)

	console.SetColor("TierMessage", color.New(color.FgGreen))

	args := ctx.Args()
	tierTypeStr := args.Get(0)
	tierType, err := madmin.NewTierType(tierTypeStr)
	fatalIf(probe.NewError(err), "Unsupported tier type")

	aliasedURL := args.Get(1)
	tierName := args.Get(2)
	if tierName == "" {
		fatalIf(errInvalidArgument(), "Tier name can't be empty")
	}

	// Create a new MinIO Admin Client
	client, cerr := newAdminClient(aliasedURL)
	fatalIf(cerr, "Unable to initialize admin connection.")

	tCfg := fetchTierConfig(ctx, strings.ToUpper(tierName), tierType)
	if err = client.AddTier(globalContext, tCfg); err != nil {
		fatalIf(probe.NewError(err).Trace(args...), "Unable to configure remote tier target")
	}

	msg := &tierMessage{
		op:     "add",
		Status: "success",
	}
	msg.SetTierConfig(tCfg)
	printMsg(msg)
	return nil
}
<|endoftext|>