<commit_before>package collector

import (
	"testing"

	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	"github.com/smartystreets/goconvey/convey"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
)

func TestScrapeGlobalVariables(t *testing.T) {
	db, mock, err := sqlmock.New()
	if err != nil {
		t.Fatalf("error opening a stub database connection: %s", err)
	}
	defer db.Close()

	columns := []string{"Variable_name", "Value"}
	rows := sqlmock.NewRows(columns).
		AddRow("wait_timeout", "28800").
		AddRow("version_compile_os", "Linux").
		AddRow("userstat", "OFF").
		AddRow("transaction_prealloc_size", "4096").
		AddRow("tx_isolation", "REPEATABLE-READ").
		AddRow("tmp_table_size", "16777216").
		AddRow("tmpdir", "/tmp").
		AddRow("sync_binlog", "0").
		AddRow("sync_frm", "ON").
		AddRow("slow_launch_time", "2").
		AddRow("innodb_version", "5.6.30-76.3").
		AddRow("version", "5.6.30-76.3-56").
		AddRow("version_comment", "Percona XtraDB Cluster...").
		AddRow("wsrep_cluster_name", "supercluster").
		AddRow("wsrep_provider_options", "base_dir = /var/lib/mysql/; base_host = 10.91.142.82; base_port = 4567; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.info_log_mask = 0; evs.install_timeout = PT7.5S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; gcache.dir = /var/lib/mysql/; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /var/lib/mysql//galera.cache; gcache.page_size = 128M; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.listen_addr = tcp://0.0.0.0:4567; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; ist.recv_addr = 10.91.142.82; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992;")
	mock.ExpectQuery(globalVariablesQuery).WillReturnRows(rows)

	ch := make(chan prometheus.Metric)
	go func() {
		if err = ScrapeGlobalVariables(db, ch); err != nil {
			t.Errorf("error calling function on test: %s", err)
		}
		close(ch)
	}()

	counterExpected := []MetricResult{
		{labels: labelMap{}, value: 28800, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 0, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 4096, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 16777216, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 0, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 1, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 2, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{"innodb_version": "5.6.30-76.3", "version": "5.6.30-76.3-56", "version_comment": "Percona XtraDB Cluster..."}, value: 1, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{"wsrep_cluster_name": "supercluster"}, value: 1, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 134217728, metricType: dto.MetricType_GAUGE},
	}
	convey.Convey("Metrics comparison", t, func() {
		for _, expect := range counterExpected {
			got := readMetric(<-ch)
			convey.So(got, convey.ShouldResemble, expect)
		}
	})

	// Ensure all SQL queries were executed
	if err := mock.ExpectationsWereMet(); err != nil {
		t.Errorf("there were unfulfilled expections: %s", err)
	}
}

func TestParseWsrepProviderOptions(t *testing.T) {
	testE := ""
	testM := "base_dir = /var/lib/mysql/; base_host = 10.91.142.82; base_port = 4567; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.info_log_mask = 0; evs.install_timeout = PT7.5S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; gcache.dir = /var/lib/mysql/; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /var/lib/mysql//galera.cache; gcache.page_size = 128M; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.listen_addr = tcp://0.0.0.0:4567; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; ist.recv_addr = 10.91.142.82; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992;"
	testG := "base_dir = /var/lib/mysql/; base_host = 10.91.194.244; base_port = 4567; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.info_log_mask = 0; evs.install_timeout = PT7.5S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; gcache.dir = /var/lib/mysql/; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /var/lib/mysql//galera.cache; gcache.page_size = 128M; gcache.size = 2G; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.listen_addr = tcp://0.0.0.0:4567; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; ist.recv_addr = 10.91.194.244; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992;"
	testB := "gcache.page_size = 128M; gcache.size = 131072; gcomm.thread_prio = ;"
	convey.Convey("Parse wsrep_provider_options", t, func() {
		convey.So(parseWsrepProviderOptions(testE), convey.ShouldEqual, 0)
		convey.So(parseWsrepProviderOptions(testM), convey.ShouldEqual, 128*1024*1024)
		convey.So(parseWsrepProviderOptions(testG), convey.ShouldEqual, 2*1024*1024*1024)
		convey.So(parseWsrepProviderOptions(testB), convey.ShouldEqual, 131072)
	})
}
<commit_msg>Fix 32bit compile issue (#273)<commit_after>package collector

import (
	"testing"

	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	"github.com/smartystreets/goconvey/convey"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
)

func TestScrapeGlobalVariables(t *testing.T) {
	db, mock, err := sqlmock.New()
	if err != nil {
		t.Fatalf("error opening a stub database connection: %s", err)
	}
	defer db.Close()

	columns := []string{"Variable_name", "Value"}
	rows := sqlmock.NewRows(columns).
		AddRow("wait_timeout", "28800").
		AddRow("version_compile_os", "Linux").
		AddRow("userstat", "OFF").
		AddRow("transaction_prealloc_size", "4096").
		AddRow("tx_isolation", "REPEATABLE-READ").
		AddRow("tmp_table_size", "16777216").
		AddRow("tmpdir", "/tmp").
		AddRow("sync_binlog", "0").
		AddRow("sync_frm", "ON").
		AddRow("slow_launch_time", "2").
		AddRow("innodb_version", "5.6.30-76.3").
		AddRow("version", "5.6.30-76.3-56").
		AddRow("version_comment", "Percona XtraDB Cluster...").
		AddRow("wsrep_cluster_name", "supercluster").
		AddRow("wsrep_provider_options", "base_dir = /var/lib/mysql/; base_host = 10.91.142.82; base_port = 4567; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.info_log_mask = 0; evs.install_timeout = PT7.5S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; gcache.dir = /var/lib/mysql/; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /var/lib/mysql//galera.cache; gcache.page_size = 128M; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.listen_addr = tcp://0.0.0.0:4567; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; ist.recv_addr = 10.91.142.82; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992;")
	mock.ExpectQuery(globalVariablesQuery).WillReturnRows(rows)

	ch := make(chan prometheus.Metric)
	go func() {
		if err = ScrapeGlobalVariables(db, ch); err != nil {
			t.Errorf("error calling function on test: %s", err)
		}
		close(ch)
	}()

	counterExpected := []MetricResult{
		{labels: labelMap{}, value: 28800, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 0, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 4096, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 16777216, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 0, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 1, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 2, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{"innodb_version": "5.6.30-76.3", "version": "5.6.30-76.3-56", "version_comment": "Percona XtraDB Cluster..."}, value: 1, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{"wsrep_cluster_name": "supercluster"}, value: 1, metricType: dto.MetricType_GAUGE},
		{labels: labelMap{}, value: 134217728, metricType: dto.MetricType_GAUGE},
	}
	convey.Convey("Metrics comparison", t, func() {
		for _, expect := range counterExpected {
			got := readMetric(<-ch)
			convey.So(got, convey.ShouldResemble, expect)
		}
	})

	// Ensure all SQL queries were executed
	if err := mock.ExpectationsWereMet(); err != nil {
		t.Errorf("there were unfulfilled expections: %s", err)
	}
}

func TestParseWsrepProviderOptions(t *testing.T) {
	testE := ""
	testM := "base_dir = /var/lib/mysql/; base_host = 10.91.142.82; base_port = 4567; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.info_log_mask = 0; evs.install_timeout = PT7.5S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; gcache.dir = /var/lib/mysql/; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /var/lib/mysql//galera.cache; gcache.page_size = 128M; gcache.size = 128M; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.listen_addr = tcp://0.0.0.0:4567; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; ist.recv_addr = 10.91.142.82; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992;"
	testG := "base_dir = /var/lib/mysql/; base_host = 10.91.194.244; base_port = 4567; cert.log_conflicts = no; debug = no; evs.auto_evict = 0; evs.causal_keepalive_period = PT1S; evs.debug_log_mask = 0x1; evs.delay_margin = PT1S; evs.delayed_keep_period = PT30S; evs.inactive_check_period = PT0.5S; evs.inactive_timeout = PT15S; evs.info_log_mask = 0; evs.install_timeout = PT7.5S; evs.join_retrans_period = PT1S; evs.keepalive_period = PT1S; evs.max_install_timeouts = 3; evs.send_window = 4; evs.stats_report_period = PT1M; evs.suspect_timeout = PT5S; evs.use_aggregate = true; evs.user_send_window = 2; evs.version = 0; evs.view_forget_timeout = P1D; gcache.dir = /var/lib/mysql/; gcache.keep_pages_count = 0; gcache.keep_pages_size = 0; gcache.mem_size = 0; gcache.name = /var/lib/mysql//galera.cache; gcache.page_size = 128M; gcache.size = 2G; gcomm.thread_prio = ; gcs.fc_debug = 0; gcs.fc_factor = 1.0; gcs.fc_limit = 16; gcs.fc_master_slave = no; gcs.max_packet_size = 64500; gcs.max_throttle = 0.25; gcs.recv_q_hard_limit = 9223372036854775807; gcs.recv_q_soft_limit = 0.25; gcs.sync_donor = no; gmcast.listen_addr = tcp://0.0.0.0:4567; gmcast.mcast_addr = ; gmcast.mcast_ttl = 1; gmcast.peer_timeout = PT3S; gmcast.segment = 0; gmcast.time_wait = PT5S; gmcast.version = 0; ist.recv_addr = 10.91.194.244; pc.announce_timeout = PT3S; pc.checksum = false; pc.ignore_quorum = false; pc.ignore_sb = false; pc.linger = PT20S; pc.npvo = false; pc.recovery = true; pc.version = 0; pc.wait_prim = true; pc.wait_prim_timeout = P30S; pc.weight = 1; protonet.backend = asio; protonet.version = 0; repl.causal_read_timeout = PT30S; repl.commit_order = 3; repl.key_format = FLAT8; repl.max_ws_size = 2147483647; repl.proto_max = 7; socket.checksum = 2; socket.recv_buf_size = 212992;"
	testB := "gcache.page_size = 128M; gcache.size = 131072; gcomm.thread_prio = ;"
	convey.Convey("Parse wsrep_provider_options", t, func() {
		convey.So(parseWsrepProviderOptions(testE), convey.ShouldEqual, 0)
		convey.So(parseWsrepProviderOptions(testM), convey.ShouldEqual, 128*1024*1024)
		convey.So(parseWsrepProviderOptions(testG), convey.ShouldEqual, int64(2*1024*1024*1024))
		convey.So(parseWsrepProviderOptions(testB), convey.ShouldEqual, 131072)
	})
}
<|endoftext|>


<commit_before><commit_msg>api/keymanager<commit_after><|endoftext|>


<commit_before>package backup

import (
	"archive/tar"
	"context"
	"emperror.dev/errors"
	"github.com/apex/log"
	gzip "github.com/klauspost/pgzip"
	"github.com/remeh/sizedwaitgroup"
	"golang.org/x/sync/errgroup"
	"io"
	"os"
	"runtime"
	"strings"
	"sync"
)

type Archive struct {
	sync.Mutex

	TrimPrefix string
	Files      *IncludedFiles
}

// Creates an archive at dst with all of the files defined in the included files struct.
func (a *Archive) Create(dst string, ctx context.Context) error {
	f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return errors.WithStackIf(err)
	}
	defer f.Close()

	maxCpu := runtime.NumCPU() / 2
	if maxCpu > 4 {
		maxCpu = 4
	}

	gzw, _ := gzip.NewWriterLevel(f, gzip.BestSpeed)
	_ = gzw.SetConcurrency(1<<20, maxCpu)

	defer gzw.Flush()
	defer gzw.Close()

	tw := tar.NewWriter(gzw)
	defer tw.Flush()
	defer tw.Close()

	wg := sizedwaitgroup.New(10)
	g, ctx := errgroup.WithContext(ctx)
	// Iterate over all of the files to be included and put them into the archive. This is
	// done as a concurrent goroutine to speed things along. If an error is encountered at
	// any step, the entire process is aborted.
	for _, p := range a.Files.All() {
		p := p
		g.Go(func() error {
			wg.Add()
			defer wg.Done()

			select {
			case <-ctx.Done():
				return errors.WithStackIf(ctx.Err())
			default:
				return a.addToArchive(p, tw)
			}
		})
	}

	// Block until the entire routine is completed.
	if err := g.Wait(); err != nil {
		f.Close()

		// Attempt to remove the archive if there is an error, report that error to
		// the logger if it fails.
		if rerr := os.Remove(dst); rerr != nil && !os.IsNotExist(rerr) {
			log.WithField("location", dst).Warn("failed to delete corrupted backup archive")
		}

		return errors.WithStackIf(err)
	}

	return nil
}

// Adds a single file to the existing tar archive writer.
func (a *Archive) addToArchive(p string, w *tar.Writer) error {
	f, err := os.Open(p)
	if err != nil {
		// If you try to backup something that no longer exists (got deleted somewhere during the process
		// but not by this process), just skip over it and don't kill the entire backup.
		if os.IsNotExist(err) {
			return nil
		}

		return errors.WithStackIf(err)
	}
	defer f.Close()

	s, err := f.Stat()
	if err != nil {
		// Same as above, don't kill the process just because the file no longer exists.
		if os.IsNotExist(err) {
			return nil
		}

		return errors.WithStackIf(err)
	}

	header := &tar.Header{
		// Trim the long server path from the name of the file so that the resulting
		// archive is exactly how the user would see it in the panel file manager.
		Name:    strings.TrimPrefix(p, a.TrimPrefix),
		Size:    s.Size(),
		Mode:    int64(s.Mode()),
		ModTime: s.ModTime(),
	}

	// These actions must occur sequentially, even if this function is called multiple
	// in parallel. You'll get some nasty panic's otherwise.
	a.Lock()
	defer a.Unlock()

	if err := w.WriteHeader(header); err != nil {
		return errors.WithStackIf(err)
	}

	buf := make([]byte, 4*1024)
	if _, err := io.CopyBuffer(w, f, buf); err != nil {
		return errors.WithStackIf(err)
	}

	return nil
}
<commit_msg>Attempt to fix 'archive/tar: write too long' error when creating a backup<commit_after>package backup

import (
	"archive/tar"
	"context"
	"emperror.dev/errors"
	"github.com/apex/log"
	gzip "github.com/klauspost/pgzip"
	"github.com/remeh/sizedwaitgroup"
	"golang.org/x/sync/errgroup"
	"io"
	"os"
	"runtime"
	"strings"
	"sync"
)

type Archive struct {
	sync.Mutex

	TrimPrefix string
	Files      *IncludedFiles
}

// Creates an archive at dst with all of the files defined in the included files struct.
func (a *Archive) Create(dst string, ctx context.Context) error {
	f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
	if err != nil {
		return errors.WithStackIf(err)
	}
	defer f.Close()

	maxCpu := runtime.NumCPU() / 2
	if maxCpu > 4 {
		maxCpu = 4
	}

	gzw, _ := gzip.NewWriterLevel(f, gzip.BestSpeed)
	_ = gzw.SetConcurrency(1<<20, maxCpu)

	defer gzw.Flush()
	defer gzw.Close()

	tw := tar.NewWriter(gzw)
	defer tw.Flush()
	defer tw.Close()

	wg := sizedwaitgroup.New(10)
	g, ctx := errgroup.WithContext(ctx)
	// Iterate over all of the files to be included and put them into the archive. This is
	// done as a concurrent goroutine to speed things along. If an error is encountered at
	// any step, the entire process is aborted.
	for _, p := range a.Files.All() {
		p := p
		g.Go(func() error {
			wg.Add()
			defer wg.Done()

			select {
			case <-ctx.Done():
				return errors.WithStackIf(ctx.Err())
			default:
				return a.addToArchive(p, tw)
			}
		})
	}

	// Block until the entire routine is completed.
	if err := g.Wait(); err != nil {
		f.Close()

		// Attempt to remove the archive if there is an error, report that error to
		// the logger if it fails.
		if rerr := os.Remove(dst); rerr != nil && !os.IsNotExist(rerr) {
			log.WithField("location", dst).Warn("failed to delete corrupted backup archive")
		}

		return errors.WithStackIf(err)
	}

	return nil
}

// Adds a single file to the existing tar archive writer.
func (a *Archive) addToArchive(p string, w *tar.Writer) error {
	f, err := os.Open(p)
	if err != nil {
		// If you try to backup something that no longer exists (got deleted somewhere during the process
		// but not by this process), just skip over it and don't kill the entire backup.
		if os.IsNotExist(err) {
			return nil
		}

		return errors.WithStackIf(err)
	}
	defer f.Close()

	s, err := f.Stat()
	if err != nil {
		// Same as above, don't kill the process just because the file no longer exists.
		if os.IsNotExist(err) {
			return nil
		}

		return errors.WithStackIf(err)
	}

	header, err := tar.FileInfoHeader(s, strings.TrimPrefix(p, a.TrimPrefix))
	if err != nil {
		return errors.WithStackIf(err)
	}

	// These actions must occur sequentially, even if this function is called multiple
	// in parallel. You'll get some nasty panic's otherwise.
	a.Lock()
	defer a.Unlock()

	if err := w.WriteHeader(header); err != nil {
		return errors.WithStackIf(err)
	}

	buf := make([]byte, 4*1024)
	if _, err := io.CopyBuffer(w, f, buf); err != nil {
		return errors.WithStackIf(err)
	}

	return nil
}
<|endoftext|>


<commit_before>package repositoriesmanager

import (
	"fmt"
	"time"

	"github.com/go-gorp/gorp"
	"github.com/mitchellh/mapstructure"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/database"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

//ReceiveEvents has to be launched as a goroutine.
func ReceiveEvents() {
	for {
		e := sdk.Event{}
		cache.Dequeue("events_repositoriesmanager", &e)
		db := database.DBMap(database.DB())
		if db != nil {
			if err := processEvent(db, e); err != nil {
				log.Error("ReceiveEvents> err while processing %s : %v", err, e)
				retryEvent(&e)
			}
			continue
		}
		retryEvent(&e)
	}
}

func retryEvent(e *sdk.Event) {
	e.Attempts++
	if e.Attempts >= 3 {
		log.Error("ReceiveEvents> Aborting event processing %v", e)
		return
	}
	time.Sleep(5 * time.Second)
	cache.Enqueue("events_repositoriesmanager", e)
}

func processEvent(db gorp.SqlExecutor, event sdk.Event) error {
	log.Debug("repositoriesmanager>processEvent> receive: type:%s all: %+v", event.EventType, event)

	if event.EventType != fmt.Sprintf("%T", sdk.EventPipelineBuild{}) {
		return nil
	}

	var eventpb sdk.EventPipelineBuild
	if err := mapstructure.Decode(event.Payload, &eventpb); err != nil {
		log.Error("Error during consumption: %s", err)
		return err
	}

	if eventpb.RepositoryManagerName == "" {
		return nil
	}

	log.Debug("repositoriesmanager>processEvent> event:%+v", event)

	c, erra := AuthorizedClient(db, eventpb.ProjectKey, eventpb.RepositoryManagerName)
	if erra != nil {
		return fmt.Errorf("repositoriesmanager>processEvent> AuthorizedClient (%s, %s) > err:%s", eventpb.ProjectKey, eventpb.RepositoryManagerName, erra)
	}

	if err := c.SetStatus(event); err != nil {
		retryEvent(&event)
		return fmt.Errorf("repositoriesmanager>processEvent> SetStatus > err:%s", err)
	}

	retryEvent(&event)

	return nil
}
<commit_msg>fix (api): remove sleep before retry (#559)<commit_after>package repositoriesmanager

import (
	"fmt"

	"github.com/go-gorp/gorp"
	"github.com/mitchellh/mapstructure"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/database"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

//ReceiveEvents has to be launched as a goroutine.
func ReceiveEvents() {
	for {
		e := sdk.Event{}
		cache.Dequeue("events_repositoriesmanager", &e)
		db := database.DBMap(database.DB())
		if db != nil {
			if err := processEvent(db, e); err != nil {
				log.Error("ReceiveEvents> err while processing %s : %v", err, e)
				retryEvent(&e)
			}
			continue
		}
		retryEvent(&e)
	}
}

func retryEvent(e *sdk.Event) {
	e.Attempts++
	if e.Attempts > 2 {
		log.Error("ReceiveEvents> Aborting event processing %v", e)
		return
	}
	cache.Enqueue("events_repositoriesmanager", e)
}

func processEvent(db gorp.SqlExecutor, event sdk.Event) error {
	log.Debug("repositoriesmanager>processEvent> receive: type:%s all: %+v", event.EventType, event)

	if event.EventType != fmt.Sprintf("%T", sdk.EventPipelineBuild{}) {
		return nil
	}

	var eventpb sdk.EventPipelineBuild
	if err := mapstructure.Decode(event.Payload, &eventpb); err != nil {
		log.Error("Error during consumption: %s", err)
		return err
	}

	if eventpb.RepositoryManagerName == "" {
		return nil
	}

	log.Debug("repositoriesmanager>processEvent> event:%+v", event)

	c, erra := AuthorizedClient(db, eventpb.ProjectKey, eventpb.RepositoryManagerName)
	if erra != nil {
		return fmt.Errorf("repositoriesmanager>processEvent> AuthorizedClient (%s, %s) > err:%s", eventpb.ProjectKey, eventpb.RepositoryManagerName, erra)
	}

	if err := c.SetStatus(event); err != nil {
		retryEvent(&event)
		return fmt.Errorf("repositoriesmanager>processEvent> SetStatus > err:%s", err)
	}

	retryEvent(&event)

	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"bufio"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"

	getter "github.com/hashicorp/go-getter"
	"github.com/mitchellh/cli"
)

func init() {
	getter.Getters["file"].(*getter.FileGetter).Copy = true
}

func ProvisionCommandFactory() (cli.Command, error) {
	return &Provision{}, nil
}

type Provision struct {
}

func (c *Provision) Help() string {
	helpText := `
Usage: nomad-e2e provision <provider> <environment>

  Uses terraform to provision a target test environment to use
  for end-to-end testing.

  The output is a list of environment variables used to configure
  various api clients such as Nomad, Consul and Vault.

Provision Options:

  -env-path
    Sets the path for where to search for test environment configuration.
	This defaults to './environments/'.

  -nomad-binary
    Sets the target nomad-binary to use when provisioning a nomad cluster.
	The binary is retrieved by go-getter and can therefore be a local file
	path, remote http url, or other support go-getter uri.

  -nomad-checksum
    If set, will ensure the binary from -nomad-binary matches the given
	checksum.

  -destroy
    If set, will destroy the target environment.

  -tf-path
    Sets the path for which terraform state files are stored. Defaults to
	the current working directory.
`
	return strings.TrimSpace(helpText)
}

func (c *Provision) Synopsis() string {
	return "Provisions the target testing environment"
}

func (c *Provision) Run(args []string) int {
	var envPath string
	var nomadBinary string
	var destroy bool
	var tfPath string
	cmdFlags := flag.NewFlagSet("provision", flag.ContinueOnError)
	cmdFlags.Usage = func() { log.Println(c.Help()) }
	cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs")
	cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "")
	cmdFlags.BoolVar(&destroy, "destroy", false, "")
	cmdFlags.StringVar(&tfPath, "tf-path", "", "")

	if err := cmdFlags.Parse(args); err != nil {
		log.Fatalf("failed to parse flags: %v", err)
	}

	args = cmdFlags.Args()
	if len(args) != 2 {
		log.Fatalf("expected 2 args, but got: %v", args)
		log.Fatal(c.Help())
	}

	env, err := newEnv(envPath, args[0], args[1], tfPath)
	if err != nil {
		log.Fatal(err)
	}

	if destroy {
		if err := env.destroy(); err != nil {
			log.Fatal(err)
			return 1
		}
		fmt.Println("Environment successfully destroyed")
		return 0
	}

	// Use go-getter to fetch the nomad binary
	nomadPath, err := c.fetchBinary(nomadBinary)
	defer os.RemoveAll(nomadPath)

	results, err := env.provision(nomadPath)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf(strings.TrimSpace(`
NOMAD_ADDR=%s
	`), results.nomadAddr)

	return 0
}

// Fetches the nomad binary and returns the temporary directory where it exists
func (c *Provision) fetchBinary(bin string) (string, error) {
	nomadBinaryDir, err := ioutil.TempDir("", "")
	if err != nil {
		return "", fmt.Errorf("failed to create temp dir: %v", err)
	}

	if err = getter.GetFile(path.Join(nomadBinaryDir, "nomad"), bin); err != nil {
		return "", fmt.Errorf("failed to get nomad binary: %v", err)
	}

	return nomadBinaryDir, nil
}

type environment struct {
	path     string
	provider string
	name     string

	tf      string
	tfPath  string
	tfState string
}

type envResults struct {
	nomadAddr string
}

func newEnv(envPath, provider, name, tfStatePath string) (*environment, error) {
	// Make sure terraform is on the PATH
	tf, err := exec.LookPath("terraform")
	if err != nil {
		return nil, fmt.Errorf("failed to lookup terraform binary: %v", err)
	}

	// set the path to the terraform module
	tfPath := path.Join(envPath, provider, name)
	log.Printf("[DEBUG] provision: using tf path %s", tfPath)
	if _, err := os.Stat(tfPath); os.IsNotExist(err) {
		return nil, fmt.Errorf("failed to lookup terraform configuration dir %s: %v", tfPath, err)
	}

	// set the path to state file
	tfState := path.Join(tfStatePath, fmt.Sprintf("e2e.%s.%s.tfstate", provider, name))

	env := &environment{
		path:     envPath,
		provider: provider,
		name:     name,
		tf:       tf,
		tfPath:   tfPath,
		tfState:  tfState,
	}
	return env, nil
}

// provision calls terraform to setup the environment with the given nomad binary
func (env *environment) provision(nomadPath string) (*envResults, error) {
	tfArgs := []string{"apply", "-auto-approve", "-input=false", "-no-color",
		"-state", env.tfState,
		"-var", fmt.Sprintf("nomad_binary=%s", path.Join(nomadPath, "nomad")),
		env.tfPath,
	}

	// Setup the 'terraform apply' command
	ctx := context.Background()
	cmd := exec.CommandContext(ctx, env.tf, tfArgs...)

	// Funnel the stdout/stderr to logging
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, fmt.Errorf("failed to get stderr pipe: %v", err)
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("failed to get stdout pipe: %v", err)
	}

	// Run 'terraform apply'
	cmd.Start()
	go tfLog("tf.stderr", stderr)
	go tfLog("tf.stdout", stdout)

	err = cmd.Wait()
	if err != nil {
		return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err)
	}

	// Setup and run 'terraform output' to get the module output
	cmd = exec.CommandContext(ctx, env.tf, "output", "-json", "-state", env.tfState)
	out, err := cmd.Output()
	if err != nil {
		return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err)
	}

	// Parse the json and pull out results
	tfOutput := make(map[string]map[string]interface{})
	err = json.Unmarshal(out, &tfOutput)
	if err != nil {
		return nil, fmt.Errorf("failed to parse terraform output: %v", err)
	}

	results := &envResults{}
	if nomadAddr, ok := tfOutput["nomad_addr"]; ok {
		results.nomadAddr = nomadAddr["value"].(string)
	}

	return results, nil
}

//destroy calls terraform to destroy the environment
func (env *environment) destroy() error {
	tfArgs := []string{"destroy", "-auto-approve", "-no-color",
		"-state", env.tfState,
		"-var", "nomad_binary=",
		env.tfPath,
	}
	cmd := exec.Command(env.tf, tfArgs...)

	// Funnel the stdout/stderr to logging
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return fmt.Errorf("failed to get stderr pipe: %v", err)
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("failed to get stdout pipe: %v", err)
	}

	// Run 'terraform destroy'
	cmd.Start()
	go tfLog("tf.stderr", stderr)
	go tfLog("tf.stdout", stdout)

	err = cmd.Wait()
	if err != nil {
		return fmt.Errorf("terraform exited with a non-zero status: %v", err)
	}

	return nil
}

func tfLog(prefix string, r io.ReadCloser) {
	defer r.Close()
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		log.Printf("[DEBUG] provision.%s: %s", prefix, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Printf("[WARN] provision.%s: %v", err)
	}

}
<commit_msg>e2e/cli: use discover utility for nomad binary<commit_after>package command

import (
	"bufio"
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"

	getter "github.com/hashicorp/go-getter"
	"github.com/hashicorp/nomad/helper/discover"
	"github.com/mitchellh/cli"
)

func init() {
	getter.Getters["file"].(*getter.FileGetter).Copy = true
}

func ProvisionCommandFactory() (cli.Command, error) {
	return &Provision{}, nil
}

type Provision struct {
}

func (c *Provision) Help() string {
	helpText := `
Usage: nomad-e2e provision <provider> <environment>

  Uses terraform to provision a target test environment to use
  for end-to-end testing.

  The output is a list of environment variables used to configure
  various api clients such as Nomad, Consul and Vault.

Provision Options:

  -env-path
    Sets the path for where to search for test environment configuration.
	This defaults to './environments/'.

  -nomad-binary
    Sets the target nomad-binary to use when provisioning a nomad cluster.
	The binary is retrieved by go-getter and can therefore be a local file
	path, remote http url, or other support go-getter uri.

  -nomad-checksum
    If set, will ensure the binary from -nomad-binary matches the given
	checksum.

  -destroy
    If set, will destroy the target environment.

  -tf-path
    Sets the path for which terraform state files are stored. Defaults to
	the current working directory.
`
	return strings.TrimSpace(helpText)
}

func (c *Provision) Synopsis() string {
	return "Provisions the target testing environment"
}

func (c *Provision) Run(args []string) int {
	var envPath string
	var nomadBinary string
	var destroy bool
	var tfPath string
	cmdFlags := flag.NewFlagSet("provision", flag.ContinueOnError)
	cmdFlags.Usage = func() { log.Println(c.Help()) }
	cmdFlags.StringVar(&envPath, "env-path", "./environments/", "Path to e2e environment terraform configs")
	cmdFlags.StringVar(&nomadBinary, "nomad-binary", "", "")
	cmdFlags.BoolVar(&destroy, "destroy", false, "")
	cmdFlags.StringVar(&tfPath, "tf-path", "", "")

	if err := cmdFlags.Parse(args); err != nil {
		log.Fatalf("failed to parse flags: %v", err)
	}

	args = cmdFlags.Args()
	if len(args) != 2 {
		log.Fatalf("expected 2 args, but got: %v", args)
		log.Fatal(c.Help())
	}

	env, err := newEnv(envPath, args[0], args[1], tfPath)
	if err != nil {
		log.Fatal(err)
	}

	if destroy {
		if err := env.destroy(); err != nil {
			log.Fatal(err)
			return 1
		}
		fmt.Println("Environment successfully destroyed")
		return 0
	}

	// Use go-getter to fetch the nomad binary
	nomadPath, err := c.fetchBinary(nomadBinary)
	defer os.RemoveAll(nomadPath)

	results, err := env.provision(nomadPath)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf(strings.TrimSpace(`
NOMAD_ADDR=%s
	`), results.nomadAddr)

	return 0
}

// Fetches the nomad binary and returns the temporary directory where it exists
func (c *Provision) fetchBinary(bin string) (string, error) {
	nomadBinaryDir, err := ioutil.TempDir("", "")
	if err != nil {
		return "", fmt.Errorf("failed to create temp dir: %v", err)
	}

	if bin == "" {
		bin, err = discover.NomadExecutable()
		if err != nil {
			return "", fmt.Errorf("failed to discover nomad binary: %v", err)
		}
	}
	if err = getter.GetFile(path.Join(nomadBinaryDir, "nomad"), bin); err != nil {
		return "", fmt.Errorf("failed to get nomad binary: %v", err)
	}

	return nomadBinaryDir, nil
}

type environment struct {
	path     string
	provider string
	name     string

	tf      string
	tfPath  string
	tfState string
}

type envResults struct {
	nomadAddr string
}

func newEnv(envPath, provider, name, tfStatePath string) (*environment, error) {
	// Make sure terraform is on the PATH
	tf, err := exec.LookPath("terraform")
	if err != nil {
		return nil, fmt.Errorf("failed to lookup terraform binary: %v", err)
	}

	// set the path to the terraform module
	tfPath := path.Join(envPath, provider, name)
	log.Printf("[DEBUG] provision: using tf path %s", tfPath)
	if _, err := os.Stat(tfPath); os.IsNotExist(err) {
		return nil, fmt.Errorf("failed to lookup terraform configuration dir %s: %v", tfPath, err)
	}

	// set the path to state file
	tfState := path.Join(tfStatePath, fmt.Sprintf("e2e.%s.%s.tfstate", provider, name))

	env := &environment{
		path:     envPath,
		provider: provider,
		name:     name,
		tf:       tf,
		tfPath:   tfPath,
		tfState:  tfState,
	}
	return env, nil
}

// provision calls terraform to setup the environment with the given nomad binary
func (env *environment) provision(nomadPath string) (*envResults, error) {
	tfArgs := []string{"apply", "-auto-approve", "-input=false", "-no-color",
		"-state", env.tfState,
		"-var", fmt.Sprintf("nomad_binary=%s", path.Join(nomadPath, "nomad")),
		env.tfPath,
	}

	// Setup the 'terraform apply' command
	ctx := context.Background()
	cmd := exec.CommandContext(ctx, env.tf, tfArgs...)

	// Funnel the stdout/stderr to logging
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, fmt.Errorf("failed to get stderr pipe: %v", err)
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, fmt.Errorf("failed to get stdout pipe: %v", err)
	}

	// Run 'terraform apply'
	cmd.Start()
	go tfLog("tf.stderr", stderr)
	go tfLog("tf.stdout", stdout)

	err = cmd.Wait()
	if err != nil {
		return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err)
	}

	// Setup and run 'terraform output' to get the module output
	cmd = exec.CommandContext(ctx, env.tf, "output", "-json", "-state", env.tfState)
	out, err := cmd.Output()
	if err != nil {
		return nil, fmt.Errorf("terraform exited with a non-zero status: %v", err)
	}

	// Parse the json and pull out results
	tfOutput := make(map[string]map[string]interface{})
	err = json.Unmarshal(out, &tfOutput)
	if err != nil {
		return nil, fmt.Errorf("failed to parse terraform output: %v", err)
	}

	results := &envResults{}
	if nomadAddr, ok := tfOutput["nomad_addr"]; ok {
		results.nomadAddr = nomadAddr["value"].(string)
	}

	return results, nil
}

//destroy calls terraform to destroy the environment
func (env *environment) destroy() error {
	tfArgs := []string{"destroy", "-auto-approve", "-no-color",
		"-state", env.tfState,
		"-var", "nomad_binary=",
		env.tfPath,
	}
	cmd := exec.Command(env.tf, tfArgs...)

	// Funnel the stdout/stderr to logging
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return fmt.Errorf("failed to get stderr pipe: %v", err)
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return fmt.Errorf("failed to get stdout pipe: %v", err)
	}

	// Run 'terraform destroy'
	cmd.Start()
	go tfLog("tf.stderr", stderr)
	go tfLog("tf.stdout", stdout)

	err = cmd.Wait()
	if err != nil {
		return fmt.Errorf("terraform exited with a non-zero status: %v", err)
	}

	return nil
}

func tfLog(prefix string, r io.ReadCloser) {
	defer r.Close()
	scanner := bufio.NewScanner(r)
	for scanner.Scan() {
		log.Printf("[DEBUG] provision.%s: %s", prefix, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		log.Printf("[WARN] provision.%s: %v", err)
	}

}
<|endoftext|>


<commit_before>package workflow

import (
	"bytes"
	"context"
	"encoding/base64"
	"io"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/environment"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/api/pipeline"
	"github.com/ovh/cds/engine/api/workflowtemplate"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/exportentities"
)

// Export a workflow
func Export(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format, u *sdk.User, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	ctx, end := observability.Span(ctx, "workflow.Export")
	defer end()

	wf, errload := Load(ctx, db, cache, proj, name, u, LoadOptions{})
	if errload != nil {
		return 0, sdk.WrapError(errload, "workflow.Export> Cannot load workflow %s", name)
	}

	// If repo is from as-code do not export WorkflowSkipIfOnlyOneRepoWebhook
	if wf.FromRepository != "" {
		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
	}

	return exportWorkflow(*wf, f, w, opts...)
}

func exportWorkflow(wf sdk.Workflow, f exportentities.Format, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	e, err := exportentities.NewWorkflow(wf, opts...)
	if err != nil {
		return 0, sdk.WrapError(err, "exportWorkflow")
	}

	// Useful to not display history_length in yaml or json if it's his default value
	if e.HistoryLength != nil && *e.HistoryLength == sdk.DefaultHistoryLength {
		e.HistoryLength = nil
	}

	// Marshal to the desired format
	b, err := exportentities.Marshal(e, f)
	if err != nil {
		return 0, sdk.WithStack(err)
	}

	return w.Write(b)
}

// Pull a workflow with all it dependencies; it writes a tar buffer in the writer
func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format,
	encryptFunc sdk.EncryptFunc, u *sdk.User, opts ...exportentities.WorkflowOptions) (exportentities.WorkflowPulled, error) {
	ctx, end := observability.Span(ctx, "workflow.Pull")
	defer end()

	var wp exportentities.WorkflowPulled

	options := LoadOptions{
		DeepPipeline: true,
	}
	wf, errload := Load(ctx, db, cache, proj, name, u, options)
	if errload != nil {
		return wp, sdk.WrapError(errload, "cannot load workflow %s", name)
	}

	i, err := workflowtemplate.GetInstanceByWorkflowID(db, wf.ID)
	if err != nil {
		return wp, err
	}
	if i != nil {
		wf.Template, err = workflowtemplate.GetByID(db, i.WorkflowTemplateID)
		if err != nil {
			return wp, err
		}
		if err := workflowtemplate.AggregateOnWorkflowTemplate(db, wf.Template); err != nil {
			return wp, err
		}
	}

	apps := wf.GetApplications()
	envs := wf.GetEnvironments()
	pips := wf.GetPipelines()

	//Reload app to retrieve secrets
	for i := range apps {
		app := &apps[i]
		vars, errv := application.GetAllVariable(db, proj.Key, app.Name, application.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load application variables %s", app.Name)
		}
		app.Variable = vars

		if err := application.LoadAllDecryptedKeys(db, app); err != nil {
			return wp, sdk.WrapError(err, "cannot load application keys %s", app.Name)
		}
	}

	//Reload env to retrieve secrets
	for i := range envs {
		env := &envs[i]
		vars, errv := environment.GetAllVariable(db, proj.Key, env.Name, environment.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load environment variables %s", env.Name)
		}
		env.Variable = vars

		if err := environment.LoadAllDecryptedKeys(db, env); err != nil {
			return wp, sdk.WrapError(err, "cannot load environment keys %s", env.Name)
		}
	}

	buffw := new(bytes.Buffer)
	// If the repository is "as-code", hide the hook
	if wf.FromRepository != "" {
		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
	}
	if _, err := exportWorkflow(*wf, f, buffw, opts...); err != nil {
		return wp, sdk.WrapError(err, "unable to export workflow")
	}
	wp.Workflow.Name = wf.Name
	wp.Workflow.Value = base64.StdEncoding.EncodeToString(buffw.Bytes())

	wp.Applications = make([]exportentities.WorkflowPulledItem, len(apps))
	for i, a := range apps {
		buff := new(bytes.Buffer)
		if _, err := application.ExportApplication(db, a, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export app %s", a.Name)
		}
		wp.Applications[i].Name = a.Name
		wp.Applications[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	wp.Environments = make([]exportentities.WorkflowPulledItem, len(envs))
	for i, e := range envs {
		buff := new(bytes.Buffer)
		if _, err := environment.ExportEnvironment(db, e, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export env %s", e.Name)
		}
		wp.Environments[i].Name = e.Name
		wp.Environments[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	wp.Pipelines = make([]exportentities.WorkflowPulledItem, len(pips))
	for i, p := range pips {
		buff := new(bytes.Buffer)
		if _, err := pipeline.ExportPipeline(p, f, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export pipeline %s", p.Name)
		}
		wp.Pipelines[i].Name = p.Name
		wp.Pipelines[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	return wp, nil
}
<commit_msg>fix(api): do not export application, pipeline and environment coming from another repository (#4174)<commit_after>package workflow

import (
	"bytes"
	"context"
	"encoding/base64"
	"io"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/environment"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/api/pipeline"
	"github.com/ovh/cds/engine/api/workflowtemplate"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/exportentities"
)

// Export a workflow
func Export(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format, u *sdk.User, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	ctx, end := observability.Span(ctx, "workflow.Export")
	defer end()

	wf, errload := Load(ctx, db, cache, proj, name, u, LoadOptions{})
	if errload != nil {
		return 0, sdk.WrapError(errload, "workflow.Export> Cannot load workflow %s", name)
	}

	// If repo is from as-code do not export WorkflowSkipIfOnlyOneRepoWebhook
	if wf.FromRepository != "" {
		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
	}

	return exportWorkflow(*wf, f, w, opts...)
}

func exportWorkflow(wf sdk.Workflow, f exportentities.Format, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	e, err := exportentities.NewWorkflow(wf, opts...)
	if err != nil {
		return 0, sdk.WrapError(err, "exportWorkflow")
	}

	// Useful to not display history_length in yaml or json if it's his default value
	if e.HistoryLength != nil && *e.HistoryLength == sdk.DefaultHistoryLength {
		e.HistoryLength = nil
	}

	// Marshal to the desired format
	b, err := exportentities.Marshal(e, f)
	if err != nil {
		return 0, sdk.WithStack(err)
	}

	return w.Write(b)
}

// Pull a workflow with all it dependencies; it writes a tar buffer in the writer
func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format,
	encryptFunc sdk.EncryptFunc, u *sdk.User, opts ...exportentities.WorkflowOptions) (exportentities.WorkflowPulled, error) {
	ctx, end := observability.Span(ctx, "workflow.Pull")
	defer end()

	var wp exportentities.WorkflowPulled

	options := LoadOptions{
		DeepPipeline: true,
	}
	wf, errload := Load(ctx, db, cache, proj, name, u, options)
	if errload != nil {
		return wp, sdk.WrapError(errload, "cannot load workflow %s", name)
	}

	i, err := workflowtemplate.GetInstanceByWorkflowID(db, wf.ID)
	if err != nil {
		return wp, err
	}
	if i != nil {
		wf.Template, err = workflowtemplate.GetByID(db, i.WorkflowTemplateID)
		if err != nil {
			return wp, err
		}
		if err := workflowtemplate.AggregateOnWorkflowTemplate(db, wf.Template); err != nil {
			return wp, err
		}
	}

	apps := wf.GetApplications()
	envs := wf.GetEnvironments()
	pips := wf.GetPipelines()

	//Reload app to retrieve secrets
	for i := range apps {
		app := &apps[i]
		vars, errv := application.GetAllVariable(db, proj.Key, app.Name, application.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load application variables %s", app.Name)
		}
		app.Variable = vars

		if err := application.LoadAllDecryptedKeys(db, app); err != nil {
			return wp, sdk.WrapError(err, "cannot load application keys %s", app.Name)
		}
	}

	//Reload env to retrieve secrets
	for i := range envs {
		env := &envs[i]
		vars, errv := environment.GetAllVariable(db, proj.Key, env.Name, environment.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load environment variables %s", env.Name)
		}
		env.Variable = vars

		if err := environment.LoadAllDecryptedKeys(db, env); err != nil {
			return wp, sdk.WrapError(err, "cannot load environment keys %s", env.Name)
		}
	}

	buffw := new(bytes.Buffer)
	// If the repository is "as-code", hide the hook
	if wf.FromRepository != "" {
		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
	}
	if _, err := exportWorkflow(*wf, f, buffw, opts...); err != nil {
		return wp, sdk.WrapError(err, "unable to export workflow")
	}
	wp.Workflow.Name = wf.Name
	wp.Workflow.Value = base64.StdEncoding.EncodeToString(buffw.Bytes())

	wp.Applications = make([]exportentities.WorkflowPulledItem, len(apps))
	for i, a := range apps {
		if a.FromRepository != wf.FromRepository { // don't export if coming from an other repository
			continue
		}
		buff := new(bytes.Buffer)
		if _, err := application.ExportApplication(db, a, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export app %s", a.Name)
		}
		wp.Applications[i].Name = a.Name
		wp.Applications[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	wp.Environments = make([]exportentities.WorkflowPulledItem, len(envs))
	for i, e := range envs {
		if e.FromRepository != wf.FromRepository { // don't export if coming from an other repository
			continue
		}
		buff := new(bytes.Buffer)
		if _, err := environment.ExportEnvironment(db, e, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export env %s", e.Name)
		}
		wp.Environments[i].Name = e.Name
		wp.Environments[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	wp.Pipelines = make([]exportentities.WorkflowPulledItem, len(pips))
	for i, p := range pips {
		if p.FromRepository != wf.FromRepository { // don't export if coming from an other repository
			continue
		}
		buff := new(bytes.Buffer)
		if _, err := pipeline.ExportPipeline(p, f, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export pipeline %s", p.Name)
		}
		wp.Pipelines[i].Name = p.Name
		wp.Pipelines[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	return wp, nil
}
<|endoftext|>


<commit_before>package workflow

import (
	"bytes"
	"context"
	"encoding/base64"
	"io"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/environment"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/api/pipeline"
	"github.com/ovh/cds/engine/api/workflowtemplate"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/exportentities"
)

// Export a workflow
func Export(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format, u *sdk.User, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	ctx, end := observability.Span(ctx, "workflow.Export")
	defer end()

	wf, errload := Load(ctx, db, cache, proj, name, u, LoadOptions{})
	if errload != nil {
		return 0, sdk.WrapError(errload, "workflow.Export> Cannot load workflow %s", name)
	}

	// If repo is from as-code do not export WorkflowSkipIfOnlyOneRepoWebhook
	if wf.FromRepository != "" {
		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
	}

	return exportWorkflow(*wf, f, w, opts...)
}

func exportWorkflow(wf sdk.Workflow, f exportentities.Format, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	e, err := exportentities.NewWorkflow(wf, opts...)
	if err != nil {
		return 0, sdk.WrapError(err, "exportWorkflow")
	}

	// Useful to not display history_length in yaml or json if it's his default value
	if e.HistoryLength != nil && *e.HistoryLength == sdk.DefaultHistoryLength {
		e.HistoryLength = nil
	}

	// Marshal to the desired format
	b, err := exportentities.Marshal(e, f)
	if err != nil {
		return 0, sdk.WithStack(err)
	}

	return w.Write(b)
}

// Pull a workflow with all it dependencies; it writes a tar buffer in the writer
func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format,
	encryptFunc sdk.EncryptFunc, u *sdk.User, opts ...exportentities.WorkflowOptions) (exportentities.WorkflowPulled, error) {
	ctx, end := observability.Span(ctx, "workflow.Pull")
	defer end()

	var wp exportentities.WorkflowPulled

	options := LoadOptions{
		DeepPipeline: true,
	}
	wf, errload := Load(ctx, db, cache, proj, name, u, options)
	if errload != nil {
		return wp, sdk.WrapError(errload, "cannot load workflow %s", name)
	}

	i, err := workflowtemplate.GetInstanceByWorkflowID(db, wf.ID)
	if err != nil {
		return wp, err
	}
	if i != nil {
		wf.Template, err = workflowtemplate.GetByID(db, i.WorkflowTemplateID)
		if err != nil {
			return wp, err
		}
		if err := workflowtemplate.AggregateOnWorkflowTemplate(db, wf.Template); err != nil {
			return wp, err
		}
	}

	apps := wf.GetApplications()
	envs := wf.GetEnvironments()
	pips := wf.GetPipelines()

	//Reload app to retrieve secrets
	for i := range apps {
		app := &apps[i]
		vars, errv := application.GetAllVariable(db, proj.Key, app.Name, application.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load application variables %s", app.Name)
		}
		app.Variable = vars

		if err := application.LoadAllDecryptedKeys(db, app); err != nil {
			return wp, sdk.WrapError(err, "cannot load application keys %s", app.Name)
		}
	}

	//Reload env to retrieve secrets
	for i := range envs {
		env := &envs[i]
		vars, errv := environment.GetAllVariable(db, proj.Key, env.Name, environment.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load environment variables %s", env.Name)
		}
		env.Variable = vars

		if err := environment.LoadAllDecryptedKeys(db, env); err != nil {
			return wp, sdk.WrapError(err, "cannot load environment keys %s", env.Name)
		}
	}

	buffw := new(bytes.Buffer)
	// If the repository is "as-code", hide the hook
	if wf.FromRepository != "" {
		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
	}
	if _, err := exportWorkflow(*wf, f, buffw, opts...); err != nil {
		return wp, sdk.WrapError(err, "unable to export workflow")
	}
	wp.Workflow.Name = wf.Name
	wp.Workflow.Value = base64.StdEncoding.EncodeToString(buffw.Bytes())

	wp.Applications = make([]exportentities.WorkflowPulledItem, len(apps))
	for i, a := range apps {
		if a.FromRepository != wf.FromRepository { // don't export if coming from an other repository
			continue
		}
		buff := new(bytes.Buffer)
		if _, err := application.ExportApplication(db, a, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export app %s", a.Name)
		}
		wp.Applications[i].Name = a.Name
		wp.Applications[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	wp.Environments = make([]exportentities.WorkflowPulledItem, len(envs))
	for i, e := range envs {
		if e.FromRepository != wf.FromRepository { // don't export if coming from an other repository
			continue
		}
		buff := new(bytes.Buffer)
		if _, err := environment.ExportEnvironment(db, e, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export env %s", e.Name)
		}
		wp.Environments[i].Name = e.Name
		wp.Environments[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	wp.Pipelines = make([]exportentities.WorkflowPulledItem, len(pips))
	for i, p := range pips {
		if p.FromRepository != wf.FromRepository { // don't export if coming from an other repository
			continue
		}
		buff := new(bytes.Buffer)
		if _, err := pipeline.ExportPipeline(p, f, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export pipeline %s", p.Name)
		}
		wp.Pipelines[i].Name = p.Name
		wp.Pipelines[i].Value = base64.StdEncoding.EncodeToString(buff.Bytes())
	}

	return wp, nil
}
<commit_msg>fix(api): workflow pull renders empty pipelines (#4193)<commit_after>package workflow

import (
	"bytes"
	"context"
	"encoding/base64"
	"io"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/application"
	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/environment"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/api/pipeline"
	"github.com/ovh/cds/engine/api/workflowtemplate"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/exportentities"
)

// Export a workflow
func Export(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format, u *sdk.User, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	ctx, end := observability.Span(ctx, "workflow.Export")
	defer end()

	wf, errload := Load(ctx, db, cache, proj, name, u, LoadOptions{})
	if errload != nil {
		return 0, sdk.WrapError(errload, "workflow.Export> Cannot load workflow %s", name)
	}

	// If repo is from as-code do not export WorkflowSkipIfOnlyOneRepoWebhook
	if wf.FromRepository != "" {
		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
	}

	return exportWorkflow(*wf, f, w, opts...)
}

func exportWorkflow(wf sdk.Workflow, f exportentities.Format, w io.Writer, opts ...exportentities.WorkflowOptions) (int, error) {
	e, err := exportentities.NewWorkflow(wf, opts...)
	if err != nil {
		return 0, sdk.WrapError(err, "exportWorkflow")
	}

	// Useful to not display history_length in yaml or json if it's his default value
	if e.HistoryLength != nil && *e.HistoryLength == sdk.DefaultHistoryLength {
		e.HistoryLength = nil
	}

	// Marshal to the desired format
	b, err := exportentities.Marshal(e, f)
	if err != nil {
		return 0, sdk.WithStack(err)
	}

	return w.Write(b)
}

// Pull a workflow with all it dependencies; it writes a tar buffer in the writer
func Pull(ctx context.Context, db gorp.SqlExecutor, cache cache.Store, proj *sdk.Project, name string, f exportentities.Format,
	encryptFunc sdk.EncryptFunc, u *sdk.User, opts ...exportentities.WorkflowOptions) (exportentities.WorkflowPulled, error) {
	ctx, end := observability.Span(ctx, "workflow.Pull")
	defer end()

	var wp exportentities.WorkflowPulled

	options := LoadOptions{
		DeepPipeline: true,
	}
	wf, errload := Load(ctx, db, cache, proj, name, u, options)
	if errload != nil {
		return wp, sdk.WrapError(errload, "cannot load workflow %s", name)
	}

	i, err := workflowtemplate.GetInstanceByWorkflowID(db, wf.ID)
	if err != nil {
		return wp, err
	}
	if i != nil {
		wf.Template, err = workflowtemplate.GetByID(db, i.WorkflowTemplateID)
		if err != nil {
			return wp, err
		}
		if err := workflowtemplate.AggregateOnWorkflowTemplate(db, wf.Template); err != nil {
			return wp, err
		}
	}

	apps := wf.GetApplications()
	envs := wf.GetEnvironments()
	pips := wf.GetPipelines()

	//Reload app to retrieve secrets
	for i := range apps {
		app := &apps[i]
		vars, errv := application.GetAllVariable(db, proj.Key, app.Name, application.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load application variables %s", app.Name)
		}
		app.Variable = vars

		if err := application.LoadAllDecryptedKeys(db, app); err != nil {
			return wp, sdk.WrapError(err, "cannot load application keys %s", app.Name)
		}
	}

	//Reload env to retrieve secrets
	for i := range envs {
		env := &envs[i]
		vars, errv := environment.GetAllVariable(db, proj.Key, env.Name, environment.WithClearPassword())
		if errv != nil {
			return wp, sdk.WrapError(errv, "cannot load environment variables %s", env.Name)
		}
		env.Variable = vars

		if err := environment.LoadAllDecryptedKeys(db, env); err != nil {
			return wp, sdk.WrapError(err, "cannot load environment keys %s", env.Name)
		}
	}

	buffw := new(bytes.Buffer)
	// If the repository is "as-code", hide the hook
	if wf.FromRepository != "" {
		opts = append(opts, exportentities.WorkflowSkipIfOnlyOneRepoWebhook)
	}
	if _, err := exportWorkflow(*wf, f, buffw, opts...); err != nil {
		return wp, sdk.WrapError(err, "unable to export workflow")
	}
	wp.Workflow.Name = wf.Name
	wp.Workflow.Value = base64.StdEncoding.EncodeToString(buffw.Bytes())

	for _, a := range apps {
		if a.FromRepository != wf.FromRepository { // don't export if coming from an other repository
			continue
		}
		buff := new(bytes.Buffer)
		if _, err := application.ExportApplication(db, a, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export app %s", a.Name)
		}
		wp.Applications = append(wp.Applications, exportentities.WorkflowPulledItem{
			Name:  a.Name,
			Value: base64.StdEncoding.EncodeToString(buff.Bytes()),
		})
	}

	for _, e := range envs {
		if e.FromRepository != wf.FromRepository { // don't export if coming from an other repository
			continue
		}
		buff := new(bytes.Buffer)
		if _, err := environment.ExportEnvironment(db, e, f, encryptFunc, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export env %s", e.Name)
		}
		wp.Environments = append(wp.Environments, exportentities.WorkflowPulledItem{
			Name:  e.Name,
			Value: base64.StdEncoding.EncodeToString(buff.Bytes()),
		})
	}

	for _, p := range pips {
		if p.FromRepository != wf.FromRepository { // don't export if coming from an other repository
			continue
		}
		buff := new(bytes.Buffer)
		if _, err := pipeline.ExportPipeline(p, f, buff); err != nil {
			return wp, sdk.WrapError(err, "unable to export pipeline %s", p.Name)
		}
		wp.Pipelines = append(wp.Pipelines, exportentities.WorkflowPulledItem{
			Name:  p.Name,
			Value: base64.StdEncoding.EncodeToString(buff.Bytes()),
		})
	}

	return wp, nil
}
<|endoftext|>


<commit_before><commit_msg>[cassandra] Fixed locstate table definition<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"runtime"
	"strings"
	"time"
)

const VERSION = "0.2.0"

var (
	help    = flag.Bool("h", false, "Helps")
	client  = flag.Int("c", 10, "Clients")
	seconds = flag.Int64("t", 60, "Seconds")
	url     = flag.String("url", "", "URL")
	file    = flag.String("f", "", "URL list file")
	urls    []string
)

func fetch(url string, c chan bool) {
	status := false
	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()
	if http.StatusOK == resp.StatusCode {
		status = true
	}

	c <- status
}

func readLines(filename string) (lines []string, err error) {
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	for _, line := range strings.Split(string(bytes), "\n") {
		if line != "" {
			lines = append(lines, line)
		}
	}
	return
}

func showHelp() {
	fmt.Println("version:", VERSION)
	fmt.Println("Usage: load [-c clients] [-t Seconds] [-url url]")
	fmt.Println("")
	fmt.Println("Options:")
	fmt.Println("-h\t:", "help")
	fmt.Println("-c\t:", "clients number")
	fmt.Println("-t\t:", "how long you want to bench")
	fmt.Println("-url\t:", "bench url")
	fmt.Println("-f\t:", "url list file, one url per line.")
}

func main() {
	flag.Parse()

	if *help {
		showHelp()
		return
	}

	filename := *file
	if *url == "" && filename == "" {
		fmt.Println("Please input url.")
		return
	}

	if filename != "" {
		lines, err := readLines(filename)
		if err != nil {
			fmt.Println(err)
			return
		}
		urls = lines
	}
	fmt.Println(*client, "clients, run", *seconds, "seconds")
	runtime.GOMAXPROCS(*client)

	success, fail := 0, 0
	c := make(chan bool)

	st := time.Now().Unix()
	for st+*seconds > time.Now().Unix() {
		if filename != "" {
			for _, uri := range urls {
				go fetch(uri, c)
			}
		} else {
			go fetch(*url, c)
		}

		status := <-c
		if status == true {
			success += 1
		} else {
			fail += 1
		}
	}

	total := success + fail
	fmt.Println("Total:", total)
	fmt.Println("Success:", success)
	fmt.Println("Fail:", fail)
}
<commit_msg>clear<commit_after>///todo: replace the flag to config.json
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
)

const VERSION = "0.2.0"

var (
	help    = flag.Bool("h", false, "Helps")
	client  = flag.Int("c", 10, "Clients")
	seconds = flag.Int64("t", 60, "Seconds")
	url     = flag.String("url", "", "URL")
	file    = flag.String("f", "", "URL list file")
	urls    []string
)

func fetch(url string, c chan bool) {
	status := false
	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
	}
	defer resp.Body.Close()
	if http.StatusOK == resp.StatusCode {
		status = true
	}

	c <- status
}

func readLines(filename string) (lines []string, err error) {
	bytes, err := ioutil.ReadFile(filename)
	if err != nil {
		return
	}

	for _, line := range strings.Split(string(bytes), "\n") {
		if line != "" {
			lines = append(lines, line)
		}
	}
	return
}

func main() {
	flag.Parse()

	fmt.Println(*client, "clients, run", seconds, "seconds")

	success, fail := 0, 0
	c := make(chan bool)

	endTime := time.Now().Add(time.Duration(*seconds) * time.Second)
	for endTime.Before(time.Now()) {
		for i := 0; i < *client; i++ {
			go fetch(*url, c)
		}
		status := <-c
		if status {
			success += 1
		} else {
			fail += 1
		}
	}

	total := success + fail
	fmt.Println("Total:", total)
	fmt.Println("Success:", success)
	fmt.Println("Fail:", fail)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase_1 "github.com/keybase/client/protocol/go"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
	"strings"
)

type CmdSearch struct {
	query string
}

func (c *CmdSearch) ParseArgv(ctx *cli.Context) error {
	c.query = strings.Join(ctx.Args(), " ")
	if c.query == "" {
		return fmt.Errorf("Search query must not be empty.")
	}
	return nil
}

func (c *CmdSearch) RunClient() (err error) {
	cli, err := GetUserClient()
	if err != nil {
		return err
	}

	protocols := []rpc2.Protocol{
		NewLogUIProtocol(),
	}
	if err = RegisterProtocols(protocols); err != nil {
		return err
	}

	results, err := cli.Search(keybase_1.SearchArg{Query: c.query})
	if err != nil {
		return err
	}

	return showResults(results)
}

func (c *CmdSearch) Run() error {
	eng := engine.NewSearchEngine(c.query)
	ctx := engine.Context{
		LogUI:    G_UI.GetLogUI(),
		SecretUI: G_UI.GetSecretUI(),
	}
	err := engine.RunEngine(eng, &ctx)
	if err != nil {
		return err
	}

	return showResults(eng.GetResults())
}

func showResults(results []keybase_1.UserSummary) error {
	for _, user := range results {
		fmt.Printf("%s", user.Username)
		for _, social := range user.Proofs.Social {
			fmt.Printf(" %s:%s", social.ProofType, social.ProofName)
		}
		for _, web := range user.Proofs.Web {
			for _, protocol := range web.Protocols {
				fmt.Printf(" %s://%s", protocol, web.Hostname)
			}
		}
		fmt.Println()
	}
	return nil
}

func NewCmdSearch(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:        "search",
		Usage:       "keybase search <query>",
		Description: "search for keybase users",
		Flags:       []cli.Flag{},
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdSearch{}, "search", c)
		},
	}
}

func (c *CmdSearch) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:     true,
		GpgKeyring: true,
		KbKeyring:  true,
		API:        true,
	}
}
<commit_msg>add --json mode for search<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase_1 "github.com/keybase/client/protocol/go"
	jsonw "github.com/keybase/go-jsonw"
	"github.com/maxtaco/go-framed-msgpack-rpc/rpc2"
	"strings"
)

type CmdSearch struct {
	query string
	json  bool
}

func (c *CmdSearch) ParseArgv(ctx *cli.Context) error {
	c.query = strings.Join(ctx.Args(), " ")
	if c.query == "" {
		return fmt.Errorf("Search query must not be empty.")
	}
	c.json = ctx.Bool("json")
	return nil
}

func (c *CmdSearch) RunClient() (err error) {
	cli, err := GetUserClient()
	if err != nil {
		return err
	}

	protocols := []rpc2.Protocol{
		NewLogUIProtocol(),
	}
	if err = RegisterProtocols(protocols); err != nil {
		return err
	}

	results, err := cli.Search(keybase_1.SearchArg{Query: c.query})
	if err != nil {
		return err
	}

	return c.showResults(results)
}

func (c *CmdSearch) Run() error {
	eng := engine.NewSearchEngine(c.query)
	ctx := engine.Context{
		LogUI:    G_UI.GetLogUI(),
		SecretUI: G_UI.GetSecretUI(),
	}
	err := engine.RunEngine(eng, &ctx)
	if err != nil {
		return err
	}

	return c.showResults(eng.GetResults())
}

func (c *CmdSearch) showResults(results []keybase_1.UserSummary) error {
	if c.json {
		return c.showJsonResults(results)
	} else {
		return c.showRegularResults(results)
	}
}

func (c *CmdSearch) showRegularResults(results []keybase_1.UserSummary) error {
	for _, user := range results {
		fmt.Printf("%s", user.Username)
		for _, social := range user.Proofs.Social {
			fmt.Printf(" %s:%s", social.ProofType, social.ProofName)
		}
		for _, web := range user.Proofs.Web {
			for _, protocol := range web.Protocols {
				fmt.Printf(" %s://%s", protocol, web.Hostname)
			}
		}
		fmt.Println()
	}
	return nil
}

func (c *CmdSearch) showJsonResults(results []keybase_1.UserSummary) error {
	output := jsonw.NewArray(len(results))
	for userIndex, user := range results {
		userBlob := jsonw.NewDictionary()
		userBlob.SetKey("username", jsonw.NewString(user.Username))
		for _, social := range user.Proofs.Social {
			userBlob.SetKey(social.ProofType, jsonw.NewString(social.ProofName))
		}
		if len(user.Proofs.Web) > 0 {
			websites := jsonw.NewArray(len(user.Proofs.Web))
			webIndex := 0
			userBlob.SetKey("websites", websites)
			for _, webProof := range user.Proofs.Web {
				for _, protocol := range webProof.Protocols {
					websites.SetIndex(webIndex, jsonw.NewString(
						fmt.Sprintf("%s://%s", protocol, webProof.Hostname)))
					webIndex++
				}
			}
		}
		output.SetIndex(userIndex, userBlob)
	}
	fmt.Println(output.MarshalPretty())
	return nil
}

func NewCmdSearch(cl *libcmdline.CommandLine) cli.Command {
	return cli.Command{
		Name:        "search",
		Usage:       "keybase search <query>",
		Description: "search for keybase users",
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "j, json",
				Usage: "output a json blob",
			},
		},
		Action: func(c *cli.Context) {
			cl.ChooseCommand(&CmdSearch{}, "search", c)
		},
	}
}

func (c *CmdSearch) GetUsage() libkb.Usage {
	return libkb.Usage{
		Config:     true,
		GpgKeyring: true,
		KbKeyring:  true,
		API:        true,
	}
}
<|endoftext|>


<commit_before>package repo

import (
	"os"
	"fmt"
	"strconv"
	"time"

	"crypto/md5"
	"encoding/base64"
	"encoding/json"
	"io/ioutil"

	"github.com/DeveloppSoft/go-ipfs-api"
)

type Ledger struct {
	Repo string

	sh *shell.Shell // IPFS api
}

func NewLedger(repo_path string, ipfs_api string) Ledger {
	// Create some files if needed
	checkAndMake(repo_path)
	checkAndMake(repo_path + "/feed")
	checkAndMake(repo_path + "/ressources")

	return Ledger{Repo: repo_path, sh: shell.NewShell(ipfs_api)}
}

// Recursively add stuff in the repo and do an `ipfs name publish`
func (l *Ledger) Sync() error {
	// First, add the repo to ipfs
	id, err := l.sh.AddDir(l.Repo)
	if err != nil {
		return err
	}

	// Do the ipfs name publish <id>
	// Publish for 365 days
	return l.sh.Publish(id, "8760h")
}

// Get a message, returned as a reader
func (l *Ledger) GetMessage(peer string, sequence string) (string, error) {
	reader, err := l.sh.Cat(peer + "/feed/" + sequence + ".json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get the last seq number, as a string (no need to convert)
func (l *Ledger) GetLastSeq(peer string) (string, error) {
	reader, err := l.sh.Cat(peer + "/lastseq")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get all messages from a peer, return a slice of them, ordered from the more recent to the oldest
func (l *Ledger) GetFeed(peer string) ([]string, error) {
	result := make([]string, 0)

	seq_str, err := l.GetLastSeq(peer)
	if err != nil {
		return result, err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return result, err
	}

	for i := seq; i > 0; i-- {
		msg, err := l.GetMessage(peer, strconv.Itoa(i))
		if err != nil {
			return result, err
		}

		result = append(result, msg)
	}

	return result, nil
}

// Return our id or ""
func (l *Ledger) Whoami() string {
	id, err := l.sh.ID()

	if err != nil {
		return ""
	} else {
		return id.ID
	}
}

// Just retrieve about.json
func (l *Ledger) About(peer string) (string, error) {
	reader, err := l.sh.Cat(peer + "/about.json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Fill the profile of our user
func (l *Ledger) SetAbout(about About) error {
	bytes, err := about.Export()
	if err != nil {
		return err
	}

	// Write that to about.json
	return ioutil.WriteFile(l.Repo+"/about.json", bytes, os.ModePerm)
}

type Message struct {
	Seq       int
	Timestamp time.Time

	Data string
}

// Add a message and increase the lastseq
func (l *Ledger) Publish(data string) error {
	seq_str, err := l.GetLastSeq(l.Whoami())
	if err != nil {
		return err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return err
	}

	seq++
	seq_str = strconv.Itoa(seq)

	// Build the message
	msg := Message{Seq: seq, Timestamp: time.Now(), Data: data}
	msg_byte, err := json.Marshal(msg)
	if err != nil {
		return err
	}

	// Just write it to the repo
	err = ioutil.WriteFile(l.Repo+"/feed", msg_byte, os.ModePerm) // TODO: better perm
	if err != nil {
		return err
	}

	// Increment lastseq
	return ioutil.WriteFile(l.Repo+"/lastseq", []byte(seq_str), os.ModePerm) // TODO: better perm
}

func (l *Ledger) AddRessource(b64 string) (string, error) {
	// Unpack data
	data, err := base64.StdEncoding.DecodeString(b64)
	if err != nil {
		return "", err
	}

	// Calculate checksum (no need for a mega high algo here, let's use md5)
	hash_bytes := md5.Sum(data)
	hash := fmt.Sprintf("%s", hash_bytes)

	err = ioutil.WriteFile(l.Repo+"/ressources/"+hash, data, os.ModePerm) // Need better perms
	return hash, err
}

func (l *Ledger) GetRessource(id string) (string, error) {
	reader, err := l.sh.Cat(id)
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	if err != nil {
		return "", err
	}

	return base64.StdEncoding.EncodeToString(bytes), nil
}
<commit_msg>[repo] now resolve names<commit_after>package repo

import (
	"fmt"
	"os"
	"strconv"
	"time"

	"crypto/md5"
	"encoding/base64"
	"encoding/json"
	"io/ioutil"

	"github.com/DeveloppSoft/go-ipfs-api"
)

type Ledger struct {
	Repo string

	sh *shell.Shell // IPFS api
}

func NewLedger(repo_path string, ipfs_api string) Ledger {
	// Create some files if needed
	checkAndMake(repo_path)
	checkAndMake(repo_path + "/feed")
	checkAndMake(repo_path + "/ressources")

	return Ledger{Repo: repo_path, sh: shell.NewShell(ipfs_api)}
}

// Recursively add stuff in the repo and do an `ipfs name publish`
func (l *Ledger) Sync() error {
	// First, add the repo to ipfs
	id, err := l.sh.AddDir(l.Repo)
	if err != nil {
		return err
	}

	// Do the ipfs name publish <id>
	// Publish for 365 days
	return l.sh.Publish(id, "8760h")
}

// Get a message, returned as a reader
func (l *Ledger) GetMessage(peer_name string, sequence string) (string, error) {
	peer, err := l.Resolve(peer_name)
	if err != nil {
		return "", err
	}

	reader, err := l.sh.Cat(peer + "/feed/" + sequence + ".json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get the last seq number, as a string (no need to convert)
func (l *Ledger) GetLastSeq(peer_name string) (string, error) {
	peer, err := l.Resolve(peer_name)
	if err != nil {
		return "", err
	}

	reader, err := l.sh.Cat(peer + "/lastseq")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Get all messages from a peer, return a slice of them, ordered from the more recent to the oldest
func (l *Ledger) GetFeed(peer_name string) ([]string, error) {
	result := make([]string, 0)

	seq_str, err := l.GetLastSeq(peer_name)
	if err != nil {
		return result, err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return result, err
	}

	for i := seq; i > 0; i-- {
		msg, err := l.GetMessage(peer_name, strconv.Itoa(i))
		if err != nil {
			return result, err
		}

		result = append(result, msg)
	}

	return result, nil
}

// Return our id or ""
func (l *Ledger) Whoami() string {
	id, err := l.sh.ID()

	if err != nil {
		return ""
	} else {
		return id.ID
	}
}

// Just retrieve about.json
func (l *Ledger) About(peer_name string) (string, error) {
	peer, err := l.Resolve(peer_name)
	if err != nil {
		return "", err
	}

	reader, err := l.sh.Cat(peer + "/about.json")
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	return string(bytes), err
}

// Fill the profile of our user
func (l *Ledger) SetAbout(about About) error {
	bytes, err := about.Export()
	if err != nil {
		return err
	}

	// Write that to about.json
	return ioutil.WriteFile(l.Repo+"/about.json", bytes, os.ModePerm)
}

type Message struct {
	Seq       int
	Timestamp time.Time

	Data string
}

// Add a message and increase the lastseq
func (l *Ledger) Publish(data string) error {
	seq_str, err := l.GetLastSeq(l.Whoami())
	if err != nil {
		return err
	}

	seq, err := strconv.Atoi(seq_str)
	if err != nil {
		return err
	}

	seq++
	seq_str = strconv.Itoa(seq)

	// Build the message
	msg := Message{Seq: seq, Timestamp: time.Now(), Data: data}
	msg_byte, err := json.Marshal(msg)
	if err != nil {
		return err
	}

	// Just write it to the repo
	err = ioutil.WriteFile(l.Repo+"/feed", msg_byte, os.ModePerm) // TODO: better perm
	if err != nil {
		return err
	}

	// Increment lastseq
	return ioutil.WriteFile(l.Repo+"/lastseq", []byte(seq_str), os.ModePerm) // TODO: better perm
}

func (l *Ledger) AddRessource(b64 string) (string, error) {
	// Unpack data
	data, err := base64.StdEncoding.DecodeString(b64)
	if err != nil {
		return "", err
	}

	// Calculate checksum (no need for a mega high algo here, let's use md5)
	hash_bytes := md5.Sum(data)
	hash := fmt.Sprintf("%s", hash_bytes)

	err = ioutil.WriteFile(l.Repo+"/ressources/"+hash, data, os.ModePerm) // Need better perms
	return hash, err
}

func (l *Ledger) GetRessource(id string) (string, error) {
	reader, err := l.sh.Cat(id)
	if err != nil {
		return "", err
	}

	bytes, err := ioutil.ReadAll(reader)
	if err != nil {
		return "", err
	}

	return base64.StdEncoding.EncodeToString(bytes), nil
}

func (l *Ledger) Resolve(name string) (string, error) {
	return l.sh.Resolve(name)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"container/list"
	"fmt"
	"strconv"
	"strings"
	"time"
)

// PullRequestInfo represents needed information for a pull request.
type PullRequestInfo struct {
	MergeBase string
	Commits   *list.List
	NumFiles  int
}

// GetMergeBase checks and returns merge base of two branches.
func (repo *Repository) GetMergeBase(base, head string) (string, error) {
	stdout, err := NewCommand("merge-base", base, head).RunInDir(repo.Path)
	if err != nil {
		if strings.HasSuffix(err.Error(), " 1") {
			return "", ErrNoMergeBase{}
		}
		return "", err
	}
	return strings.TrimSpace(stdout), nil
}

// GetPullRequestInfo generates and returns pull request information
// between base and head branches of repositories.
func (repo *Repository) GetPullRequestInfo(basePath, baseBranch, headBranch string) (_ *PullRequestInfo, err error) {
	var remoteBranch string

	// We don't need a temporary remote for same repository.
	if repo.Path != basePath {
		// Add a temporary remote
		tmpRemote := strconv.FormatInt(time.Now().UnixNano(), 10)
		if err = repo.AddRemote(tmpRemote, basePath, true); err != nil {
			return nil, fmt.Errorf("AddRemote: %v", err)
		}
		defer repo.RemoveRemote(tmpRemote)

		remoteBranch = "remotes/" + tmpRemote + "/" + baseBranch
	} else {
		remoteBranch = baseBranch
	}

	prInfo := new(PullRequestInfo)
	prInfo.MergeBase, err = repo.GetMergeBase(remoteBranch, headBranch)
	if err != nil {
		return nil, err
	}

	logs, err := NewCommand("log", prInfo.MergeBase+"..."+headBranch, _PRETTY_LOG_FORMAT).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}
	prInfo.Commits, err = repo.parsePrettyFormatLogToList(logs)
	if err != nil {
		return nil, fmt.Errorf("parsePrettyFormatLogToList: %v", err)
	}

	// Count number of changed files.
	stdout, err := NewCommand("diff", "--name-only", remoteBranch+"..."+headBranch).RunInDir(repo.Path)
	if err != nil {
		return nil, err
	}
	prInfo.NumFiles = len(strings.Split(stdout, "\n")) - 1

	return prInfo, nil
}

// GetPatch generates and returns patch data between given revisions.
func (repo *Repository) GetPatch(base, head string) ([]byte, error) {
	return NewCommand("diff", "-p", "--binary", base, head).RunInDirBytes(repo.Path)
}
<commit_msg>repo_pull: adjust merge base error detection<commit_after>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"container/list"
	"fmt"
	"strconv"
	"strings"
	"time"
)

// PullRequestInfo represents needed information for a pull request.
type PullRequestInfo struct {
	MergeBase string
	Commits   *list.List
	NumFiles  int
}

// GetMergeBase checks and returns merge base of two branches.
func (repo *Repository) GetMergeBase(base, head string) (string, error) {
	stdout, err := NewCommand("merge-base", base, head).RunInDir(repo.Path)
	if err != nil {
		if strings.Contains(err.Error(), "exit status 1") {
			return "", ErrNoMergeBase{}
		}
		return "", err
	}
	return strings.TrimSpace(stdout), nil
}

// GetPullRequestInfo generates and returns pull request information
// between base and head branches of repositories.
func (repo *Repository) GetPullRequestInfo(basePath, baseBranch, headBranch string) (_ *PullRequestInfo, err error) {
	var remoteBranch string

	// We don't need a temporary remote for same repository.
	if repo.Path != basePath {
		// Add a temporary remote
		tmpRemote := strconv.FormatInt(time.Now().UnixNano(), 10)
		if err = repo.AddRemote(tmpRemote, basePath, true); err != nil {
			return nil, fmt.Errorf("AddRemote: %v", err)
		}
		defer repo.RemoveRemote(tmpRemote)

		remoteBranch = "remotes/" + tmpRemote + "/" + baseBranch
	} else {
		remoteBranch = baseBranch
	}

	prInfo := new(PullRequestInfo)
	prInfo.MergeBase, err = repo.GetMergeBase(remoteBranch, headBranch)
	if err != nil {
		return nil, err
	}

	logs, err := NewCommand("log", prInfo.MergeBase+"..."+headBranch, _PRETTY_LOG_FORMAT).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}
	prInfo.Commits, err = repo.parsePrettyFormatLogToList(logs)
	if err != nil {
		return nil, fmt.Errorf("parsePrettyFormatLogToList: %v", err)
	}

	// Count number of changed files.
	stdout, err := NewCommand("diff", "--name-only", remoteBranch+"..."+headBranch).RunInDir(repo.Path)
	if err != nil {
		return nil, err
	}
	prInfo.NumFiles = len(strings.Split(stdout, "\n")) - 1

	return prInfo, nil
}

// GetPatch generates and returns patch data between given revisions.
func (repo *Repository) GetPatch(base, head string) ([]byte, error) {
	return NewCommand("diff", "-p", "--binary", base, head).RunInDirBytes(repo.Path)
}
<|endoftext|>


<commit_before>package walgo

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

var (
	defaultRequester Requester
)

const (
	userAgentHeader     = "User-Agent"
	authorizationHeader = "Authorization"
	bearerPrefix        = "Bearer "
	DefaultClientName   = "walgo"
)

func init() {
	defaultRequester = NewRequester(http.DefaultClient, DefaultClientName, "")
}

type Requester interface {
	// Get performs a GET reuqest to the given URL with the given parameters.
	Get(url string, p ParameterMap) (res Response, err error)

	// Post performs a POST request to the given URL with the given
	// parameters and no request body.
	Post(url string, p ParameterMap) (r Response, err error)

	// PostJson performs a POST request to the given URL with the given
	// parameters and the supplied interface type encoded as JSON.
	PostJson(url string, p ParameterMap, v interface{}) (r Response, err error)

	// PostRaw performs a POST request to the given URL with the given
	// parameters and the supplied bytes as the request body.
	PostRaw(url string, p ParameterMap, data []byte) (r Response, err error)

	// PostMultipart performs a POST request to the given URL with the given
	// parameters and the supplied multipart payload encoded as the request
	// body.
	PostMultipart(url string, p ParameterMap, m *MultipartPayload) (r Response, err error)

	// PostValues performs a POST request to the given URL with the given
	// parameters and a body consisting of the supplied values urlencoded.
	PostValues(url string, p ParameterMap, v url.Values) (r Response, err error)

	// Put performs a PUT request to the given URL with the given parameters.
	Put(url string, p ParameterMap) (r Response, err error)

	// PutJson performs a PUT request to the given URL with the given
	// parameters and the supplied interface type encoded as JSON.
	PutJson(url string, p ParameterMap, v interface{}) (r Response, err error)

	// PutRaw performs a PUT request to the given URL with the given
	// parameters and the supplied bytes as the request body.
	PutRaw(url string, p ParameterMap, data []byte) (r Response, err error)

	// PutMultipart performs a PUT request to the given URL with the given
	// parameters and the supplied multipart payload encoded as the request
	// body.
	PutMultipart(url string, p ParameterMap, m *MultipartPayload) (r Response, err error)

	// PutValues performs a PUT request to the given URL with the given
	// parameters and a body consisting of the supplied values urlencoded.
	PutValues(url string, p ParameterMap, v url.Values) (r Response, err error)

	// Delete peforms a DELETE request to the given URL with the given
	// parameters.
	Delete(url string, p ParameterMap) (r Response, err error)

	makeRequest(url string, p ParameterMap, method string, l *payload) (r Response, err error)
}

type requesterImpl struct {
	client    *http.Client
	userAgent string
	authToken string
}

// NewRequester creates a new Requester using the supplied client. Every
// request has the User-Agent header set to the given value and if the
// authentication token differs from "" it is also used as the Authorization
// header value (with the "Bearer "-prefix).
func NewRequester(c *http.Client, userAgent, authToken string) (r Requester) {
	return &requesterImpl{
		client:    c,
		userAgent: userAgent,
		authToken: authToken,
	}
}

func (f *requesterImpl) makeRequest(url string, p ParameterMap, method string, l *payload) (r Response, err error) {
	u, err := createParameterUrl(url, p)
	if err != nil {
		return nil, err
	}

	startTime := time.Now()
	code := -1
	var output []byte

	buffer := &bytes.Buffer{}

	if l != nil {
		data := l.getData()
		c, err2 := buffer.Write(data)
		if c != len(data) || err2 != nil {
			return nil, errors.New("Error creating data buffer.")
		}
	}

	req, err := http.NewRequest(method, u.String(), buffer)
	if err != nil {
		return nil, err
	}

	if l != nil {
		req.Header.Add(contentTypeHeader, l.getContentType())
	}

	req.Header.Add(userAgentHeader, f.userAgent)
	if "" != f.authToken {
		req.Header.Add(authorizationHeader, bearerPrefix+f.authToken)
	}

	resp, err := f.client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp != nil && resp.Body != nil {
		code = resp.StatusCode
		output, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
	}

	duration := time.Now().Sub(startTime)

	r = responseImpl{
		data:     output,
		code:     code,
		duration: duration,
	}

	return r, err
}

func createParameterUrl(urlStr string, p ParameterMap) (u *url.URL, err error) {
	u, err = url.Parse(urlStr)
	if err != nil {
		return nil, err
	}

	query := u.Query()

	if p != nil {
		for k, v := range p {
			query.Add(k, v)
		}
	}

	u.RawQuery = query.Encode()
	return u, nil
}
<commit_msg>Inlined function<commit_after>package walgo

import (
	"bytes"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

var (
	defaultRequester Requester
)

const (
	userAgentHeader     = "User-Agent"
	authorizationHeader = "Authorization"
	bearerPrefix        = "Bearer "
	DefaultClientName   = "walgo"
)

func init() {
	defaultRequester = NewRequester(http.DefaultClient, DefaultClientName, "")
}

type Requester interface {
	// Get performs a GET reuqest to the given URL with the given parameters.
	Get(url string, p ParameterMap) (res Response, err error)

	// Post performs a POST request to the given URL with the given
	// parameters and no request body.
	Post(url string, p ParameterMap) (r Response, err error)

	// PostJson performs a POST request to the given URL with the given
	// parameters and the supplied interface type encoded as JSON.
	PostJson(url string, p ParameterMap, v interface{}) (r Response, err error)

	// PostRaw performs a POST request to the given URL with the given
	// parameters and the supplied bytes as the request body.
	PostRaw(url string, p ParameterMap, data []byte) (r Response, err error)

	// PostMultipart performs a POST request to the given URL with the given
	// parameters and the supplied multipart payload encoded as the request
	// body.
	PostMultipart(url string, p ParameterMap, m *MultipartPayload) (r Response, err error)

	// PostValues performs a POST request to the given URL with the given
	// parameters and a body consisting of the supplied values urlencoded.
	PostValues(url string, p ParameterMap, v url.Values) (r Response, err error)

	// Put performs a PUT request to the given URL with the given parameters.
	Put(url string, p ParameterMap) (r Response, err error)

	// PutJson performs a PUT request to the given URL with the given
	// parameters and the supplied interface type encoded as JSON.
	PutJson(url string, p ParameterMap, v interface{}) (r Response, err error)

	// PutRaw performs a PUT request to the given URL with the given
	// parameters and the supplied bytes as the request body.
	PutRaw(url string, p ParameterMap, data []byte) (r Response, err error)

	// PutMultipart performs a PUT request to the given URL with the given
	// parameters and the supplied multipart payload encoded as the request
	// body.
	PutMultipart(url string, p ParameterMap, m *MultipartPayload) (r Response, err error)

	// PutValues performs a PUT request to the given URL with the given
	// parameters and a body consisting of the supplied values urlencoded.
	PutValues(url string, p ParameterMap, v url.Values) (r Response, err error)

	// Delete peforms a DELETE request to the given URL with the given
	// parameters.
	Delete(url string, p ParameterMap) (r Response, err error)

	makeRequest(url string, p ParameterMap, method string, l *payload) (r Response, err error)
}

type requesterImpl struct {
	client    *http.Client
	userAgent string
	authToken string
}

// NewRequester creates a new Requester using the supplied client. Every
// request has the User-Agent header set to the given value and if the
// authentication token differs from "" it is also used as the Authorization
// header value (with the "Bearer "-prefix).
func NewRequester(c *http.Client, userAgent, authToken string) (r Requester) {
	return &requesterImpl{
		client:    c,
		userAgent: userAgent,
		authToken: authToken,
	}
}

func (f *requesterImpl) makeRequest(urlStr string, p ParameterMap, method string, l *payload) (r Response, err error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}

	query := u.Query()

	if p != nil {
		for k, v := range p {
			query.Add(k, v)
		}
	}

	u.RawQuery = query.Encode()

	startTime := time.Now()
	code := -1
	var output []byte

	buffer := &bytes.Buffer{}

	if l != nil {
		data := l.getData()
		c, err2 := buffer.Write(data)
		if c != len(data) || err2 != nil {
			return nil, errors.New("Error creating data buffer.")
		}
	}

	req, err := http.NewRequest(method, u.String(), buffer)
	if err != nil {
		return nil, err
	}

	if l != nil {
		req.Header.Add(contentTypeHeader, l.getContentType())
	}

	req.Header.Add(userAgentHeader, f.userAgent)
	if "" != f.authToken {
		req.Header.Add(authorizationHeader, bearerPrefix+f.authToken)
	}

	resp, err := f.client.Do(req)
	if err != nil {
		return nil, err
	}
	if resp != nil && resp.Body != nil {
		code = resp.StatusCode
		output, err = ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
	}

	duration := time.Now().Sub(startTime)

	r = responseImpl{
		data:     output,
		code:     code,
		duration: duration,
	}

	return r, err
}
<|endoftext|>


<commit_before>package requestor

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

// Options contains parameters to be defined before sending the request to the server. Certain values
// can be omitted based on the request method (i.e. GET typically won't need to send a Body).
type Options struct {
	Method      string
	Body        string
	ContentType string
	Auth        []string
	Headers     map[string]string
}

// HTTPData contains the information returned from our request.
type HTTPData struct {
	Body    []byte
	Status  string
	Code    int
	Headers http.Header
	Error   error
}

// Send issues a HTTP request with the values specified in Options.
func Send(url string, options *Options) (*HTTPData, error) {
	var req *http.Request
	var data HTTPData
	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}
	body := bytes.NewReader([]byte(options.Body))
	req, _ = http.NewRequest(strings.ToUpper(options.Method), url, body)

	if len(options.Auth) > 0 {
		req.SetBasicAuth(options.Auth[0], options.Auth[1])
	}

	if len(options.ContentType) > 0 {
		req.Header.Set("Content-Type", options.ContentType)
	}

	if len(options.Headers) > 0 {
		for k, _ := range options.Headers {
			req.Header.Add(k, options.Headers[k])
		}
	}

	res, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	defer res.Body.Close()

	payload, _ := ioutil.ReadAll(res.Body)

	data.Body = payload
	data.Code = res.StatusCode
	data.Status = res.Status
	data.Headers = res.Header

	if res.StatusCode >= 400 {
		data.Error = fmt.Errorf("HTTP %d: %s", res.StatusCode, string(payload))
	}

	return &data, nil
}
<commit_msg>Renamed fields, added function to convert body<commit_after>package requestor

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
)

// Options contains parameters to be defined before sending the request to the server. Certain values
// can be omitted based on the request method (i.e. GET typically won't need to send a Body).
type Options struct {
	Method      string
	ContentType string
	Body        string
	Auth        []string
	Headers     map[string]string
}

// HTTPData contains the information returned from our request.
type HTTPData struct {
	Status  string
	Code    int
	Headers http.Header
	Payload []byte
	Error   error
}

// Send issues an HTTP request with the values specified in Options.
func Send(url string, options *Options) *HTTPData {
	var req *http.Request
	var data HTTPData
	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}
	body := bytes.NewReader([]byte(options.Body))
	req, _ = http.NewRequest(strings.ToUpper(options.Method), url, body)

	if len(options.Auth) > 0 {
		req.SetBasicAuth(options.Auth[0], options.Auth[1])
	}

	if len(options.ContentType) > 0 {
		req.Header.Set("Content-Type", options.ContentType)
	}

	if len(options.Headers) > 0 {
		for k, _ := range options.Headers {
			req.Header.Add(k, options.Headers[k])
		}
	}

	res, err := client.Do(req)
	if err != nil {
		data.Error = err

		return &data
	}

	defer res.Body.Close()

	payload, _ := ioutil.ReadAll(res.Body)

	data.Payload = payload
	data.Code = res.StatusCode
	data.Status = res.Status
	data.Headers = res.Header

	if res.StatusCode >= 400 {
		data.Error = fmt.Errorf("HTTP %d: %s", res.StatusCode, string(payload))
	}

	return &data
}

// String will convert the payload/body of the request from a []byte to a string value.
func (h *HTTPData) String() string {
	return string(h.Payload)
}
<|endoftext|>