<commit_before>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package handler

import (
	"encoding/json"
	"os"
	"os/user"
	"sync"

	"github.com/TheThingsNetwork/ttn/api/protocol/lorawan"
	"github.com/TheThingsNetwork/ttn/core/types"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"golang.org/x/net/context" // See https://github.com/grpc/grpc-go/issues/711"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
)

// ManagerClient is used to manage applications and devices on a handler
type ManagerClient struct {
	sync.RWMutex
	id                       string
	accessToken              string
	conn                     *grpc.ClientConn
	applicationManagerClient ApplicationManagerClient
}

// NewManagerClient returns a new ManagerClient for a handler on the given conn that accepts the given access token
func NewManagerClient(conn *grpc.ClientConn, accessToken string) (*ManagerClient, error) {
	applicationManagerClient := NewApplicationManagerClient(conn)

	id := "client"
	if user, err := user.Current(); err == nil {
		id += "-" + user.Username
	}
	if hostname, err := os.Hostname(); err == nil {
		id += "@" + hostname
	}

	return &ManagerClient{
		id:          id,
		accessToken: accessToken,
		conn:        conn,
		applicationManagerClient: applicationManagerClient,
	}, nil
}

// SetID sets the ID of this client
func (h *ManagerClient) SetID(id string) {
	h.Lock()
	defer h.Unlock()
	h.id = id
}

// UpdateAccessToken updates the access token that is used for running commands
func (h *ManagerClient) UpdateAccessToken(accessToken string) {
	h.Lock()
	defer h.Unlock()
	h.accessToken = accessToken
}

func (h *ManagerClient) getContext() context.Context {
	h.RLock()
	defer h.RUnlock()
	md := metadata.Pairs(
		"id", h.id,
		"token", h.accessToken,
	)
	return metadata.NewContext(context.Background(), md)
}

// GetApplication retrieves an application from the Handler
func (h *ManagerClient) GetApplication(appID string) (*Application, error) {
	res, err := h.applicationManagerClient.GetApplication(h.getContext(), &ApplicationIdentifier{AppId: appID})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not get application from Handler")
	}
	return res, nil
}

// SetApplication sets an application on the Handler
func (h *ManagerClient) SetApplication(in *Application) error {
	_, err := h.applicationManagerClient.SetApplication(h.getContext(), in)
	return errors.Wrap(errors.FromGRPCError(err), "Could not set application on Handler")
}

// RegisterApplication registers an application on the Handler
func (h *ManagerClient) RegisterApplication(appID string) error {
	_, err := h.applicationManagerClient.RegisterApplication(h.getContext(), &ApplicationIdentifier{AppId: appID})
	return errors.Wrap(errors.FromGRPCError(err), "Could not register application on Handler")
}

// DeleteApplication deletes an application and all its devices from the Handler
func (h *ManagerClient) DeleteApplication(appID string) error {
	_, err := h.applicationManagerClient.DeleteApplication(h.getContext(), &ApplicationIdentifier{AppId: appID})
	return errors.Wrap(errors.FromGRPCError(err), "Could not delete application from Handler")
}

// GetDevice retrieves a device from the Handler
func (h *ManagerClient) GetDevice(appID string, devID string) (*Device, error) {
	res, err := h.applicationManagerClient.GetDevice(h.getContext(), &DeviceIdentifier{AppId: appID, DevId: devID})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not get device from Handler")
	}
	return res, nil
}

// SetDevice sets a device on the Handler
func (h *ManagerClient) SetDevice(in *Device) error {
	_, err := h.applicationManagerClient.SetDevice(h.getContext(), in)
	return errors.Wrap(errors.FromGRPCError(err), "Could not set device on Handler")
}

// DeleteDevice deletes a device from the Handler
func (h *ManagerClient) DeleteDevice(appID string, devID string) error {
	_, err := h.applicationManagerClient.DeleteDevice(h.getContext(), &DeviceIdentifier{AppId: appID, DevId: devID})
	return errors.Wrap(errors.FromGRPCError(err), "Could not delete device from Handler")
}

// GetDevicesForApplication retrieves all devices for an application from the Handler
func (h *ManagerClient) GetDevicesForApplication(appID string) (devices []*Device, err error) {
	res, err := h.applicationManagerClient.GetDevicesForApplication(h.getContext(), &ApplicationIdentifier{AppId: appID})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not get devices for application from Handler")
	}
	for _, dev := range res.Devices {
		devices = append(devices, dev)
	}
	return
}

// GetDevAddr requests a random device address with the given constraints
func (h *ManagerClient) GetDevAddr(constraints ...string) (types.DevAddr, error) {
	devAddrManager := lorawan.NewDevAddrManagerClient(h.conn)
	resp, err := devAddrManager.GetDevAddr(h.getContext(), &lorawan.DevAddrRequest{
		Usage: constraints,
	})
	if err != nil {
		return types.DevAddr{}, errors.Wrap(errors.FromGRPCError(err), "Could not get DevAddr from Handler")
	}
	return *resp.DevAddr, nil
}

// DryUplink transforms the uplink payload with the payload functions provided
// in the app..
func (h *ManagerClient) DryUplink(payload []byte, app *Application, port uint32) (*DryUplinkResult, error) {
	res, err := h.applicationManagerClient.DryUplink(h.getContext(), &DryUplinkMessage{
		App:     app,
		Payload: payload,
		Port:    port,
	})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not dry-run uplink on Handler")
	}
	return res, nil
}

// DryDownlinkWithPayload transforms the downlink payload with the payload functions
// provided in app.
func (h *ManagerClient) DryDownlinkWithPayload(payload []byte, app *Application, port uint32) (*DryDownlinkResult, error) {
	res, err := h.applicationManagerClient.DryDownlink(h.getContext(), &DryDownlinkMessage{
		App:     app,
		Payload: payload,
		Port:    port,
	})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not dry-run downlink with payload on Handler")
	}
	return res, nil
}

// DryDownlinkWithFields transforms the downlink fields with the payload functions
// provided in app.
func (h *ManagerClient) DryDownlinkWithFields(fields map[string]interface{}, app *Application, port uint32) (*DryDownlinkResult, error) {
	marshalled, err := json.Marshal(fields)
	if err != nil {
		return nil, err
	}

	res, err := h.applicationManagerClient.DryDownlink(h.getContext(), &DryDownlinkMessage{
		App:    app,
		Fields: string(marshalled),
		Port:   port,
	})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not dry-run downlink with fields on Handler")
	}
	return res, nil
}

// Close closes the client
func (h *ManagerClient) Close() error {
	return h.conn.Close()
}
<commit_msg>Add pagination to list in Application Manager Client<commit_after>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package handler

import (
	"encoding/json"
	"os"
	"os/user"
	"strconv"
	"sync"

	"github.com/TheThingsNetwork/ttn/api/protocol/lorawan"
	"github.com/TheThingsNetwork/ttn/core/types"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"golang.org/x/net/context" // See https://github.com/grpc/grpc-go/issues/711"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
)

// ManagerClient is used to manage applications and devices on a handler
type ManagerClient struct {
	sync.RWMutex
	id                       string
	accessToken              string
	conn                     *grpc.ClientConn
	applicationManagerClient ApplicationManagerClient
}

// NewManagerClient returns a new ManagerClient for a handler on the given conn that accepts the given access token
func NewManagerClient(conn *grpc.ClientConn, accessToken string) (*ManagerClient, error) {
	applicationManagerClient := NewApplicationManagerClient(conn)

	id := "client"
	if user, err := user.Current(); err == nil {
		id += "-" + user.Username
	}
	if hostname, err := os.Hostname(); err == nil {
		id += "@" + hostname
	}

	return &ManagerClient{
		id:          id,
		accessToken: accessToken,
		conn:        conn,
		applicationManagerClient: applicationManagerClient,
	}, nil
}

// SetID sets the ID of this client
func (h *ManagerClient) SetID(id string) {
	h.Lock()
	defer h.Unlock()
	h.id = id
}

// UpdateAccessToken updates the access token that is used for running commands
func (h *ManagerClient) UpdateAccessToken(accessToken string) {
	h.Lock()
	defer h.Unlock()
	h.accessToken = accessToken
}

func (h *ManagerClient) getContext() context.Context {
	h.RLock()
	defer h.RUnlock()
	md := metadata.Pairs(
		"id", h.id,
		"token", h.accessToken,
	)
	return metadata.NewContext(context.Background(), md)
}

func (h *ManagerClient) getContextWithLimitAndOffset(limit, offset int) context.Context {
	h.RLock()
	defer h.RUnlock()
	md := metadata.Pairs(
		"id", h.id,
		"token", h.accessToken,
		"limit", strconv.Itoa(limit),
		"offset", strconv.Itoa(offset),
	)
	return metadata.NewContext(context.Background(), md)
}

// GetApplication retrieves an application from the Handler
func (h *ManagerClient) GetApplication(appID string) (*Application, error) {
	res, err := h.applicationManagerClient.GetApplication(h.getContext(), &ApplicationIdentifier{AppId: appID})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not get application from Handler")
	}
	return res, nil
}

// SetApplication sets an application on the Handler
func (h *ManagerClient) SetApplication(in *Application) error {
	_, err := h.applicationManagerClient.SetApplication(h.getContext(), in)
	return errors.Wrap(errors.FromGRPCError(err), "Could not set application on Handler")
}

// RegisterApplication registers an application on the Handler
func (h *ManagerClient) RegisterApplication(appID string) error {
	_, err := h.applicationManagerClient.RegisterApplication(h.getContext(), &ApplicationIdentifier{AppId: appID})
	return errors.Wrap(errors.FromGRPCError(err), "Could not register application on Handler")
}

// DeleteApplication deletes an application and all its devices from the Handler
func (h *ManagerClient) DeleteApplication(appID string) error {
	_, err := h.applicationManagerClient.DeleteApplication(h.getContext(), &ApplicationIdentifier{AppId: appID})
	return errors.Wrap(errors.FromGRPCError(err), "Could not delete application from Handler")
}

// GetDevice retrieves a device from the Handler
func (h *ManagerClient) GetDevice(appID string, devID string) (*Device, error) {
	res, err := h.applicationManagerClient.GetDevice(h.getContext(), &DeviceIdentifier{AppId: appID, DevId: devID})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not get device from Handler")
	}
	return res, nil
}

// SetDevice sets a device on the Handler
func (h *ManagerClient) SetDevice(in *Device) error {
	_, err := h.applicationManagerClient.SetDevice(h.getContext(), in)
	return errors.Wrap(errors.FromGRPCError(err), "Could not set device on Handler")
}

// DeleteDevice deletes a device from the Handler
func (h *ManagerClient) DeleteDevice(appID string, devID string) error {
	_, err := h.applicationManagerClient.DeleteDevice(h.getContext(), &DeviceIdentifier{AppId: appID, DevId: devID})
	return errors.Wrap(errors.FromGRPCError(err), "Could not delete device from Handler")
}

// GetDevicesForApplication retrieves all devices for an application from the Handler.
// Pass a limit to indicate the maximum number of results you want to receive, and the offset to indicate how many results should be skipped.
func (h *ManagerClient) GetDevicesForApplication(appID string, limit, offset int) (devices []*Device, err error) {
	res, err := h.applicationManagerClient.GetDevicesForApplication(h.getContextWithLimitAndOffset(limit, offset), &ApplicationIdentifier{AppId: appID})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not get devices for application from Handler")
	}
	for _, dev := range res.Devices {
		devices = append(devices, dev)
	}
	return
}

// GetDevAddr requests a random device address with the given constraints
func (h *ManagerClient) GetDevAddr(constraints ...string) (types.DevAddr, error) {
	devAddrManager := lorawan.NewDevAddrManagerClient(h.conn)
	resp, err := devAddrManager.GetDevAddr(h.getContext(), &lorawan.DevAddrRequest{
		Usage: constraints,
	})
	if err != nil {
		return types.DevAddr{}, errors.Wrap(errors.FromGRPCError(err), "Could not get DevAddr from Handler")
	}
	return *resp.DevAddr, nil
}

// DryUplink transforms the uplink payload with the payload functions provided
// in the app..
func (h *ManagerClient) DryUplink(payload []byte, app *Application, port uint32) (*DryUplinkResult, error) {
	res, err := h.applicationManagerClient.DryUplink(h.getContext(), &DryUplinkMessage{
		App:     app,
		Payload: payload,
		Port:    port,
	})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not dry-run uplink on Handler")
	}
	return res, nil
}

// DryDownlinkWithPayload transforms the downlink payload with the payload functions
// provided in app.
func (h *ManagerClient) DryDownlinkWithPayload(payload []byte, app *Application, port uint32) (*DryDownlinkResult, error) {
	res, err := h.applicationManagerClient.DryDownlink(h.getContext(), &DryDownlinkMessage{
		App:     app,
		Payload: payload,
		Port:    port,
	})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not dry-run downlink with payload on Handler")
	}
	return res, nil
}

// DryDownlinkWithFields transforms the downlink fields with the payload functions
// provided in app.
func (h *ManagerClient) DryDownlinkWithFields(fields map[string]interface{}, app *Application, port uint32) (*DryDownlinkResult, error) {
	marshalled, err := json.Marshal(fields)
	if err != nil {
		return nil, err
	}

	res, err := h.applicationManagerClient.DryDownlink(h.getContext(), &DryDownlinkMessage{
		App:    app,
		Fields: string(marshalled),
		Port:   port,
	})
	if err != nil {
		return nil, errors.Wrap(errors.FromGRPCError(err), "Could not dry-run downlink with fields on Handler")
	}
	return res, nil
}

// Close closes the client
func (h *ManagerClient) Close() error {
	return h.conn.Close()
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package repos contains information about Go source repositories.
package repos

import "fmt"

type Repo struct {
	// GoGerritProject, if non-empty, is the repo's Gerrit project
	// name, such as "go", "net", or "sys".
	GoGerritProject string

	// ImportPath is the repo's import path.
	// It is empty for the main Go repo and other repos that do not
	// contain Go code.
	ImportPath string

	// MirrorToGitHub controls whether this repo is mirrored
	// from Gerrit to GitHub. If true, GoGerritProject and
	// gitHubRepo must both be defined.
	MirrorToGitHub bool

	// MirrorToCSR controls whether this repo is mirrored from
	// Gerrit to Cloud Source Repositories. If true, GoGerritProject
	// must be defined. It will be mirrored to a CSR repo with the
	// same name as the Gerrit repo.
	MirrorToCSR bool

	// showOnDashboard is whether to show the repo on the bottom
	// of build.golang.org in the repo overview section.
	showOnDashboard bool

	// CoordinatorCanBuild reports whether this a repo that the
	// build coordinator knows how to build.
	CoordinatorCanBuild bool

	// GitHubRepo is the "org/repo" of where this repo exists on
	// GitHub. If MirrorToGitHub is true, this is the
	// destination.
	GitHubRepo string

	// WebsiteDesc is the description of the repo for showing on
	// https://golang.org/pkg/#subrepo.
	// It should be plain text. Hostnames may be auto-linkified.
	WebsiteDesc string
}

// ByGerritProject maps from a Gerrit project name ("go", "net", etc)
// to the Repo's information.
var ByGerritProject = map[string]*Repo{ /* initialized below */ }

// ByImportPath maps from an import path ("golang.org/x/net") to the
// Repo's information.
var ByImportPath = map[string]*Repo{ /* initialized below */ }

func init() {
	addMirrored("go", coordinatorCanBuild, noDash, enableCSR)
	addMirrored("dl", importPath("golang.org/dl"), coordinatorCanBuild)
	addMirrored("gddo", importPath("github.com/golang/gddo"), archivedOnGitHub)
	addMirrored("gofrontend")
	addMirrored("proposal")
	addMirrored("sublime-build")
	addMirrored("sublime-config")

	x("arch")
	x("benchmarks", desc("benchmarks to measure Go as it is developed"))
	x("blog", noDash)
	x("build", desc("build.golang.org's implementation"))
	x("crypto", desc("additional cryptography packages"))
	x("debug", desc("an experimental debugger for Go"))
	x("example", noDash)
	x("exp", desc("experimental and deprecated packages (handle with care; may change without warning)"))
	x("image", desc("additional imaging packages"))
	x("lint", noDash, archivedOnGitHub)
	x("mobile", desc("experimental support for Go on mobile platforms"))
	x("mod")
	x("net", desc("additional networking packages"))
	x("oauth2")
	x("perf", desc("packages and tools for performance measurement, storage, and analysis"))
	x("pkgsite", desc("home of the pkg.go.dev website"), noBuildAndNoDash)
	x("playground", noDash, enableCSR)
	x("review", desc("a tool for working with Gerrit code reviews"))
	x("scratch", noDash)
	x("sync", desc("additional concurrency primitives"))
	x("sys", desc("packages for making system calls"))
	x("talks")
	x("term")
	x("text", desc("packages for working with text"))
	x("time", desc("additional time packages"))
	x("tools", desc("godoc, goimports, gorename, and other tools"))
	x("tour", noDash)
	x("vgo", noDash)
	x("vulndb", desc("the Go Vulnerability Database and packages for working with it"))
	x("website", desc("home of the golang.org and go.dev websites"), enableCSR)
	x("xerrors", noDash)

	add(&Repo{GoGerritProject: "gollvm"})
	add(&Repo{GoGerritProject: "grpc-review"})

	add(&Repo{
		GoGerritProject: "protobuf",
		MirrorToGitHub:  true,
		ImportPath:      "google.golang.org/protobuf",
		GitHubRepo:      "protocolbuffers/protobuf-go",
	})

	add(&Repo{
		GoGerritProject: "vscode-go",
		MirrorToGitHub:  true,
		GitHubRepo:      "golang/vscode-go",
	})
}

type modifyRepo func(*Repo)

// noDash is an option to the x func that marks the repo as hidden on
// the https://build.golang.org/ dashboard.
func noDash(r *Repo) { r.showOnDashboard = false }

func noBuildAndNoDash(r *Repo) { r.CoordinatorCanBuild, r.showOnDashboard = false, false }

func coordinatorCanBuild(r *Repo) { r.CoordinatorCanBuild = true }

func archivedOnGitHub(r *Repo) {
	// When a repository is archived on GitHub, trying to push
	// to it will fail. So don't mirror.
	r.MirrorToGitHub = false
}

func enableCSR(r *Repo) {
	r.MirrorToCSR = true
}

func importPath(v string) modifyRepo { return func(r *Repo) { r.ImportPath = v } }

func desc(v string) modifyRepo { return func(r *Repo) { r.WebsiteDesc = v } }

// addMirrored adds a repo that's on Gerrit and mirrored to GitHub.
func addMirrored(proj string, opts ...modifyRepo) {
	repo := &Repo{
		GoGerritProject: proj,
		MirrorToGitHub:  true,
		GitHubRepo:      "golang/" + proj,
	}
	for _, o := range opts {
		o(repo)
	}
	add(repo)
}

// x adds a golang.org/x repo.
func x(proj string, opts ...modifyRepo) {
	repo := &Repo{
		GoGerritProject:     proj,
		MirrorToGitHub:      true,
		CoordinatorCanBuild: true,
		ImportPath:          "golang.org/x/" + proj,
		GitHubRepo:          "golang/" + proj,
		showOnDashboard:     true,
	}
	for _, o := range opts {
		o(repo)
	}
	add(repo)
}

func add(r *Repo) {
	if (r.MirrorToCSR || r.MirrorToGitHub || r.showOnDashboard) && r.GoGerritProject == "" {
		panic(fmt.Sprintf("project %+v sets feature(s) that require a GoGerritProject, but has none", r))
	}
	if r.MirrorToGitHub && r.GitHubRepo == "" {
		panic(fmt.Sprintf("project %+v has MirrorToGitHub but no gitHubRepo", r))
	}
	if r.showOnDashboard && !r.CoordinatorCanBuild {
		panic(fmt.Sprintf("project %+v is showOnDashboard but not marked buildable by coordinator", r))
	}

	if p := r.GoGerritProject; p != "" {
		if _, dup := ByGerritProject[p]; dup {
			panic(fmt.Sprintf("duplicate Gerrit project %q in %+v", p, r))
		}
		ByGerritProject[p] = r
	}
	if p := r.ImportPath; p != "" {
		if _, dup := ByImportPath[p]; dup {
			panic(fmt.Sprintf("duplicate import path %q in %+v", p, r))
		}
		ByImportPath[p] = r
	}
}

// ShowOnDashboard reports whether this repo should show up on build.golang.org
// in the list of repos at bottom.
//
// When this returns true, r.GoGerritProject is guaranteed to be non-empty.
func (r *Repo) ShowOnDashboard() bool { return r.showOnDashboard }
<commit_msg>repos: enable CSR mirroring for x/vulndb<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package repos contains information about Go source repositories.
package repos

import "fmt"

type Repo struct {
	// GoGerritProject, if non-empty, is the repo's Gerrit project
	// name, such as "go", "net", or "sys".
	GoGerritProject string

	// ImportPath is the repo's import path.
	// It is empty for the main Go repo and other repos that do not
	// contain Go code.
	ImportPath string

	// MirrorToGitHub controls whether this repo is mirrored
	// from Gerrit to GitHub. If true, GoGerritProject and
	// gitHubRepo must both be defined.
	MirrorToGitHub bool

	// MirrorToCSR controls whether this repo is mirrored from
	// Gerrit to Cloud Source Repositories. If true, GoGerritProject
	// must be defined. It will be mirrored to a CSR repo with the
	// same name as the Gerrit repo.
	MirrorToCSR bool

	// showOnDashboard is whether to show the repo on the bottom
	// of build.golang.org in the repo overview section.
	showOnDashboard bool

	// CoordinatorCanBuild reports whether this a repo that the
	// build coordinator knows how to build.
	CoordinatorCanBuild bool

	// GitHubRepo is the "org/repo" of where this repo exists on
	// GitHub. If MirrorToGitHub is true, this is the
	// destination.
	GitHubRepo string

	// WebsiteDesc is the description of the repo for showing on
	// https://golang.org/pkg/#subrepo.
	// It should be plain text. Hostnames may be auto-linkified.
	WebsiteDesc string
}

// ByGerritProject maps from a Gerrit project name ("go", "net", etc)
// to the Repo's information.
var ByGerritProject = map[string]*Repo{ /* initialized below */ }

// ByImportPath maps from an import path ("golang.org/x/net") to the
// Repo's information.
var ByImportPath = map[string]*Repo{ /* initialized below */ }

func init() {
	addMirrored("go", coordinatorCanBuild, noDash, enableCSR)
	addMirrored("dl", importPath("golang.org/dl"), coordinatorCanBuild)
	addMirrored("gddo", importPath("github.com/golang/gddo"), archivedOnGitHub)
	addMirrored("gofrontend")
	addMirrored("proposal")
	addMirrored("sublime-build")
	addMirrored("sublime-config")

	x("arch")
	x("benchmarks", desc("benchmarks to measure Go as it is developed"))
	x("blog", noDash)
	x("build", desc("build.golang.org's implementation"))
	x("crypto", desc("additional cryptography packages"))
	x("debug", desc("an experimental debugger for Go"))
	x("example", noDash)
	x("exp", desc("experimental and deprecated packages (handle with care; may change without warning)"))
	x("image", desc("additional imaging packages"))
	x("lint", noDash, archivedOnGitHub)
	x("mobile", desc("experimental support for Go on mobile platforms"))
	x("mod")
	x("net", desc("additional networking packages"))
	x("oauth2")
	x("perf", desc("packages and tools for performance measurement, storage, and analysis"))
	x("pkgsite", desc("home of the pkg.go.dev website"), noBuildAndNoDash)
	x("playground", noDash, enableCSR)
	x("review", desc("a tool for working with Gerrit code reviews"))
	x("scratch", noDash)
	x("sync", desc("additional concurrency primitives"))
	x("sys", desc("packages for making system calls"))
	x("talks")
	x("term")
	x("text", desc("packages for working with text"))
	x("time", desc("additional time packages"))
	x("tools", desc("godoc, goimports, gorename, and other tools"))
	x("tour", noDash)
	x("vgo", noDash)
	x("vulndb", desc("the Go Vulnerability Database and packages for working with it"), enableCSR)
	x("website", desc("home of the golang.org and go.dev websites"), enableCSR)
	x("xerrors", noDash)

	add(&Repo{GoGerritProject: "gollvm"})
	add(&Repo{GoGerritProject: "grpc-review"})

	add(&Repo{
		GoGerritProject: "protobuf",
		MirrorToGitHub:  true,
		ImportPath:      "google.golang.org/protobuf",
		GitHubRepo:      "protocolbuffers/protobuf-go",
	})

	add(&Repo{
		GoGerritProject: "vscode-go",
		MirrorToGitHub:  true,
		GitHubRepo:      "golang/vscode-go",
	})
}

type modifyRepo func(*Repo)

// noDash is an option to the x func that marks the repo as hidden on
// the https://build.golang.org/ dashboard.
func noDash(r *Repo) { r.showOnDashboard = false }

func noBuildAndNoDash(r *Repo) { r.CoordinatorCanBuild, r.showOnDashboard = false, false }

func coordinatorCanBuild(r *Repo) { r.CoordinatorCanBuild = true }

func archivedOnGitHub(r *Repo) {
	// When a repository is archived on GitHub, trying to push
	// to it will fail. So don't mirror.
	r.MirrorToGitHub = false
}

func enableCSR(r *Repo) {
	r.MirrorToCSR = true
}

func importPath(v string) modifyRepo { return func(r *Repo) { r.ImportPath = v } }

func desc(v string) modifyRepo { return func(r *Repo) { r.WebsiteDesc = v } }

// addMirrored adds a repo that's on Gerrit and mirrored to GitHub.
func addMirrored(proj string, opts ...modifyRepo) {
	repo := &Repo{
		GoGerritProject: proj,
		MirrorToGitHub:  true,
		GitHubRepo:      "golang/" + proj,
	}
	for _, o := range opts {
		o(repo)
	}
	add(repo)
}

// x adds a golang.org/x repo.
func x(proj string, opts ...modifyRepo) {
	repo := &Repo{
		GoGerritProject:     proj,
		MirrorToGitHub:      true,
		CoordinatorCanBuild: true,
		ImportPath:          "golang.org/x/" + proj,
		GitHubRepo:          "golang/" + proj,
		showOnDashboard:     true,
	}
	for _, o := range opts {
		o(repo)
	}
	add(repo)
}

func add(r *Repo) {
	if (r.MirrorToCSR || r.MirrorToGitHub || r.showOnDashboard) && r.GoGerritProject == "" {
		panic(fmt.Sprintf("project %+v sets feature(s) that require a GoGerritProject, but has none", r))
	}
	if r.MirrorToGitHub && r.GitHubRepo == "" {
		panic(fmt.Sprintf("project %+v has MirrorToGitHub but no gitHubRepo", r))
	}
	if r.showOnDashboard && !r.CoordinatorCanBuild {
		panic(fmt.Sprintf("project %+v is showOnDashboard but not marked buildable by coordinator", r))
	}

	if p := r.GoGerritProject; p != "" {
		if _, dup := ByGerritProject[p]; dup {
			panic(fmt.Sprintf("duplicate Gerrit project %q in %+v", p, r))
		}
		ByGerritProject[p] = r
	}
	if p := r.ImportPath; p != "" {
		if _, dup := ByImportPath[p]; dup {
			panic(fmt.Sprintf("duplicate import path %q in %+v", p, r))
		}
		ByImportPath[p] = r
	}
}

// ShowOnDashboard reports whether this repo should show up on build.golang.org
// in the list of repos at bottom.
//
// When this returns true, r.GoGerritProject is guaranteed to be non-empty.
func (r *Repo) ShowOnDashboard() bool { return r.showOnDashboard }
<|endoftext|>


<commit_before>package base

import (
	"fmt"
	"io"
	"strings"
	"unicode"
)

type CodeWriter struct {
	currentMargin     string
	marginStack       []string
	pendingEmptyLines int
	Out               io.Writer
}

func (w *CodeWriter) flushPending() {
	for w.pendingEmptyLines > 0 {
		w.writeLine("")
		w.pendingEmptyLines -= 1
	}
}

func (w *CodeWriter) writeLine(text string) {
	line := strings.TrimRightFunc(w.currentMargin+text, unicode.IsSpace)
	io.WriteString(w.Out, line)
	io.WriteString(w.Out, "\n")
}

func (w *CodeWriter) Line(text string) {
	w.flushPending()
	w.writeLine(text)
}

func (w *CodeWriter) Linef(format string, args ...interface{}) {
	w.Line(fmt.Sprintf(format, args...))
}

func (w *CodeWriter) EmptyLines(count int) {
	if count > w.pendingEmptyLines {
		w.pendingEmptyLines = count
	}
}

func (w *CodeWriter) PushMargin(margin string) {
	w.marginStack = append(w.marginStack, w.currentMargin)
	w.currentMargin += margin
}

func (w *CodeWriter) PopMargin() {
	top := len(w.marginStack) - 1
	w.currentMargin = w.marginStack[top]
	w.marginStack = w.marginStack[:top]
}
<commit_msg>Increase the strictness of CodeWriter.Linef.<commit_after>package base

import (
	"fmt"
	"io"
	"strings"
	"unicode"
)

type CodeWriter struct {
	currentMargin     string
	marginStack       []string
	pendingEmptyLines int
	Out               io.Writer
}

func (w *CodeWriter) flushPending() {
	for w.pendingEmptyLines > 0 {
		w.writeLine("")
		w.pendingEmptyLines -= 1
	}
}

func (w *CodeWriter) writeLine(text string) {
	line := strings.TrimRightFunc(w.currentMargin+text, unicode.IsSpace)
	io.WriteString(w.Out, line)
	io.WriteString(w.Out, "\n")
}

func (w *CodeWriter) Line(text string) {
	w.flushPending()
	w.writeLine(text)
}

// The type signature of this function is stricter than what is done in fmt.
// This helps catch refactoring bugs where types being passed to Sprintf change.
// In general, code generators should not be using format strings for turning
// non-string data types into strings. Be explicit, don't implicitly rely on how
// Go formats data.
func (w *CodeWriter) Linef(format string, args ...string) {
	// "arg..." passes the slice straight through, so the type needs to match.
	// This is a gotcha for Python programers - the argument list is not expanded.
	rewrapped := make([]interface{}, len(args))
	for i, arg := range args {
		rewrapped[i] = arg
	}
	w.Line(fmt.Sprintf(format, rewrapped...))
}

func (w *CodeWriter) EmptyLines(count int) {
	if count > w.pendingEmptyLines {
		w.pendingEmptyLines = count
	}
}

func (w *CodeWriter) PushMargin(margin string) {
	w.marginStack = append(w.marginStack, w.currentMargin)
	w.currentMargin += margin
}

func (w *CodeWriter) PopMargin() {
	top := len(w.marginStack) - 1
	w.currentMargin = w.marginStack[top]
	w.marginStack = w.marginStack[:top]
}
<|endoftext|>


<commit_before>package main

import (
	"html/template"
)

type Reporter struct {
	Project     string              `json:"project"`
	UnitTestx   UnitTest            `json:"unit_test"`
	Cyclox      map[string]Cycloi   `json:"cyclo"`
	SimpleTips  map[string][]string `json:"simple_tips"`
	CopyTips    [][]string          `json:"copy_tips"`
	ScanTips    map[string][]string `json:"scan_tips"`
	DependGraph string              `json:"depend_graph"`
	DeadCode    []string            `json:"dead_code"`
	NoTestPkg   []string            `json:"no_test_package"`
	SpellError  []string            `json:"spell_error"`
}

type UnitTest struct {
	AvgCover           string                 `json:"average_cover"`
	PackagesTestDetail map[string]PackageTest `json:"packages_test_detail"`
	PackagesRaceDetail map[string][]string    `json:"packages_race_detail"`
}

type PackageTest struct {
	IsPass   bool    `json:"is_pass"`
	Coverage string  `json:"coverage"`
	Time     float64 `json:"time"`
}

type Cycloi struct {
	Average string   `json:"average"`
	Result  []string `json:"result"`
}

type Test struct {
	Path   string  `json:path`
	Result int     `json:result`
	Time   float64 `json:time`
	Cover  float64 `json:cover`
}

type File struct {
	Color     string
	CycloVal  string
	CycloInfo string
}

type Copycode struct {
	Files string
	Path  []string
}

type Race struct {
	Pkg  string
	Len  string
	Leng string
	Info []string
}

type Simple struct {
	Path string `json:path`
	Info string `json:info`
}

type Scan struct {
	Path string `json:path`
	Info string `json:info`
}

type Deadcode struct {
	Path string `json:path`
	Info string `json:info`
}
type CycloInfo struct {
	Comp int
	Info string
}
type Cyclo struct {
	Pkg  string
	Size int
	Info []CycloInfo
}

type HtmlData struct {
	Project               string
	Score                 int
	Tests                 string
	TestSummaryCoverAvg   string
	Races                 []Race
	NoTests               string
	Simples               string
	SimpleLevel           int
	Deadcodes             string
	Copycodes             string
	Cyclos                string
	CycloSummarysCycloAvg string
	CycloSummarysCyclo50  string
	CycloSummarysCyclo15  string
	DepGraph              template.HTML
}

type ApolloMeta struct {
	Branch         string `json:"branch"`
	Project        string `json:"project"`
	CommitID       string `json:"commitid"`
	CommitUser     string `json:"commituser"`
	User           string `json:"user"`
	UnintTestCover string `json:"uninttestcover"`
	StaticCheck    string `json:"staticcheck"`
	CycloBig       string `json:"cyclobig"`
	Score          int    `json:"score"`
	Starttime      string `json:"starttime"`
	Endtime        string `json:"endtime"`
}

// type OptionMeta struct {
// 	Branch     string `json:"branch"`
// 	CommitID   string `json:"commitID"`
// 	CommitUser string `json:"commitUser"`
// 	User       string `json:"user"`
// }

// type Value struct {
// 	Filepath string     `json:"filepath"`
// 	Meta     ApolloMeta `json:"meta"`
// }
<commit_msg>add build<commit_after>package main

import (
	"html/template"
)

type Reporter struct {
	Project     string              `json:"project"`
	UnitTestx   UnitTest            `json:"unit_test"`
	Cyclox      map[string]Cycloi   `json:"cyclo"`
	SimpleTips  map[string][]string `json:"simple_tips"`
	CopyTips    [][]string          `json:"copy_tips"`
	ScanTips    map[string][]string `json:"scan_tips"`
	DependGraph string              `json:"depend_graph"`
	DeadCode    []string            `json:"dead_code"`
	NoTestPkg   []string            `json:"no_test_package"`
	SpellError  []string            `json:"spell_error"`
}

type UnitTest struct {
	AvgCover           string                 `json:"average_cover"`
	PackagesTestDetail map[string]PackageTest `json:"packages_test_detail"`
	PackagesRaceDetail map[string][]string    `json:"packages_race_detail"`
}

type PackageTest struct {
	IsPass   bool    `json:"is_pass"`
	Coverage string  `json:"coverage"`
	Time     float64 `json:"time"`
}

type Cycloi struct {
	Average string   `json:"average"`
	Result  []string `json:"result"`
}

type Test struct {
	Path   string  `json:"path"`
	Result int     `json:"result"`
	Time   float64 `json:"time"`
	Cover  float64 `json:"cover"`
}

type File struct {
	Color     string
	CycloVal  string
	CycloInfo string
}

type Copycode struct {
	Files string
	Path  []string
}

type Race struct {
	Pkg  string
	Len  string
	Leng string
	Info []string
}

type Simple struct {
	Path string `json:"path"`
	Info string `json:"info"`
}

type Scan struct {
	Path string `json:"path"`
	Info string `json:"info"`
}

type Deadcode struct {
	Path string `json:"path"`
	Info string `json:"info"`
}
type CycloInfo struct {
	Comp int
	Info string
}
type Cyclo struct {
	Pkg  string
	Size int
	Info []CycloInfo
}

type HtmlData struct {
	Project               string
	Score                 int
	Tests                 string
	TestSummaryCoverAvg   string
	Races                 []Race
	NoTests               string
	Simples               string
	SimpleLevel           int
	Deadcodes             string
	Copycodes             string
	Cyclos                string
	CycloSummarysCycloAvg string
	CycloSummarysCyclo50  string
	CycloSummarysCyclo15  string
	DepGraph              template.HTML
}

type ApolloMeta struct {
	Branch         string `json:"branch"`
	Project        string `json:"project"`
	CommitID       string `json:"commitid"`
	CommitUser     string `json:"commituser"`
	User           string `json:"user"`
	UnintTestCover string `json:"uninttestcover"`
	StaticCheck    string `json:"staticcheck"`
	CycloBig       string `json:"cyclobig"`
	Score          int    `json:"score"`
	Starttime      string `json:"starttime"`
	Endtime        string `json:"endtime"`
}

// type OptionMeta struct {
// 	Branch     string `json:"branch"`
// 	CommitID   string `json:"commitID"`
// 	CommitUser string `json:"commitUser"`
// 	User       string `json:"user"`
// }

// type Value struct {
// 	Filepath string     `json:"filepath"`
// 	Meta     ApolloMeta `json:"meta"`
// }
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"github.com/golang/glog"
	"github.com/kubernetes-incubator/service-catalog/pkg/registry/servicecatalog/server"
	"k8s.io/apiserver/pkg/registry/generic"
	"k8s.io/apiserver/pkg/registry/generic/registry"
	genericapiserver "k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/storage"
)

// EtcdConfig contains a generic API server Config along with config specific to
// the service catalog API server.
type etcdConfig struct {
	genericConfig *genericapiserver.RecommendedConfig
	extraConfig   *extraConfig
}

// extraConfig contains all additional configuration parameters for etcdConfig
type extraConfig struct {
	// BABYNETES: cargo culted from master.go
	deleteCollectionWorkers int
	storageFactory          storage.StorageFactory
}

// NewEtcdConfig returns a new server config to describe an etcd-backed API server
func NewEtcdConfig(
	genCfg *genericapiserver.RecommendedConfig,
	deleteCollWorkers int,
	factory storage.StorageFactory,
) Config {
	return &etcdConfig{
		genericConfig: genCfg,
		extraConfig: &extraConfig{
			deleteCollectionWorkers: deleteCollWorkers,
			storageFactory:          factory,
		},
	}
}

// Complete fills in any fields not set that are required to have valid data
// and can be derived from other fields.
func (c *etcdConfig) Complete() CompletedConfig {
	completedGenericConfig := completeGenericConfig(c.genericConfig)
	return completedEtcdConfig{
		genericConfig: completedGenericConfig,
		extraConfig:   c.extraConfig,
		// Not every API group compiled in is necessarily enabled by the operator
		// at runtime.
		//
		// Install the API resource config source, which describes versions of
		// which API groups are enabled.
		apiResourceConfigSource: DefaultAPIResourceConfigSource(),
	}
}

// CompletedEtcdConfig is an internal type to take advantage of typechecking in
// the type system.
type completedEtcdConfig struct {
	genericConfig           genericapiserver.CompletedConfig
	extraConfig             *extraConfig
	apiResourceConfigSource storage.APIResourceConfigSource
}

// NewServer creates a new server that can be run. Returns a non-nil error if the server couldn't
// be created
func (c completedEtcdConfig) NewServer(stopCh <-chan struct{}) (*ServiceCatalogAPIServer, error) {
	s, err := createSkeletonServer(c.genericConfig)
	if err != nil {
		return nil, err
	}
	glog.V(4).Infoln("Created skeleton API server")

	roFactory := etcdRESTOptionsFactory{
		deleteCollectionWorkers: c.extraConfig.deleteCollectionWorkers,
		// TODO https://github.com/kubernetes/kubernetes/issues/44507
		// we still need to enable it so finalizers are respected.
		enableGarbageCollection: true,
		storageFactory:          c.extraConfig.storageFactory,
		storageDecorator:        generic.UndecoratedStorage,
	}

	glog.V(4).Infoln("Installing API groups")
	// default namespace doesn't matter for etcd
	providers := restStorageProviders("" /* default namespace */, server.StorageTypeEtcd, nil)
	for _, provider := range providers {
		groupInfo, err := provider.NewRESTStorage(c.apiResourceConfigSource, roFactory)
		if IsErrAPIGroupDisabled(err) {
			glog.Warningf("Skipping API group %v because it is not enabled", provider.GroupName())
			continue
		} else if err != nil {
			return nil, err
		}

		glog.V(4).Infof("Installing API group %v", provider.GroupName())
		if err := s.GenericAPIServer.InstallAPIGroup(groupInfo); err != nil {
			glog.Fatalf("Error installing API group %v: %v", provider.GroupName(), err)
		} else {
			// we've sucessfully installed, so hook the stopCh to the destroy func of all the sucessfully installed apigroups
			for _, mappings := range groupInfo.VersionedResourcesStorageMap { // gv to resource mappings
				for _, storage := range mappings { // resource name (brokers, brokers/status) to backing storage
					go func() {
						s, ok := storage.(*registry.Store)
						if ok {
							<-stopCh
							s.DestroyFunc()
						}
					}()
				}
			}
		}
	}

	glog.Infoln("Finished installing API groups")

	return s, nil
}
<commit_msg>Call destroy function on each storage interface (#1705)<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"github.com/golang/glog"
	"github.com/kubernetes-incubator/service-catalog/pkg/registry/servicecatalog/server"
	"k8s.io/apiserver/pkg/registry/generic"
	"k8s.io/apiserver/pkg/registry/generic/registry"
	"k8s.io/apiserver/pkg/registry/rest"
	genericapiserver "k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/storage"
)

// EtcdConfig contains a generic API server Config along with config specific to
// the service catalog API server.
type etcdConfig struct {
	genericConfig *genericapiserver.RecommendedConfig
	extraConfig   *extraConfig
}

// extraConfig contains all additional configuration parameters for etcdConfig
type extraConfig struct {
	// BABYNETES: cargo culted from master.go
	deleteCollectionWorkers int
	storageFactory          storage.StorageFactory
}

// NewEtcdConfig returns a new server config to describe an etcd-backed API server
func NewEtcdConfig(
	genCfg *genericapiserver.RecommendedConfig,
	deleteCollWorkers int,
	factory storage.StorageFactory,
) Config {
	return &etcdConfig{
		genericConfig: genCfg,
		extraConfig: &extraConfig{
			deleteCollectionWorkers: deleteCollWorkers,
			storageFactory:          factory,
		},
	}
}

// Complete fills in any fields not set that are required to have valid data
// and can be derived from other fields.
func (c *etcdConfig) Complete() CompletedConfig {
	completedGenericConfig := completeGenericConfig(c.genericConfig)
	return completedEtcdConfig{
		genericConfig: completedGenericConfig,
		extraConfig:   c.extraConfig,
		// Not every API group compiled in is necessarily enabled by the operator
		// at runtime.
		//
		// Install the API resource config source, which describes versions of
		// which API groups are enabled.
		apiResourceConfigSource: DefaultAPIResourceConfigSource(),
	}
}

// CompletedEtcdConfig is an internal type to take advantage of typechecking in
// the type system.
type completedEtcdConfig struct {
	genericConfig           genericapiserver.CompletedConfig
	extraConfig             *extraConfig
	apiResourceConfigSource storage.APIResourceConfigSource
}

// NewServer creates a new server that can be run. Returns a non-nil error if the server couldn't
// be created
func (c completedEtcdConfig) NewServer(stopCh <-chan struct{}) (*ServiceCatalogAPIServer, error) {
	s, err := createSkeletonServer(c.genericConfig)
	if err != nil {
		return nil, err
	}
	glog.V(4).Infoln("Created skeleton API server")

	roFactory := etcdRESTOptionsFactory{
		deleteCollectionWorkers: c.extraConfig.deleteCollectionWorkers,
		// TODO https://github.com/kubernetes/kubernetes/issues/44507
		// we still need to enable it so finalizers are respected.
		enableGarbageCollection: true,
		storageFactory:          c.extraConfig.storageFactory,
		storageDecorator:        generic.UndecoratedStorage,
	}

	glog.V(4).Infoln("Installing API groups")
	// default namespace doesn't matter for etcd
	providers := restStorageProviders("" /* default namespace */, server.StorageTypeEtcd, nil)
	for _, provider := range providers {
		groupInfo, err := provider.NewRESTStorage(c.apiResourceConfigSource, roFactory)
		if IsErrAPIGroupDisabled(err) {
			glog.Warningf("Skipping API group %v because it is not enabled", provider.GroupName())
			continue
		} else if err != nil {
			return nil, err
		}

		glog.V(4).Infof("Installing API group %v", provider.GroupName())
		if err := s.GenericAPIServer.InstallAPIGroup(groupInfo); err != nil {
			glog.Fatalf("Error installing API group %v: %v", provider.GroupName(), err)
		} else {
			// we've sucessfully installed, so hook the stopCh to the destroy func of all the sucessfully installed apigroups
			for _, mappings := range groupInfo.VersionedResourcesStorageMap { // gv to resource mappings
				for _, storage := range mappings { // resource name (brokers, brokers/status) to backing storage
					go func(store rest.Storage) {
						s, ok := store.(*registry.Store)
						if ok {
							<-stopCh
							s.DestroyFunc()
						}
					}(storage)
				}
			}
		}
	}

	glog.Infoln("Finished installing API groups")

	return s, nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2019 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Binary extract extracts a pre-configured OpenJDK11 source tree to produce Kythe compilation units.
package main

import (
	"bufio"
	"context"
	"errors"
	"flag"
	"io"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"strings"

	"kythe.io/kythe/go/util/flagutil"

	"github.com/bazelbuild/rules_go/go/tools/bazel"
)

const (
	javaCommandVar      = "KYTHE_JAVA_COMMAND"
	wrapperExtractorVar = "KYTHE_JAVA_EXTRACTOR_JAR"

	kytheRootVar   = "KYTHE_ROOT_DIRECTORY"
	kytheOutputVar = "KYTHE_OUTPUT_DIRECTORY"
	kytheCorpusVar = "KYTHE_CORPUS"
	kytheVNameVar  = "KYTHE_VNAMES"

	javaMakeVar           = "JAVA_CMD"
	runfilesWrapperPath   = "io_kythe/kythe/extractors/openjdk11/java_wrapper"
	runfilesVNamesPath    = "io_kythe/kythe/extractors/openjdk11/vnames.json"
	runfilesExtractorPath = "io_kythe/kythe/java/com/google/devtools/kythe/extractors/java/standalone/javac9_extractor_deploy.jar"
)

var (
	makeDir       string
	makeTargets   = targetList{"clean", "jdk"}
	outputDir     string
	vNameRules    string
	wrapperPath   string
	extractorPath string
	errorPattern  = regexp.MustCompile("ERROR: extractor failure for module ([^:]*):")
)

// targetList is a simple comma-separated list of strings used
// for the -targets flag.
// flagutil.StringList always accumulates values, which we don't want.
type targetList []string

// Set implements part of the flag.Getter interface for targetList and will
// set the new value from s.
func (tl *targetList) Set(s string) error {
	*tl = strings.Split(s, ",")
	return nil
}

// String implements part of the flag.Getter interface for targetList and will
// return the value as a comma-separate string.
func (tl *targetList) String() string {
	if tl == nil {
		return ""
	}
	return strings.Join(*tl, ",")
}

func setupRunfiles() error {
	if os.Getenv("RUNFILES_DIR") != "" || os.Getenv("RUNFILES_MANIFEST_FILE") != "" {
		return nil
	}
	for _, base := range []string{os.Args[0] + ".runfiles", "."} {
		root, err := filepath.Abs(base)
		if err != nil {
			continue
		} else if _, err := os.Stat(root); err != nil {
			continue
		}
		os.Setenv("RUNFILES_DIR", root)
		manifest := filepath.Join(root, "MANIFEST")
		if _, err := os.Stat(manifest); err == nil {
			os.Setenv("RUNFILES_MANIFEST_FILE", manifest)
		}
		return nil
	}
	return errors.New("unable to setup runfiles")
}

func defaultWrapperPath() string {
	val, _ := bazel.Runfile(runfilesWrapperPath)
	return val
}

func defaultVNamesPath() string {
	val, _ := bazel.Runfile(runfilesVNamesPath)
	return val
}

func defaultExtractorPath() string {
	val, _ := bazel.Runfile(runfilesExtractorPath)
	return val
}

func defaultOutputDir() string {
	val := os.Getenv(kytheOutputVar)
	if val == "" {
		usr, err := user.Current()
		if err != nil {
			log.Fatalf("ERROR: unable to determine current user: %v", err)
		}
		val = filepath.Join(usr.HomeDir, "kythe-openjdk11-output")
	}
	return val
}

func findJavaCommand() (string, error) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	cmd := exec.CommandContext(ctx, "make", "-n", "-p")
	cmd.Dir = makeDir
	cmd.Stderr = os.Stderr
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	if err := cmd.Start(); err != nil {
		return "", err
	}
	const prefix = javaMakeVar + " := "
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		if strings.HasPrefix(scanner.Text(), prefix) {
			cancel() // Safe to call repeatedly.
			cmd.Wait()
			return strings.TrimPrefix(scanner.Text(), prefix), nil
		}
	}
	return "", cmd.Wait()
}

func mustFindJavaCommand() string {
	java, err := findJavaCommand()
	if err != nil {
		log.Fatalf("unable to determine %s: %v", javaCommandVar, err)
	}
	return java
}

func setEnvDefaultFunc(env []string, key string, value func() string) []string {
	if val := os.Getenv(key); val == "" {
		env = append(env, key+"="+value())
	}
	return env
}

func setEnvDefault(env []string, key, value string) []string {
	return setEnvDefaultFunc(env, key, func() string { return value })
}

func makeEnv() []string {
	env := os.Environ()
	env = setEnvDefaultFunc(env, javaCommandVar, mustFindJavaCommand)
	env = setEnvDefault(env, kytheCorpusVar, "openjdk11")
	if vNameRules != "" {
		env = setEnvDefault(env, kytheVNameVar, vNameRules)
	}
	env = append(env,
		kytheRootVar+"="+makeDir,
		kytheOutputVar+"="+outputDir,
		wrapperExtractorVar+"="+extractorPath)
	return env
}

func init() {
	setupRunfiles()
	flag.StringVar(&makeDir, "jdk", "", "path to the OpenJDK11 source tree (required)")
	flag.StringVar(&outputDir, "output", defaultOutputDir(), "path to which the compilations and errors should be written (optional)")
	flag.StringVar(&vNameRules, "rules", defaultVNamesPath(), "path of vnames.json file (optional)")
	flag.StringVar(&wrapperPath, "java_wrapper", defaultWrapperPath(), "path to the java_wrapper executable (optional)")
	flag.StringVar(&extractorPath, "extractor_jar", defaultExtractorPath(), "path to the javac_extractor_deployt.jar (optional)")
	flag.Var(&makeTargets, "targets", "comma-separated list of make targets to build")
	flag.Usage = flagutil.SimpleUsage("Extract a configured openjdk11 source directory", "[--java_wrapper=] [path]")
}

func main() {
	flag.Parse()
	if makeDir == "" {
		flagutil.UsageError("missing -jdk")
	}
	if wrapperPath == "" {
		flagutil.UsageError("missing -java_wrapper")
	}
	if _, err := os.Stat(wrapperPath); err != nil {
		flagutil.UsageErrorf("java_wrapper not found: %v", err)
	}

	cmd := exec.Command("make", append([]string{javaMakeVar + "=" + wrapperPath, "ENABLE_JAVAC_SERVER=no"}, makeTargets...)...)
	cmd.Dir = makeDir
	cmd.Env = makeEnv()
	cmd.Stdout = nil // Quiet, you
	stderr, err := cmd.StderrPipe()

	if err != nil {
		log.Fatal(err)
	}

	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	logCollectedErrors(io.TeeReader(stderr, os.Stderr))

	if err := cmd.Wait(); err != nil {
		log.Fatal(err)
	}
}

func collectExtractionErrors(stderr io.Reader) ([]string, error) {
	var result []string
	scanner := bufio.NewScanner(stderr)
	for scanner.Scan() {
		if subs := errorPattern.FindSubmatch(scanner.Bytes()); subs != nil {
			result = append(result, string(subs[1]))
		}
	}
	return result, scanner.Err()
}

func logCollectedErrors(stderr io.Reader) {
	errors, err := collectExtractionErrors(stderr)
	if err != nil {
		log.Println(err)
	}
	log.Println("Error extracting modules:\n\t", strings.Join(errors, "\n\t"))
}
<commit_msg>chore: fix a typo in the openjdk11 extractor script (#4125)<commit_after>/*
 * Copyright 2019 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Binary extract extracts a pre-configured OpenJDK11 source tree to produce Kythe compilation units.
package main

import (
	"bufio"
	"context"
	"errors"
	"flag"
	"io"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"strings"

	"kythe.io/kythe/go/util/flagutil"

	"github.com/bazelbuild/rules_go/go/tools/bazel"
)

const (
	javaCommandVar      = "KYTHE_JAVA_COMMAND"
	wrapperExtractorVar = "KYTHE_JAVA_EXTRACTOR_JAR"

	kytheRootVar   = "KYTHE_ROOT_DIRECTORY"
	kytheOutputVar = "KYTHE_OUTPUT_DIRECTORY"
	kytheCorpusVar = "KYTHE_CORPUS"
	kytheVNameVar  = "KYTHE_VNAMES"

	javaMakeVar           = "JAVA_CMD"
	runfilesWrapperPath   = "io_kythe/kythe/extractors/openjdk11/java_wrapper"
	runfilesVNamesPath    = "io_kythe/kythe/extractors/openjdk11/vnames.json"
	runfilesExtractorPath = "io_kythe/kythe/java/com/google/devtools/kythe/extractors/java/standalone/javac9_extractor_deploy.jar"
)

var (
	makeDir       string
	makeTargets   = targetList{"clean", "jdk"}
	outputDir     string
	vNameRules    string
	wrapperPath   string
	extractorPath string
	errorPattern  = regexp.MustCompile("ERROR: extractor failure for module ([^:]*):")
)

// targetList is a simple comma-separated list of strings used
// for the -targets flag.
// flagutil.StringList always accumulates values, which we don't want.
type targetList []string

// Set implements part of the flag.Getter interface for targetList and will
// set the new value from s.
func (tl *targetList) Set(s string) error {
	*tl = strings.Split(s, ",")
	return nil
}

// String implements part of the flag.Getter interface for targetList and will
// return the value as a comma-separate string.
func (tl *targetList) String() string {
	if tl == nil {
		return ""
	}
	return strings.Join(*tl, ",")
}

func setupRunfiles() error {
	if os.Getenv("RUNFILES_DIR") != "" || os.Getenv("RUNFILES_MANIFEST_FILE") != "" {
		return nil
	}
	for _, base := range []string{os.Args[0] + ".runfiles", "."} {
		root, err := filepath.Abs(base)
		if err != nil {
			continue
		} else if _, err := os.Stat(root); err != nil {
			continue
		}
		os.Setenv("RUNFILES_DIR", root)
		manifest := filepath.Join(root, "MANIFEST")
		if _, err := os.Stat(manifest); err == nil {
			os.Setenv("RUNFILES_MANIFEST_FILE", manifest)
		}
		return nil
	}
	return errors.New("unable to setup runfiles")
}

func defaultWrapperPath() string {
	val, _ := bazel.Runfile(runfilesWrapperPath)
	return val
}

func defaultVNamesPath() string {
	val, _ := bazel.Runfile(runfilesVNamesPath)
	return val
}

func defaultExtractorPath() string {
	val, _ := bazel.Runfile(runfilesExtractorPath)
	return val
}

func defaultOutputDir() string {
	val := os.Getenv(kytheOutputVar)
	if val == "" {
		usr, err := user.Current()
		if err != nil {
			log.Fatalf("ERROR: unable to determine current user: %v", err)
		}
		val = filepath.Join(usr.HomeDir, "kythe-openjdk11-output")
	}
	return val
}

func findJavaCommand() (string, error) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	cmd := exec.CommandContext(ctx, "make", "-n", "-p")
	cmd.Dir = makeDir
	cmd.Stderr = os.Stderr
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return "", err
	}
	if err := cmd.Start(); err != nil {
		return "", err
	}
	const prefix = javaMakeVar + " := "
	scanner := bufio.NewScanner(stdout)
	for scanner.Scan() {
		if strings.HasPrefix(scanner.Text(), prefix) {
			cancel() // Safe to call repeatedly.
			cmd.Wait()
			return strings.TrimPrefix(scanner.Text(), prefix), nil
		}
	}
	return "", cmd.Wait()
}

func mustFindJavaCommand() string {
	java, err := findJavaCommand()
	if err != nil {
		log.Fatalf("unable to determine %s: %v", javaCommandVar, err)
	}
	return java
}

func setEnvDefaultFunc(env []string, key string, value func() string) []string {
	if val := os.Getenv(key); val == "" {
		env = append(env, key+"="+value())
	}
	return env
}

func setEnvDefault(env []string, key, value string) []string {
	return setEnvDefaultFunc(env, key, func() string { return value })
}

func makeEnv() []string {
	env := os.Environ()
	env = setEnvDefaultFunc(env, javaCommandVar, mustFindJavaCommand)
	env = setEnvDefault(env, kytheCorpusVar, "openjdk11")
	if vNameRules != "" {
		env = setEnvDefault(env, kytheVNameVar, vNameRules)
	}
	env = append(env,
		kytheRootVar+"="+makeDir,
		kytheOutputVar+"="+outputDir,
		wrapperExtractorVar+"="+extractorPath)
	return env
}

func init() {
	setupRunfiles()
	flag.StringVar(&makeDir, "jdk", "", "path to the OpenJDK11 source tree (required)")
	flag.StringVar(&outputDir, "output", defaultOutputDir(), "path to which the compilations and errors should be written (optional)")
	flag.StringVar(&vNameRules, "rules", defaultVNamesPath(), "path of vnames.json file (optional)")
	flag.StringVar(&wrapperPath, "java_wrapper", defaultWrapperPath(), "path to the java_wrapper executable (optional)")
	flag.StringVar(&extractorPath, "extractor_jar", defaultExtractorPath(), "path to the javac_extractor_deploy.jar (optional)")
	flag.Var(&makeTargets, "targets", "comma-separated list of make targets to build")
	flag.Usage = flagutil.SimpleUsage("Extract a configured openjdk11 source directory", "[--java_wrapper=] [path]")
}

func main() {
	flag.Parse()
	if makeDir == "" {
		flagutil.UsageError("missing -jdk")
	}
	if wrapperPath == "" {
		flagutil.UsageError("missing -java_wrapper")
	}
	if _, err := os.Stat(wrapperPath); err != nil {
		flagutil.UsageErrorf("java_wrapper not found: %v", err)
	}

	cmd := exec.Command("make", append([]string{javaMakeVar + "=" + wrapperPath, "ENABLE_JAVAC_SERVER=no"}, makeTargets...)...)
	cmd.Dir = makeDir
	cmd.Env = makeEnv()
	cmd.Stdout = nil // Quiet, you
	stderr, err := cmd.StderrPipe()

	if err != nil {
		log.Fatal(err)
	}

	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	logCollectedErrors(io.TeeReader(stderr, os.Stderr))

	if err := cmd.Wait(); err != nil {
		log.Fatal(err)
	}
}

func collectExtractionErrors(stderr io.Reader) ([]string, error) {
	var result []string
	scanner := bufio.NewScanner(stderr)
	for scanner.Scan() {
		if subs := errorPattern.FindSubmatch(scanner.Bytes()); subs != nil {
			result = append(result, string(subs[1]))
		}
	}
	return result, scanner.Err()
}

func logCollectedErrors(stderr io.Reader) {
	errors, err := collectExtractionErrors(stderr)
	if err != nil {
		log.Println(err)
	}
	log.Println("Error extracting modules:\n\t", strings.Join(errors, "\n\t"))
}
<|endoftext|>


<commit_before>package rh

import (
	"fmt"

	"github.com/jmoiron/sqlx"
	"github.com/pkg/errors"
)

// DatabaseConfig describes database connection parameters
type DatabaseConfig struct {
	Driver   string
	URL      string
	Username string
	Password string
}

// OpenDatabase will create a new database connection. This connection must be closed when it is no longer needed.
func OpenDatabase(config *DatabaseConfig) (*sqlx.DB, error) {
	var url string

	switch config.Driver {
	case "mysql":
		url = fmt.Sprintf("%s:%s@%s", config.Username, config.Password, config.URL)
	case "postgres":
		url = fmt.Sprintf("postgres://%s:%s@%s", config.Username, config.Password, config.URL)
	default:
		return nil, fmt.Errorf("unsupported database driver %+q in OpenDatabase", config.Driver)
	}

	database, err := sqlx.Connect(config.Driver, url)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to OpenDatabase, cannot connect to %+q", config.URL)
	}

	if err := database.Ping(); err != nil {
		return nil, errors.Wrapf(err, "failed to OpenDatabase, ping failed %+q", config.URL)
	}

	return database, nil
}

// GetExportMarker returns the export marker record for the given bucket and tableName. It will return an error if the record does not exist. Before the first import you must insert the record into the import_markers table.
func GetExportMarker(db *sqlx.DB, bucket, tableName string) (*ExportMarker, error) {
	var marker ExportMarker

	q := `
SELECT bucket, table_name, year, month, hour, day
FROM   import_markers
WHERE  bucket=$1
AND    table_name=$2
`
	if err := db.Get(&marker, q, bucket, tableName); err != nil {
		return nil, errors.Wrapf(err, "getExportMarker failed to Get for bucket %+q and tableName %+q", bucket, tableName)
	}

	return &marker, nil
}

// ImportFile will load Redshift table from the file in the S3 bucket. The file is identified by the marker.
func ImportFile(db *sqlx.DB, config *S3Config, marker *ExportMarker) error {
	q := fmt.Sprintf(`
COPY %s
FROM 's3://%s/%s'
WITH CREDENTIALS 'aws_iam_role=%s'
REGION '%s'
IGNOREHEADER 1
NULL AS 'NULL'
`,
		marker.TableName,
		marker.Bucket,
		marker.FullPath(),
		config.Arn,
		config.Region,
	)

	if _, err := db.Exec(q); err != nil {
		return err
	}

	return nil
}

// UpdateExportMarker updates the marker record in import_markers table.
func UpdateExportMarker(db *sqlx.DB, marker *ExportMarker) error {
	q := `
UPDATE import_markers
SET    year = :year, month = :month, day = :day, hour = :hour
WHERE  bucket = :bucket
AND    table_name = :table_name
	`

	if _, err := db.NamedExec(q, marker); err != nil {
		return err
	}

	return nil
}
<commit_msg>Add DeleteRecordsForMarker<commit_after>package rh

import (
	"fmt"

	"github.com/jmoiron/sqlx"
	"github.com/pkg/errors"
)

// DatabaseConfig describes database connection parameters
type DatabaseConfig struct {
	Driver   string
	URL      string
	Username string
	Password string
}

// OpenDatabase will create a new database connection. This connection must be closed when it is no longer needed.
func OpenDatabase(config *DatabaseConfig) (*sqlx.DB, error) {
	var url string

	switch config.Driver {
	case "mysql":
		url = fmt.Sprintf("%s:%s@%s", config.Username, config.Password, config.URL)
	case "postgres":
		url = fmt.Sprintf("postgres://%s:%s@%s", config.Username, config.Password, config.URL)
	default:
		return nil, fmt.Errorf("unsupported database driver %+q in OpenDatabase", config.Driver)
	}

	database, err := sqlx.Connect(config.Driver, url)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to OpenDatabase, cannot connect to %+q", config.URL)
	}

	if err := database.Ping(); err != nil {
		return nil, errors.Wrapf(err, "failed to OpenDatabase, ping failed %+q", config.URL)
	}

	return database, nil
}

// GetExportMarker returns the export marker record for the given bucket and tableName. It will return an error if the record does not exist. Before the first import you must insert the record into the import_markers table.
func GetExportMarker(db *sqlx.DB, bucket, tableName string) (*ExportMarker, error) {
	var marker ExportMarker

	q := `
SELECT bucket, table_name, year, month, hour, day
FROM   import_markers
WHERE  bucket=$1
AND    table_name=$2
`
	if err := db.Get(&marker, q, bucket, tableName); err != nil {
		return nil, errors.Wrapf(err, "getExportMarker failed to Get for bucket %+q and tableName %+q", bucket, tableName)
	}

	return &marker, nil
}

// DeleteRecordsForMarker will delete database records before import to avoid duplicate values
func DeleteRecordsForMarker(db *sqlx.DB, marker *ExportMarker, timeColumn string) error {
	q := fmt.Sprintf("DELETE FROM %s WHERE %s >= $1 AND %s <= $2",
		marker.TableName,
		timeColumn,
		timeColumn,
	)

	timeRange := NewTimeRange(marker.Time())
	if _, err := db.Exec(q, timeRange.From, timeRange.To); err != nil {
		return err
	}

	return nil
}

// ImportFile will load Redshift table from the file in the S3 bucket. The file is identified by the marker.
func ImportFile(db *sqlx.DB, config *S3Config, marker *ExportMarker) error {
	q := fmt.Sprintf(`
COPY %s
FROM 's3://%s/%s'
WITH CREDENTIALS 'aws_iam_role=%s'
REGION '%s'
IGNOREHEADER 1
NULL AS 'NULL'
`,
		marker.TableName,
		marker.Bucket,
		marker.FullPath(),
		config.Arn,
		config.Region,
	)

	if _, err := db.Exec(q); err != nil {
		return err
	}

	return nil
}

// UpdateExportMarker updates the marker record in import_markers table.
func UpdateExportMarker(db *sqlx.DB, marker *ExportMarker) error {
	q := `
UPDATE import_markers
SET    year = :year, month = :month, day = :day, hour = :hour
WHERE  bucket = :bucket
AND    table_name = :table_name
	`

	if _, err := db.NamedExec(q, marker); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package controller

import (
	"encoding/json"
	"fmt"
	"reflect"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/Sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/cache"

	"github.com/zalando-incubator/postgres-operator/pkg/cluster"
	"github.com/zalando-incubator/postgres-operator/pkg/spec"
	"github.com/zalando-incubator/postgres-operator/pkg/util"
	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
	"github.com/zalando-incubator/postgres-operator/pkg/util/ringlog"
)

func (c *Controller) clusterResync(stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()
	ticker := time.NewTicker(c.opConfig.ResyncPeriod)

	for {
		select {
		case <-ticker.C:
			_, err := c.clusterListFunc(metav1.ListOptions{ResourceVersion: "0"})
			if err != nil {
				c.logger.Errorf("could not list clusters: %v", err)
			}
		case <-stopCh:
			return
		}
	}
}

func (c *Controller) clusterListFunc(options metav1.ListOptions) (runtime.Object, error) {
	var list spec.PostgresqlList
	var activeClustersCnt, failedClustersCnt int

	req := c.RestClient.
		Get().
		Namespace(c.opConfig.Namespace).
		Resource(constants.ResourceName).
		VersionedParams(&options, metav1.ParameterCodec)

	b, err := req.DoRaw()
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(b, &list)

	if time.Now().Unix()-atomic.LoadInt64(&c.lastClusterSyncTime) <= int64(c.opConfig.ResyncPeriod.Seconds()) {
		c.logger.Debugln("skipping resync of clusters")
		return &list, err
	}

	for i, pg := range list.Items {
		if pg.Error != nil {
			failedClustersCnt++
			continue
		}
		c.queueClusterEvent(nil, &list.Items[i], spec.EventSync)
		activeClustersCnt++
	}
	if len(list.Items) > 0 {
		if failedClustersCnt > 0 && activeClustersCnt == 0 {
			c.logger.Infof("there are no clusters running. %d are in the failed state", failedClustersCnt)
		} else if failedClustersCnt == 0 && activeClustersCnt > 0 {
			c.logger.Infof("there are %d clusters running", activeClustersCnt)
		} else {
			c.logger.Infof("there are %d clusters running and %d are in the failed state", activeClustersCnt, failedClustersCnt)
		}
	} else {
		c.logger.Infof("no clusters running")
	}

	atomic.StoreInt64(&c.lastClusterSyncTime, time.Now().Unix())

	return &list, err
}

type tprDecoder struct {
	dec   *json.Decoder
	close func() error
}

func (d *tprDecoder) Close() {
	d.close()
}

func (d *tprDecoder) Decode() (action watch.EventType, object runtime.Object, err error) {
	var e struct {
		Type   watch.EventType
		Object spec.Postgresql
	}
	if err := d.dec.Decode(&e); err != nil {
		return watch.Error, nil, err
	}

	return e.Type, &e.Object, nil
}

func (c *Controller) clusterWatchFunc(options metav1.ListOptions) (watch.Interface, error) {
	options.Watch = true
	r, err := c.RestClient.
		Get().
		Namespace(c.opConfig.Namespace).
		Resource(constants.ResourceName).
		VersionedParams(&options, metav1.ParameterCodec).
		FieldsSelectorParam(nil).
		Stream()

	if err != nil {
		return nil, err
	}

	return watch.NewStreamWatcher(&tprDecoder{
		dec:   json.NewDecoder(r),
		close: r.Close,
	}), nil
}

func (c *Controller) addCluster(lg *logrus.Entry, clusterName spec.NamespacedName, pgSpec *spec.Postgresql) *cluster.Cluster {
	cl := cluster.New(c.makeClusterConfig(), c.KubeClient, *pgSpec, lg)
	cl.Run(c.stopCh)
	teamName := strings.ToLower(cl.Spec.TeamID)

	defer c.clustersMu.Unlock()
	c.clustersMu.Lock()

	c.teamClusters[teamName] = append(c.teamClusters[teamName], clusterName)
	c.clusters[clusterName] = cl
	c.clusterLogs[clusterName] = ringlog.New(c.opConfig.RingLogLines)

	return cl
}

func (c *Controller) processEvent(event spec.ClusterEvent) {
	var clusterName spec.NamespacedName

	lg := c.logger.WithField("worker", event.WorkerID)

	if event.EventType == spec.EventAdd || event.EventType == spec.EventSync {
		clusterName = util.NameFromMeta(event.NewSpec.ObjectMeta)
	} else {
		clusterName = util.NameFromMeta(event.OldSpec.ObjectMeta)
	}
	lg = lg.WithField("cluster-name", clusterName)

	c.clustersMu.RLock()
	cl, clusterFound := c.clusters[clusterName]
	c.clustersMu.RUnlock()

	switch event.EventType {
	case spec.EventAdd:
		if clusterFound {
			lg.Debugf("cluster already exists")
			return
		}

		lg.Infof("creation of the cluster started")

		cl = c.addCluster(lg, clusterName, event.NewSpec)

		if err := cl.Create(); err != nil {
			cl.Error = fmt.Errorf("could not create cluster: %v", err)
			lg.Error(cl.Error)

			return
		}

		lg.Infoln("cluster has been created")
	case spec.EventUpdate:
		lg.Infoln("update of the cluster started")

		if !clusterFound {
			lg.Warnln("cluster does not exist")
			return
		}
		if err := cl.Update(event.NewSpec); err != nil {
			cl.Error = fmt.Errorf("could not update cluster: %v", err)
			lg.Error(cl.Error)

			return
		}
		cl.Error = nil
		lg.Infoln("cluster has been updated")
	case spec.EventDelete:
		if !clusterFound {
			lg.Errorf("unknown cluster: %q", clusterName)
			return
		}
		lg.Infoln("deletion of the cluster started")

		teamName := strings.ToLower(cl.Spec.TeamID)
		func() {
			defer c.clustersMu.Unlock()
			c.clustersMu.Lock()

			delete(c.clusters, clusterName)
			delete(c.clusterLogs, clusterName)
			for i, val := range c.teamClusters[teamName] {
				if val == clusterName {
					copy(c.teamClusters[teamName][i:], c.teamClusters[teamName][i+1:])
					c.teamClusters[teamName][len(c.teamClusters[teamName])-1] = spec.NamespacedName{}
					c.teamClusters[teamName] = c.teamClusters[teamName][:len(c.teamClusters[teamName])-1]
					break
				}
			}
		}()

		if err := cl.Delete(); err != nil {
			lg.Errorf("could not delete cluster: %v", err)
			return
		}

		lg.Infof("cluster has been deleted")
	case spec.EventSync:
		lg.Infof("syncing of the cluster started")

		// no race condition because a cluster is always processed by single worker
		if !clusterFound {
			cl = c.addCluster(lg, clusterName, event.NewSpec)
		}

		if err := cl.Sync(); err != nil {
			cl.Error = fmt.Errorf("could not sync cluster: %v", err)
			lg.Error(cl.Error)
			return
		}
		cl.Error = nil

		lg.Infof("cluster has been synced")
	}
}

func (c *Controller) processClusterEventsQueue(idx int, stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	go func() {
		<-stopCh
		c.clusterEventQueues[idx].Close()
	}()

	for {
		obj, err := c.clusterEventQueues[idx].Pop(cache.PopProcessFunc(func(interface{}) error { return nil }))
		if err != nil {
			if err == cache.FIFOClosedError {
				return
			}
			c.logger.Errorf("error when processing cluster events queue: %v", err)
			continue
		}
		event, ok := obj.(spec.ClusterEvent)
		if !ok {
			c.logger.Errorf("could not cast to ClusterEvent")
		}

		c.processEvent(event)
	}
}

func (c *Controller) queueClusterEvent(old, new *spec.Postgresql, eventType spec.EventType) {
	var (
		uid          types.UID
		clusterName  spec.NamespacedName
		clusterError error
	)

	if old != nil { //update, delete
		uid = old.GetUID()
		clusterName = util.NameFromMeta(old.ObjectMeta)
		if eventType == spec.EventUpdate && new.Error == nil && old.Error != nil {
			eventType = spec.EventSync
			clusterError = new.Error
		} else {
			clusterError = old.Error
		}
	} else { //add, sync
		uid = new.GetUID()
		clusterName = util.NameFromMeta(new.ObjectMeta)
		clusterError = new.Error
	}

	if clusterError != nil && eventType != spec.EventDelete {
		c.logger.
			WithField("cluster-name", clusterName).
			Debugf("skipping %q event for the invalid cluster: %v", eventType, clusterError)
		return
	}

	workerID := c.clusterWorkerID(clusterName)
	clusterEvent := spec.ClusterEvent{
		EventType: eventType,
		UID:       uid,
		OldSpec:   old,
		NewSpec:   new,
		WorkerID:  workerID,
	}

	lg := c.logger.WithField("worker", workerID).WithField("cluster-name", clusterName)
	if err := c.clusterEventQueues[workerID].Add(clusterEvent); err != nil {
		lg.Errorf("error while queueing cluster event: %v", clusterEvent)
	}
	lg.Infof("%q event has been queued", eventType)

	if eventType != spec.EventDelete {
		return
	}

	for _, evType := range []spec.EventType{spec.EventAdd, spec.EventSync, spec.EventUpdate} {
		obj, exists, err := c.clusterEventQueues[workerID].GetByKey(queueClusterKey(evType, uid))
		if err != nil {
			lg.Warningf("could not get event from the queue: %v", err)
			continue
		}

		if !exists {
			continue
		}

		err = c.clusterEventQueues[workerID].Delete(obj)
		if err != nil {
			lg.Warningf("could not delete event from the queue: %v", err)
		} else {
			lg.Debugf("event %q has been discarded for the cluster", evType)
		}
	}
}

func (c *Controller) postgresqlAdd(obj interface{}) {
	pg, ok := obj.(*spec.Postgresql)
	if !ok {
		c.logger.Errorf("could not cast to postgresql spec")
		return
	}

	// We will not get multiple Add events for the same cluster
	c.queueClusterEvent(nil, pg, spec.EventAdd)
}

func (c *Controller) postgresqlUpdate(prev, cur interface{}) {
	pgOld, ok := prev.(*spec.Postgresql)
	if !ok {
		c.logger.Errorf("could not cast to postgresql spec")
	}
	pgNew, ok := cur.(*spec.Postgresql)
	if !ok {
		c.logger.Errorf("could not cast to postgresql spec")
	}
	if pgOld.ResourceVersion == pgNew.ResourceVersion {
		return
	}
	if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) {
		return
	}

	c.queueClusterEvent(pgOld, pgNew, spec.EventUpdate)
}

func (c *Controller) postgresqlDelete(obj interface{}) {
	pg, ok := obj.(*spec.Postgresql)
	if !ok {
		c.logger.Errorf("could not cast to postgresql spec")
		return
	}

	c.queueClusterEvent(pg, nil, spec.EventDelete)
}
<commit_msg>delete cluster key from the clusters list only when delete procedure is finished<commit_after>package controller

import (
	"encoding/json"
	"fmt"
	"reflect"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/Sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/tools/cache"

	"github.com/zalando-incubator/postgres-operator/pkg/cluster"
	"github.com/zalando-incubator/postgres-operator/pkg/spec"
	"github.com/zalando-incubator/postgres-operator/pkg/util"
	"github.com/zalando-incubator/postgres-operator/pkg/util/constants"
	"github.com/zalando-incubator/postgres-operator/pkg/util/ringlog"
)

func (c *Controller) clusterResync(stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()
	ticker := time.NewTicker(c.opConfig.ResyncPeriod)

	for {
		select {
		case <-ticker.C:
			_, err := c.clusterListFunc(metav1.ListOptions{ResourceVersion: "0"})
			if err != nil {
				c.logger.Errorf("could not list clusters: %v", err)
			}
		case <-stopCh:
			return
		}
	}
}

func (c *Controller) clusterListFunc(options metav1.ListOptions) (runtime.Object, error) {
	var list spec.PostgresqlList
	var activeClustersCnt, failedClustersCnt int

	req := c.RestClient.
		Get().
		Namespace(c.opConfig.Namespace).
		Resource(constants.ResourceName).
		VersionedParams(&options, metav1.ParameterCodec)

	b, err := req.DoRaw()
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(b, &list)

	if time.Now().Unix()-atomic.LoadInt64(&c.lastClusterSyncTime) <= int64(c.opConfig.ResyncPeriod.Seconds()) {
		c.logger.Debugln("skipping resync of clusters")
		return &list, err
	}

	for i, pg := range list.Items {
		if pg.Error != nil {
			failedClustersCnt++
			continue
		}
		c.queueClusterEvent(nil, &list.Items[i], spec.EventSync)
		activeClustersCnt++
	}
	if len(list.Items) > 0 {
		if failedClustersCnt > 0 && activeClustersCnt == 0 {
			c.logger.Infof("there are no clusters running. %d are in the failed state", failedClustersCnt)
		} else if failedClustersCnt == 0 && activeClustersCnt > 0 {
			c.logger.Infof("there are %d clusters running", activeClustersCnt)
		} else {
			c.logger.Infof("there are %d clusters running and %d are in the failed state", activeClustersCnt, failedClustersCnt)
		}
	} else {
		c.logger.Infof("no clusters running")
	}

	atomic.StoreInt64(&c.lastClusterSyncTime, time.Now().Unix())

	return &list, err
}

type tprDecoder struct {
	dec   *json.Decoder
	close func() error
}

func (d *tprDecoder) Close() {
	d.close()
}

func (d *tprDecoder) Decode() (action watch.EventType, object runtime.Object, err error) {
	var e struct {
		Type   watch.EventType
		Object spec.Postgresql
	}
	if err := d.dec.Decode(&e); err != nil {
		return watch.Error, nil, err
	}

	return e.Type, &e.Object, nil
}

func (c *Controller) clusterWatchFunc(options metav1.ListOptions) (watch.Interface, error) {
	options.Watch = true
	r, err := c.RestClient.
		Get().
		Namespace(c.opConfig.Namespace).
		Resource(constants.ResourceName).
		VersionedParams(&options, metav1.ParameterCodec).
		FieldsSelectorParam(nil).
		Stream()

	if err != nil {
		return nil, err
	}

	return watch.NewStreamWatcher(&tprDecoder{
		dec:   json.NewDecoder(r),
		close: r.Close,
	}), nil
}

func (c *Controller) addCluster(lg *logrus.Entry, clusterName spec.NamespacedName, pgSpec *spec.Postgresql) *cluster.Cluster {
	cl := cluster.New(c.makeClusterConfig(), c.KubeClient, *pgSpec, lg)
	cl.Run(c.stopCh)
	teamName := strings.ToLower(cl.Spec.TeamID)

	defer c.clustersMu.Unlock()
	c.clustersMu.Lock()

	c.teamClusters[teamName] = append(c.teamClusters[teamName], clusterName)
	c.clusters[clusterName] = cl
	c.clusterLogs[clusterName] = ringlog.New(c.opConfig.RingLogLines)

	return cl
}

func (c *Controller) processEvent(event spec.ClusterEvent) {
	var clusterName spec.NamespacedName

	lg := c.logger.WithField("worker", event.WorkerID)

	if event.EventType == spec.EventAdd || event.EventType == spec.EventSync {
		clusterName = util.NameFromMeta(event.NewSpec.ObjectMeta)
	} else {
		clusterName = util.NameFromMeta(event.OldSpec.ObjectMeta)
	}
	lg = lg.WithField("cluster-name", clusterName)

	c.clustersMu.RLock()
	cl, clusterFound := c.clusters[clusterName]
	c.clustersMu.RUnlock()

	switch event.EventType {
	case spec.EventAdd:
		if clusterFound {
			lg.Debugf("cluster already exists")
			return
		}

		lg.Infof("creation of the cluster started")

		cl = c.addCluster(lg, clusterName, event.NewSpec)

		if err := cl.Create(); err != nil {
			cl.Error = fmt.Errorf("could not create cluster: %v", err)
			lg.Error(cl.Error)

			return
		}

		lg.Infoln("cluster has been created")
	case spec.EventUpdate:
		lg.Infoln("update of the cluster started")

		if !clusterFound {
			lg.Warnln("cluster does not exist")
			return
		}
		if err := cl.Update(event.NewSpec); err != nil {
			cl.Error = fmt.Errorf("could not update cluster: %v", err)
			lg.Error(cl.Error)

			return
		}
		cl.Error = nil
		lg.Infoln("cluster has been updated")
	case spec.EventDelete:
		if !clusterFound {
			lg.Errorf("unknown cluster: %q", clusterName)
			return
		}
		lg.Infoln("deletion of the cluster started")

		teamName := strings.ToLower(cl.Spec.TeamID)

		if err := cl.Delete(); err != nil {
			lg.Errorf("could not delete cluster: %v", err)
		}

		func() {
			defer c.clustersMu.Unlock()
			c.clustersMu.Lock()

			delete(c.clusters, clusterName)
			delete(c.clusterLogs, clusterName)
			for i, val := range c.teamClusters[teamName] {
				if val == clusterName {
					copy(c.teamClusters[teamName][i:], c.teamClusters[teamName][i+1:])
					c.teamClusters[teamName][len(c.teamClusters[teamName])-1] = spec.NamespacedName{}
					c.teamClusters[teamName] = c.teamClusters[teamName][:len(c.teamClusters[teamName])-1]
					break
				}
			}
		}()

		lg.Infof("cluster has been deleted")
	case spec.EventSync:
		lg.Infof("syncing of the cluster started")

		// no race condition because a cluster is always processed by single worker
		if !clusterFound {
			cl = c.addCluster(lg, clusterName, event.NewSpec)
		}

		if err := cl.Sync(); err != nil {
			cl.Error = fmt.Errorf("could not sync cluster: %v", err)
			lg.Error(cl.Error)
			return
		}
		cl.Error = nil

		lg.Infof("cluster has been synced")
	}
}

func (c *Controller) processClusterEventsQueue(idx int, stopCh <-chan struct{}, wg *sync.WaitGroup) {
	defer wg.Done()

	go func() {
		<-stopCh
		c.clusterEventQueues[idx].Close()
	}()

	for {
		obj, err := c.clusterEventQueues[idx].Pop(cache.PopProcessFunc(func(interface{}) error { return nil }))
		if err != nil {
			if err == cache.FIFOClosedError {
				return
			}
			c.logger.Errorf("error when processing cluster events queue: %v", err)
			continue
		}
		event, ok := obj.(spec.ClusterEvent)
		if !ok {
			c.logger.Errorf("could not cast to ClusterEvent")
		}

		c.processEvent(event)
	}
}

func (c *Controller) queueClusterEvent(old, new *spec.Postgresql, eventType spec.EventType) {
	var (
		uid          types.UID
		clusterName  spec.NamespacedName
		clusterError error
	)

	if old != nil { //update, delete
		uid = old.GetUID()
		clusterName = util.NameFromMeta(old.ObjectMeta)
		if eventType == spec.EventUpdate && new.Error == nil && old.Error != nil {
			eventType = spec.EventSync
			clusterError = new.Error
		} else {
			clusterError = old.Error
		}
	} else { //add, sync
		uid = new.GetUID()
		clusterName = util.NameFromMeta(new.ObjectMeta)
		clusterError = new.Error
	}

	if clusterError != nil && eventType != spec.EventDelete {
		c.logger.
			WithField("cluster-name", clusterName).
			Debugf("skipping %q event for the invalid cluster: %v", eventType, clusterError)
		return
	}

	workerID := c.clusterWorkerID(clusterName)
	clusterEvent := spec.ClusterEvent{
		EventType: eventType,
		UID:       uid,
		OldSpec:   old,
		NewSpec:   new,
		WorkerID:  workerID,
	}

	lg := c.logger.WithField("worker", workerID).WithField("cluster-name", clusterName)
	if err := c.clusterEventQueues[workerID].Add(clusterEvent); err != nil {
		lg.Errorf("error while queueing cluster event: %v", clusterEvent)
	}
	lg.Infof("%q event has been queued", eventType)

	if eventType != spec.EventDelete {
		return
	}

	for _, evType := range []spec.EventType{spec.EventAdd, spec.EventSync, spec.EventUpdate} {
		obj, exists, err := c.clusterEventQueues[workerID].GetByKey(queueClusterKey(evType, uid))
		if err != nil {
			lg.Warningf("could not get event from the queue: %v", err)
			continue
		}

		if !exists {
			continue
		}

		err = c.clusterEventQueues[workerID].Delete(obj)
		if err != nil {
			lg.Warningf("could not delete event from the queue: %v", err)
		} else {
			lg.Debugf("event %q has been discarded for the cluster", evType)
		}
	}
}

func (c *Controller) postgresqlAdd(obj interface{}) {
	pg, ok := obj.(*spec.Postgresql)
	if !ok {
		c.logger.Errorf("could not cast to postgresql spec")
		return
	}

	// We will not get multiple Add events for the same cluster
	c.queueClusterEvent(nil, pg, spec.EventAdd)
}

func (c *Controller) postgresqlUpdate(prev, cur interface{}) {
	pgOld, ok := prev.(*spec.Postgresql)
	if !ok {
		c.logger.Errorf("could not cast to postgresql spec")
	}
	pgNew, ok := cur.(*spec.Postgresql)
	if !ok {
		c.logger.Errorf("could not cast to postgresql spec")
	}
	if pgOld.ResourceVersion == pgNew.ResourceVersion {
		return
	}
	if reflect.DeepEqual(pgOld.Spec, pgNew.Spec) {
		return
	}

	c.queueClusterEvent(pgOld, pgNew, spec.EventUpdate)
}

func (c *Controller) postgresqlDelete(obj interface{}) {
	pg, ok := obj.(*spec.Postgresql)
	if !ok {
		c.logger.Errorf("could not cast to postgresql spec")
		return
	}

	c.queueClusterEvent(pg, nil, spec.EventDelete)
}
<|endoftext|>