<commit_before>package main

import (
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/materials-commons/materials"
	"github.com/materials-commons/materials/wsmaterials"
	"io"
	"archive/tar"
	"compress/gzip"
	"net/http"
	"os"
	"os/user"
	"path/filepath"
)

var mcurl = ""
var usr, _ = user.Current()
var mcuser, _ = materials.NewCurrentUser()
var commons = materials.NewMaterialsCommons(mcuser)

//var user = NewCurrentUser()

type ServerOptions struct {
	AsServer bool   `long:"server" description:"Run as webserver"`
	Port     int    `long:"port" default:"8081" description:"The port the server listens on"`
	Address  string `long:"address" default:"127.0.0.1" description:"The address to bind to"`
}

type ProjectOptions struct {
	Project   string `long:"project" description:"Specify the project"`
	Directory string `long:"directory" description:"The directory path to the project"`
	Add       bool   `long:"add" description:"Add the project to the project config file"`
	Delete    bool   `long:"delete" description:"Delete the project from the project config file"`
	List      bool   `long:"list" description:"List all known projects and their locations"`
	Upload    bool   `long:"upload" description:"Uploads a new project. Cannot be used on existing projects"`
}

type Options struct {
	Server     ServerOptions  `group:"Server Options"`
	Project    ProjectOptions `group:"Project Options"`
	Initialize bool           `long:"init" description:"Create configuration"`
}

func initialize() {
	usr, err := user.Current()
	checkError(err)

	dirPath := filepath.Join(usr.HomeDir, ".materials")
	err = os.MkdirAll(dirPath, 0777)
	checkError(err)

	if newVersionOfWebsite() {
		websiteFilepath := filepath.Join(dirPath, "website")
		os.RemoveAll(websiteFilepath)
		downloadWebsite(dirPath)
	}
}

type MaterialsWebsiteInfo struct {
	Version     string `json:"version"`
	Description string `json:"description"`
}

func newVersionOfWebsite() bool {

	/*
		resp, _ := http.Get(mcurl + "/materials_website.json")
		defer resp.Body.Close()

		body, _ := ioutil.ReadAll(resp.Body)
		var websiteInfo MaterialsWebsiteInfo
		json.Unmarshal(body, &websiteInfo)
	*/
	return true
}

func downloadWebsite(dirPath string) {
	getDownloadedVersionOfWebsite()
	websiteTarPath := filepath.Join(dirPath, "materials.tar.gz")
	out, _ := os.Create(websiteTarPath)
	defer out.Close()

	resp, _ := http.Get(mcurl + "/materials.tar.gz")
	defer resp.Body.Close()
	io.Copy(out, resp.Body)
	unpackWebsite(websiteTarPath)
}

func unpackWebsite(path string) {
	file, _ := os.Open(path)
	defer file.Close()

	zhandle, _ := gzip.NewReader(file)
	defer zhandle.Close()

	thandle := tar.NewReader(zhandle)
	for {
		hdr, err := thandle.Next()
		if err == io.EOF {
			break
		}

		if err != nil {
			break
		}

		if hdr.Typeflag == tar.TypeDir {
			dirpath := filepath.Join(mcuser.DotMaterialsPath(), hdr.Name)
			os.MkdirAll(dirpath, 0777)
		} else if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {
			filepath := filepath.Join(mcuser.DotMaterialsPath(), hdr.Name)
			out, _ := os.Create(filepath)
			if _, err := io.Copy(out, thandle); err != nil {
				fmt.Println(err)
			}
			out.Close()
		}
	}
}

func getDownloadedVersionOfWebsite() int {
	//content := ioutil.ReadFile()
	return 0
}

func checkError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Fatal error: %s\n", err.Error())
		os.Exit(1)
	}
}

func setup() {
	envMCURL := os.Getenv("MCURL")
	if envMCURL == "" {
		mcurl = "https://materialscommons.org"
	} else {
		mcurl = envMCURL
	}
}

func listProjects() {
	projects, err := materials.CurrentUserProjects()
	if err != nil {
		return
	}
	for _, p := range projects.Projects() {
		fmt.Printf("%s, %s\n", p.Name, p.Path)
	}
}

func runWebServer(address string, port int) {
	wsContainer := wsmaterials.NewRegisteredServicesContainer()
	http.Handle("/", wsContainer)
	mcwebdir := os.Getenv("MCWEBDIR")
	if mcwebdir == "" {
		mcwebdir = mcuser.DotMaterialsPath()
	}
	websiteDir := filepath.Join(mcwebdir, "website")
	dir := http.Dir(websiteDir)
	http.Handle("/materials/", http.StripPrefix("/materials/", http.FileServer(dir)))
	addr := fmt.Sprintf("%s:%d", address, port)
	fmt.Println(http.ListenAndServe(addr, nil))
}

func uploadProject(projectName string) {
	projects, _ := materials.CurrentUserProjects()
	project, _ := projects.Find(projectName)
	err := project.Upload(commons)
	if err != nil {
		fmt.Println(err)
	} else {
		project.Status = "Loaded"
		projects.Update(project)
	}
}

func main() {
	var opts Options
	_, err := flags.Parse(&opts)

	if err != nil {
		panic(err)
		os.Exit(1)
	}

	setup()

	if opts.Initialize {
		initialize()
	}

	if opts.Project.List {
		listProjects()
	}

	if opts.Server.AsServer {
		runWebServer(opts.Server.Address, opts.Server.Port)
	}

	if opts.Project.Upload {
		uploadProject(opts.Project.Project)
	}
}
<commit_msg>Ignore certificate when downloading website.<commit_after>package main

import (
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/materials-commons/materials"
	"github.com/materials-commons/materials/wsmaterials"
	"io"
	"archive/tar"
	"compress/gzip"
	"net/http"
	"os"
	"os/user"
	"path/filepath"
	"crypto/tls"
)

var mcurl = ""
var usr, _ = user.Current()
var mcuser, _ = materials.NewCurrentUser()
var commons = materials.NewMaterialsCommons(mcuser)

//var user = NewCurrentUser()

type ServerOptions struct {
	AsServer bool   `long:"server" description:"Run as webserver"`
	Port     int    `long:"port" default:"8081" description:"The port the server listens on"`
	Address  string `long:"address" default:"127.0.0.1" description:"The address to bind to"`
}

type ProjectOptions struct {
	Project   string `long:"project" description:"Specify the project"`
	Directory string `long:"directory" description:"The directory path to the project"`
	Add       bool   `long:"add" description:"Add the project to the project config file"`
	Delete    bool   `long:"delete" description:"Delete the project from the project config file"`
	List      bool   `long:"list" description:"List all known projects and their locations"`
	Upload    bool   `long:"upload" description:"Uploads a new project. Cannot be used on existing projects"`
}

type Options struct {
	Server     ServerOptions  `group:"Server Options"`
	Project    ProjectOptions `group:"Project Options"`
	Initialize bool           `long:"init" description:"Create configuration"`
}

func initialize() {
	usr, err := user.Current()
	checkError(err)

	dirPath := filepath.Join(usr.HomeDir, ".materials")
	err = os.MkdirAll(dirPath, 0777)
	checkError(err)

	if newVersionOfWebsite() {
		websiteFilepath := filepath.Join(dirPath, "website")
		os.RemoveAll(websiteFilepath)
		downloadWebsite(dirPath)
	}
}

type MaterialsWebsiteInfo struct {
	Version     string `json:"version"`
	Description string `json:"description"`
}

func newVersionOfWebsite() bool {

	/*
		resp, _ := http.Get(mcurl + "/materials_website.json")
		defer resp.Body.Close()

		body, _ := ioutil.ReadAll(resp.Body)
		var websiteInfo MaterialsWebsiteInfo
		json.Unmarshal(body, &websiteInfo)
	*/
	return true
}

func downloadWebsite(dirPath string) {
	getDownloadedVersionOfWebsite()
	websiteTarPath := filepath.Join(dirPath, "materials.tar.gz")
	out, _ := os.Create(websiteTarPath)
	defer out.Close()

	client := makeClient()

	resp, _ := client.Get(mcurl + "/materials.tar.gz")
	defer resp.Body.Close()
	io.Copy(out, resp.Body)
	unpackWebsite(websiteTarPath)
}

func makeClient() *http.Client {
	tr := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	return &http.Client{Transport: tr}
}

func unpackWebsite(path string) {
	file, _ := os.Open(path)
	defer file.Close()

	zhandle, _ := gzip.NewReader(file)
	defer zhandle.Close()

	thandle := tar.NewReader(zhandle)
	for {
		hdr, err := thandle.Next()
		if err == io.EOF {
			break
		}

		if err != nil {
			break
		}

		if hdr.Typeflag == tar.TypeDir {
			dirpath := filepath.Join(mcuser.DotMaterialsPath(), hdr.Name)
			os.MkdirAll(dirpath, 0777)
		} else if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {
			filepath := filepath.Join(mcuser.DotMaterialsPath(), hdr.Name)
			out, _ := os.Create(filepath)
			if _, err := io.Copy(out, thandle); err != nil {
				fmt.Println(err)
			}
			out.Close()
		}
	}
}

func getDownloadedVersionOfWebsite() int {
	//content := ioutil.ReadFile()
	return 0
}

func checkError(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Fatal error: %s\n", err.Error())
		os.Exit(1)
	}
}

func setup() {
	envMCURL := os.Getenv("MCURL")
	if envMCURL == "" {
		mcurl = "https://materialscommons.org"
	} else {
		mcurl = envMCURL
	}
}

func listProjects() {
	projects, err := materials.CurrentUserProjects()
	if err != nil {
		return
	}
	for _, p := range projects.Projects() {
		fmt.Printf("%s, %s\n", p.Name, p.Path)
	}
}

func runWebServer(address string, port int) {
	wsContainer := wsmaterials.NewRegisteredServicesContainer()
	http.Handle("/", wsContainer)
	mcwebdir := os.Getenv("MCWEBDIR")
	if mcwebdir == "" {
		mcwebdir = mcuser.DotMaterialsPath()
	}
	websiteDir := filepath.Join(mcwebdir, "website")
	dir := http.Dir(websiteDir)
	http.Handle("/materials/", http.StripPrefix("/materials/", http.FileServer(dir)))
	addr := fmt.Sprintf("%s:%d", address, port)
	fmt.Println(http.ListenAndServe(addr, nil))
}

func uploadProject(projectName string) {
	projects, _ := materials.CurrentUserProjects()
	project, _ := projects.Find(projectName)
	err := project.Upload(commons)
	if err != nil {
		fmt.Println(err)
	} else {
		project.Status = "Loaded"
		projects.Update(project)
	}
}

func main() {
	var opts Options
	_, err := flags.Parse(&opts)

	if err != nil {
		panic(err)
		os.Exit(1)
	}

	setup()

	if opts.Initialize {
		initialize()
	}

	if opts.Project.List {
		listProjects()
	}

	if opts.Server.AsServer {
		runWebServer(opts.Server.Address, opts.Server.Port)
	}

	if opts.Project.Upload {
		uploadProject(opts.Project.Project)
	}
}
<|endoftext|>


<commit_before>/*
 * privileges.go - Handles inserting/removing into user keyrings.
 *
 * Copyright 2017 Google Inc.
 * Author: Joe Richey (joerichey@google.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package security

import (
	"log"
	"os"
	"os/user"
	"runtime"
	"sync"

	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"github.com/google/fscrypt/util"
)

// KeyType is always logon as required by filesystem encryption.
const KeyType = "logon"

// Keyring related error values
var (
	ErrFindingKeyring = util.SystemError("could not find user keyring")
	ErrKeyringInsert  = util.SystemError("could not insert key into the keyring")
	ErrKeyringSearch  = errors.New("could not find key with descriptor")
	ErrKeyringDelete  = util.SystemError("could not delete key from the keyring")
	ErrKeyringLink    = util.SystemError("could not link keyring")
)

// KeyringsSetup configures the desired keyring linkage by linking the target
// user's keying into the privileged user's keyring.
func KeyringsSetup(target, privileged *user.User) error {
	targetKeyringID, err := userKeyringID(target)
	if err != nil {
		return err
	}
	privilegedKeyringID, err := userKeyringID(privileged)
	if err != nil {
		return err
	}
	return keyringLink(targetKeyringID, privilegedKeyringID)
}

// FindKey tries to locate a key in the kernel keyring with the provided
// description. The key ID is returned if we can find the key. An error is
// returned if the key does not exist.
func FindKey(description string, target *user.User) (int, error) {
	keyringID, err := userKeyringID(target)
	if err != nil {
		return 0, err
	}

	keyID, err := unix.KeyctlSearch(keyringID, KeyType, description, 0)
	log.Printf("KeyctlSearch(%d, %s, %s) = %d, %v", keyringID, KeyType, description, keyID, err)
	if err != nil {
		return 0, errors.Wrap(ErrKeyringSearch, err.Error())
	}
	return keyID, err
}

// RemoveKey tries to remove a policy key from the kernel keyring with the
// provided description. An error is returned if the key does not exist.
func RemoveKey(description string, target *user.User) error {
	keyID, err := FindKey(description, target)
	if err != nil {
		return err
	}

	// We use KEYCTL_INVALIDATE instead of KEYCTL_REVOKE because
	// invalidating a key immediately removes it.
	_, err = unix.KeyctlInt(unix.KEYCTL_INVALIDATE, keyID, 0, 0, 0)
	log.Printf("KeyctlInvalidate(%d) = %v", keyID, err)
	if err != nil {
		return errors.Wrap(ErrKeyringDelete, err.Error())
	}
	return nil
}

// InsertKey puts the provided data into the kernel keyring with the provided
// description.
func InsertKey(data []byte, description string, target *user.User) error {
	keyringID, err := userKeyringID(target)
	if err != nil {
		return err
	}

	keyID, err := unix.AddKey(KeyType, description, data, keyringID)
	log.Printf("KeyctlAddKey(%s, %s, <data>, %d) = %d, %v",
		KeyType, description, keyringID, keyID, err)
	if err != nil {
		return errors.Wrap(ErrKeyringInsert, err.Error())
	}
	return nil
}

var (
	keyringIDCache = make(map[int]int)
	cacheLock      sync.Mutex
)

// userKeyringID returns the key id of the target user's keyring. The returned
// keyring will also be linked into the process keyring so that it will be
// accessible thoughout the program.
func userKeyringID(target *user.User) (int, error) {
	uid := util.AtoiOrPanic(target.Uid)
	// We will cache the result of this function.
	cacheLock.Lock()
	defer cacheLock.Unlock()
	if keyringID, ok := keyringIDCache[uid]; ok {
		return keyringID, nil
	}

	// The permissions of the keyrings API is a little strange. The euid is
	// used to determine if we can access/modify a key/keyring. However, the
	// ruid is used to determine KEY_SPEC_USER_KEYRING. This means both the
	// ruid and euid must match the user's uid for the lookup to work.
	if uid == os.Getuid() && uid == os.Geteuid() {
		log.Printf("Normal keyring lookup for uid=%d", uid)
		return userKeyringIDLookup(uid)
	}

	// We drop permissions in a separate thread (guaranteed as the main
	// thread is locked) because we need to drop the real AND effective IDs.
	log.Printf("Threaded keyring lookup for uid=%d", uid)
	idChan := make(chan int)
	errChan := make(chan error)
	// OSThread locks ensure the privilege change is only for the lookup.
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()
	go func() {
		runtime.LockOSThread()
		if err := SetThreadPrivileges(target, true); err != nil {
			errChan <- err
			return
		}
		keyringID, err := userKeyringIDLookup(uid)
		if err != nil {
			errChan <- err
			return
		}
		idChan <- keyringID
	}()

	// We select so the thread will have to complete
	select {
	case err := <-errChan:
		return 0, err
	case keyringID := <-idChan:
		if uid == os.Getuid() && uid == os.Geteuid() {
			log.Print("thread privileges now incorrect")
		}
		return keyringID, nil
	}
}

func userKeyringIDLookup(uid int) (int, error) {
	// This will trigger the creation of the user keyring, if necessary.
	keyringID, err := unix.KeyctlGetKeyringID(unix.KEY_SPEC_USER_KEYRING, false)
	log.Printf("keyringID(_uid.%d) = %d, %v", uid, keyringID, err)
	if err != nil {
		return 0, errors.Wrap(ErrFindingKeyring, err.Error())
	}

	// For some silly reason, a thread does not automatically "possess" keys
	// in the user keyring. So we link it into the process keyring so that
	// we will not get "permission denied" when purging or modifying keys.
	if err := keyringLink(keyringID, unix.KEY_SPEC_PROCESS_KEYRING); err != nil {
		return 0, err
	}

	keyringIDCache[uid] = keyringID
	return keyringID, nil
}

func keyringLink(keyID int, keyringID int) error {
	_, err := unix.KeyctlInt(unix.KEYCTL_LINK, keyID, keyringID, 0, 0)
	log.Printf("KeyctlLink(%d, %d) = %v", keyID, keyringID, err)
	if err != nil {
		return errors.Wrap(ErrKeyringLink, err.Error())
	}
	return nil
}
<commit_msg>security: Error if privilege reset goes wrong<commit_after>/*
 * privileges.go - Handles inserting/removing into user keyrings.
 *
 * Copyright 2017 Google Inc.
 * Author: Joe Richey (joerichey@google.com)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package security

import (
	"log"
	"os"
	"os/user"
	"runtime"
	"sync"

	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"github.com/google/fscrypt/util"
)

// KeyType is always logon as required by filesystem encryption.
const KeyType = "logon"

// Keyring related error values
var (
	ErrFindingKeyring = util.SystemError("could not find user keyring")
	ErrKeyringInsert  = util.SystemError("could not insert key into the keyring")
	ErrKeyringSearch  = errors.New("could not find key with descriptor")
	ErrKeyringDelete  = util.SystemError("could not delete key from the keyring")
	ErrKeyringLink    = util.SystemError("could not link keyring")
)

// KeyringsSetup configures the desired keyring linkage by linking the target
// user's keying into the privileged user's keyring.
func KeyringsSetup(target, privileged *user.User) error {
	targetKeyringID, err := userKeyringID(target)
	if err != nil {
		return err
	}
	privilegedKeyringID, err := userKeyringID(privileged)
	if err != nil {
		return err
	}
	return keyringLink(targetKeyringID, privilegedKeyringID)
}

// FindKey tries to locate a key in the kernel keyring with the provided
// description. The key ID is returned if we can find the key. An error is
// returned if the key does not exist.
func FindKey(description string, target *user.User) (int, error) {
	keyringID, err := userKeyringID(target)
	if err != nil {
		return 0, err
	}

	keyID, err := unix.KeyctlSearch(keyringID, KeyType, description, 0)
	log.Printf("KeyctlSearch(%d, %s, %s) = %d, %v", keyringID, KeyType, description, keyID, err)
	if err != nil {
		return 0, errors.Wrap(ErrKeyringSearch, err.Error())
	}
	return keyID, err
}

// RemoveKey tries to remove a policy key from the kernel keyring with the
// provided description. An error is returned if the key does not exist.
func RemoveKey(description string, target *user.User) error {
	keyID, err := FindKey(description, target)
	if err != nil {
		return err
	}

	// We use KEYCTL_INVALIDATE instead of KEYCTL_REVOKE because
	// invalidating a key immediately removes it.
	_, err = unix.KeyctlInt(unix.KEYCTL_INVALIDATE, keyID, 0, 0, 0)
	log.Printf("KeyctlInvalidate(%d) = %v", keyID, err)
	if err != nil {
		return errors.Wrap(ErrKeyringDelete, err.Error())
	}
	return nil
}

// InsertKey puts the provided data into the kernel keyring with the provided
// description.
func InsertKey(data []byte, description string, target *user.User) error {
	keyringID, err := userKeyringID(target)
	if err != nil {
		return err
	}

	keyID, err := unix.AddKey(KeyType, description, data, keyringID)
	log.Printf("KeyctlAddKey(%s, %s, <data>, %d) = %d, %v",
		KeyType, description, keyringID, keyID, err)
	if err != nil {
		return errors.Wrap(ErrKeyringInsert, err.Error())
	}
	return nil
}

var (
	keyringIDCache = make(map[int]int)
	cacheLock      sync.Mutex
)

// userKeyringID returns the key id of the target user's keyring. The returned
// keyring will also be linked into the process keyring so that it will be
// accessible thoughout the program.
func userKeyringID(target *user.User) (int, error) {
	uid := util.AtoiOrPanic(target.Uid)
	// We will cache the result of this function.
	cacheLock.Lock()
	defer cacheLock.Unlock()
	if keyringID, ok := keyringIDCache[uid]; ok {
		return keyringID, nil
	}

	// The permissions of the keyrings API is a little strange. The euid is
	// used to determine if we can access/modify a key/keyring. However, the
	// ruid is used to determine KEY_SPEC_USER_KEYRING. This means both the
	// ruid and euid must match the user's uid for the lookup to work.
	if uid == os.Getuid() && uid == os.Geteuid() {
		log.Printf("Normal keyring lookup for uid=%d", uid)
		return userKeyringIDLookup(uid)
	}

	// We drop permissions in a separate thread (guaranteed as the main
	// thread is locked) because we need to drop the real AND effective IDs.
	log.Printf("Threaded keyring lookup for uid=%d", uid)
	idChan := make(chan int)
	errChan := make(chan error)
	// OSThread locks ensure the privilege change is only for the lookup.
	runtime.LockOSThread()
	defer runtime.UnlockOSThread()
	go func() {
		runtime.LockOSThread()
		if err := SetThreadPrivileges(target, true); err != nil {
			errChan <- err
			return
		}
		keyringID, err := userKeyringIDLookup(uid)
		if err != nil {
			errChan <- err
			return
		}
		idChan <- keyringID
	}()

	// We select so the thread will have to complete
	select {
	case err := <-errChan:
		return 0, err
	case keyringID := <-idChan:
		if uid == os.Getuid() && uid == os.Geteuid() {
			return 0, util.SystemError("thread privileges now incorrect")
		}
		return keyringID, nil
	}
}

func userKeyringIDLookup(uid int) (int, error) {
	// This will trigger the creation of the user keyring, if necessary.
	keyringID, err := unix.KeyctlGetKeyringID(unix.KEY_SPEC_USER_KEYRING, false)
	log.Printf("keyringID(_uid.%d) = %d, %v", uid, keyringID, err)
	if err != nil {
		return 0, errors.Wrap(ErrFindingKeyring, err.Error())
	}

	// For some silly reason, a thread does not automatically "possess" keys
	// in the user keyring. So we link it into the process keyring so that
	// we will not get "permission denied" when purging or modifying keys.
	if err := keyringLink(keyringID, unix.KEY_SPEC_PROCESS_KEYRING); err != nil {
		return 0, err
	}

	keyringIDCache[uid] = keyringID
	return keyringID, nil
}

func keyringLink(keyID int, keyringID int) error {
	_, err := unix.KeyctlInt(unix.KEYCTL_LINK, keyID, keyringID, 0, 0)
	log.Printf("KeyctlLink(%d, %d) = %v", keyID, keyringID, err)
	if err != nil {
		return errors.Wrap(ErrKeyringLink, err.Error())
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"

	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/validation"

	"github.com/golang/glog"
	"github.com/pkg/errors"
)

const (
	defaultKeepAlivePeriod = 3 * time.Minute
)

// serveSecurely runs the secure http server. It fails only if certificates cannot
// be loaded or the initial listen call fails. The actual server loop (stoppable by closing
// stopCh) runs in a go routine, i.e. serveSecurely does not block.
func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
	secureServer := &http.Server{
		Addr:           s.SecureServingInfo.BindAddress,
		Handler:        s.Handler,
		MaxHeaderBytes: 1 << 20,
		TLSConfig: &tls.Config{
			NameToCertificate: s.SecureServingInfo.SNICerts,
			// Can't use SSLv3 because of POODLE and BEAST
			// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
			// Can't use TLSv1.1 because of RC4 cipher usage
			MinVersion: tls.VersionTLS12,
			// enable HTTP2 for go's 1.7 HTTP Server
			NextProtos: []string{"h2", "http/1.1"},
		},
	}

	if s.SecureServingInfo.Cert != nil {
		secureServer.TLSConfig.Certificates = []tls.Certificate{*s.SecureServingInfo.Cert}
	}

	// append all named certs. Otherwise, the go tls stack will think no SNI processing
	// is necessary because there is only one cert anyway.
	// Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI
	// cert will become the default cert. That's what we expect anyway.
	for _, c := range s.SecureServingInfo.SNICerts {
		secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c)
	}

	if s.SecureServingInfo.ClientCA != nil {
		// Populate PeerCertificates in requests, but don't reject connections without certificates
		// This allows certificates to be validated by authenticators, while still allowing other auth types
		secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
		// Specify allowed CAs for client certificates
		secureServer.TLSConfig.ClientCAs = s.SecureServingInfo.ClientCA
	}

	glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
	var err error
	s.effectiveSecurePort, err = runServer(secureServer, s.SecureServingInfo.BindNetwork, stopCh)
	return err
}

// serveInsecurely run the insecure http server. It fails only if the initial listen
// call fails. The actual server loop (stoppable by closing stopCh) runs in a go
// routine, i.e. serveInsecurely does not block.
func (s *GenericAPIServer) serveInsecurely(stopCh <-chan struct{}) error {
	insecureServer := &http.Server{
		Addr:           s.InsecureServingInfo.BindAddress,
		Handler:        s.InsecureHandler,
		MaxHeaderBytes: 1 << 20,
	}
	glog.Infof("Serving insecurely on %s", s.InsecureServingInfo.BindAddress)
	var err error
	s.effectiveInsecurePort, err = runServer(insecureServer, s.InsecureServingInfo.BindNetwork, stopCh)
	return err
}

// runServer listens on the given port, then spawns a go-routine continuously serving
// until the stopCh is closed. The port is returned. This function does not block.
func runServer(server *http.Server, network string, stopCh <-chan struct{}) (int, error) {
	if len(server.Addr) == 0 {
		return 0, errors.New("address cannot be empty")
	}

	if len(network) == 0 {
		network = "tcp"
	}

	// first listen is synchronous (fail early!)
	ln, err := net.Listen(network, server.Addr)
	if err != nil {
		return 0, fmt.Errorf("failed to listen on %v: %v", server.Addr, err)
	}

	// get port
	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
	if !ok {
		ln.Close()
		return 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
	}

	lock := sync.Mutex{} // to avoid we close an old listener during a listen retry
	go func() {
		<-stopCh
		lock.Lock()
		defer lock.Unlock()
		ln.Close()
	}()

	go func() {
		defer utilruntime.HandleCrash()

		for {
			var listener net.Listener
			listener = tcpKeepAliveListener{ln.(*net.TCPListener)}
			if server.TLSConfig != nil {
				listener = tls.NewListener(listener, server.TLSConfig)
			}

			err := server.Serve(listener)
			glog.Errorf("Error serving %v (%v); will try again.", server.Addr, err)

			// listen again
			func() {
				lock.Lock()
				defer lock.Unlock()
				for {
					time.Sleep(15 * time.Second)

					ln, err = net.Listen("tcp", server.Addr)
					if err == nil {
						return
					}
					select {
					case <-stopCh:
						return
					default:
					}
					glog.Errorf("Error listening on %v (%v); will try again.", server.Addr, err)
				}
			}()

			select {
			case <-stopCh:
				return
			default:
			}
		}
	}()

	return tcpAddr.Port, nil
}

type NamedTLSCert struct {
	TLSCert tls.Certificate

	// names is a list of domain patterns: fully qualified domain names, possibly prefixed with
	// wildcard segments.
	Names []string
}

// getNamedCertificateMap returns a map of *tls.Certificate by name. It's is
// suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs
// cannot be loaded. Returns nil if len(certs) == 0
func GetNamedCertificateMap(certs []NamedTLSCert) (map[string]*tls.Certificate, error) {
	// register certs with implicit names first, reverse order such that earlier trump over the later
	byName := map[string]*tls.Certificate{}
	for i := len(certs) - 1; i >= 0; i-- {
		if len(certs[i].Names) > 0 {
			continue
		}
		cert := &certs[i].TLSCert

		// read names from certificate common names and DNS names
		if len(cert.Certificate) == 0 {
			return nil, fmt.Errorf("empty SNI certificate, skipping")
		}
		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
		if err != nil {
			return nil, fmt.Errorf("parse error for SNI certificate: %v", err)
		}
		cn := x509Cert.Subject.CommonName
		if cn == "*" || len(validation.IsDNS1123Subdomain(strings.TrimPrefix(cn, "*."))) == 0 {
			byName[cn] = cert
		}
		for _, san := range x509Cert.DNSNames {
			byName[san] = cert
		}
		// intentionally all IPs in the cert are ignored as SNI forbids passing IPs
		// to select a cert. Before go 1.6 the tls happily passed IPs as SNI values.
	}

	// register certs with explicit names last, overwriting every of the implicit ones,
	// again in reverse order.
	for i := len(certs) - 1; i >= 0; i-- {
		namedCert := &certs[i]
		for _, name := range namedCert.Names {
			byName[name] = &certs[i].TLSCert
		}
	}

	return byName, nil
}

// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
// connections. It's used by ListenAndServe and ListenAndServeTLS so
// dead TCP connections (e.g. closing laptop mid-download) eventually
// go away.
//
// Copied from Go 1.7.2 net/http/server.go
type tcpKeepAliveListener struct {
	*net.TCPListener
}

func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
	tc, err := ln.AcceptTCP()
	if err != nil {
		return nil, err
	}
	tc.SetKeepAlive(true)
	tc.SetKeepAlivePeriod(defaultKeepAlivePeriod)
	return tc, nil
}
<commit_msg>fix bug of runServer function in server.go<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"

	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/validation"

	"github.com/golang/glog"
	"github.com/pkg/errors"
)

const (
	defaultKeepAlivePeriod = 3 * time.Minute
)

// serveSecurely runs the secure http server. It fails only if certificates cannot
// be loaded or the initial listen call fails. The actual server loop (stoppable by closing
// stopCh) runs in a go routine, i.e. serveSecurely does not block.
func (s *GenericAPIServer) serveSecurely(stopCh <-chan struct{}) error {
	secureServer := &http.Server{
		Addr:           s.SecureServingInfo.BindAddress,
		Handler:        s.Handler,
		MaxHeaderBytes: 1 << 20,
		TLSConfig: &tls.Config{
			NameToCertificate: s.SecureServingInfo.SNICerts,
			// Can't use SSLv3 because of POODLE and BEAST
			// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
			// Can't use TLSv1.1 because of RC4 cipher usage
			MinVersion: tls.VersionTLS12,
			// enable HTTP2 for go's 1.7 HTTP Server
			NextProtos: []string{"h2", "http/1.1"},
		},
	}

	if s.SecureServingInfo.Cert != nil {
		secureServer.TLSConfig.Certificates = []tls.Certificate{*s.SecureServingInfo.Cert}
	}

	// append all named certs. Otherwise, the go tls stack will think no SNI processing
	// is necessary because there is only one cert anyway.
	// Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI
	// cert will become the default cert. That's what we expect anyway.
	for _, c := range s.SecureServingInfo.SNICerts {
		secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c)
	}

	if s.SecureServingInfo.ClientCA != nil {
		// Populate PeerCertificates in requests, but don't reject connections without certificates
		// This allows certificates to be validated by authenticators, while still allowing other auth types
		secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
		// Specify allowed CAs for client certificates
		secureServer.TLSConfig.ClientCAs = s.SecureServingInfo.ClientCA
	}

	glog.Infof("Serving securely on %s", s.SecureServingInfo.BindAddress)
	var err error
	s.effectiveSecurePort, err = runServer(secureServer, s.SecureServingInfo.BindNetwork, stopCh)
	return err
}

// serveInsecurely run the insecure http server. It fails only if the initial listen
// call fails. The actual server loop (stoppable by closing stopCh) runs in a go
// routine, i.e. serveInsecurely does not block.
func (s *GenericAPIServer) serveInsecurely(stopCh <-chan struct{}) error {
	insecureServer := &http.Server{
		Addr:           s.InsecureServingInfo.BindAddress,
		Handler:        s.InsecureHandler,
		MaxHeaderBytes: 1 << 20,
	}
	glog.Infof("Serving insecurely on %s", s.InsecureServingInfo.BindAddress)
	var err error
	s.effectiveInsecurePort, err = runServer(insecureServer, s.InsecureServingInfo.BindNetwork, stopCh)
	return err
}

// runServer listens on the given port, then spawns a go-routine continuously serving
// until the stopCh is closed. The port is returned. This function does not block.
func runServer(server *http.Server, network string, stopCh <-chan struct{}) (int, error) {
	if len(server.Addr) == 0 {
		return 0, errors.New("address cannot be empty")
	}

	if len(network) == 0 {
		network = "tcp"
	}

	// first listen is synchronous (fail early!)
	ln, err := net.Listen(network, server.Addr)
	if err != nil {
		return 0, fmt.Errorf("failed to listen on %v: %v", server.Addr, err)
	}

	// get port
	tcpAddr, ok := ln.Addr().(*net.TCPAddr)
	if !ok {
		ln.Close()
		return 0, fmt.Errorf("invalid listen address: %q", ln.Addr().String())
	}

	lock := sync.Mutex{} // to avoid we close an old listener during a listen retry
	go func() {
		<-stopCh
		lock.Lock()
		defer lock.Unlock()
		ln.Close()
	}()

	go func() {
		defer utilruntime.HandleCrash()

		for {
			var listener net.Listener
			listener = tcpKeepAliveListener{ln.(*net.TCPListener)}
			if server.TLSConfig != nil {
				listener = tls.NewListener(listener, server.TLSConfig)
			}

			err := server.Serve(listener)
			glog.Errorf("Error serving %v (%v); will try again.", server.Addr, err)

			// listen again
			func() {
				lock.Lock()
				defer lock.Unlock()
				for {
					time.Sleep(15 * time.Second)

					ln, err = net.Listen(network, server.Addr)
					if err == nil {
						return
					}
					select {
					case <-stopCh:
						return
					default:
					}
					glog.Errorf("Error listening on %v (%v); will try again.", server.Addr, err)
				}
			}()

			select {
			case <-stopCh:
				return
			default:
			}
		}
	}()

	return tcpAddr.Port, nil
}

type NamedTLSCert struct {
	TLSCert tls.Certificate

	// names is a list of domain patterns: fully qualified domain names, possibly prefixed with
	// wildcard segments.
	Names []string
}

// getNamedCertificateMap returns a map of *tls.Certificate by name. It's is
// suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs
// cannot be loaded. Returns nil if len(certs) == 0
func GetNamedCertificateMap(certs []NamedTLSCert) (map[string]*tls.Certificate, error) {
	// register certs with implicit names first, reverse order such that earlier trump over the later
	byName := map[string]*tls.Certificate{}
	for i := len(certs) - 1; i >= 0; i-- {
		if len(certs[i].Names) > 0 {
			continue
		}
		cert := &certs[i].TLSCert

		// read names from certificate common names and DNS names
		if len(cert.Certificate) == 0 {
			return nil, fmt.Errorf("empty SNI certificate, skipping")
		}
		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
		if err != nil {
			return nil, fmt.Errorf("parse error for SNI certificate: %v", err)
		}
		cn := x509Cert.Subject.CommonName
		if cn == "*" || len(validation.IsDNS1123Subdomain(strings.TrimPrefix(cn, "*."))) == 0 {
			byName[cn] = cert
		}
		for _, san := range x509Cert.DNSNames {
			byName[san] = cert
		}
		// intentionally all IPs in the cert are ignored as SNI forbids passing IPs
		// to select a cert. Before go 1.6 the tls happily passed IPs as SNI values.
	}

	// register certs with explicit names last, overwriting every of the implicit ones,
	// again in reverse order.
	for i := len(certs) - 1; i >= 0; i-- {
		namedCert := &certs[i]
		for _, name := range namedCert.Names {
			byName[name] = &certs[i].TLSCert
		}
	}

	return byName, nil
}

// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
// connections. It's used by ListenAndServe and ListenAndServeTLS so
// dead TCP connections (e.g. closing laptop mid-download) eventually
// go away.
//
// Copied from Go 1.7.2 net/http/server.go
type tcpKeepAliveListener struct {
	*net.TCPListener
}

func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
	tc, err := ln.AcceptTCP()
	if err != nil {
		return nil, err
	}
	tc.SetKeepAlive(true)
	tc.SetKeepAlivePeriod(defaultKeepAlivePeriod)
	return tc, nil
}
<|endoftext|>


<commit_before>package iguazio

import (
	"bytes"
	"context"
	"crypto/tls"
	"net/http"

	"github.com/nuclio/nuclio/pkg/dashboard/auth"

	"github.com/nuclio/errors"
	"github.com/nuclio/logger"
	"github.com/nuclio/nuclio-sdk-go"
	"k8s.io/apimachinery/pkg/util/cache"
)

type Auth struct {
	logger     logger.Logger
	config     *auth.Config
	httpClient *http.Client
	cache      *cache.LRUExpireCache
}

func NewAuth(logger logger.Logger, config *auth.Config) auth.Auth {
	return &Auth{
		logger: logger.GetChild("iguazio-auth"),
		config: config,
		httpClient: &http.Client{
			Timeout: config.Iguazio.Timeout,
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
			},
		},
		cache: cache.NewLRUExpireCache(config.Iguazio.CacheSize),
	}
}

// Authenticate will ask IguazioConfig session verification endpoint to verify the request session
// and enrich with session metadata
func (a *Auth) Authenticate(request *http.Request) (auth.Session, error) {
	authorization := request.Header.Get("authorization")
	cookie := request.Header.Get("cookie")
	cacheKey := authorization + cookie

	// try resolve from cache
	if cacheData, found := a.cache.Get(cacheKey); found {
		return cacheData.(*auth.IguazioSession), nil
	}

	authHeaders := map[string]string{
		"authorization": authorization,
		"cookie":        cookie,
	}

	response, err := a.performHTTPRequest(http.MethodPost,
		a.config.Iguazio.VerificationURL,
		nil,
		map[string]string{
			"authorization": authorization,
			"cookie":        cookie,
		})
	if err != nil {
		a.logger.WarnWith("Failed to perform http authentication request",
			"err", err,
		)
		return nil, errors.Wrap(err, "Failed to perform http POST request")
	}

	// auth failed
	if response.StatusCode == http.StatusUnauthorized {
		a.logger.InfoWith("Authentication failed",
			"authorizationHeaderLength", len(authHeaders["authorization"]),
			"cookieHeaderLength", len(authHeaders["cookie"]),
		)
		return nil, nuclio.NewErrUnauthorized("Authentication failed")
	}

	// not within range of 200
	if !(response.StatusCode >= http.StatusOK && response.StatusCode < 300) {
		a.logger.WarnWith("Unexpected authentication status code",
			"authorizationHeaderLength", len(authHeaders["authorization"]),
			"cookieHeaderLength", len(authHeaders["cookie"]),
			"statusCode", response.StatusCode,
		)
		return nil, nuclio.NewErrUnauthorized("Authentication failed")
	}

	authInfo := &auth.IguazioSession{
		Username:   response.Header.Get("x-remote-user"),
		SessionKey: response.Header.Get("x-v3io-session-key"),
		UserID:     response.Header.Get("x-user-id"),
		GroupIDs:   response.Header.Values("x-user-group-ids"),
	}
	a.cache.Add(authorization+cookie, authInfo, a.config.Iguazio.CacheExpirationTimeout)
	a.logger.InfoWith("Authentication succeeded", "username", authInfo.GetUsername())
	return authInfo, nil
}

// Middleware will authenticate the incoming request and store the session within the request context
func (a *Auth) Middleware() func(next http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			session, err := a.Authenticate(r)
			if err != nil {
				a.iguazioAuthenticationFailed(w)
				return
			}
			a.logger.DebugWith("Successfully authenticated incoming request",
				"sessionUsername", session.GetUsername())
			ctx := context.WithValue(r.Context(), auth.IguazioContextKey, session)
			next.ServeHTTP(w, r.WithContext(ctx))
		})
	}
}

func (a *Auth) Kind() auth.Kind {
	return a.config.Kind
}

func (a *Auth) iguazioAuthenticationFailed(w http.ResponseWriter) {
	w.WriteHeader(http.StatusUnauthorized)
}

func (a *Auth) performHTTPRequest(method string,
	url string,
	body []byte,
	headers map[string]string) (*http.Response, error) {

	// create request
	req, err := http.NewRequest(method, url, bytes.NewBuffer(body))
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create http request")
	}

	// attach headers
	for headerKey, headerValue := range headers {
		req.Header.Set(headerKey, headerValue)
	}

	// fire request
	resp, err := a.httpClient.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to send HTTP request")
	}

	return resp, nil
}
<commit_msg>Auth - Fix populating group ids (#2262)<commit_after>package iguazio

import (
	"bytes"
	"context"
	"crypto/tls"
	"net/http"

	"github.com/nuclio/nuclio/pkg/dashboard/auth"

	"github.com/nuclio/errors"
	"github.com/nuclio/logger"
	"github.com/nuclio/nuclio-sdk-go"
	"k8s.io/apimachinery/pkg/util/cache"
)

type Auth struct {
	logger     logger.Logger
	config     *auth.Config
	httpClient *http.Client
	cache      *cache.LRUExpireCache
}

func NewAuth(logger logger.Logger, config *auth.Config) auth.Auth {
	return &Auth{
		logger: logger.GetChild("iguazio-auth"),
		config: config,
		httpClient: &http.Client{
			Timeout: config.Iguazio.Timeout,
			Transport: &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
			},
		},
		cache: cache.NewLRUExpireCache(config.Iguazio.CacheSize),
	}
}

// Authenticate will ask IguazioConfig session verification endpoint to verify the request session
// and enrich with session metadata
func (a *Auth) Authenticate(request *http.Request) (auth.Session, error) {
	authorization := request.Header.Get("authorization")
	cookie := request.Header.Get("cookie")
	cacheKey := authorization + cookie

	// try resolve from cache
	if cacheData, found := a.cache.Get(cacheKey); found {
		return cacheData.(*auth.IguazioSession), nil
	}

	authHeaders := map[string]string{
		"authorization": authorization,
		"cookie":        cookie,
	}

	response, err := a.performHTTPRequest(http.MethodPost,
		a.config.Iguazio.VerificationURL,
		nil,
		map[string]string{
			"authorization": authorization,
			"cookie":        cookie,
		})
	if err != nil {
		a.logger.WarnWith("Failed to perform http authentication request",
			"err", err,
		)
		return nil, errors.Wrap(err, "Failed to perform http POST request")
	}

	// auth failed
	if response.StatusCode == http.StatusUnauthorized {
		a.logger.InfoWith("Authentication failed",
			"authorizationHeaderLength", len(authHeaders["authorization"]),
			"cookieHeaderLength", len(authHeaders["cookie"]),
		)
		return nil, nuclio.NewErrUnauthorized("Authentication failed")
	}

	// not within range of 200
	if !(response.StatusCode >= http.StatusOK && response.StatusCode < 300) {
		a.logger.WarnWith("Unexpected authentication status code",
			"authorizationHeaderLength", len(authHeaders["authorization"]),
			"cookieHeaderLength", len(authHeaders["cookie"]),
			"statusCode", response.StatusCode,
		)
		return nil, nuclio.NewErrUnauthorized("Authentication failed")
	}

	authInfo := &auth.IguazioSession{
		Username:   response.Header.Get("x-remote-user"),
		SessionKey: response.Header.Get("x-v3io-session-key"),
		UserID:     response.Header.Get("x-user-id"),
	}

	for _, groupID := range response.Header.Values("x-user-group-ids") {
		if groupID != "" {
			authInfo.GroupIDs = append(authInfo.GroupIDs, groupID)
		}
	}

	a.cache.Add(authorization+cookie, authInfo, a.config.Iguazio.CacheExpirationTimeout)
	a.logger.InfoWith("Authentication succeeded", "username", authInfo.GetUsername())
	return authInfo, nil
}

// Middleware will authenticate the incoming request and store the session within the request context
func (a *Auth) Middleware() func(next http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			session, err := a.Authenticate(r)
			if err != nil {
				a.iguazioAuthenticationFailed(w)
				return
			}
			a.logger.DebugWith("Successfully authenticated incoming request",
				"sessionUsername", session.GetUsername())
			ctx := context.WithValue(r.Context(), auth.IguazioContextKey, session)
			next.ServeHTTP(w, r.WithContext(ctx))
		})
	}
}

func (a *Auth) Kind() auth.Kind {
	return a.config.Kind
}

func (a *Auth) iguazioAuthenticationFailed(w http.ResponseWriter) {
	w.WriteHeader(http.StatusUnauthorized)
}

func (a *Auth) performHTTPRequest(method string,
	url string,
	body []byte,
	headers map[string]string) (*http.Response, error) {

	// create request
	req, err := http.NewRequest(method, url, bytes.NewBuffer(body))
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create http request")
	}

	// attach headers
	for headerKey, headerValue := range headers {
		req.Header.Set(headerKey, headerValue)
	}

	// fire request
	resp, err := a.httpClient.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to send HTTP request")
	}

	return resp, nil
}
<|endoftext|>


<commit_before>package acme

import (
	"fmt"
	"sync"
	"time"

	"github.com/go-acme/lego/v4/challenge/tlsalpn01"
	"github.com/traefik/traefik/v2/pkg/config/dynamic"
	"github.com/traefik/traefik/v2/pkg/log"
	"github.com/traefik/traefik/v2/pkg/safe"
	traefiktls "github.com/traefik/traefik/v2/pkg/tls"
	"github.com/traefik/traefik/v2/pkg/types"
)

const providerNameALPN = "tlsalpn.acme"

// ChallengeTLSALPN TLSALPN challenge provider implements challenge.Provider.
type ChallengeTLSALPN struct {
	Timeout time.Duration

	chans   map[string]chan struct{}
	muChans sync.Mutex

	certs   map[string]*Certificate
	muCerts sync.Mutex

	configurationChan chan<- dynamic.Message
}

// NewChallengeTLSALPN creates a new ChallengeTLSALPN.
func NewChallengeTLSALPN(timeout time.Duration) *ChallengeTLSALPN {
	return &ChallengeTLSALPN{
		Timeout: timeout,
		chans:   make(map[string]chan struct{}),
		certs:   make(map[string]*Certificate),
	}
}

// Present presents a challenge to obtain new ACME certificate.
func (c *ChallengeTLSALPN) Present(domain, _, keyAuth string) error {
	logger := log.WithoutContext().WithField(log.ProviderName, providerNameALPN)
	logger.Debugf("TLS Challenge Present temp certificate for %s", domain)

	certPEMBlock, keyPEMBlock, err := tlsalpn01.ChallengeBlocks(domain, keyAuth)
	if err != nil {
		return err
	}

	cert := &Certificate{Certificate: certPEMBlock, Key: keyPEMBlock, Domain: types.Domain{Main: "TEMP-" + domain}}

	c.muChans.Lock()
	ch := make(chan struct{})
	c.chans[string(certPEMBlock)] = ch
	c.muChans.Unlock()

	c.muCerts.Lock()
	c.certs[keyAuth] = cert
	conf := createMessage(c.certs)
	c.muCerts.Unlock()

	c.configurationChan <- conf

	timer := time.NewTimer(c.Timeout)

	var errC error
	select {
	case t := <-timer.C:
		timer.Stop()
		close(c.chans[string(certPEMBlock)])

		err = c.CleanUp(domain, "", keyAuth)
		if err != nil {
			logger.Errorf("Failed to clean up TLS challenge: %v", err)
		}

		errC = fmt.Errorf("timeout %s", t)
	case <-ch:
		// noop
	}

	c.muChans.Lock()
	delete(c.chans, string(certPEMBlock))
	c.muChans.Unlock()

	return errC
}

// CleanUp cleans the challenges when certificate is obtained.
func (c *ChallengeTLSALPN) CleanUp(domain, _, keyAuth string) error {
	log.WithoutContext().WithField(log.ProviderName, providerNameALPN).
		Debugf("TLS Challenge CleanUp temp certificate for %s", domain)

	c.muCerts.Lock()
	delete(c.certs, keyAuth)
	conf := createMessage(c.certs)
	c.muCerts.Unlock()

	c.configurationChan <- conf

	return nil
}

// Init the provider.
func (c *ChallengeTLSALPN) Init() error {
	return nil
}

// Provide allows the provider to provide configurations to traefik using the given configuration channel.
func (c *ChallengeTLSALPN) Provide(configurationChan chan<- dynamic.Message, _ *safe.Pool) error {
	c.configurationChan = configurationChan

	return nil
}

// ListenConfiguration sets a new Configuration into the configurationChan.
func (c *ChallengeTLSALPN) ListenConfiguration(conf dynamic.Configuration) {
	for _, certificate := range conf.TLS.Certificates {
		if !containsACMETLS1(certificate.Stores) {
			continue
		}

		c.muChans.Lock()
		if _, ok := c.chans[certificate.CertFile.String()]; ok {
			close(c.chans[certificate.CertFile.String()])
		}
		c.muChans.Unlock()
	}
}

func createMessage(certs map[string]*Certificate) dynamic.Message {
	conf := dynamic.Message{
		ProviderName: providerNameALPN,
		Configuration: &dynamic.Configuration{
			HTTP: &dynamic.HTTPConfiguration{
				Routers:     map[string]*dynamic.Router{},
				Middlewares: map[string]*dynamic.Middleware{},
				Services:    map[string]*dynamic.Service{},
			},
			TLS: &dynamic.TLSConfiguration{},
		},
	}

	for _, cert := range certs {
		certConf := &traefiktls.CertAndStores{
			Certificate: traefiktls.Certificate{
				CertFile: traefiktls.FileOrContent(cert.Certificate),
				KeyFile:  traefiktls.FileOrContent(cert.Key),
			},
			Stores: []string{tlsalpn01.ACMETLS1Protocol},
		}
		conf.Configuration.TLS.Certificates = append(conf.Configuration.TLS.Certificates, certConf)
	}

	return conf
}

func containsACMETLS1(stores []string) bool {
	for _, store := range stores {
		if store == tlsalpn01.ACMETLS1Protocol {
			return true
		}
	}

	return false
}
<commit_msg>fix: double close chan on TLS challenge<commit_after>package acme

import (
	"fmt"
	"sync"
	"time"

	"github.com/go-acme/lego/v4/challenge/tlsalpn01"
	"github.com/traefik/traefik/v2/pkg/config/dynamic"
	"github.com/traefik/traefik/v2/pkg/log"
	"github.com/traefik/traefik/v2/pkg/safe"
	traefiktls "github.com/traefik/traefik/v2/pkg/tls"
	"github.com/traefik/traefik/v2/pkg/types"
)

const providerNameALPN = "tlsalpn.acme"

// ChallengeTLSALPN TLSALPN challenge provider implements challenge.Provider.
type ChallengeTLSALPN struct {
	Timeout time.Duration

	chans   map[string]chan struct{}
	muChans sync.Mutex

	certs   map[string]*Certificate
	muCerts sync.Mutex

	configurationChan chan<- dynamic.Message
}

// NewChallengeTLSALPN creates a new ChallengeTLSALPN.
func NewChallengeTLSALPN(timeout time.Duration) *ChallengeTLSALPN {
	return &ChallengeTLSALPN{
		Timeout: timeout,
		chans:   make(map[string]chan struct{}),
		certs:   make(map[string]*Certificate),
	}
}

// Present presents a challenge to obtain new ACME certificate.
func (c *ChallengeTLSALPN) Present(domain, _, keyAuth string) error {
	logger := log.WithoutContext().WithField(log.ProviderName, providerNameALPN)
	logger.Debugf("TLS Challenge Present temp certificate for %s", domain)

	certPEMBlock, keyPEMBlock, err := tlsalpn01.ChallengeBlocks(domain, keyAuth)
	if err != nil {
		return err
	}

	cert := &Certificate{Certificate: certPEMBlock, Key: keyPEMBlock, Domain: types.Domain{Main: "TEMP-" + domain}}

	c.muChans.Lock()
	ch := make(chan struct{})
	c.chans[string(certPEMBlock)] = ch
	c.muChans.Unlock()

	c.muCerts.Lock()
	c.certs[keyAuth] = cert
	conf := createMessage(c.certs)
	c.muCerts.Unlock()

	c.configurationChan <- conf

	timer := time.NewTimer(c.Timeout)

	select {
	case t := <-timer.C:
		timer.Stop()

		c.muChans.Lock()
		c.cleanChan(string(certPEMBlock))
		c.muChans.Unlock()

		err = c.CleanUp(domain, "", keyAuth)
		if err != nil {
			logger.Errorf("Failed to clean up TLS challenge: %v", err)
		}

		return fmt.Errorf("timeout %s", t)
	case <-ch:
		// noop
		return nil
	}
}

// CleanUp cleans the challenges when certificate is obtained.
func (c *ChallengeTLSALPN) CleanUp(domain, _, keyAuth string) error {
	log.WithoutContext().WithField(log.ProviderName, providerNameALPN).
		Debugf("TLS Challenge CleanUp temp certificate for %s", domain)

	c.muCerts.Lock()
	delete(c.certs, keyAuth)
	conf := createMessage(c.certs)
	c.muCerts.Unlock()

	c.configurationChan <- conf

	return nil
}

// Init the provider.
func (c *ChallengeTLSALPN) Init() error {
	return nil
}

// Provide allows the provider to provide configurations to traefik using the given configuration channel.
func (c *ChallengeTLSALPN) Provide(configurationChan chan<- dynamic.Message, _ *safe.Pool) error {
	c.configurationChan = configurationChan

	return nil
}

// ListenConfiguration sets a new Configuration into the configurationChan.
func (c *ChallengeTLSALPN) ListenConfiguration(conf dynamic.Configuration) {
	c.muChans.Lock()

	for _, certificate := range conf.TLS.Certificates {
		if !containsACMETLS1(certificate.Stores) {
			continue
		}

		c.cleanChan(certificate.CertFile.String())
	}

	c.muChans.Unlock()
}

func (c *ChallengeTLSALPN) cleanChan(key string) {
	if _, ok := c.chans[key]; ok {
		close(c.chans[key])
		delete(c.chans, key)
	}
}

func createMessage(certs map[string]*Certificate) dynamic.Message {
	conf := dynamic.Message{
		ProviderName: providerNameALPN,
		Configuration: &dynamic.Configuration{
			HTTP: &dynamic.HTTPConfiguration{
				Routers:     map[string]*dynamic.Router{},
				Middlewares: map[string]*dynamic.Middleware{},
				Services:    map[string]*dynamic.Service{},
			},
			TLS: &dynamic.TLSConfiguration{},
		},
	}

	for _, cert := range certs {
		certConf := &traefiktls.CertAndStores{
			Certificate: traefiktls.Certificate{
				CertFile: traefiktls.FileOrContent(cert.Certificate),
				KeyFile:  traefiktls.FileOrContent(cert.Key),
			},
			Stores: []string{tlsalpn01.ACMETLS1Protocol},
		}
		conf.Configuration.TLS.Certificates = append(conf.Configuration.TLS.Certificates, certConf)
	}

	return conf
}

func containsACMETLS1(stores []string) bool {
	for _, store := range stores {
		if store == tlsalpn01.ACMETLS1Protocol {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gang

import (
	"fmt"
	"github.com/golang/glog"

	arbcorev1 "github.com/kubernetes-sigs/kube-batch/pkg/apis/scheduling/v1alpha1"
	"github.com/kubernetes-sigs/kube-batch/pkg/scheduler/api"
	"github.com/kubernetes-sigs/kube-batch/pkg/scheduler/framework"
)

type gangPlugin struct {
	args *framework.PluginArgs
}

func New(args *framework.PluginArgs) framework.Plugin {
	return &gangPlugin{
		args: args,
	}
}

func readyTaskNum(job *api.JobInfo) int32 {
	occupid := 0
	for status, tasks := range job.TaskStatusIndex {
		if api.AllocatedStatus(status) || status == api.Succeeded {
			occupid = occupid + len(tasks)
		}
	}

	return int32(occupid)
}

func validTaskNum(job *api.JobInfo) int32 {
	occupid := 0
	for status, tasks := range job.TaskStatusIndex {
		if api.AllocatedStatus(status) ||
			status == api.Succeeded ||
			status == api.Pending {
			occupid = occupid + len(tasks)
		}
	}

	return int32(occupid)
}

func jobReady(obj interface{}) bool {
	job := obj.(*api.JobInfo)

	occupid := readyTaskNum(job)

	return occupid >= job.MinAvailable
}

func (gp *gangPlugin) OnSessionOpen(ssn *framework.Session) {
	for _, job := range ssn.Jobs {
		if validTaskNum(job) < job.MinAvailable {
			ssn.Backoff(job, arbcorev1.UnschedulableEvent, "not enough valid tasks for gang-scheduling")
		}
	}

	preemptableFn := func(preemptor *api.TaskInfo, preemptees []*api.TaskInfo) []*api.TaskInfo {
		var victims []*api.TaskInfo

		for _, preemptee := range preemptees {
			job := ssn.JobIndex[preemptee.Job]
			occupid := readyTaskNum(job)
			preemptable := job.MinAvailable <= occupid-1

			if !preemptable {
				glog.V(3).Infof("Can not preempt task <%v/%v> because of gang-scheduling",
					preemptee.Namespace, preemptee.Name)
			} else {
				victims = append(victims, preemptee)
			}
		}

		glog.V(3).Infof("Victims from Gang plugins are %+v", victims)

		return victims
	}
	if gp.args.PreemptableFnEnabled {
		ssn.AddPreemptableFn(preemptableFn)
	}

	jobOrderFn := func(l, r interface{}) int {
		lv := l.(*api.JobInfo)
		rv := r.(*api.JobInfo)

		lReady := jobReady(lv)
		rReady := jobReady(rv)

		glog.V(3).Infof("Gang JobOrderFn: <%v/%v> is ready: %t, <%v/%v> is ready: %t",
			lv.Namespace, lv.Name, lReady, rv.Namespace, rv.Name, rReady)

		if lReady && rReady {
			return 0
		}

		if lReady {
			return 1
		}

		if rReady {
			return -1
		}

		if !lReady && !rReady {
			if lv.CreationTimestamp.Equal(&rv.CreationTimestamp) {
				if lv.UID < rv.UID {
					return -1
				}
			} else if lv.CreationTimestamp.Before(&rv.CreationTimestamp) {
				return -1
			}
			return 1
		}

		return 0
	}

	if gp.args.JobOrderFnEnabled {
		ssn.AddJobOrderFn(jobOrderFn)
	}

	if gp.args.JobReadyFnEnabled {
		ssn.AddJobReadyFn(jobReady)
	}
}

func (gp *gangPlugin) OnSessionClose(ssn *framework.Session) {
	for _, job := range ssn.Jobs {
		if len(job.TaskStatusIndex[api.Pending]) != 0 {
			glog.V(3).Infof("Gang: <%v/%v> allocated: %v, pending: %v", job.Namespace, job.Name, len(job.TaskStatusIndex[api.Allocated]), len(job.TaskStatusIndex[api.Pending]))
			msg := fmt.Sprintf("%v/%v tasks in gang unschedulable: %v", len(job.TaskStatusIndex[api.Pending]), len(job.Tasks), job.FitError())
			ssn.Backoff(job, arbcorev1.UnschedulableEvent, msg)
		}
	}
}
<commit_msg>Added Pipelined Pods as valid tasks.<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gang

import (
	"fmt"
	"github.com/golang/glog"

	arbcorev1 "github.com/kubernetes-sigs/kube-batch/pkg/apis/scheduling/v1alpha1"
	"github.com/kubernetes-sigs/kube-batch/pkg/scheduler/api"
	"github.com/kubernetes-sigs/kube-batch/pkg/scheduler/framework"
)

type gangPlugin struct {
	args *framework.PluginArgs
}

func New(args *framework.PluginArgs) framework.Plugin {
	return &gangPlugin{
		args: args,
	}
}

// readyTaskNum return the number of tasks that are ready to run.
func readyTaskNum(job *api.JobInfo) int32 {
	occupid := 0
	for status, tasks := range job.TaskStatusIndex {
		if api.AllocatedStatus(status) || status == api.Succeeded {
			occupid = occupid + len(tasks)
		}
	}

	return int32(occupid)
}

// validTaskNum return the number of tasks that are valid.
func validTaskNum(job *api.JobInfo) int32 {
	occupid := 0
	for status, tasks := range job.TaskStatusIndex {
		if api.AllocatedStatus(status) ||
			status == api.Succeeded ||
			status == api.Pipelined ||
			status == api.Pending {
			occupid = occupid + len(tasks)
		}
	}

	return int32(occupid)
}

func jobReady(obj interface{}) bool {
	job := obj.(*api.JobInfo)

	occupid := readyTaskNum(job)

	return occupid >= job.MinAvailable
}

func (gp *gangPlugin) OnSessionOpen(ssn *framework.Session) {
	for _, job := range ssn.Jobs {
		if validTaskNum(job) < job.MinAvailable {
			ssn.Backoff(job, arbcorev1.UnschedulableEvent, "not enough valid tasks for gang-scheduling")
		}
	}

	preemptableFn := func(preemptor *api.TaskInfo, preemptees []*api.TaskInfo) []*api.TaskInfo {
		var victims []*api.TaskInfo

		for _, preemptee := range preemptees {
			job := ssn.JobIndex[preemptee.Job]
			occupid := readyTaskNum(job)
			preemptable := job.MinAvailable <= occupid-1

			if !preemptable {
				glog.V(3).Infof("Can not preempt task <%v/%v> because of gang-scheduling",
					preemptee.Namespace, preemptee.Name)
			} else {
				victims = append(victims, preemptee)
			}
		}

		glog.V(3).Infof("Victims from Gang plugins are %+v", victims)

		return victims
	}
	if gp.args.PreemptableFnEnabled {
		ssn.AddPreemptableFn(preemptableFn)
	}

	jobOrderFn := func(l, r interface{}) int {
		lv := l.(*api.JobInfo)
		rv := r.(*api.JobInfo)

		lReady := jobReady(lv)
		rReady := jobReady(rv)

		glog.V(3).Infof("Gang JobOrderFn: <%v/%v> is ready: %t, <%v/%v> is ready: %t",
			lv.Namespace, lv.Name, lReady, rv.Namespace, rv.Name, rReady)

		if lReady && rReady {
			return 0
		}

		if lReady {
			return 1
		}

		if rReady {
			return -1
		}

		if !lReady && !rReady {
			if lv.CreationTimestamp.Equal(&rv.CreationTimestamp) {
				if lv.UID < rv.UID {
					return -1
				}
			} else if lv.CreationTimestamp.Before(&rv.CreationTimestamp) {
				return -1
			}
			return 1
		}

		return 0
	}

	if gp.args.JobOrderFnEnabled {
		ssn.AddJobOrderFn(jobOrderFn)
	}

	if gp.args.JobReadyFnEnabled {
		ssn.AddJobReadyFn(jobReady)
	}
}

func (gp *gangPlugin) OnSessionClose(ssn *framework.Session) {
	for _, job := range ssn.Jobs {
		if len(job.TaskStatusIndex[api.Pending]) != 0 {
			glog.V(3).Infof("Gang: <%v/%v> allocated: %v, pending: %v", job.Namespace, job.Name, len(job.TaskStatusIndex[api.Allocated]), len(job.TaskStatusIndex[api.Pending]))
			msg := fmt.Sprintf("%v/%v tasks in gang unschedulable: %v", len(job.TaskStatusIndex[api.Pending]), len(job.Tasks), job.FitError())
			ssn.Backoff(job, arbcorev1.UnschedulableEvent, msg)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package local

import (
	"context"
	"io"
	"os"
	"os/exec"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/warnings"
	"github.com/pkg/errors"
)

func (b *Builder) buildDocker(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, tag string) (string, error) {
	if err := b.pullCacheFromImages(ctx, out, a); err != nil {
		return "", errors.Wrap(err, "pulling cache-from images")
	}

	var (
		imageID string
		err     error
	)

	if b.cfg.UseDockerCLI || b.cfg.UseBuildkit {
		imageID, err = b.dockerCLIBuild(ctx, out, workspace, a, tag)
	} else {
		imageID, err = b.localDocker.Build(ctx, out, workspace, a, tag)
	}

	if b.pushImages {
		return b.localDocker.Push(ctx, out, tag)
	}

	return imageID, err
}

func (b *Builder) dockerCLIBuild(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, tag string) (string, error) {
	dockerfilePath, err := docker.NormalizeDockerfilePath(workspace, a.DockerfilePath)
	if err != nil {
		return "", errors.Wrap(err, "normalizing dockerfile path")
	}

	args := []string{"build", workspace, "--file", dockerfilePath, "-t", tag}
	args = append(args, docker.GetBuildArgs(a)...)

	cmd := exec.CommandContext(ctx, "docker", args...)
	if b.cfg.UseBuildkit {
		cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1")
	}
	cmd.Stdout = out
	cmd.Stderr = out

	if err := util.RunCmd(cmd); err != nil {
		return "", errors.Wrap(err, "running build")
	}

	return b.localDocker.ImageID(ctx, tag)
}

func (b *Builder) pullCacheFromImages(ctx context.Context, out io.Writer, a *latest.DockerArtifact) error {
	if len(a.CacheFrom) == 0 {
		return nil
	}

	for _, image := range a.CacheFrom {
		imageID, err := b.localDocker.ImageID(ctx, image)
		if err != nil {
			return errors.Wrapf(err, "getting imageID for %s", image)
		}
		if imageID != "" {
			// already pulled
			continue
		}

		if err := b.localDocker.Pull(ctx, out, image); err != nil {
			warnings.Printf("Cache-From image couldn't be pulled: %s\n", image)
		}
	}

	return nil
}
<commit_msg>Don't push on build error.<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package local

import (
	"context"
	"io"
	"os"
	"os/exec"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/warnings"
	"github.com/pkg/errors"
)

func (b *Builder) buildDocker(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, tag string) (string, error) {
	if err := b.pullCacheFromImages(ctx, out, a); err != nil {
		return "", errors.Wrap(err, "pulling cache-from images")
	}

	var (
		imageID string
		err     error
	)

	if b.cfg.UseDockerCLI || b.cfg.UseBuildkit {
		imageID, err = b.dockerCLIBuild(ctx, out, workspace, a, tag)
	} else {
		imageID, err = b.localDocker.Build(ctx, out, workspace, a, tag)
	}

	if err == nil && b.pushImages {
		return b.localDocker.Push(ctx, out, tag)
	}

	return imageID, err
}

func (b *Builder) dockerCLIBuild(ctx context.Context, out io.Writer, workspace string, a *latest.DockerArtifact, tag string) (string, error) {
	dockerfilePath, err := docker.NormalizeDockerfilePath(workspace, a.DockerfilePath)
	if err != nil {
		return "", errors.Wrap(err, "normalizing dockerfile path")
	}

	args := []string{"build", workspace, "--file", dockerfilePath, "-t", tag}
	args = append(args, docker.GetBuildArgs(a)...)

	cmd := exec.CommandContext(ctx, "docker", args...)
	if b.cfg.UseBuildkit {
		cmd.Env = append(os.Environ(), "DOCKER_BUILDKIT=1")
	}
	cmd.Stdout = out
	cmd.Stderr = out

	if err := util.RunCmd(cmd); err != nil {
		return "", errors.Wrap(err, "running build")
	}

	return b.localDocker.ImageID(ctx, tag)
}

func (b *Builder) pullCacheFromImages(ctx context.Context, out io.Writer, a *latest.DockerArtifact) error {
	if len(a.CacheFrom) == 0 {
		return nil
	}

	for _, image := range a.CacheFrom {
		imageID, err := b.localDocker.ImageID(ctx, image)
		if err != nil {
			return errors.Wrapf(err, "getting imageID for %s", image)
		}
		if imageID != "" {
			// already pulled
			continue
		}

		if err := b.localDocker.Pull(ctx, out, image); err != nil {
			warnings.Printf("Cache-From image couldn't be pulled: %s\n", image)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package visibility

import (
	"github.com/Frostman/aptomi/pkg/slinga"
)

type rLink struct {
	Name string
	Link string
}

type rEndpoint struct {
	Service    string
	Context    string
	Allocation string
	Component  string
	Links      []rLink
}

type endpointsView struct {
	Endpoints map[string][]rEndpoint
}

func Endpoints(username string, users map[string]*slinga.User, state slinga.ServiceUsageState) endpointsView {
	uR := endpointsView{make(map[string][]rEndpoint)}

	for userId, user := range users {
		if username != "" && user.Name != username {
			continue
		}
		r := make([]rEndpoint, 0)

		endpoints := state.Endpoints(userId)

		for key, links := range endpoints {
			service, context, allocation, component := slinga.ParseServiceUsageKey(key)
			rLinks := make([]rLink, 0)

			for linkName, link := range links {
				rLinks = append(rLinks, rLink{linkName, link})
			}

			r = append(r, rEndpoint{service, context, allocation, component, rLinks})
		}

		uR.Endpoints[userId] = r
	}

	return uR
}
<commit_msg>server: improve endpoints impl<commit_after>package visibility

import (
	"github.com/Frostman/aptomi/pkg/slinga"
)

type rLink struct {
	Name string
	Link string
}

type rEndpoint struct {
	Service    string
	Context    string
	Allocation string
	Component  string
	Links      []rLink
}

type userEndpoints struct {
	User      *slinga.User
	Endpoints []rEndpoint
}

type endpointsView struct {
	Endpoints []userEndpoints
}

func Endpoints(username string, users map[string]*slinga.User, state slinga.ServiceUsageState) endpointsView {
	uR := endpointsView{make([]userEndpoints, 0)}

	isGlobalOp := false
	for _, user := range users {
		if user.Name == username && user.Labels["global_ops"] == "true" {
			isGlobalOp = true
			break
		}
	}

	for userId, user := range users {
		if !isGlobalOp && username != "" && user.Name != username {
			continue
		}
		r := make([]rEndpoint, 0)

		endpoints := state.Endpoints(userId)

		for key, links := range endpoints {
			service, context, allocation, component := slinga.ParseServiceUsageKey(key)
			rLinks := make([]rLink, 0)

			for linkName, link := range links {
				rLinks = append(rLinks, rLink{linkName, link})
			}

			r = append(r, rEndpoint{service, context, allocation, component, rLinks})
		}

		uR.Endpoints = append(uR.Endpoints, userEndpoints{user, r})
	}

	return uR
}
<|endoftext|>


<commit_before>package kcp

import (
	"container/heap"
	"runtime"
	"sync"
	"time"
)

// SystemTimedSched is the library level timed-scheduler
var SystemTimedSched *TimedSched = NewTimedSched(runtime.NumCPU())

type timedFunc struct {
	execute func()
	ts      time.Time
}

// a heap for sorted timed function
type timedFuncHeap []timedFunc

func (h timedFuncHeap) Len() int            { return len(h) }
func (h timedFuncHeap) Less(i, j int) bool  { return h[i].ts.Before(h[j].ts) }
func (h timedFuncHeap) Swap(i, j int)       { h[i], h[j] = h[j], h[i] }
func (h *timedFuncHeap) Push(x interface{}) { *h = append(*h, x.(timedFunc)) }
func (h *timedFuncHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	old[n-1].execute = nil // avoid memory leak
	*h = old[0 : n-1]
	return x
}

// TimedSched represents the control struct for timed parallel scheduler
type TimedSched struct {
	// prepending tasks
	prependTasks    []timedFunc
	prependLock     sync.Mutex
	chPrependNotify chan struct{}

	// tasks will be distributed through chTask
	chTask chan timedFunc

	dieOnce sync.Once
	die     chan struct{}
}

// NewTimedSched creates a parallel-scheduler with given parallelization
func NewTimedSched(parallel int) *TimedSched {
	ts := new(TimedSched)
	ts.chTask = make(chan timedFunc)
	ts.die = make(chan struct{})
	ts.chPrependNotify = make(chan struct{}, 1)

	for i := 0; i < parallel; i++ {
		go ts.sched()
	}
	go ts.prepend()
	return ts
}

func (ts *TimedSched) sched() {
	var tasks timedFuncHeap
	timer := time.NewTimer(0)
	for {
		select {
		case task := <-ts.chTask:
			now := time.Now()
			if now.After(task.ts) {
				// already delayed! execute immediately
				task.execute()
			} else {
				heap.Push(&tasks, task)
				// activate timer if timer has hibernated due to 0 tasks.
				if tasks.Len() == 1 {
					timer.Reset(task.ts.Sub(now))
				}
			}
		case <-timer.C:
			for tasks.Len() > 0 {
				now := time.Now()
				if now.After(tasks[0].ts) {
					heap.Pop(&tasks).(timedFunc).execute()
				} else {
					timer.Reset(tasks[0].ts.Sub(now))
					break
				}
			}
		case <-ts.die:
			return
		}
	}
}

func (ts *TimedSched) prepend() {
	var tasks []timedFunc
	for {
		select {
		case <-ts.chPrependNotify:
			ts.prependLock.Lock()
			// keep cap to reuse slice
			if cap(tasks) < cap(ts.prependTasks) {
				tasks = make([]timedFunc, 0, cap(ts.prependTasks))
			}
			tasks = tasks[:len(ts.prependTasks)]
			copy(tasks, ts.prependTasks)
			for k := range ts.prependTasks {
				ts.prependTasks[k].execute = nil // avoid memory leak
			}
			ts.prependTasks = ts.prependTasks[:0]
			ts.prependLock.Unlock()

			for k := range tasks {
				select {
				case ts.chTask <- tasks[k]:
					tasks[k].execute = nil // avoid memory leak
				case <-ts.die:
					return
				}
			}
			tasks = tasks[:0]
		case <-ts.die:
			return
		}
	}
}

// Put a function 'f' awaiting to be executed at 'deadline'
func (ts *TimedSched) Put(f func(), deadline time.Time) {
	ts.prependLock.Lock()
	ts.prependTasks = append(ts.prependTasks, timedFunc{f, deadline})
	ts.prependLock.Unlock()

	select {
	case ts.chPrependNotify <- struct{}{}:
	default:
	}
}

// Close terminates this scheduler
func (ts *TimedSched) Close() { ts.dieOnce.Do(func() { close(ts.die) }) }
<commit_msg>bugfix in timer<commit_after>package kcp

import (
	"container/heap"
	"runtime"
	"sync"
	"time"
)

// SystemTimedSched is the library level timed-scheduler
var SystemTimedSched *TimedSched = NewTimedSched(runtime.NumCPU())

type timedFunc struct {
	execute func()
	ts      time.Time
}

// a heap for sorted timed function
type timedFuncHeap []timedFunc

func (h timedFuncHeap) Len() int            { return len(h) }
func (h timedFuncHeap) Less(i, j int) bool  { return h[i].ts.Before(h[j].ts) }
func (h timedFuncHeap) Swap(i, j int)       { h[i], h[j] = h[j], h[i] }
func (h *timedFuncHeap) Push(x interface{}) { *h = append(*h, x.(timedFunc)) }
func (h *timedFuncHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	old[n-1].execute = nil // avoid memory leak
	*h = old[0 : n-1]
	return x
}

// TimedSched represents the control struct for timed parallel scheduler
type TimedSched struct {
	// prepending tasks
	prependTasks    []timedFunc
	prependLock     sync.Mutex
	chPrependNotify chan struct{}

	// tasks will be distributed through chTask
	chTask chan timedFunc

	dieOnce sync.Once
	die     chan struct{}
}

// NewTimedSched creates a parallel-scheduler with given parallelization
func NewTimedSched(parallel int) *TimedSched {
	ts := new(TimedSched)
	ts.chTask = make(chan timedFunc)
	ts.die = make(chan struct{})
	ts.chPrependNotify = make(chan struct{}, 1)

	for i := 0; i < parallel; i++ {
		go ts.sched()
	}
	go ts.prepend()
	return ts
}

func (ts *TimedSched) sched() {
	var tasks timedFuncHeap
	timer := time.NewTimer(0)
	for {
		select {
		case task := <-ts.chTask:
			now := time.Now()
			if now.After(task.ts) {
				// already delayed! execute immediately
				task.execute()
			} else {
				heap.Push(&tasks, task)
				// reset timer to trigger based on the top element
				timer.Reset(tasks[0].ts.Sub(now))
			}
		case <-timer.C:
			for tasks.Len() > 0 {
				now := time.Now()
				if now.After(tasks[0].ts) {
					heap.Pop(&tasks).(timedFunc).execute()
				} else {
					timer.Reset(tasks[0].ts.Sub(now))
					break
				}
			}
		case <-ts.die:
			return
		}
	}
}

func (ts *TimedSched) prepend() {
	var tasks []timedFunc
	for {
		select {
		case <-ts.chPrependNotify:
			ts.prependLock.Lock()
			// keep cap to reuse slice
			if cap(tasks) < cap(ts.prependTasks) {
				tasks = make([]timedFunc, 0, cap(ts.prependTasks))
			}
			tasks = tasks[:len(ts.prependTasks)]
			copy(tasks, ts.prependTasks)
			for k := range ts.prependTasks {
				ts.prependTasks[k].execute = nil // avoid memory leak
			}
			ts.prependTasks = ts.prependTasks[:0]
			ts.prependLock.Unlock()

			for k := range tasks {
				select {
				case ts.chTask <- tasks[k]:
					tasks[k].execute = nil // avoid memory leak
				case <-ts.die:
					return
				}
			}
			tasks = tasks[:0]
		case <-ts.die:
			return
		}
	}
}

// Put a function 'f' awaiting to be executed at 'deadline'
func (ts *TimedSched) Put(f func(), deadline time.Time) {
	ts.prependLock.Lock()
	ts.prependTasks = append(ts.prependTasks, timedFunc{f, deadline})
	ts.prependLock.Unlock()

	select {
	case ts.chPrependNotify <- struct{}{}:
	default:
	}
}

// Close terminates this scheduler
func (ts *TimedSched) Close() { ts.dieOnce.Do(func() { close(ts.die) }) }
<|endoftext|>


<commit_before>// Copyright (c) 2014 Dataence, LLC. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package timex

import (
	"fmt"
	"regexp"
	"testing"
	"time"

	"github.com/dataence/assert"
)

var (
	re1 = regexp.MustCompile("_")
	re2 = regexp.MustCompile("Z")
)

func TestTimeFormats(t *testing.T) {
	for _, f := range TimeFormats {
		tx := re2.ReplaceAllString(re1.ReplaceAllString(f, " "), "+")
		expected, err := time.Parse(f, tx)
		assert.NoError(t, true, err)
		actual, err := Parse(tx)
		assert.NoError(t, true, err)
		assert.Equal(t, true, expected.UnixNano(), actual.UnixNano())
	}
}

func ExampleTimexParse() {
	t1, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05+07:00")
	t2, err := Parse("2006-01-02T15:04:05+07:00")
	if err != nil {
		fmt.Println(err)
	} else if t1.UnixNano() != t2.UnixNano() {
		fmt.Println("%d != %d", t1.UnixNano(), t2.UnixNano())
	} else {
		fmt.Println(t2)
	}
}
<commit_msg>fixed example<commit_after>// Copyright (c) 2014 Dataence, LLC. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package timex

import (
	"fmt"
	"regexp"
	"testing"
	"time"

	"github.com/dataence/assert"
)

var (
	re1 = regexp.MustCompile("_")
	re2 = regexp.MustCompile("Z")
)

func TestTimeFormats(t *testing.T) {
	for _, f := range TimeFormats {
		tx := re2.ReplaceAllString(re1.ReplaceAllString(f, " "), "+")
		expected, err := time.Parse(f, tx)
		assert.NoError(t, true, err)
		actual, err := Parse(tx)
		assert.NoError(t, true, err)
		assert.Equal(t, true, expected.UnixNano(), actual.UnixNano())
	}
}

func ExampleParse() {
	t1, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05+07:00")
	t2, err := Parse("2006-01-02T15:04:05+07:00")
	if err != nil {
		fmt.Println(err)
	} else if t1.UnixNano() != t2.UnixNano() {
		fmt.Println("%d != %d", t1.UnixNano(), t2.UnixNano())
	} else {
		fmt.Println(t2)
	}
}
<|endoftext|>


<commit_before>package cloudflare

import (
	"encoding/json"

	"github.com/pkg/errors"
)

// OriginCA resource
// This is the origin_ca resource definition (as defined here:
// https://api.cloudflare.com/#origin-ca || https://api.cloudflare.com/#cloudflare-ca
type OriginCA struct {
	ID              string   `json:"id"`
	Certificate     string   `json:"certificate"`
	Hostnames       []string `json:"hostnames"`
	ExpiresOn       string   `json:"expires_on"`
	RequestType     string   `json:"request_type"`
	RequestValidity string   `json:"request_validity"`
	Csr             string   `json:"csr"`
}

// CreateOriginCertificate will create an origin certificate for a User
// API reference: https://api.cloudflare.com/#origin-ca-create-certificate
func (api *API) CreateOriginCertificate(certificate OriginCA) (*OriginCA, error) {
	uri := "/certificates"
	res, err := api.makeRequest("POST", uri, certificate)

	if err != nil {
		return nil, errors.Wrap(err, errMakeRequestError)
	}

	var createdCert *OriginCA

	err = json.Unmarshal(res, &createdCert)

	if err != nil {
		return nil, errors.Wrap(err, errUnmarshalError)
	}

	return createdCert, nil
}
<commit_msg>fix requested_validity json property<commit_after>package cloudflare

import (
	"encoding/json"

	"github.com/pkg/errors"
)

// OriginCA resource
// This is the origin_ca resource definition (as defined here:
// https://api.cloudflare.com/#origin-ca || https://api.cloudflare.com/#cloudflare-ca
type OriginCA struct {
	ID              string   `json:"id"`
	Certificate     string   `json:"certificate"`
	Hostnames       []string `json:"hostnames"`
	ExpiresOn       string   `json:"expires_on"`
	RequestType     string   `json:"request_type"`
	RequestValidity string   `json:"requested_validity"`
	Csr             string   `json:"csr"`
}

// CreateOriginCertificate will create an origin certificate for a User
// API reference: https://api.cloudflare.com/#origin-ca-create-certificate
func (api *API) CreateOriginCertificate(certificate OriginCA) (*OriginCA, error) {
	uri := "/certificates"
	res, err := api.makeRequest("POST", uri, certificate)

	if err != nil {
		return nil, errors.Wrap(err, errMakeRequestError)
	}

	var createdCert *OriginCA

	err = json.Unmarshal(res, &createdCert)

	if err != nil {
		return nil, errors.Wrap(err, errUnmarshalError)
	}

	return createdCert, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
)

type tree struct {
	left  *tree
	value int
	right *tree
}

func stringSliceToIntSlice(sString []string) []int {
	var sInt = []int{}
	for _, s := range sString {
		i, err := strconv.Atoi(s)
		if err != nil {
			log.Fatal(err)
		}
		sInt = append(sInt, i)
	}
	return sInt
}

func getMatrixFromDataFile(filePath string) ([][]int, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var matrix = [][]int{}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		r := strings.Fields(scanner.Text())
		intSlice := stringSliceToIntSlice(r)
		matrix = append(matrix, intSlice)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return matrix, nil
}

func displayMatrix(matrix [][]int) {
	for _, r := range matrix {
		fmt.Println(r)
	}
}

func insert(matrix [][]int, i int, j int) {
	if i == len(matrix) {
		return
	}
	for idx := 0; idx < len(matrix[i]); idx++ {
		fmt.Println(matrix[i][idx])
		insert(matrix, i+1, j)
	}

	//insert(matrix, i+1, j+1)
}

func main() {
	matrix, err := getMatrixFromDataFile("./data")
	if err != nil {
		log.Fatal(err)
	}

	displayMatrix(matrix)

	insert(matrix, 0, 0)

}
<commit_msg>update<commit_after>package main

import (
	"bufio"
	"fmt"
	"log"
	"os"
	"strconv"
	"strings"
)

type tree struct {
	left  *tree
	value int
	right *tree
}

func stringSliceToIntSlice(sString []string) []int {
	var sInt = []int{}
	for _, s := range sString {
		i, err := strconv.Atoi(s)
		if err != nil {
			log.Fatal(err)
		}
		sInt = append(sInt, i)
	}
	return sInt
}

func getMatrixFromDataFile(filePath string) ([][]int, error) {
	file, err := os.Open(filePath)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var matrix = [][]int{}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		r := strings.Fields(scanner.Text())
		intSlice := stringSliceToIntSlice(r)
		matrix = append(matrix, intSlice)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return matrix, nil
}

func displayMatrix(matrix [][]int) {
	for _, r := range matrix {
		fmt.Println(r)
	}
}

func main() {
	matrix, err := getMatrixFromDataFile("./data")
	if err != nil {
		log.Fatal(err)
	}

	//displayMatrix(matrix)

	sum, index := 0, 0

	for i, r := range matrix {

		if i == 0 {
			sum += r[index]
			continue
		}

		if r[index] >= r[index+1] {
			sum += r[index]
			fmt.Println(r[index])
		} else {
			sum += r[index+1]
			fmt.Println(r[index+1])
			index++
		}
	}

	fmt.Println(sum)

}
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/toolkit"
	"time"
)

var conn dbox.IConnection
var count int

var (
	t0     time.Time
	gtable string
)

var mastercostcenter = toolkit.M{}
var masterbranch = toolkit.M{}

// var masterbranchgroup = toolkit.M{}
var masterbranchgroup = toolkit.M{}
var masteraccountgroup = toolkit.M{}

func prepdatacostcenter() {
	toolkit.Println("--> Get Data cost center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("costcenter").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		mastercostcenter.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranch() {
	toolkit.Println("--> Get branch center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranch").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranch.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranchgroup() {
	toolkit.Println("--> Get branch group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranchgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranchgroup.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdataaccountgroup() {
	toolkit.Println("--> Get account group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masteraccountgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masteraccountgroup.Set(tkm.GetString("accountdescription"), tkm.GetString("accountgroup"))
	}
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func getstep(count int) int {
	v := count / 100
	if v == 0 {
		return 1
	}
	return v
}

func main() {
	t0 = time.Now()
	flag.StringVar(&gtable, "table", "", "tablename")
	flag.Parse()

	setinitialconnection()

	prepdatabranch()
	// prepdatacostcenter()
	// prepdataaccountgroup()
	// prepdatabranchgroup()

	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	toolkit.Println("Start data query...")
	csr, _ := workerconn.NewQuery().Select().From(gtable).Cursor(nil)
	defer csr.Close()

	scount := csr.Count()

	jobs := make(chan toolkit.M, scount)
	result := make(chan int, scount)
	for wi := 0; wi < 10; wi++ {
		go workersave(wi, jobs, result)
	}

	iscount := 0
	step := getstep(scount) * 5

	for {
		iscount++
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		jobs <- tkm

		if iscount%step == 0 {
			toolkit.Printfn("Sending %d of %d (%d) in %s", iscount, scount, iscount*100/scount,
				time.Since(t0).String())
		}

	}

	close(jobs)

	for ri := 0; ri < scount; ri++ {
		<-result

		if ri%step == 0 {
			toolkit.Printfn("Saving %d of %d (%d pct) in %s",
				ri, scount, ri*100/scount, time.Since(t0).String())
		}
	}

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func workersave(wi int, jobs <-chan toolkit.M, result chan<- int) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	qSave := workerconn.NewQuery().
		From(toolkit.Sprintf("%s-res", gtable)).
		SetConfig("multiexec", true).
		Save()

	trx := toolkit.M{}
	for trx = range jobs {
		key := trx.Get("key", toolkit.M{}).(toolkit.M)
		// trx.Set("key", key)
		// trx.Set("_id", toolkit.Sprintf("%d|%s|%s|%s|%s|%s|%s|%s", key.GetInt("year"),
		// 	key.GetString("branchid"),
		// 	key.GetString("branchname"),
		// 	key.GetString("brancharea"),
		// 	key.GetString("account"),
		// 	key.GetString("accountdescription"),
		// 	key.GetString("costgroup"),
		// 	key.GetString("src")))

		// tdate := time.Date(trx.GetInt("year"), time.Month(trx.GetInt("period")), 1, 0, 0, 0, 0, time.UTC).
		// 	AddDate(0, 3, 0)
		// gdrjdate := gdrj.SetDate(tdate)

		// trx.Set("gdrjdate", gdrjdate)

		// cc := trx.GetString("ccid")
		// trx.Set("branchid", "CD00")
		// trx.Set("branchname", "OTHER")
		// trx.Set("brancharea", "OTHER")
		// trx.Set("costgroup", "OTHER")
		// trx.Set("accountgroup", "OTHER")
		// trx.Set("branchgroup", "OTHER")

		key.Set("customer_branchgroup", "OTHER")

		// trx.Set("min_amountinidr", -trx.GetFloat64("amountinidr"))

		// if mastercostcenter.Has(cc) {
		// 	mcc := mastercostcenter[cc].(toolkit.M)
		// 	brid := mcc.GetString("branchid")

		// 	trx.Set("branchid", brid)
		// 	trx.Set("costgroup", mcc.GetString("costgroup01"))

		// 	if masterbranch.Has(brid) {
		// 		trx.Set("branchname", masterbranch[brid].(toolkit.M).GetString("name"))
		// 		trx.Set("brancharea", masterbranch[brid].(toolkit.M).GetString("area"))
		// 	}
		// }

		// branchid := trx.GetString("branchid")
		branchid := key.GetString("customer_branchid")

		branchgroup := masterbranch.Get(branchid, toolkit.M{}).(toolkit.M)
		key.Set("customer_branchgroup", branchgroup.GetString("branchgroup"))

		// accdesc := trx.GetString("accountdescription")
		// trx.Set("accountgroup", masteraccountgroup.GetString(accdesc))

		// if trx.GetString("costgroup") == "" {
		// 	trx.Set("costgroup", "OTHER")
		// }

		// if trx.GetString("branchname") == "" {
		// 	trx.Set("branchname", "OTHER")
		// }

		// if trx.GetString("brancharea") == "" {
		// 	trx.Set("brancharea", "OTHER")
		// }

		// if trx.GetString("accountgroup") == "" {
		// 	trx.Set("accountgroup", "OTHER")
		// }

		// if trx.GetString("branchgroup") == "" {
		// 	trx.Set("branchgroup", "OTHER")
		// }

		if key.GetString("customer_branchgroup") == "" {
			key.Set("customer_branchgroup", "OTHER")
		}

		trx.Set("key", key)

		err := qSave.Exec(toolkit.M{}.Set("data", trx))
		if err != nil {
			toolkit.Println(err)
		}

		result <- 1
	}
}
<commit_msg>cogs consolidate update raw<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"flag"
	"github.com/eaciit/dbox"
	"github.com/eaciit/toolkit"
	"time"
)

var conn dbox.IConnection
var count int

var (
	t0     time.Time
	gtable string
)

var mastercostcenter = toolkit.M{}
var masterbranch = toolkit.M{}

// var masterbranchgroup = toolkit.M{}
var masterbranchgroup = toolkit.M{}
var masteraccountgroup = toolkit.M{}

func prepdatacostcenter() {
	toolkit.Println("--> Get Data cost center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("costcenter").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		mastercostcenter.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranch() {
	toolkit.Println("--> Get branch center")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranch").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranch.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdatabranchgroup() {
	toolkit.Println("--> Get branch group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masterbranchgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masterbranchgroup.Set(tkm.GetString("_id"), tkm)
	}
}

func prepdataaccountgroup() {
	toolkit.Println("--> Get account group")

	// filter := dbox.Eq("key.date_fiscal", toolkit.Sprintf("%d-%d", fiscalyear-1, fiscalyear))
	csr, _ := conn.NewQuery().Select().From("masteraccountgroup").Cursor(nil)
	defer csr.Close()

	for {
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			break
		}

		masteraccountgroup.Set(tkm.GetString("accountdescription"), tkm.GetString("accountgroup"))
	}
}

func setinitialconnection() {
	var err error
	conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func getstep(count int) int {
	v := count / 100
	if v == 0 {
		return 1
	}
	return v
}

func main() {
	t0 = time.Now()
	flag.StringVar(&gtable, "table", "", "tablename")
	// flag.IntVar(&year, "year", 2014, "2014 year")
	flag.Parse()

	setinitialconnection()

	// prepdatabranch()
	// prepdatacostcenter()
	// prepdataaccountgroup()
	// prepdatabranchgroup()

	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	toolkit.Println("Start data query...")
	arrfilter := []*dbox.Filter{}
	for i := 4; i < 10; i++ {
		toolkit.Printfn("%d - 2014", i)
		f := dbox.And(dbox.Eq("year", 2014), dbox.Eq("month", i))
		arrfilter = append(arrfilter, f)
	}

	f := dbox.Or(arrfilter...)

	csr, _ := workerconn.NewQuery().Select().Where(f).From(gtable).Cursor(nil)
	defer csr.Close()

	scount := csr.Count()

	jobs := make(chan toolkit.M, scount)
	result := make(chan int, scount)
	for wi := 0; wi < 10; wi++ {
		go workersave(wi, jobs, result)
	}

	iscount := 0
	step := getstep(scount) * 5

	for {
		iscount++
		tkm := toolkit.M{}
		e := csr.Fetch(&tkm, 1, false)
		if e != nil {
			toolkit.Println("EOF")
			break
		}

		jobs <- tkm

		if iscount%step == 0 {
			toolkit.Printfn("Sending %d of %d (%d) in %s", iscount, scount, iscount*100/scount,
				time.Since(t0).String())
		}

	}

	close(jobs)

	for ri := 0; ri < scount; ri++ {
		<-result

		if ri%step == 0 {
			toolkit.Printfn("Saving %d of %d (%d pct) in %s",
				ri, scount, ri*100/scount, time.Since(t0).String())
		}
	}

	toolkit.Printfn("Processing done in %s",
		time.Since(t0).String())
}

func workersave(wi int, jobs <-chan toolkit.M, result chan<- int) {
	workerconn, _ := modules.GetDboxIConnection("db_godrej")
	defer workerconn.Close()

	qSave := workerconn.NewQuery().
		From(toolkit.Sprintf("%s-res", gtable)).
		SetConfig("multiexec", true).
		Save()

	trx := toolkit.M{}
	for trx = range jobs {
		// key := trx.Get("key", toolkit.M{}).(toolkit.M)
		// trx.Set("key", key)
		// trx.Set("_id", toolkit.Sprintf("%d|%s|%s|%s|%s|%s|%s|%s", key.GetInt("year"),
		// 	key.GetString("branchid"),
		// 	key.GetString("branchname"),
		// 	key.GetString("brancharea"),
		// 	key.GetString("account"),
		// 	key.GetString("accountdescription"),
		// 	key.GetString("costgroup"),
		// 	key.GetString("src")))

		// tdate := time.Date(trx.GetInt("year"), time.Month(trx.GetInt("period")), 1, 0, 0, 0, 0, time.UTC).
		// 	AddDate(0, 3, 0)
		// gdrjdate := gdrj.SetDate(tdate)

		// trx.Set("gdrjdate", gdrjdate)

		// cc := trx.GetString("ccid")
		// trx.Set("branchid", "CD00")
		// trx.Set("branchname", "OTHER")
		// trx.Set("brancharea", "OTHER")
		// trx.Set("costgroup", "OTHER")
		// trx.Set("accountgroup", "OTHER")
		// trx.Set("branchgroup", "OTHER")

		// key.Set("customer_branchgroup", "OTHER")

		// trx.Set("min_amountinidr", -trx.GetFloat64("amountinidr"))

		// if mastercostcenter.Has(cc) {
		// 	mcc := mastercostcenter[cc].(toolkit.M)
		// 	brid := mcc.GetString("branchid")

		// 	trx.Set("branchid", brid)
		// 	trx.Set("costgroup", mcc.GetString("costgroup01"))

		// 	if masterbranch.Has(brid) {
		// 		trx.Set("branchname", masterbranch[brid].(toolkit.M).GetString("name"))
		// 		trx.Set("brancharea", masterbranch[brid].(toolkit.M).GetString("area"))
		// 	}
		// }

		// branchid := trx.GetString("branchid")
		// branchid := key.GetString("customer_branchid")

		// branchgroup := masterbranch.Get(branchid, toolkit.M{}).(toolkit.M)
		// key.Set("customer_branchgroup", branchgroup.GetString("branchgroup"))

		// accdesc := trx.GetString("accountdescription")
		// trx.Set("accountgroup", masteraccountgroup.GetString(accdesc))

		// if trx.GetString("costgroup") == "" {
		// 	trx.Set("costgroup", "OTHER")
		// }

		// if trx.GetString("branchname") == "" {
		// 	trx.Set("branchname", "OTHER")
		// }

		// if trx.GetString("brancharea") == "" {
		// 	trx.Set("brancharea", "OTHER")
		// }

		// if trx.GetString("accountgroup") == "" {
		// 	trx.Set("accountgroup", "OTHER")
		// }

		// if trx.GetString("branchgroup") == "" {
		// 	trx.Set("branchgroup", "OTHER")
		// }

		// if key.GetString("customer_branchgroup") == "" {
		// 	key.Set("customer_branchgroup", "OTHER")
		// }

		// trx.Set("key", key)

		// For cogs consolidate
		arrstr := []string{"rm_perunit", "lc_perunit", "pf_perunit", "other_perunit", "fixed_perunit", "depre_perunit", "cogs_perunit"}
		for _, v := range arrstr {
			xval := trx.GetFloat64(v) * 6
			trx.Set(v, xval)
		}
		// ====================

		err := qSave.Exec(toolkit.M{}.Set("data", trx))
		if err != nil {
			toolkit.Println(err)
		}

		result <- 1
	}
}
<|endoftext|>