<commit_before>//  Copyright (c) 2015 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package objectserver

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"math/big"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/openstack/swift/go/hummingbird"
)

/*This hash is used to represent a zero byte async file that is
  created for an expiring object*/
const zeroByteHash = "d41d8cd98f00b204e9800998ecf8427e"
const deleteAtAccount = ".expiring_objects"
const waitForContainerUpdate = time.Second / 4

func headerToMap(headers http.Header) map[string]string {
	ret := make(map[string]string)
	for key, value := range headers {
		if len(value) > 0 {
			ret[key] = headers.Get(key)
		}
	}
	return ret
}

func splitHeader(header string) []string {
	if header == "" {
		return []string{}
	}
	return strings.Split(header, ",")
}

func (server *ObjectServer) hashPath(account, container, obj string) string {
	h := md5.New()
	io.WriteString(h, server.hashPathPrefix+"/"+account+"/"+container+"/"+obj+server.hashPathSuffix)
	return hex.EncodeToString(h.Sum(nil))
}

func (server *ObjectServer) expirerContainer(deleteAt time.Time, account, container, obj string) string {
	i := new(big.Int)
	fmt.Sscanf(server.hashPath(account, container, obj), "%x", i)
	shardInt := i.Mod(i, big.NewInt(100)).Int64()
	timestamp := (deleteAt.Unix()/server.expiringDivisor)*server.expiringDivisor - shardInt
	if timestamp < 0 {
		timestamp = 0
	} else if timestamp > 9999999999 {
		timestamp = 9999999999
	}
	return fmt.Sprintf("%010d", timestamp)
}

func (server *ObjectServer) sendContainerUpdate(host, device, method, partition, account, container, obj string, headers http.Header) bool {
	obj_url := fmt.Sprintf("http://%s/%s/%s/%s/%s/%s", host, device, partition,
		hummingbird.Urlencode(account), hummingbird.Urlencode(container), hummingbird.Urlencode(obj))
	if req, err := http.NewRequest(method, obj_url, nil); err == nil {
		req.Header = headers
		if resp, err := server.updateClient.Do(req); err == nil {
			resp.Body.Close()
			if resp.StatusCode/100 == 2 {
				return true
			}
		}
	}
	return false
}

func (server *ObjectServer) saveAsync(method, account, container, obj, localDevice string, headers http.Header) {
	hash := server.hashPath(account, container, obj)
	asyncFile := filepath.Join(server.driveRoot, localDevice, "async_pending", hash[29:32], hash+"-"+headers.Get("X-Timestamp"))
	data := map[string]interface{}{
		"op":        method,
		"account":   account,
		"container": container,
		"obj":       obj,
		"headers":   headerToMap(headers),
	}
	if os.MkdirAll(filepath.Dir(asyncFile), 0755) == nil {
		hummingbird.WriteFileAtomic(asyncFile, hummingbird.PickleDumps(data), 0660)
	}
}

func (server *ObjectServer) updateContainer(metadata map[string]string, request *http.Request, vars map[string]string) {
	partition := request.Header.Get("X-Container-Partition")
	hosts := splitHeader(request.Header.Get("X-Container-Host"))
	devices := splitHeader(request.Header.Get("X-Container-Device"))
	if partition == "" || len(hosts) == 0 || len(devices) == 0 {
		return
	}
	requestHeaders := http.Header{
		"Referer":     {hummingbird.GetDefault(request.Header, "Referer", "-")},
		"User-Agent":  {hummingbird.GetDefault(request.Header, "User-Agent", "-")},
		"X-Trans-Id":  {hummingbird.GetDefault(request.Header, "X-Trans-Id", "-")},
		"X-Timestamp": {request.Header.Get("X-Timestamp")},
	}
	if request.Method != "DELETE" {
		requestHeaders.Add("X-Content-Type", metadata["Content-Type"])
		requestHeaders.Add("X-Size", metadata["Content-Length"])
		requestHeaders.Add("X-Etag", metadata["ETag"])
	}
	failures := 0
	for index := range hosts {
		if !server.sendContainerUpdate(hosts[index], devices[index], request.Method, partition, vars["account"], vars["container"], vars["obj"], requestHeaders) {
			failures++
		}
	}
	if failures > 0 {
		server.saveAsync(request.Method, vars["account"], vars["container"], vars["obj"], vars["device"], requestHeaders)
	}
}

func (server *ObjectServer) updateDeleteAt(request *http.Request, deleteAtStr string, vars map[string]string) {
	deleteAt, err := hummingbird.ParseDate(deleteAtStr)
	if err != nil {
		return
	}
	container := hummingbird.GetDefault(request.Header, "X-Delete-At-Container", "")
	if container == "" {
		container = server.expirerContainer(deleteAt, vars["account"], vars["container"], vars["obj"])
	}
	obj := fmt.Sprintf("%010d-%s/%s/%s", deleteAt.Unix(), vars["account"], vars["container"], vars["obj"])
	partition := hummingbird.GetDefault(request.Header, "X-Delete-At-Partition", "")
	hosts := splitHeader(request.Header.Get("X-Delete-At-Host"))
	devices := splitHeader(request.Header.Get("X-Delete-At-Device"))
	requestHeaders := http.Header{
		"Referer":     {hummingbird.GetDefault(request.Header, "Referer", "-")},
		"User-Agent":  {hummingbird.GetDefault(request.Header, "User-Agent", "-")},
		"X-Trans-Id":  {hummingbird.GetDefault(request.Header, "X-Trans-Id", "-")},
		"X-Timestamp": {request.Header.Get("X-Timestamp")},
	}
	if request.Method != "DELETE" {
		requestHeaders.Add("X-Content-Type", "text/plain")
		requestHeaders.Add("X-Size", "0")
		requestHeaders.Add("X-Etag", zeroByteHash)
	}
	failures := 0
	for index := range hosts {
		if !server.sendContainerUpdate(hosts[index], devices[index], request.Method, partition, deleteAtAccount, container, obj, requestHeaders) {
			failures++
		}
	}
	if failures > 0 || len(hosts) == 0 {
		server.saveAsync(request.Method, deleteAtAccount, container, obj, vars["device"], requestHeaders)
	}
}

func (server *ObjectServer) containerUpdates(request *http.Request, metadata map[string]string, deleteAt string, vars map[string]string) {
	if deleteAt != "" {
		go server.updateDeleteAt(request, deleteAt, vars)
	}

	firstDone := make(chan struct{}, 1)
	go func() {
		server.updateContainer(metadata, request, vars)
		firstDone <- struct{}{}
	}()
	go func() {
		time.Sleep(waitForContainerUpdate)
		firstDone <- struct{}{}
	}()
	<-firstDone
}
<commit_msg>go: make error log when creating asyncs<commit_after>//  Copyright (c) 2015 Rackspace
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
//  implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package objectserver

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"math/big"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/openstack/swift/go/hummingbird"
)

/*This hash is used to represent a zero byte async file that is
  created for an expiring object*/
const zeroByteHash = "d41d8cd98f00b204e9800998ecf8427e"
const deleteAtAccount = ".expiring_objects"
const waitForContainerUpdate = time.Second / 4

func headerToMap(headers http.Header) map[string]string {
	ret := make(map[string]string)
	for key, value := range headers {
		if len(value) > 0 {
			ret[key] = headers.Get(key)
		}
	}
	return ret
}

func splitHeader(header string) []string {
	if header == "" {
		return []string{}
	}
	return strings.Split(header, ",")
}

func (server *ObjectServer) hashPath(account, container, obj string) string {
	h := md5.New()
	io.WriteString(h, server.hashPathPrefix+"/"+account+"/"+container+"/"+obj+server.hashPathSuffix)
	return hex.EncodeToString(h.Sum(nil))
}

func (server *ObjectServer) expirerContainer(deleteAt time.Time, account, container, obj string) string {
	i := new(big.Int)
	fmt.Sscanf(server.hashPath(account, container, obj), "%x", i)
	shardInt := i.Mod(i, big.NewInt(100)).Int64()
	timestamp := (deleteAt.Unix()/server.expiringDivisor)*server.expiringDivisor - shardInt
	if timestamp < 0 {
		timestamp = 0
	} else if timestamp > 9999999999 {
		timestamp = 9999999999
	}
	return fmt.Sprintf("%010d", timestamp)
}

func (server *ObjectServer) sendContainerUpdate(host, device, method, partition, account, container, obj string, headers http.Header) bool {
	obj_url := fmt.Sprintf("http://%s/%s/%s/%s/%s/%s", host, device, partition,
		hummingbird.Urlencode(account), hummingbird.Urlencode(container), hummingbird.Urlencode(obj))
	if req, err := http.NewRequest(method, obj_url, nil); err == nil {
		req.Header = headers
		if resp, err := server.updateClient.Do(req); err == nil {
			resp.Body.Close()
			if resp.StatusCode/100 == 2 {
				return true
			}
		}
	}
	return false
}

func (server *ObjectServer) saveAsync(method, account, container, obj, localDevice string, headers http.Header) {
	hash := server.hashPath(account, container, obj)
	asyncFile := filepath.Join(server.driveRoot, localDevice, "async_pending", hash[29:32], hash+"-"+headers.Get("X-Timestamp"))
	data := map[string]interface{}{
		"op":        method,
		"account":   account,
		"container": container,
		"obj":       obj,
		"headers":   headerToMap(headers),
	}
	if os.MkdirAll(filepath.Dir(asyncFile), 0755) == nil {
		hummingbird.WriteFileAtomic(asyncFile, hummingbird.PickleDumps(data), 0660)
	}
}

func (server *ObjectServer) updateContainer(metadata map[string]string, request *http.Request, vars map[string]string) {
	partition := request.Header.Get("X-Container-Partition")
	hosts := splitHeader(request.Header.Get("X-Container-Host"))
	devices := splitHeader(request.Header.Get("X-Container-Device"))
	if partition == "" || len(hosts) == 0 || len(devices) == 0 {
		return
	}
	requestHeaders := http.Header{
		"Referer":     {hummingbird.GetDefault(request.Header, "Referer", "-")},
		"User-Agent":  {hummingbird.GetDefault(request.Header, "User-Agent", "-")},
		"X-Trans-Id":  {hummingbird.GetDefault(request.Header, "X-Trans-Id", "-")},
		"X-Timestamp": {request.Header.Get("X-Timestamp")},
	}
	if request.Method != "DELETE" {
		requestHeaders.Add("X-Content-Type", metadata["Content-Type"])
		requestHeaders.Add("X-Size", metadata["Content-Length"])
		requestHeaders.Add("X-Etag", metadata["ETag"])
	}
	failures := 0
	for index := range hosts {
		if !server.sendContainerUpdate(hosts[index], devices[index], request.Method, partition, vars["account"], vars["container"], vars["obj"], requestHeaders) {
			hummingbird.GetLogger(request).LogError("ERROR container update failed with %s/%s (saving for async update later)", hosts[index], devices[index])
			failures++
		}
	}
	if failures > 0 {
		server.saveAsync(request.Method, vars["account"], vars["container"], vars["obj"], vars["device"], requestHeaders)
	}
}

func (server *ObjectServer) updateDeleteAt(request *http.Request, deleteAtStr string, vars map[string]string) {
	deleteAt, err := hummingbird.ParseDate(deleteAtStr)
	if err != nil {
		return
	}
	container := hummingbird.GetDefault(request.Header, "X-Delete-At-Container", "")
	if container == "" {
		container = server.expirerContainer(deleteAt, vars["account"], vars["container"], vars["obj"])
	}
	obj := fmt.Sprintf("%010d-%s/%s/%s", deleteAt.Unix(), vars["account"], vars["container"], vars["obj"])
	partition := hummingbird.GetDefault(request.Header, "X-Delete-At-Partition", "")
	hosts := splitHeader(request.Header.Get("X-Delete-At-Host"))
	devices := splitHeader(request.Header.Get("X-Delete-At-Device"))
	requestHeaders := http.Header{
		"Referer":     {hummingbird.GetDefault(request.Header, "Referer", "-")},
		"User-Agent":  {hummingbird.GetDefault(request.Header, "User-Agent", "-")},
		"X-Trans-Id":  {hummingbird.GetDefault(request.Header, "X-Trans-Id", "-")},
		"X-Timestamp": {request.Header.Get("X-Timestamp")},
	}
	if request.Method != "DELETE" {
		requestHeaders.Add("X-Content-Type", "text/plain")
		requestHeaders.Add("X-Size", "0")
		requestHeaders.Add("X-Etag", zeroByteHash)
	}
	failures := 0
	for index := range hosts {
		if !server.sendContainerUpdate(hosts[index], devices[index], request.Method, partition, deleteAtAccount, container, obj, requestHeaders) {
			hummingbird.GetLogger(request).LogError("ERROR container update failed with %s/%s (saving for async update later)", hosts[index], devices[index])
			failures++
		}
	}
	if failures > 0 || len(hosts) == 0 {
		server.saveAsync(request.Method, deleteAtAccount, container, obj, vars["device"], requestHeaders)
	}
}

func (server *ObjectServer) containerUpdates(request *http.Request, metadata map[string]string, deleteAt string, vars map[string]string) {
	if deleteAt != "" {
		go server.updateDeleteAt(request, deleteAt, vars)
	}

	firstDone := make(chan struct{}, 1)
	go func() {
		server.updateContainer(metadata, request, vars)
		firstDone <- struct{}{}
	}()
	go func() {
		time.Sleep(waitForContainerUpdate)
		firstDone <- struct{}{}
	}()
	<-firstDone
}
<|endoftext|>


<commit_before>package space

// Planet names
type Planet string

const secondsInEarthYear = 31557600

var orbitalPeriods = map[Planet]float64{"Mercury": 0.2408467, "Venus": 0.61519726}

// Age returns the age in years of someone on planet given their age in seconds.
func Age(ageInSeconds float64, planet Planet) float64 {
	if planet == "Earth" {
		return ageOnEarth(ageInSeconds)
	} else if planet == "Mercury" {
		return ageOnEarth(ageInSeconds) / orbitalPeriods[planet]
	} else if planet == "Venus" {
		return ageOnEarth(ageInSeconds) / orbitalPeriods[planet]
	}
	return 0
}

// AgeOnEarth returns the age of somone on Earth given their age in seconds.
func ageOnEarth(ageInSeconds float64) float64 {
	return ageInSeconds / secondsInEarthYear
}
<commit_msg>Populate orbital periods map<commit_after>package space

// Planet names
type Planet string

const secondsInEarthYear = 31557600

var orbitalPeriods = map[Planet]float64{
	"Mercury": 0.2408467,
	"Venus":   0.61519726,
	"Mars":    1.8808158,
	"Jupiter": 11.862615,
	"Saturn":  29.447498,
	"Uranus":  84.016846,
	"Neptune": 164.79132,
}

// Age returns the age in years of someone on planet given their age in seconds.
func Age(ageInSeconds float64, planet Planet) float64 {
	if planet == "Earth" {
		return ageOnEarth(ageInSeconds)
	}
	return ageOnEarth(ageInSeconds) / orbitalPeriods[planet]
}

// AgeOnEarth returns the age of somone on Earth given their age in seconds.
func ageOnEarth(ageInSeconds float64) float64 {
	return ageInSeconds / secondsInEarthYear
}
<|endoftext|>


<commit_before>// Copyright 2013-2014 go-diameter authors.  All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package avp

// AVP types. Auto-generated from our dictionaries.
const (
	// List of AVP codes auto-generated from our dictionaries:
	// cat ../dict/testdata/*.xml | sed -n 's/.*avp name="\(.*\)" code="\([0-9]*\)".*/\1 = \2/p' | tr -d '-' | sort -u

	AccountingRealtimeRequired    = 483
	AccountingRecordNumber        = 485
	AccountingRecordType          = 480
	AccountingSessionId           = 44
	AccountingSubSessionId        = 287
	AcctApplicationId             = 259
	AcctInterimInterval           = 85
	AcctMultiSessionId            = 50
	AuthApplicationId             = 258
	AuthGracePeriod               = 276
	AuthRequestType               = 274
	AuthSessionState              = 277
	AuthorizationLifetime         = 291
	CCCorrelationId               = 411
	CCInputOctets                 = 412
	CCMoney                       = 413
	CCOutputOctets                = 414
	CCRequestNumber               = 415
	CCRequestType                 = 416
	CCServiceSpecificUnits        = 417
	CCSessionFailover             = 418
	CCSubSessionId                = 419
	CCTime                        = 420
	CCTotalOctets                 = 421
	CCUnitType                    = 454
	CheckBalanceResult            = 422
	Class                         = 25
	CostInformation               = 423
	CostUnit                      = 424
	CreditControl                 = 426
	CreditControlFailureHandling  = 427
	CurrencyCode                  = 425
	DestinationHost               = 293
	DestinationRealm              = 283
	DirectDebitingFailureHandling = 428
	DisconnectCause               = 273
	ErrorMessage                  = 281
	ErrorReportingHost            = 294
	EventTimestamp                = 55
	ExperimentalResult            = 297
	ExperimentalResultCode        = 298
	Exponent                      = 429
	FailedAVP                     = 279
	FinalUnitAction               = 449
	FinalUnitIndication           = 430
	FirmwareRevision              = 267
	GSUPoolIdentifier             = 453
	GSUPoolReference              = 457
	GrantedServiceUnit            = 431
	HostIPAddress                 = 257
	InbandSecurityId              = 299
	MultiRoundTimeOut             = 272
	MultipleServicesCreditControl = 456
	MultipleServicesIndicator     = 455
	OriginHost                    = 264
	OriginRealm                   = 296
	OriginStateId                 = 278
	ProductName                   = 269
	ProxyHost                     = 280
	ProxyInfo                     = 284
	ProxyState                    = 33
	RatingGroup                   = 432
	ReAuthRequestType             = 285
	RedirectAddressType           = 433
	RedirectHost                  = 292
	RedirectHostUsage             = 261
	RedirectMaxCacheTime          = 262
	RedirectServer                = 434
	RedirectServerAddress         = 435
	RequestedAction               = 436
	RequestedServiceUnit          = 437
	RestrictionFilterRule         = 438
	ResultCode                    = 268
	RouteRecord                   = 282
	ServiceContextId              = 461
	ServiceIdentifier             = 439
	ServiceParameterInfo          = 440
	ServiceParameterType          = 441
	ServiceParameterValue         = 442
	SessionBinding                = 270
	SessionId                     = 263
	SessionServerFailover         = 271
	SessionTimeout                = 27
	SubscriptionId                = 443
	SubscriptionIdData            = 444
	SubscriptionIdType            = 450
	SupportedVendorId             = 265
	TariffChangeUsage             = 452
	TariffTimeChange              = 451
	TerminationCause              = 295
	UnitValue                     = 445
	UsedServiceUnit               = 446
	UserEquipmentInfo             = 458
	UserEquipmentInfoType         = 459
	UserEquipmentInfoValue        = 460
	UserName                      = 1
	ValidityTime                  = 448
	ValueDigits                   = 447
	VendorId                      = 266
	VendorSpecificApplicationId   = 260
)
<commit_msg>Added CalledStationId<commit_after>// Copyright 2013-2014 go-diameter authors.  All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package avp

// AVP types. Auto-generated from our dictionaries.
const (
	// List of AVP codes auto-generated from our dictionaries:
	// cat ../dict/testdata/*.xml | sed -n 's/.*avp name="\(.*\)" code="\([0-9]*\)".*/\1 = \2/p' | tr -d '-' | sort -u

	AccountingRealtimeRequired    = 483
	AccountingRecordNumber        = 485
	AccountingRecordType          = 480
	AccountingSessionId           = 44
	AccountingSubSessionId        = 287
	AcctApplicationId             = 259
	AcctInterimInterval           = 85
	AcctMultiSessionId            = 50
	AuthApplicationId             = 258
	AuthGracePeriod               = 276
	AuthRequestType               = 274
	AuthSessionState              = 277
	AuthorizationLifetime         = 291
	CalledStationId               = 30
	CCCorrelationId               = 411
	CCInputOctets                 = 412
	CCMoney                       = 413
	CCOutputOctets                = 414
	CCRequestNumber               = 415
	CCRequestType                 = 416
	CCServiceSpecificUnits        = 417
	CCSessionFailover             = 418
	CCSubSessionId                = 419
	CCTime                        = 420
	CCTotalOctets                 = 421
	CCUnitType                    = 454
	CheckBalanceResult            = 422
	Class                         = 25
	CostInformation               = 423
	CostUnit                      = 424
	CreditControl                 = 426
	CreditControlFailureHandling  = 427
	CurrencyCode                  = 425
	DestinationHost               = 293
	DestinationRealm              = 283
	DirectDebitingFailureHandling = 428
	DisconnectCause               = 273
	ErrorMessage                  = 281
	ErrorReportingHost            = 294
	EventTimestamp                = 55
	ExperimentalResult            = 297
	ExperimentalResultCode        = 298
	Exponent                      = 429
	FailedAVP                     = 279
	FinalUnitAction               = 449
	FinalUnitIndication           = 430
	FirmwareRevision              = 267
	GSUPoolIdentifier             = 453
	GSUPoolReference              = 457
	GrantedServiceUnit            = 431
	HostIPAddress                 = 257
	InbandSecurityId              = 299
	MultiRoundTimeOut             = 272
	MultipleServicesCreditControl = 456
	MultipleServicesIndicator     = 455
	OriginHost                    = 264
	OriginRealm                   = 296
	OriginStateId                 = 278
	ProductName                   = 269
	ProxyHost                     = 280
	ProxyInfo                     = 284
	ProxyState                    = 33
	RatingGroup                   = 432
	ReAuthRequestType             = 285
	RedirectAddressType           = 433
	RedirectHost                  = 292
	RedirectHostUsage             = 261
	RedirectMaxCacheTime          = 262
	RedirectServer                = 434
	RedirectServerAddress         = 435
	RequestedAction               = 436
	RequestedServiceUnit          = 437
	RestrictionFilterRule         = 438
	ResultCode                    = 268
	RouteRecord                   = 282
	ServiceContextId              = 461
	ServiceIdentifier             = 439
	ServiceParameterInfo          = 440
	ServiceParameterType          = 441
	ServiceParameterValue         = 442
	SessionBinding                = 270
	SessionId                     = 263
	SessionServerFailover         = 271
	SessionTimeout                = 27
	SubscriptionId                = 443
	SubscriptionIdData            = 444
	SubscriptionIdType            = 450
	SupportedVendorId             = 265
	TariffChangeUsage             = 452
	TariffTimeChange              = 451
	TerminationCause              = 295
	UnitValue                     = 445
	UsedServiceUnit               = 446
	UserEquipmentInfo             = 458
	UserEquipmentInfoType         = 459
	UserEquipmentInfoValue        = 460
	UserName                      = 1
	ValidityTime                  = 448
	ValueDigits                   = 447
	VendorId                      = 266
	VendorSpecificApplicationId   = 260
)
<|endoftext|>


<commit_before>package common

// Interfaces takes /etc/network/interfaces config file
// Every filed are needed
type Interfaces struct {
	InternalNIC     string `json:"InternalNIC"`     // default is eth0
	InternalIP      string `json:"InternalIP"`      // the internal ip address
	InternalMask    string `json:"InternalMask"`    // the netmask for internal ip
	InternalGateway string `json:"InternalGateway"` // the gateway for internal ip
	InternalDNS1    string `json:"InternalDNS1"`    // the dns-nameservers
	InternalDNS2    string `json:"InternalDNS2"`
	ExternalNIC     string `json:"ExternalNIC"` // default is eth1
	Restart         bool   `json:"Restart"`     // restart the system
	Changed         bool   `json:"Changed"`     // changed status
}

// InitInterfaces set the Interfaces struct
func (i *Interfaces) InitInterfaces(k, v string) {
	switch k {
	case "InternalNIC":
		i.InternalNIC = v
	case "InternalIP":
		i.InternalIP = v
	case "InternalMask":
		i.InternalMask = v
	case "InternalGateway":
		i.InternalGateway = v
	case "InternalDNS1":
		i.InternalDNS1 = v
	case "InternalDNS2":
		i.InternalDNS2 = v
	case "ExternalNIC":
		i.ExternalNIC = v
	}
}

// EtcNetworkInterface is /etc/network/interfaces file
const etcNetworkInterface = `
# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).

# The loopback network interface
auto lo
iface lo inet loopback

# Source interfaces
# Please check /etc/network/interfaces.d before changing this file
# as interfaces may have been defined in /etc/network/interfaces.d
# NOTE: the primary ethernet device is defined in
# /etc/network/interfaces.d/eth0
# See LP: #1262951
source /etc/network/interfaces.d/*.cfg
`

// PurgeMainConf use the interfaces.d dir instead of /etc/network/interfaces
func (i *Interfaces) PurgeMainConf() {
	ParseTmpl(i, etcNetworkInterface, "interface", "/etc/network/interfaces", 0644)
}

const internalNIC = `
auto {{.InternalNIC}}
iface {{.InternalNIC}} inet static
address {{.InternalIP}}
netmask {{.InternalMask}}
gateway {{.InternalGateway}}
dns-nameservers {{.InternalDNS1}} {{.InternalDNS2}}
`

// SetInternalNIC sets the internal nic
func (i *Interfaces) SetInternalNIC() {
	ParseTmpl(i, internalNIC, "internal", "/etc/network/interfaces.d/"+i.InternalNIC+".cfg", 0644)
}

const externalNIC = `
# The public network interface
auto {{.ExternalNIC}}
iface  {{.ExternalNIC}} inet manual
up ip link set dev $IFACE up
down ip link set dev $IFACE down
`

// SetExternalNIC sets the external nic
func (i *Interfaces) SetExternalNIC() {
	ParseTmpl(i, externalNIC, "external", "/etc/network/interfaces.d/"+i.ExternalNIC+".cfg", 0644)
}
<commit_msg>Change k v to interface type<commit_after>package common

// Interfaces takes /etc/network/interfaces config file
// Every filed are needed
type Interfaces struct {
	InternalNIC     string `json:"InternalNIC"`     // default is eth0
	InternalIP      string `json:"InternalIP"`      // the internal ip address
	InternalMask    string `json:"InternalMask"`    // the netmask for internal ip
	InternalGateway string `json:"InternalGateway"` // the gateway for internal ip
	InternalDNS1    string `json:"InternalDNS1"`    // the dns-nameservers
	InternalDNS2    string `json:"InternalDNS2"`
	ExternalNIC     string `json:"ExternalNIC"` // default is eth1
	Restart         bool   `json:"Restart"`     // restart the system
	Changed         bool   `json:"Changed"`     // changed status
}

// InitInterfaces set the Interfaces struct
func (i *Interfaces) InitInterfaces(k, v interface{}) {
	switch k {
	case "InternalNIC":
		i.InternalNIC = v.(string)
	case "InternalIP":
		i.InternalIP = v.(string)
	case "InternalMask":
		i.InternalMask = v.(string)
	case "InternalGateway":
		i.InternalGateway = v.(string)
	case "InternalDNS1":
		i.InternalDNS1 = v.(string)
	case "InternalDNS2":
		i.InternalDNS2 = v.(string)
	case "ExternalNIC":
		i.ExternalNIC = v.(string)
	case "Restart":
		i.Restart = v.(bool)
	}
}

// EtcNetworkInterface is /etc/network/interfaces file
const etcNetworkInterface = `
# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).

# The loopback network interface
auto lo
iface lo inet loopback

# Source interfaces
# Please check /etc/network/interfaces.d before changing this file
# as interfaces may have been defined in /etc/network/interfaces.d
# NOTE: the primary ethernet device is defined in
# /etc/network/interfaces.d/eth0
# See LP: #1262951
source /etc/network/interfaces.d/*.cfg
`

// PurgeMainConf use the interfaces.d dir instead of /etc/network/interfaces
func (i *Interfaces) PurgeMainConf() {
	ParseTmpl(i, etcNetworkInterface, "interface", "/etc/network/interfaces", 0644)
}

const internalNIC = `
auto {{.InternalNIC}}
iface {{.InternalNIC}} inet static
address {{.InternalIP}}
netmask {{.InternalMask}}
gateway {{.InternalGateway}}
dns-nameservers {{.InternalDNS1}} {{.InternalDNS2}}
`

// SetInternalNIC sets the internal nic
func (i *Interfaces) SetInternalNIC() {
	ParseTmpl(i, internalNIC, "internal", "/etc/network/interfaces.d/"+i.InternalNIC+".cfg", 0644)
}

const externalNIC = `
# The public network interface
auto {{.ExternalNIC}}
iface  {{.ExternalNIC}} inet manual
up ip link set dev $IFACE up
down ip link set dev $IFACE down
`

// SetExternalNIC sets the external nic
func (i *Interfaces) SetExternalNIC() {
	ParseTmpl(i, externalNIC, "external", "/etc/network/interfaces.d/"+i.ExternalNIC+".cfg", 0644)
}
<|endoftext|>


<commit_before>package teams

import (
	"fmt"
	"strings"
	"testing"

	"golang.org/x/net/context"

	"github.com/keybase/client/go/externals"
	"github.com/keybase/client/go/kbtest"
	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/stretchr/testify/require"
)

func TestLookupImplicitTeams(t *testing.T) {
	tc := SetupTest(t, "team", 1)
	defer tc.Cleanup()

	numKBUsers := 3
	var users []*kbtest.FakeUser
	var usernames []string
	for i := 0; i < numKBUsers; i++ {
		u, err := kbtest.CreateAndSignupFakeUser("t", tc.G)
		require.NoError(t, err)
		users = append(users, u)
		usernames = append(usernames, u.Username)
	}

	tc.G.SetServices(externals.GetServices())
	lookupAndCreate := func(displayName string) {
		_, _, err := LookupImplicitTeam(context.TODO(), tc.G, displayName, false)
		require.Error(t, err)
		require.IsType(t, TeamDoesNotExistError{}, err)

		createdTeamID, impTeamName, err := LookupOrCreateImplicitTeam(context.TODO(), tc.G, displayName, false)
		require.NoError(t, err)

		lookupTeamID, impTeamName, err := LookupImplicitTeam(context.TODO(), tc.G, displayName, false)
		require.NoError(t, err)
		require.Equal(t, createdTeamID, lookupTeamID)

		team, err := Load(context.TODO(), tc.G, keybase1.LoadTeamArg{
			ID: createdTeamID,
		})
		require.NoError(t, err)
		teamDisplay, err := team.ImplicitTeamDisplayName(context.TODO())
		require.NoError(t, err)
		formatName, err := FormatImplicitTeamName(context.TODO(), tc.G, impTeamName)
		require.NoError(t, err)
		require.Equal(t, teamDisplay, formatName)
	}

	displayName := strings.Join(usernames, ",")
	t.Logf("displayName: %s", displayName)
	lookupAndCreate(displayName)
	displayName = fmt.Sprintf("mike@twitter,%s,james@github", displayName)
	t.Logf("displayName: %s", displayName)
	lookupAndCreate(displayName)

	_, _, err := LookupOrCreateImplicitTeam(context.TODO(), tc.G, "dksjdskjs/sxs?", false)
	require.Error(t, err)
}
<commit_msg>fix create private iteams (2)<commit_after>package teams

import (
	"fmt"
	"strings"
	"testing"

	"golang.org/x/net/context"

	"github.com/keybase/client/go/externals"
	"github.com/keybase/client/go/kbtest"
	"github.com/keybase/client/go/protocol/keybase1"
	"github.com/stretchr/testify/require"
)

func TestLookupImplicitTeams(t *testing.T) {
	tc := SetupTest(t, "team", 1)
	defer tc.Cleanup()

	numKBUsers := 3
	var users []*kbtest.FakeUser
	var usernames []string
	for i := 0; i < numKBUsers; i++ {
		u, err := kbtest.CreateAndSignupFakeUser("t", tc.G)
		require.NoError(t, err)
		users = append(users, u)
		usernames = append(usernames, u.Username)
	}

	tc.G.SetServices(externals.GetServices())
	lookupAndCreate := func(displayName string, public bool) {
		t.Logf("displayName:%v public:%v", displayName, public)
		_, _, err := LookupImplicitTeam(context.TODO(), tc.G, displayName, public)
		require.Error(t, err)
		require.IsType(t, TeamDoesNotExistError{}, err)

		createdTeamID, impTeamName, err := LookupOrCreateImplicitTeam(context.TODO(), tc.G, displayName, public)
		require.NoError(t, err)
		require.Equal(t, public, !impTeamName.IsPrivate)

		// second time, LookupOrCreate should Lookup the team just created.
		createdTeamID2, impTeamName2, err := LookupOrCreateImplicitTeam(context.TODO(), tc.G, displayName, public)
		require.NoError(t, err)
		require.Equal(t, createdTeamID, createdTeamID2)
		require.Equal(t, impTeamName, impTeamName2, "public: %v", public)

		lookupTeamID, impTeamName, err := LookupImplicitTeam(context.TODO(), tc.G, displayName, public)
		require.NoError(t, err)
		require.Equal(t, createdTeamID, lookupTeamID)

		team, err := Load(context.TODO(), tc.G, keybase1.LoadTeamArg{
			ID: createdTeamID,
		})
		require.NoError(t, err)
		teamDisplay, err := team.ImplicitTeamDisplayName(context.TODO())
		require.NoError(t, err)
		formatName, err := FormatImplicitTeamName(context.TODO(), tc.G, impTeamName)
		require.NoError(t, err)
		require.Equal(t, teamDisplay, formatName)
	}

	displayName := strings.Join(usernames, ",")
	lookupAndCreate(displayName, false)
	lookupAndCreate(displayName, true)
	displayName = fmt.Sprintf("mike@twitter,%s,james@github", displayName)
	lookupAndCreate(displayName, false)
	lookupAndCreate(displayName, true)

	_, _, err := LookupOrCreateImplicitTeam(context.TODO(), tc.G, "dksjdskjs/sxs?", false)
	require.Error(t, err)
	_, _, err = LookupOrCreateImplicitTeam(context.TODO(), tc.G, "dksjdskjs/sxs?", true)
	require.Error(t, err)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package factotum encapsulates crypto operations on user's public/private keys.
package factotum

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/sha256"
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"path/filepath"
	"strings"

	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/upspin"
)

type factotumKey struct {
	keyHash      []byte
	public       upspin.PublicKey
	private      string
	ecdsaKeyPair ecdsa.PrivateKey // ecdsa form of key pair
	curveName    string
}

type keyHashArray [sha256.Size]byte

type factotum struct {
	current  keyHashArray
	previous keyHashArray
	keys     map[keyHashArray]factotumKey
}

var _ upspin.Factotum = factotum{}

var sig0 upspin.Signature // for returning nil

// KeyHash returns the hash of a key, given in string format.
func KeyHash(p upspin.PublicKey) []byte {
	keyHash := sha256.Sum256([]byte(p))
	return keyHash[:]
}

// New returns a new Factotum providing all needed private key operations,
// loading keys from dir/*.upspinkey.
// Our desired end state is that Factotum is implemented on each platform by the
// best local means of protecting private keys.  Please do not break the abstraction
// by hand coding direct generation or use of private keys.
func New(dir string) (upspin.Factotum, error) {
	op := "NewFactotum"
	privBytes, err := readFile(op, dir, "secret.upspinkey")
	if err != nil {
		return nil, err
	}
	privBytes = stripCR(privBytes)
	pubBytes, err := readFile(op, dir, "public.upspinkey")
	if err != nil {
		return nil, err
	}
	pubBytes = stripCR(pubBytes)
	pfk, err := makeKey(upspin.PublicKey(pubBytes), string(privBytes))
	if err != nil {
		return nil, err
	}
	fm := make(map[keyHashArray]factotumKey)
	var h keyHashArray
	copy(h[:], pfk.keyHash)
	log.Debug.Printf("factotum %x %q\n", h, pubBytes)
	fm[h] = *pfk
	f := &factotum{
		current:  h,
		previous: h,
		keys:     fm,
	}

	// Read older key pairs.
	// Current file format is "# EE date" concatenated with old public.upspinkey
	// then old secret.upspinkey, and repeat.  This should be cleaned up someday
	// when we have a better idea of what other kinds of keys we need to save.
	// For now, it is cavalier about bailing out at first little mistake.
	s2, err := readFile(op, dir, "secret2.upspinkey")
	if err != nil {
		return f, nil
	}
	s2 = stripCR(s2)
	lines := strings.Split(string(s2), "\n")
	for {
		if len(lines) < 5 {
			break // This is not enough for a complete key pair.
		}
		if lines[0] != "# EE " {
			break // This is not a kind of key we recognize.
		}
		// lines[0] "# EE "     Joe's key
		// lines[1] "p256"
		// lines[2] "1042...6334" public X
		// lines[3] "2694...192"  public Y
		// lines[4] "8220...5934" private D
		pfk, err := makeKey(upspin.PublicKey(lines[1]+"\n"+lines[2]+"\n"+lines[3]+"\n"), lines[4])
		if err != nil {
			break
		}
		var h keyHashArray
		copy(h[:], pfk.keyHash)
		log.Debug.Printf("factotum %x %q\n", h, lines[1]+"\n"+lines[2]+"\n"+lines[3]+"\n")
		_, ok := f.keys[h]
		if ok { // Duplicate.
			continue // TODO Should we warn?
		}
		f.keys[h] = *pfk
		f.previous = h
		lines = lines[5:]
	}
	return f, err
}

// stripCR removes \r.
func stripCR(b []byte) []byte {
	return bytes.Replace(b, []byte("\r"), []byte(""), -1)
}

// makeKey creates a factotumKey by filling in the derived fields.
func makeKey(pub upspin.PublicKey, priv string) (*factotumKey, error) {
	ePublicKey, curveName, err := ParsePublicKey(pub)
	// TODO(ehg) sanity check that priv is consistent with pub
	if err != nil {
		return nil, err
	}
	ecdsaKeyPair, err := parsePrivateKey(ePublicKey, priv)
	if err != nil {
		return nil, err
	}
	fk := factotumKey{
		keyHash:      KeyHash(pub),
		public:       pub,
		private:      priv,
		ecdsaKeyPair: *ecdsaKeyPair,
		curveName:    curveName,
	}
	return &fk, nil
}

// FileSign ECDSA-signs c|n|t|dkey|hash, as required for EEPack.
func (f factotum) FileSign(n upspin.PathName, t upspin.Time, dkey, hash []byte) (upspin.Signature, error) {
	fk := f.keys[f.current]
	r, s, err := ecdsa.Sign(rand.Reader, &fk.ecdsaKeyPair, VerHash(fk.curveName, n, t, dkey, hash))
	if err != nil {
		return sig0, err
	}
	return upspin.Signature{R: r, S: s}, nil
}

// ScalarMult is the bare private key operator, used in unwrapping packed data.
func (f factotum) ScalarMult(keyHash []byte, curve elliptic.Curve, x, y *big.Int) (sx, sy *big.Int, err error) {
	const op = "factotum.ScalarMult"
	var h keyHashArray
	copy(h[:], keyHash)
	fk, ok := f.keys[h]
	if !ok {
		err = errors.E(op, errors.Errorf("no such key %x", keyHash))
	} else {
		sx, sy = curve.ScalarMult(x, y, fk.ecdsaKeyPair.D.Bytes())
	}
	return
}

// UserSign assists in authenticating to Upspin servers.
func (f factotum) UserSign(hash []byte) (upspin.Signature, error) {
	// no logging or constraining hash, because will change to TokenBinding anyway
	fk := f.keys[f.current]
	r, s, err := ecdsa.Sign(rand.Reader, &fk.ecdsaKeyPair, hash)
	if err != nil {
		return sig0, err
	}
	return upspin.Signature{R: r, S: s}, nil
}

// Pop derives a Factotum by switching default from the current to the previous key.
func (f factotum) Pop() upspin.Factotum {
	// Arbitrarily keep f.previous unchanged, so Pop() is idempotent.
	// We don't yet have any need to go further back in time.
	return &factotum{current: f.previous, previous: f.previous, keys: f.keys}
}

// PublicKey returns the user's latest public key.
func (f factotum) PublicKey() upspin.PublicKey {
	return f.keys[f.current].public
}

// PublicKeyFromHash returns the user's public key with matching keyHash.
func (f factotum) PublicKeyFromHash(keyHash []byte) (upspin.PublicKey, error) {
	const op = "factotum.PublicKeyFromHash"
	if keyHash == nil || len(keyHash) == 0 {
		return "", errors.E(op, errors.Invalid, errors.Errorf("invalid keyHash"))
	}
	var h keyHashArray
	copy(h[:], keyHash)
	fk, ok := f.keys[h]
	if !ok {
		return "", errors.E(op, errors.NotExist, errors.Errorf("no such key"))
	}
	return fk.public, nil
}

// VerHash provides the basis for signing and verifying files.
func VerHash(curveName string, pathname upspin.PathName, time upspin.Time, dkey, cipherSum []byte) []byte {
	b := sha256.Sum256([]byte(fmt.Sprintf("%02x:%s:%d:%x:%x", curveName, pathname, time, dkey, cipherSum)))
	return b[:]
}

// parsePrivateKey returns an ECDSA private key given a user's ECDSA public key and a
// string representation of the private key.
func parsePrivateKey(publicKey *ecdsa.PublicKey, privateKey string) (priv *ecdsa.PrivateKey, err error) {
	privateKey = strings.TrimSpace(string(privateKey))
	var d big.Int
	err = d.UnmarshalText([]byte(privateKey))
	if err != nil {
		return nil, err
	}
	return &ecdsa.PrivateKey{PublicKey: *publicKey, D: &d}, nil
}

// ParsePublicKey takes an Upspin representation of a public key and converts it into an ECDSA public key, returning its type.
// The Upspin string representation uses \n as newline no matter what native OS it runs on.
func ParsePublicKey(public upspin.PublicKey) (*ecdsa.PublicKey, string, error) {
	const op = "factotum.ParsePublicKey"
	fields := strings.Split(string(public), "\n")
	if len(fields) != 4 { // 4 is because string should be terminated by \n, hence fields[3]==""
		return nil, "", errors.E(op, errors.Invalid, errors.Errorf("expected keytype, two big ints and a newline; got %d %v", len(fields), fields))
	}
	keyType := fields[0]
	var x, y big.Int
	_, ok := x.SetString(fields[1], 10)
	if !ok {
		return nil, "", errors.E(op, errors.Invalid, errors.Errorf("%s is not a big int", fields[1]))
	}
	_, ok = y.SetString(fields[2], 10)
	if !ok {
		return nil, "", errors.E(op, errors.Invalid, errors.Errorf("%s is not a big int", fields[2]))
	}

	var curve elliptic.Curve
	switch keyType {
	case "p256":
		curve = elliptic.P256()
	case "p521":
		curve = elliptic.P521()
	case "p384":
		curve = elliptic.P384()
	default:
		return nil, "", errors.E(op, errors.Invalid, errors.Errorf("unknown key type: %q", keyType))
	}
	return &ecdsa.PublicKey{Curve: curve, X: &x, Y: &y}, keyType, nil
}

func readFile(op, dir, name string) ([]byte, error) {
	b, err := ioutil.ReadFile(filepath.Join(dir, name))
	if os.IsNotExist(err) {
		return nil, errors.E(op, errors.NotExist, err)
	}
	if err != nil {
		return nil, errors.E(op, errors.IO, err)
	}
	return b, nil
}
<commit_msg>factotum: improve error handling and logging<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package factotum encapsulates crypto operations on user's public/private keys.
package factotum

import (
	"bytes"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/sha256"
	"fmt"
	"io/ioutil"
	"math/big"
	"os"
	"path/filepath"
	"strings"

	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/upspin"
)

type factotumKey struct {
	keyHash      []byte
	public       upspin.PublicKey
	private      string
	ecdsaKeyPair ecdsa.PrivateKey // ecdsa form of key pair
	curveName    string
}

type keyHashArray [sha256.Size]byte

type factotum struct {
	current  keyHashArray
	previous keyHashArray
	keys     map[keyHashArray]factotumKey
}

var _ upspin.Factotum = factotum{}

var sig0 upspin.Signature // for returning nil

// KeyHash returns the hash of a key, given in string format.
func KeyHash(p upspin.PublicKey) []byte {
	keyHash := sha256.Sum256([]byte(p))
	return keyHash[:]
}

// New returns a new Factotum providing all needed private key operations,
// loading keys from dir/*.upspinkey.
// Our desired end state is that Factotum is implemented on each platform by the
// best local means of protecting private keys.  Please do not break the abstraction
// by hand coding direct generation or use of private keys.
func New(dir string) (upspin.Factotum, error) {
	const op = "factotum.New"
	privBytes, err := readFile(op, dir, "secret.upspinkey")
	if err != nil {
		return nil, errors.E(op, err)
	}
	privBytes = stripCR(privBytes)
	pubBytes, err := readFile(op, dir, "public.upspinkey")
	if err != nil {
		return nil, errors.E(op, err)
	}
	pubBytes = stripCR(pubBytes)
	pfk, err := makeKey(upspin.PublicKey(pubBytes), string(privBytes))
	if err != nil {
		return nil, errors.E(op, err)
	}
	fm := make(map[keyHashArray]factotumKey)
	var h keyHashArray
	copy(h[:], pfk.keyHash)
	log.Debug.Printf("%s(%q): %x", op, dir, h)
	fm[h] = *pfk
	f := &factotum{
		current:  h,
		previous: h,
		keys:     fm,
	}

	// Read older key pairs.
	// Current file format is "# EE date" concatenated with old public.upspinkey
	// then old secret.upspinkey, and repeat.  This should be cleaned up someday
	// when we have a better idea of what other kinds of keys we need to save.
	// For now, it is cavalier about bailing out at first little mistake.
	s2, err := readFile(op, dir, "secret2.upspinkey")
	if err != nil {
		return f, nil
	}
	s2 = stripCR(s2)
	lines := strings.Split(string(s2), "\n")
	for {
		if len(lines) < 5 {
			break // This is not enough for a complete key pair.
		}
		if lines[0] != "# EE " {
			break // This is not a kind of key we recognize.
		}
		// lines[0] "# EE "     Joe's key
		// lines[1] "p256"
		// lines[2] "1042...6334" public X
		// lines[3] "2694...192"  public Y
		// lines[4] "8220...5934" private D
		pfk, err := makeKey(upspin.PublicKey(lines[1]+"\n"+lines[2]+"\n"+lines[3]+"\n"), lines[4])
		if err != nil {
			return f, errors.E(op, err)
		}
		var h keyHashArray
		copy(h[:], pfk.keyHash)
		log.Debug.Printf("%s(%q): %x (older)", op, dir, h)
		_, ok := f.keys[h]
		if ok { // Duplicate.
			continue // TODO Should we warn?
		}
		f.keys[h] = *pfk
		f.previous = h
		lines = lines[5:]
	}
	return f, nil
}

// stripCR removes \r.
func stripCR(b []byte) []byte {
	return bytes.Replace(b, []byte("\r"), []byte(""), -1)
}

// makeKey creates a factotumKey by filling in the derived fields.
func makeKey(pub upspin.PublicKey, priv string) (*factotumKey, error) {
	ePublicKey, curveName, err := ParsePublicKey(pub)
	// TODO(ehg) sanity check that priv is consistent with pub
	if err != nil {
		return nil, err
	}
	ecdsaKeyPair, err := parsePrivateKey(ePublicKey, priv)
	if err != nil {
		return nil, err
	}
	fk := factotumKey{
		keyHash:      KeyHash(pub),
		public:       pub,
		private:      priv,
		ecdsaKeyPair: *ecdsaKeyPair,
		curveName:    curveName,
	}
	return &fk, nil
}

// FileSign ECDSA-signs c|n|t|dkey|hash, as required for EEPack.
func (f factotum) FileSign(n upspin.PathName, t upspin.Time, dkey, hash []byte) (upspin.Signature, error) {
	fk := f.keys[f.current]
	r, s, err := ecdsa.Sign(rand.Reader, &fk.ecdsaKeyPair, VerHash(fk.curveName, n, t, dkey, hash))
	if err != nil {
		return sig0, err
	}
	return upspin.Signature{R: r, S: s}, nil
}

// ScalarMult is the bare private key operator, used in unwrapping packed data.
func (f factotum) ScalarMult(keyHash []byte, curve elliptic.Curve, x, y *big.Int) (sx, sy *big.Int, err error) {
	const op = "factotum.ScalarMult"
	var h keyHashArray
	copy(h[:], keyHash)
	fk, ok := f.keys[h]
	if !ok {
		err = errors.E(op, errors.Errorf("no such key %x", keyHash))
	} else {
		sx, sy = curve.ScalarMult(x, y, fk.ecdsaKeyPair.D.Bytes())
	}
	return
}

// UserSign assists in authenticating to Upspin servers.
func (f factotum) UserSign(hash []byte) (upspin.Signature, error) {
	// no logging or constraining hash, because will change to TokenBinding anyway
	fk := f.keys[f.current]
	r, s, err := ecdsa.Sign(rand.Reader, &fk.ecdsaKeyPair, hash)
	if err != nil {
		return sig0, err
	}
	return upspin.Signature{R: r, S: s}, nil
}

// Pop derives a Factotum by switching default from the current to the previous key.
func (f factotum) Pop() upspin.Factotum {
	// Arbitrarily keep f.previous unchanged, so Pop() is idempotent.
	// We don't yet have any need to go further back in time.
	return &factotum{current: f.previous, previous: f.previous, keys: f.keys}
}

// PublicKey returns the user's latest public key.
func (f factotum) PublicKey() upspin.PublicKey {
	return f.keys[f.current].public
}

// PublicKeyFromHash returns the user's public key with matching keyHash.
func (f factotum) PublicKeyFromHash(keyHash []byte) (upspin.PublicKey, error) {
	const op = "factotum.PublicKeyFromHash"
	if keyHash == nil || len(keyHash) == 0 {
		return "", errors.E(op, errors.Invalid, errors.Errorf("invalid keyHash"))
	}
	var h keyHashArray
	copy(h[:], keyHash)
	fk, ok := f.keys[h]
	if !ok {
		return "", errors.E(op, errors.NotExist, errors.Errorf("no such key"))
	}
	return fk.public, nil
}

// VerHash provides the basis for signing and verifying files.
func VerHash(curveName string, pathname upspin.PathName, time upspin.Time, dkey, cipherSum []byte) []byte {
	b := sha256.Sum256([]byte(fmt.Sprintf("%02x:%s:%d:%x:%x", curveName, pathname, time, dkey, cipherSum)))
	return b[:]
}

// parsePrivateKey returns an ECDSA private key given a user's ECDSA public key and a
// string representation of the private key.
func parsePrivateKey(publicKey *ecdsa.PublicKey, privateKey string) (priv *ecdsa.PrivateKey, err error) {
	privateKey = strings.TrimSpace(string(privateKey))
	var d big.Int
	err = d.UnmarshalText([]byte(privateKey))
	if err != nil {
		return nil, err
	}
	return &ecdsa.PrivateKey{PublicKey: *publicKey, D: &d}, nil
}

// ParsePublicKey takes an Upspin representation of a public key and converts it into an ECDSA public key, returning its type.
// The Upspin string representation uses \n as newline no matter what native OS it runs on.
func ParsePublicKey(public upspin.PublicKey) (*ecdsa.PublicKey, string, error) {
	const op = "factotum.ParsePublicKey"
	fields := strings.Split(string(public), "\n")
	if len(fields) != 4 { // 4 is because string should be terminated by \n, hence fields[3]==""
		return nil, "", errors.E(op, errors.Invalid, errors.Errorf("expected keytype, two big ints and a newline; got %d %v", len(fields), fields))
	}
	keyType := fields[0]
	var x, y big.Int
	_, ok := x.SetString(fields[1], 10)
	if !ok {
		return nil, "", errors.E(op, errors.Invalid, errors.Errorf("%s is not a big int", fields[1]))
	}
	_, ok = y.SetString(fields[2], 10)
	if !ok {
		return nil, "", errors.E(op, errors.Invalid, errors.Errorf("%s is not a big int", fields[2]))
	}

	var curve elliptic.Curve
	switch keyType {
	case "p256":
		curve = elliptic.P256()
	case "p521":
		curve = elliptic.P521()
	case "p384":
		curve = elliptic.P384()
	default:
		return nil, "", errors.E(op, errors.Invalid, errors.Errorf("unknown key type: %q", keyType))
	}
	return &ecdsa.PublicKey{Curve: curve, X: &x, Y: &y}, keyType, nil
}

func readFile(op, dir, name string) ([]byte, error) {
	b, err := ioutil.ReadFile(filepath.Join(dir, name))
	if os.IsNotExist(err) {
		return nil, errors.E(op, errors.NotExist, err)
	}
	if err != nil {
		return nil, errors.E(op, errors.IO, err)
	}
	return b, nil
}
<|endoftext|>


<commit_before>package diff_test

import (
	"bytes"
	"testing"

	"github.com/schemalex/schemalex/diff"
	"github.com/stretchr/testify/assert"
)

func TestDiff(t *testing.T) {
	type Spec struct {
		Before string
		After  string
		Expect string
	}

	specs := []Spec{
		// drop table
		{
			Before: "CREATE TABLE `hoge` ( `id` integer not null ); CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "DROP TABLE `hoge`;",
		},
		// create table
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `hoge` ( `id` INTEGER NOT NULL ) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COMMENT 'table comment'; CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "CREATE TABLE `hoge` (\n`id` INT (11) NOT NULL\n) ENGINE = InnoDB, DEFAULT CHARACTER SET = utf8mb4, COMMENT = 'table comment';",
		},
		// drop column
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL, `c` VARCHAR (20) NOT NULL DEFAULT 'xxx' );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "ALTER TABLE `fuga` DROP COLUMN `c`;",
		},
		// add column
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL, `c` VARCHAR (20) NOT NULL DEFAULT 'xxx');",
			Expect: "ALTER TABLE `fuga` ADD COLUMN `c` VARCHAR (20) NOT NULL DEFAULT 'xxx';",
		},
		// change column
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` BIGINT NOT NULL );",
			Expect: "ALTER TABLE `fuga` CHANGE COLUMN `id` `id` BIGINT (20) NOT NULL;",
		},
		// change column with comment
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL COMMENT 'fuga is good' );",
			Expect: "ALTER TABLE `fuga` CHANGE COLUMN `id` `id` INT (11) NOT NULL COMMENT 'fuga is good';",
		},
		// drop primary key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, PRIMARY KEY (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			Expect: "ALTER TABLE `fuga` DROP INDEX PRIMARY KEY;",
		},
		// add primary key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, PRIMARY KEY (`id`) );",
			Expect: "ALTER TABLE `fuga` ADD PRIMARY KEY (`id`);",
		},
		// drop unique key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			Expect: "ALTER TABLE `fuga` DROP INDEX `uniq_id`;",
		},
		// add unique key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			Expect: "ALTER TABLE `fuga` ADD CONSTRAINT `symbol` UNIQUE INDEX `uniq_id` USING BTREE (`id`);",
		},
		// not change index
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			Expect: "",
		},
		// not change FOREIGN KEY
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			Expect: "",
		},
		// change CONSTRAINT symbol naml
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, CONSTRAINT `fsym` FOREIGN KEY (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, CONSTRAINT `ksym` FOREIGN KEY (fid) REFERENCES f (id) );",
			Expect: "ALTER TABLE `fuga` DROP FOREIGN KEY `fsym`;\nALTER TABLE `fuga` ADD CONSTRAINT `ksym` FOREIGN KEY (`fid`) REFERENCES `f` (`id`);",
		},
		// remove FOREIGN KEY
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, INDEX fid (fid) );",
			Expect: "ALTER TABLE `fuga` DROP FOREIGN KEY `fk`;\nALTER TABLE `fuga` ADD INDEX `fid` (`fid`);",
		},
		// multi modify
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `aid` INTEGER NOT NULL, `bid` INTEGER NOT NULL, INDEX `ab` (`aid`, `bid`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `aid` INTEGER NOT NULL, `cid` INTEGER NOT NULL, INDEX `ac` (`aid`, `cid`) );",
			Expect: "ALTER TABLE `fuga` DROP INDEX `ab`;\nALTER TABLE `fuga` DROP COLUMN `bid`;\nALTER TABLE `fuga` ADD COLUMN `cid` INT (11) NOT NULL;\nALTER TABLE `fuga` ADD INDEX `ac` (`aid`, `cid`);",
		},
		// not change to query what generated by show create table
		{
			// human input
			Before: `
create table foo (
   tinyints tinyint,
   tinyintu tinyint unsigned,
   smallints smallint,
   smallintu smallint unsigned,
   mediumints mediumint,
   mediumintu mediumint unsigned,
   ints int comment 'this is sined int nullable',
   intu int unsigned,
   integers integer null default null,
   integeru integer unsigned null,
   bigins bigint,
   bigintu bigint unsigned,
   floats float,
   floaru float unsigned,
   doubles double,
   doubleu double unsigned,
   decimals decimal,
   decimalu decimal unsigned,
   varcharn varchar (10) null,
   varcharnn varchar (10) not null,
   textn text,
   textnn text not null,
   blobn blob,
   blobnn blob,
   intsd int default 0,
   intud int unsigned default 0,
   INDEX foo_idx (ints)
);
			`,
			// show create table foo
			After: `
CREATE TABLE foo (
  tinyints tinyint(4) DEFAULT NULL,
  tinyintu tinyint(3) unsigned DEFAULT NULL,
  smallints smallint(6) DEFAULT NULL,
  smallintu smallint(5) unsigned DEFAULT NULL,
  mediumints mediumint(9) DEFAULT NULL,
  mediumintu mediumint(8) unsigned DEFAULT NULL,
  ints int(11) DEFAULT NULL COMMENT 'this is sined int nullable',
  intu int(10) unsigned DEFAULT NULL,
  integers int(11) DEFAULT NULL,
  integeru int(10) unsigned DEFAULT NULL,
  bigins bigint(20) DEFAULT NULL,
  bigintu bigint(20) unsigned DEFAULT NULL,
  floats float DEFAULT NULL,
  floaru float unsigned DEFAULT NULL,
  doubles double DEFAULT NULL,
  doubleu double unsigned DEFAULT NULL,
  decimals decimal(10,0) DEFAULT NULL,
  decimalu decimal(10,0) unsigned DEFAULT NULL,
  varcharn varchar(10) DEFAULT NULL,
  varcharnn varchar(10) NOT NULL,
  textn text,
  textnn text NOT NULL,
  blobn blob,
  blobnn blob,
  intsd int(11) DEFAULT '0',
  intud int(10) unsigned DEFAULT '0',
  KEY foo_idx (ints)
)
			`,
			Expect: "",
		},
	}

	var buf bytes.Buffer
	for _, spec := range specs {
		buf.Reset()

		if !assert.NoError(t, diff.Strings(&buf, spec.Before, spec.After), "diff.String should succeed") {
			return
		}
		if !assert.Equal(t, spec.Expect, buf.String(), "result SQL should match") {
			return
		}
	}
}
<commit_msg>I home pass this test case<commit_after>package diff_test

import (
	"bytes"
	"testing"

	"github.com/schemalex/schemalex/diff"
	"github.com/stretchr/testify/assert"
)

func TestDiff(t *testing.T) {
	type Spec struct {
		Before string
		After  string
		Expect string
	}

	specs := []Spec{
		// drop table
		{
			Before: "CREATE TABLE `hoge` ( `id` integer not null ); CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "DROP TABLE `hoge`;",
		},
		// create table
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `hoge` ( `id` INTEGER NOT NULL ) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COMMENT 'table comment'; CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "CREATE TABLE `hoge` (\n`id` INT (11) NOT NULL\n) ENGINE = InnoDB, DEFAULT CHARACTER SET = utf8mb4, COMMENT = 'table comment';",
		},
		// drop column
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL, `c` VARCHAR (20) NOT NULL DEFAULT 'xxx' );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			Expect: "ALTER TABLE `fuga` DROP COLUMN `c`;",
		},
		// add column
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL, `c` VARCHAR (20) NOT NULL DEFAULT 'xxx');",
			Expect: "ALTER TABLE `fuga` ADD COLUMN `c` VARCHAR (20) NOT NULL DEFAULT 'xxx';",
		},
		// change column
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` BIGINT NOT NULL );",
			Expect: "ALTER TABLE `fuga` CHANGE COLUMN `id` `id` BIGINT (20) NOT NULL;",
		},
		// change column with comment
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL COMMENT 'fuga is good' );",
			Expect: "ALTER TABLE `fuga` CHANGE COLUMN `id` `id` INT (11) NOT NULL COMMENT 'fuga is good';",
		},
		// drop primary key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, PRIMARY KEY (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			Expect: "ALTER TABLE `fuga` DROP INDEX PRIMARY KEY;",
		},
		// add primary key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, PRIMARY KEY (`id`) );",
			Expect: "ALTER TABLE `fuga` ADD PRIMARY KEY (`id`);",
		},
		// drop unique key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			Expect: "ALTER TABLE `fuga` DROP INDEX `uniq_id`;",
		},
		// add unique key
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			Expect: "ALTER TABLE `fuga` ADD CONSTRAINT `symbol` UNIQUE INDEX `uniq_id` USING BTREE (`id`);",
		},
		// not change index
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, CONSTRAINT `symbol` UNIQUE KEY `uniq_id` USING BTREE (`id`) );",
			Expect: "",
		},
		// not change FOREIGN KEY
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			Expect: "",
		},
		// change CONSTRAINT symbol naml
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, CONSTRAINT `fsym` FOREIGN KEY (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, CONSTRAINT `ksym` FOREIGN KEY (fid) REFERENCES f (id) );",
			Expect: "ALTER TABLE `fuga` DROP FOREIGN KEY `fsym`;\nALTER TABLE `fuga` ADD CONSTRAINT `ksym` FOREIGN KEY (`fid`) REFERENCES `f` (`id`);",
		},
		// remove FOREIGN KEY
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, FOREIGN KEY fk (fid) REFERENCES f (id) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `fid` INTEGER NOT NULL, INDEX fid (fid) );",
			Expect: "ALTER TABLE `fuga` DROP FOREIGN KEY `fk`;\nALTER TABLE `fuga` ADD INDEX `fid` (`fid`);",
		},
		// multi modify
		{
			Before: "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `aid` INTEGER NOT NULL, `bid` INTEGER NOT NULL, INDEX `ab` (`aid`, `bid`) );",
			After:  "CREATE TABLE `fuga` ( `id` INTEGER NOT NULL AUTO_INCREMENT, `aid` INTEGER NOT NULL, `cid` INTEGER NOT NULL, INDEX `ac` (`aid`, `cid`) );",
			Expect: "ALTER TABLE `fuga` DROP INDEX `ab`;\nALTER TABLE `fuga` DROP COLUMN `bid`;\nALTER TABLE `fuga` ADD COLUMN `cid` INT (11) NOT NULL;\nALTER TABLE `fuga` ADD INDEX `ac` (`aid`, `cid`);",
		},
		// not change to query what generated by show create table
		{
			// human input
			Before: `
create table foo (
   id int not null AUTO_INCREMENT PRIMARY KEY,
   tinyints tinyint,
   tinyintu tinyint unsigned,
   smallints smallint,
   smallintu smallint unsigned,
   mediumints mediumint,
   mediumintu mediumint unsigned,
   ints int comment 'this is sined int nullable',
   intu int unsigned,
   integers integer null default null,
   integeru integer unsigned null,
   bigins bigint UNIQUE KEY,
   bigintu bigint unsigned,
   floats float,
   floaru float unsigned,
   doubles double,
   doubleu double unsigned,
   decimals decimal,
   decimalu decimal unsigned,
   varcharn varchar (10) null,
   varcharnn varchar (10) not null,
   textn text,
   textnn text not null,
   blobn blob,
   blobnn blob,
   intsd int default 0,
   intud int unsigned default 0,
   CONSTRAINT bar_fk FOREIGN KEY (integers) REFERENCES bar (id),
   INDEX foo_idx (ints)
);
			`,
			// show create table foo
			After: `
CREATE TABLE foo (
  id int(11) NOT NULL AUTO_INCREMENT,
  tinyints tinyint(4) DEFAULT NULL,
  tinyintu tinyint(3) unsigned DEFAULT NULL,
  smallints smallint(6) DEFAULT NULL,
  smallintu smallint(5) unsigned DEFAULT NULL,
  mediumints mediumint(9) DEFAULT NULL,
  mediumintu mediumint(8) unsigned DEFAULT NULL,
  ints int(11) DEFAULT NULL COMMENT 'this is sined int nullable',
  intu int(10) unsigned DEFAULT NULL,
  integers int(11) DEFAULT NULL,
  integeru int(10) unsigned DEFAULT NULL,
  bigins bigint(20) DEFAULT NULL,
  bigintu bigint(20) unsigned DEFAULT NULL,
  floats float DEFAULT NULL,
  floaru float unsigned DEFAULT NULL,
  doubles double DEFAULT NULL,
  doubleu double unsigned DEFAULT NULL,
  decimals decimal(10,0) DEFAULT NULL,
  decimalu decimal(10,0) unsigned DEFAULT NULL,
  varcharn varchar(10) DEFAULT NULL,
  varcharnn varchar(10) NOT NULL,
  textn text,
  textnn text NOT NULL,
  blobn blob,
  blobnn blob,
  intsd int(11) DEFAULT '0',
  intud int(10) unsigned DEFAULT '0',
  PRIMARY KEY (id),
  UNIQUE KEY bigins (bigins),
  KEY bar_fk (integers),
  KEY foo_idx (ints),
  CONSTRAINT bar_fk FOREIGN KEY (integers) REFERENCES bar (id)
)
			`,
			Expect: "",
		},
	}

	var buf bytes.Buffer
	for _, spec := range specs {
		buf.Reset()

		if !assert.NoError(t, diff.Strings(&buf, spec.Before, spec.After), "diff.String should succeed") {
			return
		}
		if !assert.Equal(t, spec.Expect, buf.String(), "result SQL should match") {
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package argo

import (
	"fmt"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
	"sqlflow.org/sqlflow/go/log"
	pb "sqlflow.org/sqlflow/go/proto"
	wfrsp "sqlflow.org/sqlflow/go/workflow/response"
)

func getStepIdx(wf *v1alpha1.Workflow, targetStepGroup string) (int, error) {
	stepIdx := 1
	stepGroupName, e := getFirstStepGroup(wf, wf.ObjectMeta.Name)
	if e != nil {
		return -1, e
	}
	for {
		if stepGroupName == targetStepGroup {
			return stepIdx, nil
		}
		stepGroupName, e = getNextStepGroup(wf, stepGroupName)
		stepIdx++
		if e != nil {
			return -1, e
		}
	}
}

func logViewURL(ns, wfID, podID string) (string, error) {
	ep := os.Getenv("SQLFLOW_WORKFLOW_LOGVIEW_ENDPOINT")
	// argo UI log view panel
	return fmt.Sprintf("%s/workflows/%s/%s?tab=workflow&nodeId=%s&sidePanel=logs:%s:main", ep, ns, wfID, podID, podID), nil
}

// Fetch fetches the workflow log and status,
// design doc: https://github.com/sql-machine-learning/sqlflow/blob/develop/doc/design/argo_workflow_on_sqlflow.md
//
// An example of Fetched Responses from the server logs:
// Step [1/3] Execute Code: echo hello1
// Step [1/3] Log: http://localhost:8001/workflows/default/steps-bdpff?nodeId=steps-bdpff-xx1
// Step [1/3] Status: Pending
// Step [1/3] Status: Running
// Step [1/3] Status: Succeed/Failed
// Step [2/3] Execute Code: echo hello2
// Step [2/3] Log: http://localhost:8001/workflows/default/steps-bdpff?nodeId=steps-bdpff-xx2
// ...
func (w *Workflow) Fetch(req *pb.FetchRequest) (*pb.FetchResponse, error) {
	logger := log.WithFields(log.Fields{
		"requestID": log.UUID(),
		"jobID":     req.Job.Id,
		"stepID":    req.StepId,
		"event":     "fetch",
	})

	wf, err := k8sReadWorkflow(req.Job.Id)
	if err != nil {
		logger.Errorf("workflowFailed/k8sRead, error: %v", err)
		return nil, err
	}
	logger.Infof("phase:%s", wf.Status.Phase)

	if isWorkflowPending(wf) {
		return wfrsp.New().Response(req.Job.Id, "", "", false), nil
	}
	stepGroupName, err := getStepGroup(wf, req.Job.Id, req.StepId)
	if err != nil {
		return nil, err
	}

	stepCnt := len(wf.Spec.Templates[0].Steps)
	stepIdx, err := getStepIdx(wf, stepGroupName)
	if err != nil {
		return nil, err
	}
	logPrefix := fmt.Sprintf("SQLFlow Step: [%d/%d]", stepIdx, stepCnt)

	pod, err := getPodByStepGroup(wf, stepGroupName)
	if err != nil {
		return nil, err
	}
	eof := false // true if finish fetching the workflow logs
	r := wfrsp.New()
	newStepPhase := req.StepPhase
	logURL, e := logViewURL(wf.ObjectMeta.Namespace, wf.ObjectMeta.Name, pod.ObjectMeta.Name)
	if e != nil {
		return nil, e
	}

	if req.StepPhase == "" {
		// the 1st container execute `argoexec wait` to wait the priority step, so package the 2nd container's command code.
		execCode := fmt.Sprintf("%s %s", strings.Join(pod.Spec.Containers[1].Command, " "), strings.Join(pod.Spec.Containers[1].Args, " "))
		r.AppendMessage(fmt.Sprintf("%s Execute Code: %s", logPrefix, execCode))
		r.AppendMessage(fmt.Sprintf("%s Log: %s", logPrefix, logURL))
	}

	// note(yancey1989): record the Pod phase to avoid output the duplicated logs at the next fetch request.
	if req.StepPhase != string(pod.Status.Phase) {
		r.AppendMessage(fmt.Sprintf("%s Status: %s", logPrefix, pod.Status.Phase))
		newStepPhase = string(pod.Status.Phase)
	}

	if isPodCompleted(pod) {
		// snip the pod logs when it complete
		// TODO(yancey1989): fetch the pod logs using an iteration way
		// to avoid the memory overflow
		podLogs, e := k8sReadPodLogs(pod.ObjectMeta.Name, "main", "", false)
		if e != nil {
			return nil, e
		}

		if e := r.AppendProtoMessages(podLogs); e != nil {
			return nil, e
		}

		// TODO(yancey1989): add duration time for the eoeResponse
		// eoe just used to simplify the client code which can be consistent with non-argo mode.
		if isPodFailed(pod) {
			e = fmt.Errorf("%s Failed, Log: %s\n%s", logPrefix, logURL, r.ErrorMessage())
			logger.Errorf("workflowFailed, %v, spent:%d", e, time.Now().Second()-wf.CreationTimestamp.Second())
			return r.ResponseWithStepComplete(req.Job.Id, "", newStepPhase, eof), e
		}
		logger.Infof("workflowSucceed, spent:%d", time.Now().Second()-wf.CreationTimestamp.Second())

		// move to the next step
		nextStepGroup, err := getNextStepGroup(wf, stepGroupName)
		if err != nil {
			return nil, err
		}
		// set the EOF to true if no next step in the workflow
		if nextStepGroup == "" && stepIdx == stepCnt {
			eof = true
		}

		if nextStepGroup != "" {
			newStepPhase = ""
			stepGroupName = nextStepGroup
		}
		return r.ResponseWithStepComplete(req.Job.Id, stepGroupName, newStepPhase, eof), nil
	}
	return r.Response(req.Job.Id, stepGroupName, newStepPhase, eof), nil
}

func parseOffset(content string) (string, string, error) {
	reTimestamps := regexp.MustCompile(`([^\s]+)\s(.*)$`)
	msg := reTimestamps.FindStringSubmatch(content)
	if len(msg) != 3 {
		return "", "", fmt.Errorf("Parse offset failed: %s", content)
	}
	return msg[1], msg[2], nil
}

func getOffsetAndContentFromLogs(logs []string, oldOffset string) ([]string, string, error) {
	// NOTE(yancey1989): using `kubectl --since-time <offset>` to get logs
	// from the offset which is the timestamps. the accuracy can only be achieved at the second level,
	// `kubectl` may return some duplicated logs as the previous fetch, and we need to skip them.
	buffer := []string{}
	skipOlderLogs := false
	offset := oldOffset
	for _, msg := range logs {
		newOffset, content, e := parseOffset(msg)
		if e != nil {
			break
		}
		if newOffset == oldOffset {
			skipOlderLogs = true
		} else {
			if skipOlderLogs || oldOffset == "" {
				buffer = append(buffer, content)
				offset = newOffset
			} else {
				// skip the duplicated logs as the previous fetch
				continue
			}
		}
	}
	return buffer, offset, nil
}

func getPodLogs(podName string, offset string) ([]string, string, error) {
	// NOTE(tony): A workflow pod usually contains two container: main and wait
	// I believe wait is used for management by Argo, so we only need to care about main.
	logs, err := k8sReadPodLogs(podName, "main", offset, true)
	if err != nil {
		return nil, "", err
	}

	return getOffsetAndContentFromLogs(logs, offset)
}
<commit_msg>Add logview url for dtm log collection (#2938)<commit_after>// Copyright 2020 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package argo

import (
	"fmt"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/argoproj/argo/pkg/apis/workflow/v1alpha1"
	"sqlflow.org/sqlflow/go/log"
	pb "sqlflow.org/sqlflow/go/proto"
	wfrsp "sqlflow.org/sqlflow/go/workflow/response"
)

func getStepIdx(wf *v1alpha1.Workflow, targetStepGroup string) (int, error) {
	stepIdx := 1
	stepGroupName, e := getFirstStepGroup(wf, wf.ObjectMeta.Name)
	if e != nil {
		return -1, e
	}
	for {
		if stepGroupName == targetStepGroup {
			return stepIdx, nil
		}
		stepGroupName, e = getNextStepGroup(wf, stepGroupName)
		stepIdx++
		if e != nil {
			return -1, e
		}
	}
}

func logViewURL(ns, wfID, podID string) (string, error) {
	// if we are using DTM log collection, construct a different url pattern
	dtmEp := os.Getenv("SQLFLOW_WORKFLOW_LOGVIEW_DTM_ENDPOINT")
	if dtmEp != "" {
		return fmt.Sprintf("%s?jobName=%s&taskName=%s&__envName=PROD", dtmEp, wfID, podID), nil
	}
	// argo UI log view panel
	ep := os.Getenv("SQLFLOW_WORKFLOW_LOGVIEW_ENDPOINT")
	return fmt.Sprintf("%s/workflows/%s/%s?tab=workflow&nodeId=%s&sidePanel=logs:%s:main", ep, ns, wfID, podID, podID), nil
}

// Fetch fetches the workflow log and status,
// design doc: https://github.com/sql-machine-learning/sqlflow/blob/develop/doc/design/argo_workflow_on_sqlflow.md
//
// An example of Fetched Responses from the server logs:
// Step [1/3] Execute Code: echo hello1
// Step [1/3] Log: http://localhost:8001/workflows/default/steps-bdpff?nodeId=steps-bdpff-xx1
// Step [1/3] Status: Pending
// Step [1/3] Status: Running
// Step [1/3] Status: Succeed/Failed
// Step [2/3] Execute Code: echo hello2
// Step [2/3] Log: http://localhost:8001/workflows/default/steps-bdpff?nodeId=steps-bdpff-xx2
// ...
func (w *Workflow) Fetch(req *pb.FetchRequest) (*pb.FetchResponse, error) {
	logger := log.WithFields(log.Fields{
		"requestID": log.UUID(),
		"jobID":     req.Job.Id,
		"stepID":    req.StepId,
		"event":     "fetch",
	})

	wf, err := k8sReadWorkflow(req.Job.Id)
	if err != nil {
		logger.Errorf("workflowFailed/k8sRead, error: %v", err)
		return nil, err
	}
	logger.Infof("phase:%s", wf.Status.Phase)

	if isWorkflowPending(wf) {
		return wfrsp.New().Response(req.Job.Id, "", "", false), nil
	}
	stepGroupName, err := getStepGroup(wf, req.Job.Id, req.StepId)
	if err != nil {
		return nil, err
	}

	stepCnt := len(wf.Spec.Templates[0].Steps)
	stepIdx, err := getStepIdx(wf, stepGroupName)
	if err != nil {
		return nil, err
	}
	logPrefix := fmt.Sprintf("SQLFlow Step: [%d/%d]", stepIdx, stepCnt)

	pod, err := getPodByStepGroup(wf, stepGroupName)
	if err != nil {
		return nil, err
	}
	eof := false // true if finish fetching the workflow logs
	r := wfrsp.New()
	newStepPhase := req.StepPhase
	logURL, e := logViewURL(wf.ObjectMeta.Namespace, wf.ObjectMeta.Name, pod.ObjectMeta.Name)
	if e != nil {
		return nil, e
	}

	if req.StepPhase == "" {
		// the 1st container execute `argoexec wait` to wait the priority step, so package the 2nd container's command code.
		execCode := fmt.Sprintf("%s %s", strings.Join(pod.Spec.Containers[1].Command, " "), strings.Join(pod.Spec.Containers[1].Args, " "))
		r.AppendMessage(fmt.Sprintf("%s Execute Code: %s", logPrefix, execCode))
		r.AppendMessage(fmt.Sprintf("%s Log: %s", logPrefix, logURL))
	}

	// note(yancey1989): record the Pod phase to avoid output the duplicated logs at the next fetch request.
	if req.StepPhase != string(pod.Status.Phase) {
		r.AppendMessage(fmt.Sprintf("%s Status: %s", logPrefix, pod.Status.Phase))
		newStepPhase = string(pod.Status.Phase)
	}

	if isPodCompleted(pod) {
		// snip the pod logs when it complete
		// TODO(yancey1989): fetch the pod logs using an iteration way
		// to avoid the memory overflow
		podLogs, e := k8sReadPodLogs(pod.ObjectMeta.Name, "main", "", false)
		if e != nil {
			return nil, e
		}

		if e := r.AppendProtoMessages(podLogs); e != nil {
			return nil, e
		}

		// TODO(yancey1989): add duration time for the eoeResponse
		// eoe just used to simplify the client code which can be consistent with non-argo mode.
		if isPodFailed(pod) {
			e = fmt.Errorf("%s Failed, Log: %s\n%s", logPrefix, logURL, r.ErrorMessage())
			logger.Errorf("workflowFailed, %v, spent:%d", e, time.Now().Second()-wf.CreationTimestamp.Second())
			return r.ResponseWithStepComplete(req.Job.Id, "", newStepPhase, eof), e
		}
		logger.Infof("workflowSucceed, spent:%d", time.Now().Second()-wf.CreationTimestamp.Second())

		// move to the next step
		nextStepGroup, err := getNextStepGroup(wf, stepGroupName)
		if err != nil {
			return nil, err
		}
		// set the EOF to true if no next step in the workflow
		if nextStepGroup == "" && stepIdx == stepCnt {
			eof = true
		}

		if nextStepGroup != "" {
			newStepPhase = ""
			stepGroupName = nextStepGroup
		}
		return r.ResponseWithStepComplete(req.Job.Id, stepGroupName, newStepPhase, eof), nil
	}
	return r.Response(req.Job.Id, stepGroupName, newStepPhase, eof), nil
}

func parseOffset(content string) (string, string, error) {
	reTimestamps := regexp.MustCompile(`([^\s]+)\s(.*)$`)
	msg := reTimestamps.FindStringSubmatch(content)
	if len(msg) != 3 {
		return "", "", fmt.Errorf("Parse offset failed: %s", content)
	}
	return msg[1], msg[2], nil
}

func getOffsetAndContentFromLogs(logs []string, oldOffset string) ([]string, string, error) {
	// NOTE(yancey1989): using `kubectl --since-time <offset>` to get logs
	// from the offset which is the timestamps. the accuracy can only be achieved at the second level,
	// `kubectl` may return some duplicated logs as the previous fetch, and we need to skip them.
	buffer := []string{}
	skipOlderLogs := false
	offset := oldOffset
	for _, msg := range logs {
		newOffset, content, e := parseOffset(msg)
		if e != nil {
			break
		}
		if newOffset == oldOffset {
			skipOlderLogs = true
		} else {
			if skipOlderLogs || oldOffset == "" {
				buffer = append(buffer, content)
				offset = newOffset
			} else {
				// skip the duplicated logs as the previous fetch
				continue
			}
		}
	}
	return buffer, offset, nil
}

func getPodLogs(podName string, offset string) ([]string, string, error) {
	// NOTE(tony): A workflow pod usually contains two container: main and wait
	// I believe wait is used for management by Argo, so we only need to care about main.
	logs, err := k8sReadPodLogs(podName, "main", offset, true)
	if err != nil {
		return nil, "", err
	}

	return getOffsetAndContentFromLogs(logs, offset)
}
<|endoftext|>


<commit_before>// +build windows

/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package lcow

import (
	"context"
	"io"
	"os"
	"path"
	"time"

	"github.com/Microsoft/go-winio/pkg/security"
	"github.com/Microsoft/hcsshim/ext4/tar2ext4"
	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/diff"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/metadata"
	"github.com/containerd/containerd/mount"
	"github.com/containerd/containerd/plugin"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	// maxLcowVhdSizeGB is the max size in GB of any layer
	maxLcowVhdSizeGB = 128 * 1024 * 1024 * 1024
)

func init() {
	plugin.Register(&plugin.Registration{
		Type: plugin.DiffPlugin,
		ID:   "windows-lcow",
		Requires: []plugin.Type{
			plugin.MetadataPlugin,
		},
		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
			md, err := ic.Get(plugin.MetadataPlugin)
			if err != nil {
				return nil, err
			}

			ic.Meta.Platforms = append(ic.Meta.Platforms, ocispec.Platform{
				OS:           "linux",
				Architecture: "amd64",
			})
			return NewWindowsLcowDiff(md.(*metadata.DB).ContentStore())
		},
	})
}

// CompareApplier handles both comparison and
// application of layer diffs.
type CompareApplier interface {
	diff.Applier
	diff.Comparer
}

// windowsLcowDiff does filesystem comparison and application
// for Windows specific Linux layer diffs.
type windowsLcowDiff struct {
	store content.Store
}

var emptyDesc = ocispec.Descriptor{}

// NewWindowsLcowDiff is the Windows LCOW container layer implementation
// for comparing and applying Linux filesystem layers on Windows
func NewWindowsLcowDiff(store content.Store) (CompareApplier, error) {
	return windowsLcowDiff{
		store: store,
	}, nil
}

// Apply applies the content associated with the provided digests onto the
// provided mounts. Archive content will be extracted and decompressed if
// necessary.
func (s windowsLcowDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts []mount.Mount, opts ...diff.ApplyOpt) (d ocispec.Descriptor, err error) {
	t1 := time.Now()
	defer func() {
		if err == nil {
			log.G(ctx).WithFields(logrus.Fields{
				"d":     time.Since(t1),
				"dgst":  desc.Digest,
				"size":  desc.Size,
				"media": desc.MediaType,
			}).Debugf("diff applied")
		}
	}()

	var config diff.ApplyConfig
	for _, o := range opts {
		if err := o(ctx, desc, &config); err != nil {
			return emptyDesc, errors.Wrap(err, "failed to apply config opt")
		}
	}

	layer, _, err := mountsToLayerAndParents(mounts)
	if err != nil {
		return emptyDesc, err
	}

	ra, err := s.store.ReaderAt(ctx, desc)
	if err != nil {
		return emptyDesc, errors.Wrap(err, "failed to get reader from content store")
	}
	defer ra.Close()

	processor := diff.NewProcessorChain(desc.MediaType, content.NewReader(ra))
	for {
		if processor, err = diff.GetProcessor(ctx, processor, config.ProcessorPayloads); err != nil {
			return emptyDesc, errors.Wrapf(err, "failed to get stream processor for %s", desc.MediaType)
		}
		if processor.MediaType() == ocispec.MediaTypeImageLayer {
			break
		}
	}
	defer processor.Close()

	// Calculate the Digest as we go
	digester := digest.Canonical.Digester()
	rc := &readCounter{
		r: io.TeeReader(processor, digester.Hash()),
	}

	layerPath := path.Join(layer, "layer.vhd")
	outFile, err := os.Create(layerPath)
	if err != nil {
		return emptyDesc, err
	}
	defer func() {
		if err != nil {
			outFile.Close()
			os.Remove(layerPath)
		}
	}()

	err = tar2ext4.Convert(rc, outFile, tar2ext4.ConvertWhiteout, tar2ext4.AppendVhdFooter, tar2ext4.MaximumDiskSize(maxLcowVhdSizeGB))
	if err != nil {
		return emptyDesc, errors.Wrapf(err, "failed to convert tar2ext4 vhd")
	}
	err = outFile.Sync()
	if err != nil {
		return emptyDesc, errors.Wrapf(err, "failed to sync tar2ext4 vhd to disk")
	}
	outFile.Close()

	err = security.GrantVmGroupAccess(layerPath)
	if err != nil {
		return emptyDesc, errors.Wrapf(err, "failed GrantVmGroupAccess on layer vhd: %v", layerPath)
	}

	return ocispec.Descriptor{
		MediaType: ocispec.MediaTypeImageLayer,
		Size:      rc.c,
		Digest:    digester.Digest(),
	}, nil
}

// Compare creates a diff between the given mounts and uploads the result
// to the content store.
func (s windowsLcowDiff) Compare(ctx context.Context, lower, upper []mount.Mount, opts ...diff.Opt) (d ocispec.Descriptor, err error) {
	return emptyDesc, errors.Wrap(errdefs.ErrNotImplemented, "windowsLcowDiff does not implement Compare method")
}

type readCounter struct {
	r io.Reader
	c int64
}

func (rc *readCounter) Read(p []byte) (n int, err error) {
	n, err = rc.r.Read(p)
	rc.c += int64(n)
	return
}

func mountsToLayerAndParents(mounts []mount.Mount) (string, []string, error) {
	if len(mounts) != 1 {
		return "", nil, errors.Wrap(errdefs.ErrInvalidArgument, "number of mounts should always be 1 for Windows lcow-layers")
	}
	mnt := mounts[0]
	if mnt.Type != "lcow-layer" {
		return "", nil, errors.Wrap(errdefs.ErrInvalidArgument, "mount layer type must be lcow-layer")
	}

	parentLayerPaths, err := mnt.GetParentPaths()
	if err != nil {
		return "", nil, err
	}

	return mnt.Source, parentLayerPaths, nil
}
<commit_msg>Read trailing data from tar reader<commit_after>// +build windows

/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package lcow

import (
	"context"
	"io"
	"io/ioutil"
	"os"
	"path"
	"time"

	"github.com/Microsoft/go-winio/pkg/security"
	"github.com/Microsoft/hcsshim/ext4/tar2ext4"
	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/diff"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/metadata"
	"github.com/containerd/containerd/mount"
	"github.com/containerd/containerd/plugin"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	// maxLcowVhdSizeGB is the max size in GB of any layer
	maxLcowVhdSizeGB = 128 * 1024 * 1024 * 1024
)

func init() {
	plugin.Register(&plugin.Registration{
		Type: plugin.DiffPlugin,
		ID:   "windows-lcow",
		Requires: []plugin.Type{
			plugin.MetadataPlugin,
		},
		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
			md, err := ic.Get(plugin.MetadataPlugin)
			if err != nil {
				return nil, err
			}

			ic.Meta.Platforms = append(ic.Meta.Platforms, ocispec.Platform{
				OS:           "linux",
				Architecture: "amd64",
			})
			return NewWindowsLcowDiff(md.(*metadata.DB).ContentStore())
		},
	})
}

// CompareApplier handles both comparison and
// application of layer diffs.
type CompareApplier interface {
	diff.Applier
	diff.Comparer
}

// windowsLcowDiff does filesystem comparison and application
// for Windows specific Linux layer diffs.
type windowsLcowDiff struct {
	store content.Store
}

var emptyDesc = ocispec.Descriptor{}

// NewWindowsLcowDiff is the Windows LCOW container layer implementation
// for comparing and applying Linux filesystem layers on Windows
func NewWindowsLcowDiff(store content.Store) (CompareApplier, error) {
	return windowsLcowDiff{
		store: store,
	}, nil
}

// Apply applies the content associated with the provided digests onto the
// provided mounts. Archive content will be extracted and decompressed if
// necessary.
func (s windowsLcowDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts []mount.Mount, opts ...diff.ApplyOpt) (d ocispec.Descriptor, err error) {
	t1 := time.Now()
	defer func() {
		if err == nil {
			log.G(ctx).WithFields(logrus.Fields{
				"d":     time.Since(t1),
				"dgst":  desc.Digest,
				"size":  desc.Size,
				"media": desc.MediaType,
			}).Debugf("diff applied")
		}
	}()

	var config diff.ApplyConfig
	for _, o := range opts {
		if err := o(ctx, desc, &config); err != nil {
			return emptyDesc, errors.Wrap(err, "failed to apply config opt")
		}
	}

	layer, _, err := mountsToLayerAndParents(mounts)
	if err != nil {
		return emptyDesc, err
	}

	ra, err := s.store.ReaderAt(ctx, desc)
	if err != nil {
		return emptyDesc, errors.Wrap(err, "failed to get reader from content store")
	}
	defer ra.Close()

	processor := diff.NewProcessorChain(desc.MediaType, content.NewReader(ra))
	for {
		if processor, err = diff.GetProcessor(ctx, processor, config.ProcessorPayloads); err != nil {
			return emptyDesc, errors.Wrapf(err, "failed to get stream processor for %s", desc.MediaType)
		}
		if processor.MediaType() == ocispec.MediaTypeImageLayer {
			break
		}
	}
	defer processor.Close()

	// Calculate the Digest as we go
	digester := digest.Canonical.Digester()
	rc := &readCounter{
		r: io.TeeReader(processor, digester.Hash()),
	}

	layerPath := path.Join(layer, "layer.vhd")
	outFile, err := os.Create(layerPath)
	if err != nil {
		return emptyDesc, err
	}
	defer func() {
		if err != nil {
			outFile.Close()
			os.Remove(layerPath)
		}
	}()

	err = tar2ext4.Convert(rc, outFile, tar2ext4.ConvertWhiteout, tar2ext4.AppendVhdFooter, tar2ext4.MaximumDiskSize(maxLcowVhdSizeGB))
	if err != nil {
		return emptyDesc, errors.Wrapf(err, "failed to convert tar2ext4 vhd")
	}
	err = outFile.Sync()
	if err != nil {
		return emptyDesc, errors.Wrapf(err, "failed to sync tar2ext4 vhd to disk")
	}
	outFile.Close()

	// Read any trailing data
	if _, err := io.Copy(ioutil.Discard, rc); err != nil {
		return emptyDesc, err
	}

	err = security.GrantVmGroupAccess(layerPath)
	if err != nil {
		return emptyDesc, errors.Wrapf(err, "failed GrantVmGroupAccess on layer vhd: %v", layerPath)
	}

	return ocispec.Descriptor{
		MediaType: ocispec.MediaTypeImageLayer,
		Size:      rc.c,
		Digest:    digester.Digest(),
	}, nil
}

// Compare creates a diff between the given mounts and uploads the result
// to the content store.
func (s windowsLcowDiff) Compare(ctx context.Context, lower, upper []mount.Mount, opts ...diff.Opt) (d ocispec.Descriptor, err error) {
	return emptyDesc, errors.Wrap(errdefs.ErrNotImplemented, "windowsLcowDiff does not implement Compare method")
}

type readCounter struct {
	r io.Reader
	c int64
}

func (rc *readCounter) Read(p []byte) (n int, err error) {
	n, err = rc.r.Read(p)
	rc.c += int64(n)
	return
}

func mountsToLayerAndParents(mounts []mount.Mount) (string, []string, error) {
	if len(mounts) != 1 {
		return "", nil, errors.Wrap(errdefs.ErrInvalidArgument, "number of mounts should always be 1 for Windows lcow-layers")
	}
	mnt := mounts[0]
	if mnt.Type != "lcow-layer" {
		return "", nil, errors.Wrap(errdefs.ErrInvalidArgument, "mount layer type must be lcow-layer")
	}

	parentLayerPaths, err := mnt.GetParentPaths()
	if err != nil {
		return "", nil, err
	}

	return mnt.Source, parentLayerPaths, nil
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package piazza

import (
	"errors"
	"log"
	"net/http"
	"syscall"
	"time"

	"github.com/gin-gonic/gin"
)

const ginHammerTime = 3 * time.Second

// GenericServer is the basic framework for standing up a gin-based server. It has methods
// to Start and Stop as well as to define the routing paths.
type GenericServer struct {
	Sys    *SystemConfig
	pid    int
	router http.Handler
}

// RouteData describes one server route: which http verb, the path string, and the handler to use.
type RouteData struct {
	Verb    string
	Path    string
	Handler gin.HandlerFunc
}

// Stop will request the server to shutdown. It will wait for the service to die before returning.
func (server *GenericServer) Stop() error {
	sys := server.Sys

	err := syscall.Kill(server.pid, syscall.SIGINT)
	if err != nil {
		return err
	}

	err = sys.WaitForServiceToDieByAddress(sys.Name, sys.BindTo)
	return err
}

// Start will start the service. You must call Configure first.
func (server *GenericServer) Start() (chan error, error) {

	sys := server.Sys

	done := make(chan error)

	ginServer := http.Server{Addr: server.Sys.BindTo, Handler: server.router}

	server.pid = syscall.Getpid()
	if sys.BindTo == "" {
		sys.BindTo = ":http"
	}

	go func() {
		err := ginServer.ListenAndServe()
		done <- err
	}()

	err := sys.WaitForServiceByAddress(sys.Name, sys.BindTo)
	if err != nil {
		return nil, err
	}

	log.Printf("Server %s started on %s (%s)", sys.Name, sys.Address, sys.BindTo)

	sys.AddService(sys.Name, sys.BindTo)

	return done, nil
}

// Configure will take the give RouteData and register them with the server.
func (server *GenericServer) Configure(routeData []RouteData) error {
	gin.SetMode(gin.ReleaseMode)

	router := gin.New()

	//router.Use(gin.Logger())
	//router.Use(gin.Recovery())

	for _, data := range routeData {
		switch data.Verb {
		case "GET":
			router.GET(data.Path, data.Handler)
		case "POST":
			router.POST(data.Path, data.Handler)
		case "PUT":
			router.PUT(data.Path, data.Handler)
		case "DELETE":
			router.DELETE(data.Path, data.Handler)
		default:
			return errors.New("Invalid verb: " + data.Verb)
		}
	}

	server.router = router

	return nil
}
<commit_msg>Now works on windows<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package piazza

import (
	"errors"
	"log"
	"net/http"
	"os"
	"syscall"
	"time"

	"github.com/gin-gonic/gin"
)

const ginHammerTime = 3 * time.Second

// GenericServer is the basic framework for standing up a gin-based server. It has methods
// to Start and Stop as well as to define the routing paths.
type GenericServer struct {
	Sys    *SystemConfig
	pid    int
	router http.Handler
}

// RouteData describes one server route: which http verb, the path string, and the handler to use.
type RouteData struct {
	Verb    string
	Path    string
	Handler gin.HandlerFunc
}

// Stop will request the server to shutdown. It will wait for the service to die before returning.
func (server *GenericServer) Stop() error {
	kp, err := os.FindProcess(server.pid)
	if err != nil {
		return err
	}
	err = kp.Kill()
	if err != nil {
		return err
	}

	sys := server.Sys
	err = sys.WaitForServiceToDieByAddress(sys.Name, sys.BindTo)
	return err
}

// Start will start the service. You must call Configure first.
func (server *GenericServer) Start() (chan error, error) {

	sys := server.Sys

	done := make(chan error)

	ginServer := http.Server{Addr: server.Sys.BindTo, Handler: server.router}

	server.pid = syscall.Getpid()
	if sys.BindTo == "" {
		sys.BindTo = ":http"
	}

	go func() {
		err := ginServer.ListenAndServe()
		done <- err
	}()

	err := sys.WaitForServiceByAddress(sys.Name, sys.BindTo)
	if err != nil {
		return nil, err
	}

	log.Printf("Server %s started on %s (%s)", sys.Name, sys.Address, sys.BindTo)

	sys.AddService(sys.Name, sys.BindTo)

	return done, nil
}

// Configure will take the give RouteData and register them with the server.
func (server *GenericServer) Configure(routeData []RouteData) error {
	gin.SetMode(gin.ReleaseMode)

	router := gin.New()

	//router.Use(gin.Logger())
	//router.Use(gin.Recovery())

	for _, data := range routeData {
		switch data.Verb {
		case "GET":
			router.GET(data.Path, data.Handler)
		case "POST":
			router.POST(data.Path, data.Handler)
		case "PUT":
			router.PUT(data.Path, data.Handler)
		case "DELETE":
			router.DELETE(data.Path, data.Handler)
		default:
			return errors.New("Invalid verb: " + data.Verb)
		}
	}

	server.router = router

	return nil
}
<|endoftext|>