<commit_before>package irc

import (
	"fmt"
	"strings"
	"time"
)

func joinedLen(names []string) int {
	var l = len(names) - 1 // " " between names
	for _, name := range names {
		l += len(name)
	}
	return l
}

type BaseReply struct {
	source  Identifier
	message string
}

func (reply *BaseReply) Source() Identifier {
	return reply.source
}

type StringReply struct {
	*BaseReply
	code string
}

func NewStringReply(source Identifier, code string,
	format string, args ...interface{}) *StringReply {
	message := fmt.Sprintf(format, args...)
	fullMessage := fmt.Sprintf(":%s %s %s", source.Id(), code, message)
	return &StringReply{
		BaseReply: &BaseReply{source, fullMessage},
		code:      code,
	}
}

func (reply *StringReply) Format(client *Client, write chan<- string) {
	write <- reply.message
}

func (reply *StringReply) String() string {
	return fmt.Sprintf("Reply(source=%s, code=%s, message=%s)",
		reply.source, reply.code, reply.message)
}

type NumericReply struct {
	*BaseReply
	code int
}

func NewNumericReply(source Identifier, code int, format string,
	args ...interface{}) *NumericReply {
	return &NumericReply{
		BaseReply: &BaseReply{source, fmt.Sprintf(format, args...)},
		code:      code,
	}
}

func (reply *NumericReply) Format(client *Client, write chan<- string) {
	write <- reply.FormatString(client)
}

func (reply *NumericReply) FormatString(client *Client) string {
	return fmt.Sprintf(":%s %03d %s %s", reply.Source().Id(), reply.code,
		client.Nick(), reply.message)
}

func (reply *NumericReply) String() string {
	return fmt.Sprintf("Reply(source=%s, code=%d, message=%s)",
		reply.source, reply.code, reply.message)
}

// names reply

type NamesReply struct {
	*BaseReply
	channel *Channel
}

func NewNamesReply(channel *Channel) Reply {
	return &NamesReply{
		BaseReply: &BaseReply{
			source: channel,
		},
		channel: channel,
	}
}

func (reply *NamesReply) Format(client *Client, write chan<- string) {
	base := RplNamReply(reply.channel, []string{})
	baseLen := len(base.FormatString(client))
	tooLong := func(names []string) bool {
		return (baseLen + joinedLen(names)) > MAX_REPLY_LEN
	}
	from, to := 0, 1
	nicks := reply.channel.Nicks()
	for to < len(nicks) {
		if (from < (to - 1)) && tooLong(nicks[from:to]) {
			RplNamReply(reply.channel, nicks[from:to-1]).Format(client, write)
			from, to = to-1, to
		} else {
			to += 1
		}
	}
	if from < len(nicks) {
		RplNamReply(reply.channel, nicks[from:]).Format(client, write)
	}
	RplEndOfNames(reply.channel).Format(client, write)
}

func (reply *NamesReply) String() string {
	return fmt.Sprintf("NamesReply(channel=%s, names=%s)",
		reply.channel, reply.channel.Nicks())
}

// messaging replies

func RplPrivMsg(source Identifier, target Identifier, message string) Reply {
	return NewStringReply(source, RPL_PRIVMSG, "%s :%s", target.Nick(), message)
}

func RplNick(source Identifier, newNick string) Reply {
	return NewStringReply(source, RPL_NICK, newNick)
}

func RplJoin(client *Client, channel *Channel) Reply {
	return NewStringReply(client, RPL_JOIN, channel.name)
}

func RplPart(client *Client, channel *Channel, message string) Reply {
	return NewStringReply(client, RPL_PART, "%s :%s", channel.name, message)
}

func RplPing(server *Server, target Identifier) Reply {
	return NewStringReply(server, RPL_PING, target.Nick())
}

func RplPong(server *Server, client *Client) Reply {
	return NewStringReply(server, RPL_PONG, client.Nick())
}

func RplQuit(client *Client, message string) Reply {
	return NewStringReply(client, RPL_QUIT, ":%s", message)
}

func RplError(server *Server, target Identifier) Reply {
	return NewStringReply(server, RPL_ERROR, target.Nick())
}

func RplInviteMsg(channel *Channel, inviter *Client) Reply {
	return NewStringReply(inviter, RPL_INVITE, channel.name)
}

// numeric replies

func RplWelcome(source Identifier, client *Client) Reply {
	return NewNumericReply(source, RPL_WELCOME,
		"Welcome to the Internet Relay Network %s", client.Id())
}

func RplYourHost(server *Server) Reply {
	return NewNumericReply(server, RPL_YOURHOST,
		"Your host is %s, running version %s", server.name, VERSION)
}

func RplCreated(server *Server) Reply {
	return NewNumericReply(server, RPL_CREATED,
		"This server was created %s", server.ctime.Format(time.RFC1123))
}

func RplMyInfo(server *Server) Reply {
	return NewNumericReply(server, RPL_MYINFO,
		"%s %s aiOorsw abeIikmntpqrsl", server.name, VERSION)
}

func RplUModeIs(server *Server, client *Client) Reply {
	return NewNumericReply(server, RPL_UMODEIS, client.ModeString())
}

func RplNoTopic(channel *Channel) Reply {
	return NewNumericReply(channel.server, RPL_NOTOPIC,
		"%s :No topic is set", channel.name)
}

func RplTopic(channel *Channel) Reply {
	return NewNumericReply(channel.server, RPL_TOPIC,
		"%s :%s", channel.name, channel.topic)
}

func RplInvitingMsg(channel *Channel, invitee *Client) Reply {
	return NewNumericReply(channel.server, RPL_INVITING,
		"%s %s", channel.name, invitee.Nick())
}

func RplNamReply(channel *Channel, names []string) *NumericReply {
	return NewNumericReply(channel.server, RPL_NAMREPLY, "= %s :%s",
		channel.name, strings.Join(names, " "))
}

func RplEndOfNames(channel *Channel) Reply {
	return NewNumericReply(channel, RPL_ENDOFNAMES,
		"%s :End of NAMES list", channel.name)
}

// :You are now an IRC operator
func RplYoureOper(server *Server) Reply {
	return NewNumericReply(server, RPL_YOUREOPER, ":You are now an IRC operator")
}

func RplWhoisUser(server *Server, client *Client) Reply {
	return NewNumericReply(server, RPL_WHOISUSER, "%s %s %s * :%s",
		client.nick, client.username, client.hostname, client.realname)
}

func RplEndOfWhois(server *Server) Reply {
	return NewNumericReply(server, RPL_ENDOFWHOIS, ":End of WHOIS list")
}

func RplChannelModeIs(channel *Channel) Reply {
	return NewNumericReply(channel.server, RPL_CHANNELMODEIS, "%s %s",
		channel.name, channel.ModeString())
}

// <channel> <user> <host> <server> <nick> ( "H" / "G" ) ["*"] [ ( "@" / "+" ) ]
// :<hopcount> <real name>
func RplWhoReply(server *Server, channel *Channel, client *Client) Reply {
	return NewNumericReply(server, RPL_WHOREPLY, "%s %s %s %s %s H :0 %s",
		channel.name, client.username, client.hostname, server.name, client.nick,
		client.realname)
}

// <name> :End of WHO list
func RplEndOfWho(server *Server, name string) Reply {
	return NewNumericReply(server, RPL_ENDOFWHO, "%s :End of WHO list", name)
}

func RplBanList(channel *Channel, ban UserMask) Reply {
	return NewNumericReply(channel.server, RPL_BANLIST, "%s %s", channel.name, ban)
}

func RplEndOfBanList(channel *Channel) Reply {
	return NewNumericReply(channel.server, RPL_ENDOFBANLIST, "%s :End of channel ban list",
		channel.name)
}

// errors (also numeric)

func ErrAlreadyRegistered(source Identifier) Reply {
	return NewNumericReply(source, ERR_ALREADYREGISTRED,
		":You may not reregister")
}

func ErrNickNameInUse(source Identifier, nick string) Reply {
	return NewNumericReply(source, ERR_NICKNAMEINUSE,
		"%s :Nickname is already in use", nick)
}

func ErrUnknownCommand(source Identifier, command string) Reply {
	return NewNumericReply(source, ERR_UNKNOWNCOMMAND,
		"%s :Unknown command", command)
}

func ErrUsersDontMatch(source Identifier) Reply {
	return NewNumericReply(source, ERR_USERSDONTMATCH,
		":Cannot change mode for other users")
}

func ErrNeedMoreParams(source Identifier, command string) Reply {
	return NewNumericReply(source, ERR_NEEDMOREPARAMS,
		"%s :Not enough parameters", command)
}

func ErrNoSuchChannel(server *Server, channel string) Reply {
	return NewNumericReply(server, ERR_NOSUCHCHANNEL,
		"%s :No such channel", channel)
}

func ErrUserOnChannel(channel *Channel, member *Client) Reply {
	return NewNumericReply(channel.server, ERR_USERONCHANNEL,
		"%s %s :is already on channel", member.nick, channel.name)
}

func ErrNotOnChannel(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_NOTONCHANNEL,
		"%s :You're not on that channel", channel.name)
}

func ErrInviteOnlyChannel(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_INVITEONLYCHAN,
		"%s :Cannot join channel (+i)", channel.name)
}

func ErrBadChannelKey(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_BADCHANNELKEY,
		"%s :Cannot join channel (+k)", channel.name)
}

func ErrNoSuchNick(source Identifier, nick string) Reply {
	return NewNumericReply(source, ERR_NOSUCHNICK,
		"%s :No such nick/channel", nick)
}

func ErrPasswdMismatch(server *Server) Reply {
	return NewNumericReply(server, ERR_PASSWDMISMATCH, ":Password incorrect")
}

func ErrNoChanModes(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_NOCHANMODES,
		"%s :Channel doesn't support modes", channel.name)
}

func ErrNoPrivileges(server *Server) Reply {
	return NewNumericReply(server, ERR_NOPRIVILEGES, ":Permission Denied")
}

func ErrRestricted(server *Server) Reply {
	return NewNumericReply(server, ERR_RESTRICTED, ":Your connection is restricted!")
}

func ErrNoSuchServer(server *Server, target string) Reply {
	return NewNumericReply(server, ERR_NOSUCHSERVER, "%s :No such server", target)
}

func ErrUserNotInChannel(server *Server, nick string, channel *Channel) Reply {
	return NewNumericReply(server, ERR_USERNOTINCHANNEL,
		"%s %s :They aren't on that channel", nick, channel.name)
}

func ErrCannotSendToChan(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_CANNOTSENDTOCHAN,
		"%s :Cannot send to channel", channel.name)
}

// <channel> :You're not channel operator
func ErrChanOPrivIsNeeded(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_CHANOPRIVSNEEDED,
		"%s :You're not channel operator", channel.name)
}
<commit_msg>avoid nil reference in who reply<commit_after>package irc

import (
	"fmt"
	"strings"
	"time"
)

func joinedLen(names []string) int {
	var l = len(names) - 1 // " " between names
	for _, name := range names {
		l += len(name)
	}
	return l
}

type BaseReply struct {
	source  Identifier
	message string
}

func (reply *BaseReply) Source() Identifier {
	return reply.source
}

type StringReply struct {
	*BaseReply
	code string
}

func NewStringReply(source Identifier, code string,
	format string, args ...interface{}) *StringReply {
	message := fmt.Sprintf(format, args...)
	fullMessage := fmt.Sprintf(":%s %s %s", source.Id(), code, message)
	return &StringReply{
		BaseReply: &BaseReply{source, fullMessage},
		code:      code,
	}
}

func (reply *StringReply) Format(client *Client, write chan<- string) {
	write <- reply.message
}

func (reply *StringReply) String() string {
	return fmt.Sprintf("Reply(source=%s, code=%s, message=%s)",
		reply.source, reply.code, reply.message)
}

type NumericReply struct {
	*BaseReply
	code int
}

func NewNumericReply(source Identifier, code int, format string,
	args ...interface{}) *NumericReply {
	return &NumericReply{
		BaseReply: &BaseReply{source, fmt.Sprintf(format, args...)},
		code:      code,
	}
}

func (reply *NumericReply) Format(client *Client, write chan<- string) {
	write <- reply.FormatString(client)
}

func (reply *NumericReply) FormatString(client *Client) string {
	return fmt.Sprintf(":%s %03d %s %s", reply.Source().Id(), reply.code,
		client.Nick(), reply.message)
}

func (reply *NumericReply) String() string {
	return fmt.Sprintf("Reply(source=%s, code=%d, message=%s)",
		reply.source, reply.code, reply.message)
}

// names reply

type NamesReply struct {
	*BaseReply
	channel *Channel
}

func NewNamesReply(channel *Channel) Reply {
	return &NamesReply{
		BaseReply: &BaseReply{
			source: channel,
		},
		channel: channel,
	}
}

func (reply *NamesReply) Format(client *Client, write chan<- string) {
	base := RplNamReply(reply.channel, []string{})
	baseLen := len(base.FormatString(client))
	tooLong := func(names []string) bool {
		return (baseLen + joinedLen(names)) > MAX_REPLY_LEN
	}
	from, to := 0, 1
	nicks := reply.channel.Nicks()
	for to < len(nicks) {
		if (from < (to - 1)) && tooLong(nicks[from:to]) {
			RplNamReply(reply.channel, nicks[from:to-1]).Format(client, write)
			from, to = to-1, to
		} else {
			to += 1
		}
	}
	if from < len(nicks) {
		RplNamReply(reply.channel, nicks[from:]).Format(client, write)
	}
	RplEndOfNames(reply.channel).Format(client, write)
}

func (reply *NamesReply) String() string {
	return fmt.Sprintf("NamesReply(channel=%s, names=%s)",
		reply.channel, reply.channel.Nicks())
}

// messaging replies

func RplPrivMsg(source Identifier, target Identifier, message string) Reply {
	return NewStringReply(source, RPL_PRIVMSG, "%s :%s", target.Nick(), message)
}

func RplNick(source Identifier, newNick string) Reply {
	return NewStringReply(source, RPL_NICK, newNick)
}

func RplJoin(client *Client, channel *Channel) Reply {
	return NewStringReply(client, RPL_JOIN, channel.name)
}

func RplPart(client *Client, channel *Channel, message string) Reply {
	return NewStringReply(client, RPL_PART, "%s :%s", channel.name, message)
}

func RplPing(server *Server, target Identifier) Reply {
	return NewStringReply(server, RPL_PING, target.Nick())
}

func RplPong(server *Server, client *Client) Reply {
	return NewStringReply(server, RPL_PONG, client.Nick())
}

func RplQuit(client *Client, message string) Reply {
	return NewStringReply(client, RPL_QUIT, ":%s", message)
}

func RplError(server *Server, target Identifier) Reply {
	return NewStringReply(server, RPL_ERROR, target.Nick())
}

func RplInviteMsg(channel *Channel, inviter *Client) Reply {
	return NewStringReply(inviter, RPL_INVITE, channel.name)
}

// numeric replies

func RplWelcome(source Identifier, client *Client) Reply {
	return NewNumericReply(source, RPL_WELCOME,
		"Welcome to the Internet Relay Network %s", client.Id())
}

func RplYourHost(server *Server) Reply {
	return NewNumericReply(server, RPL_YOURHOST,
		"Your host is %s, running version %s", server.name, VERSION)
}

func RplCreated(server *Server) Reply {
	return NewNumericReply(server, RPL_CREATED,
		"This server was created %s", server.ctime.Format(time.RFC1123))
}

func RplMyInfo(server *Server) Reply {
	return NewNumericReply(server, RPL_MYINFO,
		"%s %s aiOorsw abeIikmntpqrsl", server.name, VERSION)
}

func RplUModeIs(server *Server, client *Client) Reply {
	return NewNumericReply(server, RPL_UMODEIS, client.ModeString())
}

func RplNoTopic(channel *Channel) Reply {
	return NewNumericReply(channel.server, RPL_NOTOPIC,
		"%s :No topic is set", channel.name)
}

func RplTopic(channel *Channel) Reply {
	return NewNumericReply(channel.server, RPL_TOPIC,
		"%s :%s", channel.name, channel.topic)
}

func RplInvitingMsg(channel *Channel, invitee *Client) Reply {
	return NewNumericReply(channel.server, RPL_INVITING,
		"%s %s", channel.name, invitee.Nick())
}

func RplNamReply(channel *Channel, names []string) *NumericReply {
	return NewNumericReply(channel.server, RPL_NAMREPLY, "= %s :%s",
		channel.name, strings.Join(names, " "))
}

func RplEndOfNames(channel *Channel) Reply {
	return NewNumericReply(channel, RPL_ENDOFNAMES,
		"%s :End of NAMES list", channel.name)
}

// :You are now an IRC operator
func RplYoureOper(server *Server) Reply {
	return NewNumericReply(server, RPL_YOUREOPER, ":You are now an IRC operator")
}

func RplWhoisUser(server *Server, client *Client) Reply {
	return NewNumericReply(server, RPL_WHOISUSER, "%s %s %s * :%s",
		client.nick, client.username, client.hostname, client.realname)
}

func RplEndOfWhois(server *Server) Reply {
	return NewNumericReply(server, RPL_ENDOFWHOIS, ":End of WHOIS list")
}

func RplChannelModeIs(channel *Channel) Reply {
	return NewNumericReply(channel.server, RPL_CHANNELMODEIS, "%s %s",
		channel.name, channel.ModeString())
}

// <channel> <user> <host> <server> <nick> ( "H" / "G" ) ["*"] [ ( "@" / "+" ) ]
// :<hopcount> <real name>
func RplWhoReply(server *Server, channel *Channel, client *Client) Reply {
	channelName := "*"
	if channel != nil {
		channelName = channel.name
	}
	return NewNumericReply(server, RPL_WHOREPLY, "%s %s %s %s %s H :0 %s",
		channelName, client.username, client.hostname, server.name, client.nick,
		client.realname)
}

// <name> :End of WHO list
func RplEndOfWho(server *Server, name string) Reply {
	return NewNumericReply(server, RPL_ENDOFWHO, "%s :End of WHO list", name)
}

func RplBanList(channel *Channel, ban UserMask) Reply {
	return NewNumericReply(channel.server, RPL_BANLIST, "%s %s", channel.name, ban)
}

func RplEndOfBanList(channel *Channel) Reply {
	return NewNumericReply(channel.server, RPL_ENDOFBANLIST, "%s :End of channel ban list",
		channel.name)
}

// errors (also numeric)

func ErrAlreadyRegistered(source Identifier) Reply {
	return NewNumericReply(source, ERR_ALREADYREGISTRED,
		":You may not reregister")
}

func ErrNickNameInUse(source Identifier, nick string) Reply {
	return NewNumericReply(source, ERR_NICKNAMEINUSE,
		"%s :Nickname is already in use", nick)
}

func ErrUnknownCommand(source Identifier, command string) Reply {
	return NewNumericReply(source, ERR_UNKNOWNCOMMAND,
		"%s :Unknown command", command)
}

func ErrUsersDontMatch(source Identifier) Reply {
	return NewNumericReply(source, ERR_USERSDONTMATCH,
		":Cannot change mode for other users")
}

func ErrNeedMoreParams(source Identifier, command string) Reply {
	return NewNumericReply(source, ERR_NEEDMOREPARAMS,
		"%s :Not enough parameters", command)
}

func ErrNoSuchChannel(server *Server, channel string) Reply {
	return NewNumericReply(server, ERR_NOSUCHCHANNEL,
		"%s :No such channel", channel)
}

func ErrUserOnChannel(channel *Channel, member *Client) Reply {
	return NewNumericReply(channel.server, ERR_USERONCHANNEL,
		"%s %s :is already on channel", member.nick, channel.name)
}

func ErrNotOnChannel(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_NOTONCHANNEL,
		"%s :You're not on that channel", channel.name)
}

func ErrInviteOnlyChannel(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_INVITEONLYCHAN,
		"%s :Cannot join channel (+i)", channel.name)
}

func ErrBadChannelKey(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_BADCHANNELKEY,
		"%s :Cannot join channel (+k)", channel.name)
}

func ErrNoSuchNick(source Identifier, nick string) Reply {
	return NewNumericReply(source, ERR_NOSUCHNICK,
		"%s :No such nick/channel", nick)
}

func ErrPasswdMismatch(server *Server) Reply {
	return NewNumericReply(server, ERR_PASSWDMISMATCH, ":Password incorrect")
}

func ErrNoChanModes(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_NOCHANMODES,
		"%s :Channel doesn't support modes", channel.name)
}

func ErrNoPrivileges(server *Server) Reply {
	return NewNumericReply(server, ERR_NOPRIVILEGES, ":Permission Denied")
}

func ErrRestricted(server *Server) Reply {
	return NewNumericReply(server, ERR_RESTRICTED, ":Your connection is restricted!")
}

func ErrNoSuchServer(server *Server, target string) Reply {
	return NewNumericReply(server, ERR_NOSUCHSERVER, "%s :No such server", target)
}

func ErrUserNotInChannel(server *Server, nick string, channel *Channel) Reply {
	return NewNumericReply(server, ERR_USERNOTINCHANNEL,
		"%s %s :They aren't on that channel", nick, channel.name)
}

func ErrCannotSendToChan(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_CANNOTSENDTOCHAN,
		"%s :Cannot send to channel", channel.name)
}

// <channel> :You're not channel operator
func ErrChanOPrivIsNeeded(channel *Channel) Reply {
	return NewNumericReply(channel.server, ERR_CHANOPRIVSNEEDED,
		"%s :You're not channel operator", channel.name)
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"path"
	"strings"
	"sync"
	"time"
)

// updateUploadsJSON - update `uploads.json` with new uploadsJSON for all disks.
func (xl xlObjects) updateUploadsJSON(bucket, object string, uploadsJSON uploadsV1) (err error) {
	uploadsPath := path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)
	uniqueID := getUUID()
	tmpUploadsPath := path.Join(tmpMetaPrefix, uniqueID)
	var errs = make([]error, len(xl.storageDisks))
	var wg = &sync.WaitGroup{}

	// Update `uploads.json` for all the disks.
	for index, disk := range xl.storageDisks {
		if disk == nil {
			errs[index] = errDiskNotFound
			continue
		}
		wg.Add(1)
		// Update `uploads.json` in routine.
		go func(index int, disk StorageAPI) {
			defer wg.Done()
			uploadsBytes, wErr := json.Marshal(uploadsJSON)
			if wErr != nil {
				errs[index] = wErr
				return
			}
			if wErr = disk.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsBytes); wErr != nil {
				errs[index] = wErr
				return
			}
			if wErr = disk.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath); wErr != nil {
				errs[index] = wErr
				return
			}
		}(index, disk)
	}

	// Wait for all the routines to finish updating `uploads.json`
	wg.Wait()

	// Count all the errors and validate if we have write quorum.
	if !isQuorum(errs, xl.writeQuorum) {
		// Rename `uploads.json` left over back to tmp location.
		for index, disk := range xl.storageDisks {
			if disk == nil {
				continue
			}
			// Undo rename `uploads.json` in parallel.
			wg.Add(1)
			go func(index int, disk StorageAPI) {
				defer wg.Done()
				if errs[index] != nil {
					return
				}
				_ = disk.RenameFile(minioMetaBucket, uploadsPath, minioMetaBucket, tmpUploadsPath)
			}(index, disk)
		}
		wg.Wait()
		return errXLWriteQuorum
	}
	return nil
}

// writeUploadJSON - create `uploads.json` or update it with new uploadID.
func (xl xlObjects) writeUploadJSON(bucket, object, uploadID string, initiated time.Time) (err error) {
	uploadsPath := path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)
	uniqueID := getUUID()
	tmpUploadsPath := path.Join(tmpMetaPrefix, uniqueID)

	var errs = make([]error, len(xl.storageDisks))
	var wg = &sync.WaitGroup{}

	var uploadsJSON uploadsV1
	for _, disk := range xl.storageDisks {
		if disk == nil {
			continue
		}
		uploadsJSON, err = readUploadsJSON(bucket, object, disk)
		break
	}
	if err != nil {
		// For any other errors.
		if err != errFileNotFound {
			return err
		}
		// Set uploads format to `xl` otherwise.
		uploadsJSON = newUploadsV1("xl")
	}
	// Add a new upload id.
	uploadsJSON.AddUploadID(uploadID, initiated)

	// Update `uploads.json` on all disks.
	for index, disk := range xl.storageDisks {
		if disk == nil {
			errs[index] = errDiskNotFound
			continue
		}
		wg.Add(1)
		// Update `uploads.json` in a routine.
		go func(index int, disk StorageAPI) {
			defer wg.Done()
			uploadsJSONBytes, wErr := json.Marshal(&uploadsJSON)
			if wErr != nil {
				errs[index] = wErr
				return
			}
			// Write `uploads.json` to disk.
			if wErr = disk.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsJSONBytes); wErr != nil {
				errs[index] = wErr
				return
			}
			wErr = disk.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath)
			if wErr != nil {
				if dErr := disk.DeleteFile(minioMetaBucket, tmpUploadsPath); dErr != nil {
					errs[index] = dErr
					return
				}
				errs[index] = wErr
				return
			}
			errs[index] = nil
		}(index, disk)
	}

	// Wait here for all the writes to finish.
	wg.Wait()

	// Count all the errors and validate if we have write quorum.
	if !isQuorum(errs, xl.writeQuorum) {
		// Rename `uploads.json` left over back to tmp location.
		for index, disk := range xl.storageDisks {
			if disk == nil {
				continue
			}
			// Undo rename `uploads.json` in parallel.
			wg.Add(1)
			go func(index int, disk StorageAPI) {
				defer wg.Done()
				if errs[index] != nil {
					return
				}
				_ = disk.RenameFile(minioMetaBucket, uploadsPath, minioMetaBucket, tmpUploadsPath)
			}(index, disk)
		}
		wg.Wait()
		return errXLWriteQuorum
	}
	return nil
}

// Returns if the prefix is a multipart upload.
func (xl xlObjects) isMultipartUpload(bucket, prefix string) bool {
	for _, disk := range xl.getLoadBalancedQuorumDisks() {
		if disk == nil {
			continue
		}
		_, err := disk.StatFile(bucket, pathJoin(prefix, uploadsJSONFile))
		if err != nil {
			// For any reason disk was deleted or goes offline, continue
			if err == errDiskNotFound || err == errFaultyDisk {
				continue
			}
			return false
		}
		break
	}
	return true
}

// listUploadsInfo - list all uploads info.
func (xl xlObjects) listUploadsInfo(prefixPath string) (uploadsInfo []uploadInfo, err error) {
	for _, disk := range xl.getLoadBalancedQuorumDisks() {
		if disk == nil {
			continue
		}
		splitPrefixes := strings.SplitN(prefixPath, "/", 3)
		var uploadsJSON uploadsV1
		uploadsJSON, err = readUploadsJSON(splitPrefixes[1], splitPrefixes[2], disk)
		if err != nil {
			// For any reason disk was deleted or goes offline, continue
			if err == errDiskNotFound || err == errFaultyDisk {
				continue
			}
			if err == errFileNotFound {
				return []uploadInfo{}, nil
			}
			return nil, err
		}
		uploadsInfo = uploadsJSON.Uploads
		break
	}
	return uploadsInfo, nil
}

// isUploadIDExists - verify if a given uploadID exists and is valid.
func (xl xlObjects) isUploadIDExists(bucket, object, uploadID string) bool {
	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
	return xl.isObject(minioMetaBucket, uploadIDPath)
}

// Removes part given by partName belonging to a mulitpart upload from minioMetaBucket
func (xl xlObjects) removeObjectPart(bucket, object, uploadID, partName string) {
	curpartPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, partName)
	wg := sync.WaitGroup{}
	for i, disk := range xl.storageDisks {
		if disk == nil {
			continue
		}
		wg.Add(1)
		go func(index int, disk StorageAPI) {
			defer wg.Done()
			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
			// requests. xl.json is the authoritative source of truth on which parts constitute
			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
			_ = disk.DeleteFile(minioMetaBucket, curpartPath)
		}(i, disk)
	}
	wg.Wait()
}

// statPart - returns fileInfo structure for a successful stat on part file.
func (xl xlObjects) statPart(bucket, object, uploadID, partName string) (fileInfo FileInfo, err error) {
	partNamePath := path.Join(mpartMetaPrefix, bucket, object, uploadID, partName)
	for _, disk := range xl.getLoadBalancedQuorumDisks() {
		if disk == nil {
			continue
		}
		fileInfo, err = disk.StatFile(minioMetaBucket, partNamePath)
		if err != nil {
			// For any reason disk was deleted or goes offline, continue
			if err == errDiskNotFound {
				continue
			}
			return FileInfo{}, err
		}
		break
	}
	return fileInfo, nil
}

// commitXLMetadata - commit `xl.json` from source prefix to destination prefix.
func (xl xlObjects) commitXLMetadata(srcPrefix, dstPrefix string) error {
	var wg = &sync.WaitGroup{}
	var mErrs = make([]error, len(xl.storageDisks))

	srcJSONFile := path.Join(srcPrefix, xlMetaJSONFile)
	dstJSONFile := path.Join(dstPrefix, xlMetaJSONFile)

	// Rename `xl.json` to all disks in parallel.
	for index, disk := range xl.storageDisks {
		if disk == nil {
			mErrs[index] = errDiskNotFound
			continue
		}
		wg.Add(1)
		// Rename `xl.json` in a routine.
		go func(index int, disk StorageAPI) {
			defer wg.Done()
			// Renames `xl.json` from source prefix to destination prefix.
			rErr := disk.RenameFile(minioMetaBucket, srcJSONFile, minioMetaBucket, dstJSONFile)
			if rErr != nil {
				mErrs[index] = rErr
				return
			}
			// Delete any dangling directories.
			dErr := disk.DeleteFile(minioMetaBucket, srcPrefix)
			if dErr != nil {
				mErrs[index] = dErr
				return
			}
			mErrs[index] = nil
		}(index, disk)
	}
	// Wait for all the routines.
	wg.Wait()

	// Do we have write quorum?.
	if !isQuorum(mErrs, xl.writeQuorum) {
		return errXLWriteQuorum
	}
	// For all other errors return.
	for _, err := range mErrs {
		if err != nil && err != errDiskNotFound {
			return err
		}
	}
	return nil
}
<commit_msg>Ignored errors in cleanup in commitXLMetadata (#2044)<commit_after>/*
 * Minio Cloud Storage, (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"encoding/json"
	"path"
	"strings"
	"sync"
	"time"
)

// updateUploadsJSON - update `uploads.json` with new uploadsJSON for all disks.
func (xl xlObjects) updateUploadsJSON(bucket, object string, uploadsJSON uploadsV1) (err error) {
	uploadsPath := path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)
	uniqueID := getUUID()
	tmpUploadsPath := path.Join(tmpMetaPrefix, uniqueID)
	var errs = make([]error, len(xl.storageDisks))
	var wg = &sync.WaitGroup{}

	// Update `uploads.json` for all the disks.
	for index, disk := range xl.storageDisks {
		if disk == nil {
			errs[index] = errDiskNotFound
			continue
		}
		wg.Add(1)
		// Update `uploads.json` in routine.
		go func(index int, disk StorageAPI) {
			defer wg.Done()
			uploadsBytes, wErr := json.Marshal(uploadsJSON)
			if wErr != nil {
				errs[index] = wErr
				return
			}
			if wErr = disk.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsBytes); wErr != nil {
				errs[index] = wErr
				return
			}
			if wErr = disk.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath); wErr != nil {
				errs[index] = wErr
				return
			}
		}(index, disk)
	}

	// Wait for all the routines to finish updating `uploads.json`
	wg.Wait()

	// Count all the errors and validate if we have write quorum.
	if !isQuorum(errs, xl.writeQuorum) {
		// Rename `uploads.json` left over back to tmp location.
		for index, disk := range xl.storageDisks {
			if disk == nil {
				continue
			}
			// Undo rename `uploads.json` in parallel.
			wg.Add(1)
			go func(index int, disk StorageAPI) {
				defer wg.Done()
				if errs[index] != nil {
					return
				}
				_ = disk.RenameFile(minioMetaBucket, uploadsPath, minioMetaBucket, tmpUploadsPath)
			}(index, disk)
		}
		wg.Wait()
		return errXLWriteQuorum
	}
	return nil
}

// writeUploadJSON - create `uploads.json` or update it with new uploadID.
func (xl xlObjects) writeUploadJSON(bucket, object, uploadID string, initiated time.Time) (err error) {
	uploadsPath := path.Join(mpartMetaPrefix, bucket, object, uploadsJSONFile)
	uniqueID := getUUID()
	tmpUploadsPath := path.Join(tmpMetaPrefix, uniqueID)

	var errs = make([]error, len(xl.storageDisks))
	var wg = &sync.WaitGroup{}

	var uploadsJSON uploadsV1
	for _, disk := range xl.storageDisks {
		if disk == nil {
			continue
		}
		uploadsJSON, err = readUploadsJSON(bucket, object, disk)
		break
	}
	if err != nil {
		// For any other errors.
		if err != errFileNotFound {
			return err
		}
		// Set uploads format to `xl` otherwise.
		uploadsJSON = newUploadsV1("xl")
	}
	// Add a new upload id.
	uploadsJSON.AddUploadID(uploadID, initiated)

	// Update `uploads.json` on all disks.
	for index, disk := range xl.storageDisks {
		if disk == nil {
			errs[index] = errDiskNotFound
			continue
		}
		wg.Add(1)
		// Update `uploads.json` in a routine.
		go func(index int, disk StorageAPI) {
			defer wg.Done()
			uploadsJSONBytes, wErr := json.Marshal(&uploadsJSON)
			if wErr != nil {
				errs[index] = wErr
				return
			}
			// Write `uploads.json` to disk.
			if wErr = disk.AppendFile(minioMetaBucket, tmpUploadsPath, uploadsJSONBytes); wErr != nil {
				errs[index] = wErr
				return
			}
			wErr = disk.RenameFile(minioMetaBucket, tmpUploadsPath, minioMetaBucket, uploadsPath)
			if wErr != nil {
				if dErr := disk.DeleteFile(minioMetaBucket, tmpUploadsPath); dErr != nil {
					errs[index] = dErr
					return
				}
				errs[index] = wErr
				return
			}
			errs[index] = nil
		}(index, disk)
	}

	// Wait here for all the writes to finish.
	wg.Wait()

	// Count all the errors and validate if we have write quorum.
	if !isQuorum(errs, xl.writeQuorum) {
		// Rename `uploads.json` left over back to tmp location.
		for index, disk := range xl.storageDisks {
			if disk == nil {
				continue
			}
			// Undo rename `uploads.json` in parallel.
			wg.Add(1)
			go func(index int, disk StorageAPI) {
				defer wg.Done()
				if errs[index] != nil {
					return
				}
				_ = disk.RenameFile(minioMetaBucket, uploadsPath, minioMetaBucket, tmpUploadsPath)
			}(index, disk)
		}
		wg.Wait()
		return errXLWriteQuorum
	}
	return nil
}

// Returns if the prefix is a multipart upload.
func (xl xlObjects) isMultipartUpload(bucket, prefix string) bool {
	for _, disk := range xl.getLoadBalancedQuorumDisks() {
		if disk == nil {
			continue
		}
		_, err := disk.StatFile(bucket, pathJoin(prefix, uploadsJSONFile))
		if err != nil {
			// For any reason disk was deleted or goes offline, continue
			if err == errDiskNotFound || err == errFaultyDisk {
				continue
			}
			return false
		}
		break
	}
	return true
}

// listUploadsInfo - list all uploads info.
func (xl xlObjects) listUploadsInfo(prefixPath string) (uploadsInfo []uploadInfo, err error) {
	for _, disk := range xl.getLoadBalancedQuorumDisks() {
		if disk == nil {
			continue
		}
		splitPrefixes := strings.SplitN(prefixPath, "/", 3)
		var uploadsJSON uploadsV1
		uploadsJSON, err = readUploadsJSON(splitPrefixes[1], splitPrefixes[2], disk)
		if err != nil {
			// For any reason disk was deleted or goes offline, continue
			if err == errDiskNotFound || err == errFaultyDisk {
				continue
			}
			if err == errFileNotFound {
				return []uploadInfo{}, nil
			}
			return nil, err
		}
		uploadsInfo = uploadsJSON.Uploads
		break
	}
	return uploadsInfo, nil
}

// isUploadIDExists - verify if a given uploadID exists and is valid.
func (xl xlObjects) isUploadIDExists(bucket, object, uploadID string) bool {
	uploadIDPath := path.Join(mpartMetaPrefix, bucket, object, uploadID)
	return xl.isObject(minioMetaBucket, uploadIDPath)
}

// Removes part given by partName belonging to a mulitpart upload from minioMetaBucket
func (xl xlObjects) removeObjectPart(bucket, object, uploadID, partName string) {
	curpartPath := path.Join(mpartMetaPrefix, bucket, object, uploadID, partName)
	wg := sync.WaitGroup{}
	for i, disk := range xl.storageDisks {
		if disk == nil {
			continue
		}
		wg.Add(1)
		go func(index int, disk StorageAPI) {
			defer wg.Done()
			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
			// requests. xl.json is the authoritative source of truth on which parts constitute
			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
			_ = disk.DeleteFile(minioMetaBucket, curpartPath)
		}(i, disk)
	}
	wg.Wait()
}

// statPart - returns fileInfo structure for a successful stat on part file.
func (xl xlObjects) statPart(bucket, object, uploadID, partName string) (fileInfo FileInfo, err error) {
	partNamePath := path.Join(mpartMetaPrefix, bucket, object, uploadID, partName)
	for _, disk := range xl.getLoadBalancedQuorumDisks() {
		if disk == nil {
			continue
		}
		fileInfo, err = disk.StatFile(minioMetaBucket, partNamePath)
		if err != nil {
			// For any reason disk was deleted or goes offline, continue
			if err == errDiskNotFound {
				continue
			}
			return FileInfo{}, err
		}
		break
	}
	return fileInfo, nil
}

// commitXLMetadata - commit `xl.json` from source prefix to destination prefix.
func (xl xlObjects) commitXLMetadata(srcPrefix, dstPrefix string) error {
	var wg = &sync.WaitGroup{}
	var mErrs = make([]error, len(xl.storageDisks))

	srcJSONFile := path.Join(srcPrefix, xlMetaJSONFile)
	dstJSONFile := path.Join(dstPrefix, xlMetaJSONFile)

	// Rename `xl.json` to all disks in parallel.
	for index, disk := range xl.storageDisks {
		if disk == nil {
			mErrs[index] = errDiskNotFound
			continue
		}
		wg.Add(1)
		// Rename `xl.json` in a routine.
		go func(index int, disk StorageAPI) {
			defer wg.Done()
			// Delete any dangling directories.
			defer disk.DeleteFile(minioMetaBucket, srcPrefix)

			// Renames `xl.json` from source prefix to destination prefix.
			rErr := disk.RenameFile(minioMetaBucket, srcJSONFile, minioMetaBucket, dstJSONFile)
			if rErr != nil {
				mErrs[index] = rErr
				return
			}
			mErrs[index] = nil
		}(index, disk)
	}
	// Wait for all the routines.
	wg.Wait()

	// Do we have write quorum?.
	if !isQuorum(mErrs, xl.writeQuorum) {
		return errXLWriteQuorum
	}
	// For all other errors return.
	for _, err := range mErrs {
		if err != nil && err != errDiskNotFound {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package xmmsclient

import (
	"encoding/binary"
	"fmt"
	"io"
	"math"
)

type listConsumer func(value XmmsValue)

func deserializeInt(r io.Reader) (value XmmsInt, err error) {
	err = binary.Read(r, binary.BigEndian, &value)
	return
}

func deserializeFloat(r io.Reader) (value XmmsFloat, err error) {
	var mantissaInt int32
	var exponent int32
	var mantissa float64

	err = binary.Read(r, binary.BigEndian, &mantissaInt)
	if err != nil {
		return
	}

	err = binary.Read(r, binary.BigEndian, &exponent)
	if err != nil {
		return
	}

	if mantissaInt > 0 {
		mantissa = float64(mantissaInt) / float64(math.MaxInt32)
	} else {
		mantissa = float64(mantissaInt) / float64(math.Abs(math.MinInt32))
	}

	value = XmmsFloat(math.Ldexp(mantissa, int(exponent)))

	return
}

func deserializeRawString(r io.Reader) (value string, err error) {
	var length uint32
	err = binary.Read(r, binary.BigEndian, &length)
	if err != nil {
		return
	}
	data := make([]byte, length)
	err = binary.Read(r, binary.BigEndian, &data)
	if err != nil {
		return
	}
	value = string(data[:length-1])

	return
}

func deserializeString(r io.Reader) (value XmmsString, err error) {
	data, err := deserializeRawString(r)
	if err == nil {
		value = XmmsString(data)
	}
	return
}

func deserializeError(r io.Reader) (value XmmsError, err error) {
	data, err := deserializeRawString(r)
	if err == nil {
		value = XmmsError(data)
	}
	return
}

func deserializeAnyList(r io.Reader, consumer listConsumer) error {
	var restrict uint32
	err := binary.Read(r, binary.BigEndian, &restrict) // TODO: respect restrict
	if err != nil {
		return err
	}
	var length uint32
	err = binary.Read(r, binary.BigEndian, &length)
	if err != nil {
		return err
	}
	if restrict != TypeNone {
		for i := uint32(0); i < length; i++ {
			entry, err := deserializeXmmsValueOfType(restrict, r)
			if err != nil {
				return err
			}
			consumer(entry)
		}
	} else {
		for i := uint32(0); i < length; i++ {
			entry, err := deserializeXmmsValue(r)
			if err != nil {
				return err
			}
			consumer(entry)
		}
	}
	return nil
}

func deserializeList(r io.Reader) (value XmmsList, err error) {
	list := XmmsList{}
	err = deserializeAnyList(r, func(value XmmsValue) {
		list = append(list, value)
	})
	value = list
	return
}

func deserializeDict(r io.Reader) (value XmmsDict, err error) {
	var length uint32
	err = binary.Read(r, binary.BigEndian, &length)
	if err != nil {
		return
	}
	dict := XmmsDict{}
	for i := uint32(0); i < length; i++ {
		var entry XmmsValue
		var key string

		key, err = deserializeRawString(r)
		if err != nil {
			return
		}

		entry, err = deserializeXmmsValue(r)
		if err != nil {
			return
		}

		dict[string(key)] = entry
	}
	value = dict
	return
}

func deserializeColl(r io.Reader) (result XmmsColl, err error) {
	err = binary.Read(r, binary.BigEndian, &result.Type)
	if err != nil {
		return
	}

	result.Attributes, err = deserializeDict(r)
	if err != nil {
		return
	}

	err = deserializeAnyList(r, func(raw XmmsValue) {
		if value, ok := raw.(XmmsInt); ok {
			result.IDList = append(result.IDList, int(value))
		}
	})
	if err != nil {
		return
	}

	err = deserializeAnyList(r, func(raw XmmsValue) {
		if value, ok := raw.(XmmsColl); ok {
			result.Operands = append(result.Operands, value)
		}
	})

	return
}

func deserializeXmmsValueOfType(valueType uint32, r io.Reader) (result XmmsValue, err error) {
	switch valueType {
	case TypeInt64:
		result, err = deserializeInt(r)
	case TypeFloat:
		result, err = deserializeFloat(r)
	case TypeError:
		result, err = deserializeError(r)
	case TypeString:
		result, err = deserializeString(r)
	case TypeList:
		result, err = deserializeList(r)
	case TypeDict:
		result, err = deserializeDict(r)
	case TypeColl:
		result, err = deserializeColl(r)
	}
	return
}

func deserializeXmmsValue(r io.Reader) (XmmsValue, error) {
	var valueType uint32
	err := binary.Read(r, binary.BigEndian, &valueType)
	if err != nil {
		return nil, err
	}
	return deserializeXmmsValueOfType(valueType, r)
}

func tryDeserialize(r io.Reader) (XmmsValue, error) {
	value, err := deserializeXmmsValue(r)
	if err != nil {
		return nil, err
	}

	errorMessage, ok := value.(XmmsError)
	if ok {
		return nil, fmt.Errorf("%s", errorMessage)
	}

	return value, nil
}

func tryDeserializeList(r io.Reader, consumer listConsumer) error {
	var valueType uint32

	err := binary.Read(r, binary.BigEndian, &valueType)
	if err != nil {
		return err
	}

	switch valueType {
	case TypeError:
		value, err := deserializeError(r)
		if err != nil {
			return err
		}
		return fmt.Errorf("%s", value)
	case TypeList:
		return deserializeAnyList(r, consumer)
	default:
		return fmt.Errorf("Trying to parse non-list as list (%v)", valueType)
	}
}

func tryDeserializeIntList(r io.Reader) ([]int, error) {
	var list []int
	err := tryDeserializeList(r, func(raw XmmsValue) {
		if value, ok := raw.(XmmsInt); ok {
			list = append(list, int(value))
		}
	})
	if err != nil {
		return nil, err
	}
	return list, nil
}

func tryDeserializeStringList(r io.Reader) ([]string, error) {
	var list []string
	err := tryDeserializeList(r, func(raw XmmsValue) {
		if value, ok := raw.(XmmsString); ok {
			list = append(list, string(value))
		}
	})
	if err != nil {
		return nil, err
	}
	return list, nil
}

func tryDeserializeDictList(r io.Reader) ([]XmmsDict, error) {
	var list []XmmsDict
	err := tryDeserializeList(r, func(raw XmmsValue) {
		if value, ok := raw.(XmmsDict); ok {
			list = append(list, value)
		}
	})
	if err != nil {
		return nil, err
	}
	return list, nil
}
<commit_msg>Drop some useless typecasts.<commit_after>package xmmsclient

import (
	"encoding/binary"
	"fmt"
	"io"
	"math"
)

type listConsumer func(value XmmsValue)

func deserializeInt(r io.Reader) (value XmmsInt, err error) {
	err = binary.Read(r, binary.BigEndian, &value)
	return
}

func deserializeFloat(r io.Reader) (value XmmsFloat, err error) {
	var mantissaInt int32
	var exponent int32
	var mantissa float64

	err = binary.Read(r, binary.BigEndian, &mantissaInt)
	if err != nil {
		return
	}

	err = binary.Read(r, binary.BigEndian, &exponent)
	if err != nil {
		return
	}

	if mantissaInt > 0 {
		mantissa = float64(mantissaInt) / float64(math.MaxInt32)
	} else {
		mantissa = float64(mantissaInt) / math.Abs(math.MinInt32)
	}

	value = XmmsFloat(math.Ldexp(mantissa, int(exponent)))

	return
}

func deserializeRawString(r io.Reader) (value string, err error) {
	var length uint32
	err = binary.Read(r, binary.BigEndian, &length)
	if err != nil {
		return
	}
	data := make([]byte, length)
	err = binary.Read(r, binary.BigEndian, &data)
	if err != nil {
		return
	}
	value = string(data[:length-1])

	return
}

func deserializeString(r io.Reader) (value XmmsString, err error) {
	data, err := deserializeRawString(r)
	if err == nil {
		value = XmmsString(data)
	}
	return
}

func deserializeError(r io.Reader) (value XmmsError, err error) {
	data, err := deserializeRawString(r)
	if err == nil {
		value = XmmsError(data)
	}
	return
}

func deserializeAnyList(r io.Reader, consumer listConsumer) error {
	var restrict uint32
	err := binary.Read(r, binary.BigEndian, &restrict) // TODO: respect restrict
	if err != nil {
		return err
	}
	var length uint32
	err = binary.Read(r, binary.BigEndian, &length)
	if err != nil {
		return err
	}
	if restrict != TypeNone {
		for i := uint32(0); i < length; i++ {
			entry, err := deserializeXmmsValueOfType(restrict, r)
			if err != nil {
				return err
			}
			consumer(entry)
		}
	} else {
		for i := uint32(0); i < length; i++ {
			entry, err := deserializeXmmsValue(r)
			if err != nil {
				return err
			}
			consumer(entry)
		}
	}
	return nil
}

func deserializeList(r io.Reader) (value XmmsList, err error) {
	list := XmmsList{}
	err = deserializeAnyList(r, func(value XmmsValue) {
		list = append(list, value)
	})
	value = list
	return
}

func deserializeDict(r io.Reader) (value XmmsDict, err error) {
	var length uint32
	err = binary.Read(r, binary.BigEndian, &length)
	if err != nil {
		return
	}
	dict := XmmsDict{}
	for i := uint32(0); i < length; i++ {
		var entry XmmsValue
		var key string

		key, err = deserializeRawString(r)
		if err != nil {
			return
		}

		entry, err = deserializeXmmsValue(r)
		if err != nil {
			return
		}

		dict[key] = entry
	}
	value = dict
	return
}

func deserializeColl(r io.Reader) (result XmmsColl, err error) {
	err = binary.Read(r, binary.BigEndian, &result.Type)
	if err != nil {
		return
	}

	result.Attributes, err = deserializeDict(r)
	if err != nil {
		return
	}

	err = deserializeAnyList(r, func(raw XmmsValue) {
		if value, ok := raw.(XmmsInt); ok {
			result.IDList = append(result.IDList, int(value))
		}
	})
	if err != nil {
		return
	}

	err = deserializeAnyList(r, func(raw XmmsValue) {
		if value, ok := raw.(XmmsColl); ok {
			result.Operands = append(result.Operands, value)
		}
	})

	return
}

func deserializeXmmsValueOfType(valueType uint32, r io.Reader) (result XmmsValue, err error) {
	switch valueType {
	case TypeInt64:
		result, err = deserializeInt(r)
	case TypeFloat:
		result, err = deserializeFloat(r)
	case TypeError:
		result, err = deserializeError(r)
	case TypeString:
		result, err = deserializeString(r)
	case TypeList:
		result, err = deserializeList(r)
	case TypeDict:
		result, err = deserializeDict(r)
	case TypeColl:
		result, err = deserializeColl(r)
	}
	return
}

func deserializeXmmsValue(r io.Reader) (XmmsValue, error) {
	var valueType uint32
	err := binary.Read(r, binary.BigEndian, &valueType)
	if err != nil {
		return nil, err
	}
	return deserializeXmmsValueOfType(valueType, r)
}

func tryDeserialize(r io.Reader) (XmmsValue, error) {
	value, err := deserializeXmmsValue(r)
	if err != nil {
		return nil, err
	}

	errorMessage, ok := value.(XmmsError)
	if ok {
		return nil, fmt.Errorf("%s", errorMessage)
	}

	return value, nil
}

func tryDeserializeList(r io.Reader, consumer listConsumer) error {
	var valueType uint32

	err := binary.Read(r, binary.BigEndian, &valueType)
	if err != nil {
		return err
	}

	switch valueType {
	case TypeError:
		value, err := deserializeError(r)
		if err != nil {
			return err
		}
		return fmt.Errorf("%s", value)
	case TypeList:
		return deserializeAnyList(r, consumer)
	default:
		return fmt.Errorf("Trying to parse non-list as list (%v)", valueType)
	}
}

func tryDeserializeIntList(r io.Reader) ([]int, error) {
	var list []int
	err := tryDeserializeList(r, func(raw XmmsValue) {
		if value, ok := raw.(XmmsInt); ok {
			list = append(list, int(value))
		}
	})
	if err != nil {
		return nil, err
	}
	return list, nil
}

func tryDeserializeStringList(r io.Reader) ([]string, error) {
	var list []string
	err := tryDeserializeList(r, func(raw XmmsValue) {
		if value, ok := raw.(XmmsString); ok {
			list = append(list, string(value))
		}
	})
	if err != nil {
		return nil, err
	}
	return list, nil
}

func tryDeserializeDictList(r io.Reader) ([]XmmsDict, error) {
	var list []XmmsDict
	err := tryDeserializeList(r, func(raw XmmsValue) {
		if value, ok := raw.(XmmsDict); ok {
			list = append(list, value)
		}
	})
	if err != nil {
		return nil, err
	}
	return list, nil
}
<|endoftext|>


<commit_before>package mount

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	log "github.com/sirupsen/logrus"
)

const (
	procMountsPath = "/proc/self/mountinfo"
	// procMountsRetries is number of times to retry for a consistent
	// read of procMountsPath.
	procMountsRetries = 3
)

var (
	bindRemountOpts = []string{"remount"}
)

func init() {
	if _, err := os.Stat(procSelfMountInfo); !os.IsNotExist(err) {
		MountNamespaceSupportEnabled = true
	}
}

// getDiskFormat uses 'lsblk' to see if the given disk is unformatted
func getDiskFormat(disk string) (string, error) {
	args := []string{"-n", "-o", "FSTYPE", disk}

	f := log.Fields{
		"disk": disk,
	}
	log.WithFields(f).WithField("args", args).Info(
		"checking if disk is formatted using lsblk")
	buf, err := exec.Command("lsblk", args...).CombinedOutput()
	out := string(buf)
	log.WithField("output", out).Debug("lsblk output")

	if err != nil {
		log.WithFields(f).WithError(err).Error(
			"Could not determine if disk is formatted")
		return "", err
	}

	// Split lsblk output into lines. Unformatted devices should contain only
	// "\n". Beware of "\n\n", that's a device with one empty partition.
	out = strings.TrimSuffix(out, "\n") // Avoid last empty line
	lines := strings.Split(out, "\n")
	if lines[0] != "" {
		// The device is formatted
		return lines[0], nil
	}

	if len(lines) == 1 {
		// The device is unformatted and has no dependent devices
		return "", nil
	}

	// The device has dependent devices, most probably partitions (LVM, LUKS
	// and MD RAID are reported as FSTYPE and caught above).
	return "unknown data, probably partitions", nil
}

// formatAndMount uses unix utils to format and mount the given disk
func formatAndMount(source, target, fsType string, options []string) error {

	options = append(options, "defaults")
	f := log.Fields{
		"source":  source,
		"target":  target,
		"fsType":  fsType,
		"options": options,
	}

	// Try to mount the disk
	log.WithFields(f).Info("attempting to mount disk")
	mountErr := mount(source, target, fsType, options)
	if mountErr == nil {
		return nil
	}

	// Mount failed. This indicates either that the disk is unformatted or
	// it contains an unexpected filesystem.
	existingFormat, err := getDiskFormat(source)
	if err != nil {
		return err
	}
	if existingFormat == "" {
		// Disk is unformatted so format it.
		args := []string{source}
		// Use 'ext4' as the default
		if len(fsType) == 0 {
			fsType = "ext4"
		}

		if fsType == "ext4" || fsType == "ext3" {
			args = []string{"-F", source}
		}
		f["fsType"] = fsType
		log.WithFields(f).Info(
			"disk appears unformatted, attempting format")

		mkfsCmd := fmt.Sprintf("mkfs.%s", fsType)
		if err := exec.Command(mkfsCmd, args...).Run(); err != nil {
			log.WithFields(f).WithError(err).Error(
				"format of disk failed")
		}

		// the disk has been formatted successfully try to mount it again.
		log.WithFields(f).Info(
			"disk successfully formatted")
		return mount(source, target, fsType, options)
	}

	// Disk is already formatted and failed to mount
	if len(fsType) == 0 || fsType == existingFormat {
		// This is mount error
		return mountErr
	}

	// Block device is formatted with unexpected filesystem
	return fmt.Errorf(
		"failed to mount volume as %q; already contains %s: error: %v",
		fsType, existingFormat, mountErr)
}

// bindMount performs a bind mount
func bindMount(source, target string, options []string) error {
	err := doMount("mount", source, target, "", []string{"bind"})
	if err != nil {
		return err
	}
	return doMount("mount", source, target, "", options)
}

// getMounts returns a slice of all the mounted filesystems
func getMounts() ([]*Info, error) {
	_, hash1, err := readProcMounts(procMountsPath, false)
	if err != nil {
		return nil, err
	}

	for i := 0; i < procMountsRetries; i++ {
		mps, hash2, err := readProcMounts(procMountsPath, true)
		if err != nil {
			return nil, err
		}
		if hash1 == hash2 {
			// Success
			return mps, nil
		}
		hash1 = hash2
	}
	return nil, fmt.Errorf(
		"failed to get a consistent snapshot of %v after %d tries",
		procMountsPath, procMountsRetries)
}

// readProcMounts reads procMountsInfo and produce a hash
// of the contents and a list of the mounts as Info objects.
func readProcMounts(path string, info bool) ([]*Info, uint32, error) {

	file, err := os.Open(path)
	if err != nil {
		return nil, 0, err
	}
	defer file.Close()
	return readProcMountsFrom(file, info, procMountsFields)
}
<commit_msg>Fix for Linux Mount Package<commit_after>package mount

import (
	"fmt"
	"os"
	"os/exec"
	"strings"

	log "github.com/sirupsen/logrus"
)

const (
	procMountsPath = "/proc/self/mountinfo"
	// procMountsRetries is number of times to retry for a consistent
	// read of procMountsPath.
	procMountsRetries = 3
)

var (
	bindRemountOpts = []string{"remount"}
)

// getDiskFormat uses 'lsblk' to see if the given disk is unformatted
func getDiskFormat(disk string) (string, error) {
	args := []string{"-n", "-o", "FSTYPE", disk}

	f := log.Fields{
		"disk": disk,
	}
	log.WithFields(f).WithField("args", args).Info(
		"checking if disk is formatted using lsblk")
	buf, err := exec.Command("lsblk", args...).CombinedOutput()
	out := string(buf)
	log.WithField("output", out).Debug("lsblk output")

	if err != nil {
		log.WithFields(f).WithError(err).Error(
			"Could not determine if disk is formatted")
		return "", err
	}

	// Split lsblk output into lines. Unformatted devices should contain only
	// "\n". Beware of "\n\n", that's a device with one empty partition.
	out = strings.TrimSuffix(out, "\n") // Avoid last empty line
	lines := strings.Split(out, "\n")
	if lines[0] != "" {
		// The device is formatted
		return lines[0], nil
	}

	if len(lines) == 1 {
		// The device is unformatted and has no dependent devices
		return "", nil
	}

	// The device has dependent devices, most probably partitions (LVM, LUKS
	// and MD RAID are reported as FSTYPE and caught above).
	return "unknown data, probably partitions", nil
}

// formatAndMount uses unix utils to format and mount the given disk
func formatAndMount(source, target, fsType string, options []string) error {

	options = append(options, "defaults")
	f := log.Fields{
		"source":  source,
		"target":  target,
		"fsType":  fsType,
		"options": options,
	}

	// Try to mount the disk
	log.WithFields(f).Info("attempting to mount disk")
	mountErr := mount(source, target, fsType, options)
	if mountErr == nil {
		return nil
	}

	// Mount failed. This indicates either that the disk is unformatted or
	// it contains an unexpected filesystem.
	existingFormat, err := getDiskFormat(source)
	if err != nil {
		return err
	}
	if existingFormat == "" {
		// Disk is unformatted so format it.
		args := []string{source}
		// Use 'ext4' as the default
		if len(fsType) == 0 {
			fsType = "ext4"
		}

		if fsType == "ext4" || fsType == "ext3" {
			args = []string{"-F", source}
		}
		f["fsType"] = fsType
		log.WithFields(f).Info(
			"disk appears unformatted, attempting format")

		mkfsCmd := fmt.Sprintf("mkfs.%s", fsType)
		if err := exec.Command(mkfsCmd, args...).Run(); err != nil {
			log.WithFields(f).WithError(err).Error(
				"format of disk failed")
		}

		// the disk has been formatted successfully try to mount it again.
		log.WithFields(f).Info(
			"disk successfully formatted")
		return mount(source, target, fsType, options)
	}

	// Disk is already formatted and failed to mount
	if len(fsType) == 0 || fsType == existingFormat {
		// This is mount error
		return mountErr
	}

	// Block device is formatted with unexpected filesystem
	return fmt.Errorf(
		"failed to mount volume as %q; already contains %s: error: %v",
		fsType, existingFormat, mountErr)
}

// bindMount performs a bind mount
func bindMount(source, target string, options []string) error {
	err := doMount("mount", source, target, "", []string{"bind"})
	if err != nil {
		return err
	}
	return doMount("mount", source, target, "", options)
}

// getMounts returns a slice of all the mounted filesystems
func getMounts() ([]*Info, error) {
	_, hash1, err := readProcMounts(procMountsPath, false)
	if err != nil {
		return nil, err
	}

	for i := 0; i < procMountsRetries; i++ {
		mps, hash2, err := readProcMounts(procMountsPath, true)
		if err != nil {
			return nil, err
		}
		if hash1 == hash2 {
			// Success
			return mps, nil
		}
		hash1 = hash2
	}
	return nil, fmt.Errorf(
		"failed to get a consistent snapshot of %v after %d tries",
		procMountsPath, procMountsRetries)
}

// readProcMounts reads procMountsInfo and produce a hash
// of the contents and a list of the mounts as Info objects.
func readProcMounts(path string, info bool) ([]*Info, uint32, error) {

	file, err := os.Open(path)
	if err != nil {
		return nil, 0, err
	}
	defer file.Close()
	return readProcMountsFrom(file, info, procMountsFields)
}
<|endoftext|>


<commit_before>package containerd

import (
	"context"
	"syscall"

	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/linux/runcopts"
	"github.com/containerd/containerd/mount"
)

// NewTaskOpts allows the caller to set options on a new task
type NewTaskOpts func(context.Context, *Client, *TaskInfo) error

// WithRootFS allows a task to be created without a snapshot being allocated to its container
func WithRootFS(mounts []mount.Mount) NewTaskOpts {
	return func(ctx context.Context, c *Client, ti *TaskInfo) error {
		ti.RootFS = mounts
		return nil
	}
}

// WithExit causes the task to exit after a successful checkpoint
func WithExit(r *CheckpointTaskInfo) error {
	r.Options = &runcopts.CheckpointOptions{
		Exit: true,
	}
	return nil
}

// ProcessDeleteOpts allows the caller to set options for the deletion of a task
type ProcessDeleteOpts func(context.Context, Process) error

// WithProcessKill will forcefully kill and delete a process
func WithProcessKill(ctx context.Context, p Process) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	// ignore errors to wait and kill as we are forcefully killing
	// the process and don't care about the exit status
	s, err := p.Wait(ctx)
	if err != nil {
		return err
	}
	if err := p.Kill(ctx, syscall.SIGKILL); err != nil {
		if errdefs.IsFailedPrecondition(err) || errdefs.IsNotFound(err) {
			return nil
		}
		return err
	}
	// wait for the process to fully stop before letting the rest of the deletion complete
	<-s
	return nil
}

type KillInfo struct {
	// All kills all processes inside the task
	// only valid on tasks, ignored on processes
	All bool
}

type KillOpts func(context.Context, Process, *KillInfo) error

// WithKillAll kills all processes for a task
func WithKillAll(ctx context.Context, p Process, i *KillInfo) error {
	i.All = true
	return nil
}
<commit_msg>WithProcessKill should kill all processes.<commit_after>package containerd

import (
	"context"
	"syscall"

	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/linux/runcopts"
	"github.com/containerd/containerd/mount"
)

// NewTaskOpts allows the caller to set options on a new task
type NewTaskOpts func(context.Context, *Client, *TaskInfo) error

// WithRootFS allows a task to be created without a snapshot being allocated to its container
func WithRootFS(mounts []mount.Mount) NewTaskOpts {
	return func(ctx context.Context, c *Client, ti *TaskInfo) error {
		ti.RootFS = mounts
		return nil
	}
}

// WithExit causes the task to exit after a successful checkpoint
func WithExit(r *CheckpointTaskInfo) error {
	r.Options = &runcopts.CheckpointOptions{
		Exit: true,
	}
	return nil
}

// ProcessDeleteOpts allows the caller to set options for the deletion of a task
type ProcessDeleteOpts func(context.Context, Process) error

// WithProcessKill will forcefully kill and delete a process
func WithProcessKill(ctx context.Context, p Process) error {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	// ignore errors to wait and kill as we are forcefully killing
	// the process and don't care about the exit status
	s, err := p.Wait(ctx)
	if err != nil {
		return err
	}
	if err := p.Kill(ctx, syscall.SIGKILL, WithKillAll); err != nil {
		if errdefs.IsFailedPrecondition(err) || errdefs.IsNotFound(err) {
			return nil
		}
		return err
	}
	// wait for the process to fully stop before letting the rest of the deletion complete
	<-s
	return nil
}

type KillInfo struct {
	// All kills all processes inside the task
	// only valid on tasks, ignored on processes
	All bool
}

type KillOpts func(context.Context, Process, *KillInfo) error

// WithKillAll kills all processes for a task
func WithKillAll(ctx context.Context, p Process, i *KillInfo) error {
	i.All = true
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package interpodaffinity

import (
	"context"
	"fmt"
	"sync/atomic"

	v1 "k8s.io/api/core/v1"
	framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
	"k8s.io/kubernetes/pkg/scheduler/internal/parallelize"
	schedutil "k8s.io/kubernetes/pkg/scheduler/util"
)

// preScoreStateKey is the key in CycleState to InterPodAffinity pre-computed data for Scoring.
const preScoreStateKey = "PreScore" + Name

type scoreMap map[string]map[string]int64

// preScoreState computed at PreScore and used at Score.
type preScoreState struct {
	topologyScore scoreMap
	podInfo       *framework.PodInfo
}

// Clone implements the mandatory Clone interface. We don't really copy the data since
// there is no need for that.
func (s *preScoreState) Clone() framework.StateData {
	return s
}

func (m scoreMap) processTerm(
	term *framework.WeightedAffinityTerm,
	podToCheck *v1.Pod,
	fixedNode *v1.Node,
	multiplier int,
) {
	if len(fixedNode.Labels) == 0 {
		return
	}

	match := schedutil.PodMatchesTermsNamespaceAndSelector(podToCheck, term.Namespaces, term.Selector)
	tpValue, tpValueExist := fixedNode.Labels[term.TopologyKey]
	if match && tpValueExist {
		if m[term.TopologyKey] == nil {
			m[term.TopologyKey] = make(map[string]int64)
		}
		m[term.TopologyKey][tpValue] += int64(term.Weight * int32(multiplier))
	}
	return
}

func (m scoreMap) processTerms(terms []framework.WeightedAffinityTerm, podToCheck *v1.Pod, fixedNode *v1.Node, multiplier int) {
	for _, term := range terms {
		m.processTerm(&term, podToCheck, fixedNode, multiplier)
	}
}

func (m scoreMap) append(other scoreMap) {
	for topology, oScores := range other {
		scores := m[topology]
		if scores == nil {
			m[topology] = oScores
			continue
		}
		for k, v := range oScores {
			scores[k] += v
		}
	}
}

func (pl *InterPodAffinity) processExistingPod(
	state *preScoreState,
	existingPod *framework.PodInfo,
	existingPodNodeInfo *framework.NodeInfo,
	incomingPod *v1.Pod,
	topoScore scoreMap,
) {
	existingPodNode := existingPodNodeInfo.Node()

	// For every soft pod affinity term of <pod>, if <existingPod> matches the term,
	// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPods>`s node by the term`s weight.
	topoScore.processTerms(state.podInfo.PreferredAffinityTerms, existingPod.Pod, existingPodNode, 1)

	// For every soft pod anti-affinity term of <pod>, if <existingPod> matches the term,
	// decrement <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPod>`s node by the term`s weight.
	topoScore.processTerms(state.podInfo.PreferredAntiAffinityTerms, existingPod.Pod, existingPodNode, -1)

	// For every hard pod affinity term of <existingPod>, if <pod> matches the term,
	// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPod>'s node by the constant <args.hardPodAffinityWeight>
	if pl.args.HardPodAffinityWeight > 0 {
		for _, term := range existingPod.RequiredAffinityTerms {
			t := framework.WeightedAffinityTerm{AffinityTerm: term, Weight: pl.args.HardPodAffinityWeight}
			topoScore.processTerm(&t, incomingPod, existingPodNode, 1)
		}
	}

	// For every soft pod affinity term of <existingPod>, if <pod> matches the term,
	// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPod>'s node by the term's weight.
	topoScore.processTerms(existingPod.PreferredAffinityTerms, incomingPod, existingPodNode, 1)

	// For every soft pod anti-affinity term of <existingPod>, if <pod> matches the term,
	// decrement <pm.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPod>'s node by the term's weight.
	topoScore.processTerms(existingPod.PreferredAntiAffinityTerms, incomingPod, existingPodNode, -1)
}

// PreScore builds and writes cycle state used by Score and NormalizeScore.
func (pl *InterPodAffinity) PreScore(
	pCtx context.Context,
	cycleState *framework.CycleState,
	pod *v1.Pod,
	nodes []*v1.Node,
) *framework.Status {
	if len(nodes) == 0 {
		// No nodes to score.
		return nil
	}

	if pl.sharedLister == nil {
		return framework.NewStatus(framework.Error, fmt.Sprintf("BuildTopologyPairToScore with empty shared lister"))
	}

	affinity := pod.Spec.Affinity
	hasAffinityConstraints := affinity != nil && affinity.PodAffinity != nil
	hasAntiAffinityConstraints := affinity != nil && affinity.PodAntiAffinity != nil

	// Unless the pod being scheduled has affinity terms, we only
	// need to process nodes hosting pods with affinity.
	var allNodes []*framework.NodeInfo
	var err error
	if hasAffinityConstraints || hasAntiAffinityConstraints {
		allNodes, err = pl.sharedLister.NodeInfos().List()
		if err != nil {
			framework.NewStatus(framework.Error, fmt.Sprintf("get all nodes from shared lister error, err: %v", err))
		}
	} else {
		allNodes, err = pl.sharedLister.NodeInfos().HavePodsWithAffinityList()
		if err != nil {
			framework.NewStatus(framework.Error, fmt.Sprintf("get pods with affinity list error, err: %v", err))
		}
	}

	state := &preScoreState{
		topologyScore: make(map[string]map[string]int64),
		podInfo:       framework.NewPodInfo(pod),
	}

	topoScores := make([]scoreMap, len(allNodes))
	index := int32(-1)
	processNode := func(i int) {
		nodeInfo := allNodes[i]
		if nodeInfo.Node() == nil {
			return
		}
		// Unless the pod being scheduled has affinity terms, we only
		// need to process pods with affinity in the node.
		podsToProcess := nodeInfo.PodsWithAffinity
		if hasAffinityConstraints || hasAntiAffinityConstraints {
			// We need to process all the pods.
			podsToProcess = nodeInfo.Pods
		}

		topoScore := make(scoreMap)
		for _, existingPod := range podsToProcess {
			pl.processExistingPod(state, existingPod, nodeInfo, pod, topoScore)
		}
		if len(topoScore) > 0 {
			topoScores[atomic.AddInt32(&index, 1)] = topoScore
		}
	}
	parallelize.Until(context.Background(), len(allNodes), processNode)

	for i := 0; i <= int(index); i++ {
		state.topologyScore.append(topoScores[i])
	}

	cycleState.Write(preScoreStateKey, state)
	return nil
}

func getPreScoreState(cycleState *framework.CycleState) (*preScoreState, error) {
	c, err := cycleState.Read(preScoreStateKey)
	if err != nil {
		return nil, fmt.Errorf("Error reading %q from cycleState: %v", preScoreStateKey, err)
	}

	s, ok := c.(*preScoreState)
	if !ok {
		return nil, fmt.Errorf("%+v  convert to interpodaffinity.preScoreState error", c)
	}
	return s, nil
}

// Score invoked at the Score extension point.
// The "score" returned in this function is the matching number of pods on the `nodeName`,
// it is normalized later.
func (pl *InterPodAffinity) Score(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
	nodeInfo, err := pl.sharedLister.NodeInfos().Get(nodeName)
	if err != nil || nodeInfo.Node() == nil {
		return 0, framework.NewStatus(framework.Error, fmt.Sprintf("getting node %q from Snapshot: %v, node is nil: %v", nodeName, err, nodeInfo.Node() == nil))
	}
	node := nodeInfo.Node()

	s, err := getPreScoreState(cycleState)
	if err != nil {
		return 0, framework.NewStatus(framework.Error, err.Error())
	}
	var score int64
	for tpKey, tpValues := range s.topologyScore {
		if v, exist := node.Labels[tpKey]; exist {
			score += tpValues[v]
		}
	}

	return score, nil
}

// NormalizeScore normalizes the score for each filteredNode.
// The basic rule is: the bigger the score(matching number of pods) is, the smaller the
// final normalized score will be.
func (pl *InterPodAffinity) NormalizeScore(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status {
	s, err := getPreScoreState(cycleState)
	if err != nil {
		return framework.NewStatus(framework.Error, err.Error())
	}
	if len(s.topologyScore) == 0 {
		return nil
	}

	var maxCount, minCount int64
	for i := range scores {
		score := scores[i].Score
		if score > maxCount {
			maxCount = score
		}
		if score < minCount {
			minCount = score
		}
	}

	maxMinDiff := maxCount - minCount
	for i := range scores {
		fScore := float64(0)
		if maxMinDiff > 0 {
			fScore = float64(framework.MaxNodeScore) * (float64(scores[i].Score-minCount) / float64(maxMinDiff))
		}

		scores[i].Score = int64(fScore)
	}

	return nil
}

// ScoreExtensions of the Score plugin.
func (pl *InterPodAffinity) ScoreExtensions() framework.ScoreExtensions {
	return pl
}
<commit_msg>Fix the incorrect error message of `InterPodAffinity` plugin<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package interpodaffinity

import (
	"context"
	"fmt"
	"sync/atomic"

	v1 "k8s.io/api/core/v1"
	framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
	"k8s.io/kubernetes/pkg/scheduler/internal/parallelize"
	schedutil "k8s.io/kubernetes/pkg/scheduler/util"
)

// preScoreStateKey is the key in CycleState to InterPodAffinity pre-computed data for Scoring.
const preScoreStateKey = "PreScore" + Name

type scoreMap map[string]map[string]int64

// preScoreState computed at PreScore and used at Score.
type preScoreState struct {
	topologyScore scoreMap
	podInfo       *framework.PodInfo
}

// Clone implements the mandatory Clone interface. We don't really copy the data since
// there is no need for that.
func (s *preScoreState) Clone() framework.StateData {
	return s
}

func (m scoreMap) processTerm(
	term *framework.WeightedAffinityTerm,
	podToCheck *v1.Pod,
	fixedNode *v1.Node,
	multiplier int,
) {
	if len(fixedNode.Labels) == 0 {
		return
	}

	match := schedutil.PodMatchesTermsNamespaceAndSelector(podToCheck, term.Namespaces, term.Selector)
	tpValue, tpValueExist := fixedNode.Labels[term.TopologyKey]
	if match && tpValueExist {
		if m[term.TopologyKey] == nil {
			m[term.TopologyKey] = make(map[string]int64)
		}
		m[term.TopologyKey][tpValue] += int64(term.Weight * int32(multiplier))
	}
	return
}

func (m scoreMap) processTerms(terms []framework.WeightedAffinityTerm, podToCheck *v1.Pod, fixedNode *v1.Node, multiplier int) {
	for _, term := range terms {
		m.processTerm(&term, podToCheck, fixedNode, multiplier)
	}
}

func (m scoreMap) append(other scoreMap) {
	for topology, oScores := range other {
		scores := m[topology]
		if scores == nil {
			m[topology] = oScores
			continue
		}
		for k, v := range oScores {
			scores[k] += v
		}
	}
}

func (pl *InterPodAffinity) processExistingPod(
	state *preScoreState,
	existingPod *framework.PodInfo,
	existingPodNodeInfo *framework.NodeInfo,
	incomingPod *v1.Pod,
	topoScore scoreMap,
) {
	existingPodNode := existingPodNodeInfo.Node()

	// For every soft pod affinity term of <pod>, if <existingPod> matches the term,
	// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPods>`s node by the term`s weight.
	topoScore.processTerms(state.podInfo.PreferredAffinityTerms, existingPod.Pod, existingPodNode, 1)

	// For every soft pod anti-affinity term of <pod>, if <existingPod> matches the term,
	// decrement <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPod>`s node by the term`s weight.
	topoScore.processTerms(state.podInfo.PreferredAntiAffinityTerms, existingPod.Pod, existingPodNode, -1)

	// For every hard pod affinity term of <existingPod>, if <pod> matches the term,
	// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPod>'s node by the constant <args.hardPodAffinityWeight>
	if pl.args.HardPodAffinityWeight > 0 {
		for _, term := range existingPod.RequiredAffinityTerms {
			t := framework.WeightedAffinityTerm{AffinityTerm: term, Weight: pl.args.HardPodAffinityWeight}
			topoScore.processTerm(&t, incomingPod, existingPodNode, 1)
		}
	}

	// For every soft pod affinity term of <existingPod>, if <pod> matches the term,
	// increment <p.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPod>'s node by the term's weight.
	topoScore.processTerms(existingPod.PreferredAffinityTerms, incomingPod, existingPodNode, 1)

	// For every soft pod anti-affinity term of <existingPod>, if <pod> matches the term,
	// decrement <pm.counts> for every node in the cluster with the same <term.TopologyKey>
	// value as that of <existingPod>'s node by the term's weight.
	topoScore.processTerms(existingPod.PreferredAntiAffinityTerms, incomingPod, existingPodNode, -1)
}

// PreScore builds and writes cycle state used by Score and NormalizeScore.
func (pl *InterPodAffinity) PreScore(
	pCtx context.Context,
	cycleState *framework.CycleState,
	pod *v1.Pod,
	nodes []*v1.Node,
) *framework.Status {
	if len(nodes) == 0 {
		// No nodes to score.
		return nil
	}

	if pl.sharedLister == nil {
		return framework.NewStatus(framework.Error, fmt.Sprintf("InterPodAffinity PreScore with empty shared lister found"))
	}

	affinity := pod.Spec.Affinity
	hasAffinityConstraints := affinity != nil && affinity.PodAffinity != nil
	hasAntiAffinityConstraints := affinity != nil && affinity.PodAntiAffinity != nil

	// Unless the pod being scheduled has affinity terms, we only
	// need to process nodes hosting pods with affinity.
	var allNodes []*framework.NodeInfo
	var err error
	if hasAffinityConstraints || hasAntiAffinityConstraints {
		allNodes, err = pl.sharedLister.NodeInfos().List()
		if err != nil {
			framework.NewStatus(framework.Error, fmt.Sprintf("get all nodes from shared lister error, err: %v", err))
		}
	} else {
		allNodes, err = pl.sharedLister.NodeInfos().HavePodsWithAffinityList()
		if err != nil {
			framework.NewStatus(framework.Error, fmt.Sprintf("get pods with affinity list error, err: %v", err))
		}
	}

	state := &preScoreState{
		topologyScore: make(map[string]map[string]int64),
		podInfo:       framework.NewPodInfo(pod),
	}

	topoScores := make([]scoreMap, len(allNodes))
	index := int32(-1)
	processNode := func(i int) {
		nodeInfo := allNodes[i]
		if nodeInfo.Node() == nil {
			return
		}
		// Unless the pod being scheduled has affinity terms, we only
		// need to process pods with affinity in the node.
		podsToProcess := nodeInfo.PodsWithAffinity
		if hasAffinityConstraints || hasAntiAffinityConstraints {
			// We need to process all the pods.
			podsToProcess = nodeInfo.Pods
		}

		topoScore := make(scoreMap)
		for _, existingPod := range podsToProcess {
			pl.processExistingPod(state, existingPod, nodeInfo, pod, topoScore)
		}
		if len(topoScore) > 0 {
			topoScores[atomic.AddInt32(&index, 1)] = topoScore
		}
	}
	parallelize.Until(context.Background(), len(allNodes), processNode)

	for i := 0; i <= int(index); i++ {
		state.topologyScore.append(topoScores[i])
	}

	cycleState.Write(preScoreStateKey, state)
	return nil
}

func getPreScoreState(cycleState *framework.CycleState) (*preScoreState, error) {
	c, err := cycleState.Read(preScoreStateKey)
	if err != nil {
		return nil, fmt.Errorf("Error reading %q from cycleState: %v", preScoreStateKey, err)
	}

	s, ok := c.(*preScoreState)
	if !ok {
		return nil, fmt.Errorf("%+v  convert to interpodaffinity.preScoreState error", c)
	}
	return s, nil
}

// Score invoked at the Score extension point.
// The "score" returned in this function is the matching number of pods on the `nodeName`,
// it is normalized later.
func (pl *InterPodAffinity) Score(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
	nodeInfo, err := pl.sharedLister.NodeInfos().Get(nodeName)
	if err != nil || nodeInfo.Node() == nil {
		return 0, framework.NewStatus(framework.Error, fmt.Sprintf("getting node %q from Snapshot: %v, node is nil: %v", nodeName, err, nodeInfo.Node() == nil))
	}
	node := nodeInfo.Node()

	s, err := getPreScoreState(cycleState)
	if err != nil {
		return 0, framework.NewStatus(framework.Error, err.Error())
	}
	var score int64
	for tpKey, tpValues := range s.topologyScore {
		if v, exist := node.Labels[tpKey]; exist {
			score += tpValues[v]
		}
	}

	return score, nil
}

// NormalizeScore normalizes the score for each filteredNode.
// The basic rule is: the bigger the score(matching number of pods) is, the smaller the
// final normalized score will be.
func (pl *InterPodAffinity) NormalizeScore(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status {
	s, err := getPreScoreState(cycleState)
	if err != nil {
		return framework.NewStatus(framework.Error, err.Error())
	}
	if len(s.topologyScore) == 0 {
		return nil
	}

	var maxCount, minCount int64
	for i := range scores {
		score := scores[i].Score
		if score > maxCount {
			maxCount = score
		}
		if score < minCount {
			minCount = score
		}
	}

	maxMinDiff := maxCount - minCount
	for i := range scores {
		fScore := float64(0)
		if maxMinDiff > 0 {
			fScore = float64(framework.MaxNodeScore) * (float64(scores[i].Score-minCount) / float64(maxMinDiff))
		}

		scores[i].Score = int64(fScore)
	}

	return nil
}

// ScoreExtensions of the Score plugin.
func (pl *InterPodAffinity) ScoreExtensions() framework.ScoreExtensions {
	return pl
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"path"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	. "github.com/jacobsa/ogletest"
)

func TestMount(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type MountTest struct {
	ctx   context.Context
	clock timeutil.SimulatedClock
	conn  gcs.Conn

	// A temporary directory that is cleaned up at the end of the test run.
	dir string
}

var _ SetUpInterface = &MountTest{}
var _ TearDownInterface = &MountTest{}

func init() { RegisterTestSuite(&MountTest{}) }

func (t *MountTest) SetUp(ti *TestInfo) {
	var err error

	t.ctx = ti.Ctx
	t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))
	t.conn = gcsfake.NewConn(&t.clock)

	// Set up the temporary directory.
	t.dir, err = ioutil.TempDir("", "run_test")
	AssertEq(nil, err)
}

func (t *MountTest) TearDown() {
	var err error

	err = os.RemoveAll(t.dir)
	AssertEq(nil, err)
}

func (t *MountTest) mount(
	args []string) (mfs *fuse.MountedFileSystem, err error) {
	mfs, err = mount(
		args,
		new(flag.FlagSet),
		t.conn)

	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *MountTest) BasicUsage() {
	var err error
	const fileName = "foo"

	// Grab a bucket.
	bucket := t.conn.GetBucket("some_bucket")

	// Mount that bucket.
	mfs, err := t.mount([]string{
		bucket.Name(),
		t.dir,
	})

	AssertEq(nil, err)

	// Create a file.
	err = ioutil.WriteFile(path.Join(t.dir, fileName), []byte("taco"), 0400)
	AssertEq(nil, err)

	// Read the object from the bucket.
	contents, err := gcsutil.ReadObject(t.ctx, bucket, fileName)
	AssertEq(nil, err)
	ExpectEq("taco", string(contents))

	// Read the file.
	contents, err = ioutil.ReadFile(path.Join(t.dir, fileName))
	AssertEq(nil, err)
	ExpectEq("taco", string(contents))

	// Unmount and join.
	err = fuse.Unmount(t.dir)
	AssertEq(nil, err)

	err = mfs.Join(t.ctx)
	AssertEq(nil, err)
}
<commit_msg>Fixed resource busy issues.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strings"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsfake"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	. "github.com/jacobsa/ogletest"
)

func TestMount(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type MountTest struct {
	ctx   context.Context
	clock timeutil.SimulatedClock
	conn  gcs.Conn

	// A temporary directory that is cleaned up at the end of the test run.
	dir string
}

var _ SetUpInterface = &MountTest{}
var _ TearDownInterface = &MountTest{}

func init() { RegisterTestSuite(&MountTest{}) }

func (t *MountTest) SetUp(ti *TestInfo) {
	var err error

	t.ctx = ti.Ctx
	t.clock.SetTime(time.Date(2012, 8, 15, 22, 56, 0, 0, time.Local))
	t.conn = gcsfake.NewConn(&t.clock)

	// Set up the temporary directory.
	t.dir, err = ioutil.TempDir("", "run_test")
	AssertEq(nil, err)
}

func (t *MountTest) TearDown() {
	var err error

	err = os.RemoveAll(t.dir)
	AssertEq(nil, err)
}

func (t *MountTest) mount(
	args []string) (mfs *fuse.MountedFileSystem, err error) {
	mfs, err = mount(
		args,
		new(flag.FlagSet),
		t.conn)

	return
}

// Unmount the file system. Try again on "resource busy" errors.
func (t *MountTest) unmount() (err error) {
	delay := 10 * time.Millisecond
	for {
		err = fuse.Unmount(t.dir)
		if err == nil {
			return
		}

		if strings.Contains(err.Error(), "resource busy") {
			log.Println("Resource busy error while unmounting; trying again")
			time.Sleep(delay)
			delay = time.Duration(1.3 * float64(delay))
			continue
		}

		err = fmt.Errorf("Unmount: %v", err)
		return
	}
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *MountTest) BasicUsage() {
	var err error
	const fileName = "foo"

	// Grab a bucket.
	bucket := t.conn.GetBucket("some_bucket")

	// Mount that bucket.
	mfs, err := t.mount([]string{
		bucket.Name(),
		t.dir,
	})

	AssertEq(nil, err)

	// Create a file.
	err = ioutil.WriteFile(path.Join(t.dir, fileName), []byte("taco"), 0400)
	AssertEq(nil, err)

	// Read the object from the bucket.
	contents, err := gcsutil.ReadObject(t.ctx, bucket, fileName)
	AssertEq(nil, err)
	ExpectEq("taco", string(contents))

	// Read the file.
	contents, err = ioutil.ReadFile(path.Join(t.dir, fileName))
	AssertEq(nil, err)
	ExpectEq("taco", string(contents))

	// Unmount and join.
	err = t.unmount()
	AssertEq(nil, err)

	err = mfs.Join(t.ctx)
	AssertEq(nil, err)
}
<|endoftext|>