<commit_before>package atirador

import (
	"time"

	"github.com/rafaeljusto/atiradorfrequente/núcleo/protocolo"
)

// validarCR garante que a frequência referente ao ID bate com o CR informado
// pelo usuário.
func validarCR(cr string, frequência frequência) protocolo.Mensagens {
	if cr != frequência.CR {
		return protocolo.NovasMensagens(
			protocolo.NovaMensagemComValor(protocolo.MensagemCódigoCRInválido, cr),
		)
	}

	return nil
}

// validarNúmeroControle garante que a frequência referente ao ID bate com o
// número de controle (ID + número aleatório).
func validarNúmeroControle(númeroControle protocolo.NúmeroControle, frequência frequência) protocolo.Mensagens {
	if númeroControle.ID() != frequência.ID || númeroControle.Controle() != frequência.Controle {
		return protocolo.NovasMensagens(
			protocolo.NovaMensagemComValor(protocolo.MensagemCódigoNúmeroControleInválido, númeroControle.String()),
		)
	}

	return nil
}

// validarIntervaloMáximoConfirmação verifique se o prazo máximo para envio da
// confirmação já expirou. No caso de expirado a mensagem de erro retornada
// informa qual foi a data limite.
func validarIntervaloMáximoConfirmação(frequência frequência, prazoConfirmação time.Duration) protocolo.Mensagens {
	if data := frequência.DataCriação.Add(prazoConfirmação); data.Before(time.Now()) {
		return protocolo.NovasMensagens(
			protocolo.NovaMensagemComValor(protocolo.MensagemCódigoPrazoConfirmaçãoExpirado, data.Format(time.RFC3339)),
		)
	}

	return nil
}

// TODO(rafaeljusto): Validar imagem de confirmação. Podemos validar o formato.
<commit_msg>Adiciona lembrete de regra na imagem de confirmação<commit_after>package atirador

import (
	"time"

	"github.com/rafaeljusto/atiradorfrequente/núcleo/protocolo"
)

// validarCR garante que a frequência referente ao ID bate com o CR informado
// pelo usuário.
func validarCR(cr string, frequência frequência) protocolo.Mensagens {
	if cr != frequência.CR {
		return protocolo.NovasMensagens(
			protocolo.NovaMensagemComValor(protocolo.MensagemCódigoCRInválido, cr),
		)
	}

	return nil
}

// validarNúmeroControle garante que a frequência referente ao ID bate com o
// número de controle (ID + número aleatório).
func validarNúmeroControle(númeroControle protocolo.NúmeroControle, frequência frequência) protocolo.Mensagens {
	if númeroControle.ID() != frequência.ID || númeroControle.Controle() != frequência.Controle {
		return protocolo.NovasMensagens(
			protocolo.NovaMensagemComValor(protocolo.MensagemCódigoNúmeroControleInválido, númeroControle.String()),
		)
	}

	return nil
}

// validarIntervaloMáximoConfirmação verifique se o prazo máximo para envio da
// confirmação já expirou. No caso de expirado a mensagem de erro retornada
// informa qual foi a data limite.
func validarIntervaloMáximoConfirmação(frequência frequência, prazoConfirmação time.Duration) protocolo.Mensagens {
	if data := frequência.DataCriação.Add(prazoConfirmação); data.Before(time.Now()) {
		return protocolo.NovasMensagens(
			protocolo.NovaMensagemComValor(protocolo.MensagemCódigoPrazoConfirmaçãoExpirado, data.Format(time.RFC3339)),
		)
	}

	return nil
}

// TODO(rafaeljusto): Validar imagem de confirmação. Podemos validar o formato.
// Não deve ser igual a imagem do número de controle (MD5?)
<|endoftext|>


<commit_before>package identify

import (
	"sync"
	"testing"
	"time"

	net "github.com/libp2p/go-libp2p-net"
	ma "github.com/multiformats/go-multiaddr"
)

// TestObsAddrSet
func TestObsAddrSet(t *testing.T) {
	m := func(s string) ma.Multiaddr {
		m, err := ma.NewMultiaddr(s)
		if err != nil {
			t.Error(err)
		}
		return m
	}

	addrsMarch := func(a, b []ma.Multiaddr) bool {
		if len(a) != len(b) {
			return false
		}

		for _, aa := range a {
			found := false
			for _, bb := range b {
				if aa.Equal(bb) {
					found = true
					break
				}
			}
			if !found {
				return false
			}
		}
		return true
	}

	a1 := m("/ip4/1.2.3.4/tcp/1231")
	a2 := m("/ip4/1.2.3.4/tcp/1232")
	a3 := m("/ip4/1.2.3.4/tcp/1233")
	a4 := m("/ip4/1.2.3.4/tcp/1234")
	a5 := m("/ip4/1.2.3.4/tcp/1235")

	b1 := m("/ip4/1.2.3.6/tcp/1236")
	b2 := m("/ip4/1.2.3.7/tcp/1237")
	b3 := m("/ip4/1.2.3.8/tcp/1237")
	b4 := m("/ip4/1.2.3.9/tcp/1237")
	b5 := m("/ip4/1.2.3.10/tcp/1237")

	oas := &ObservedAddrSet{}

	if !addrsMarch(oas.Addrs(), nil) {
		t.Error("addrs should be empty")
	}

	add := func(oas *ObservedAddrSet, observed, observer ma.Multiaddr) {
		dummyLocal := m("/ip4/127.0.0.1/tcp/10086")
		dummyDirection := net.DirOutbound

		oas.Add(observed, dummyLocal, observer, dummyDirection)
	}

	add(oas, a1, a4)
	add(oas, a2, a4)
	add(oas, a3, a4)

	// these are all different so we should not yet get them.
	if !addrsMarch(oas.Addrs(), nil) {
		t.Error("addrs should _still_ be empty (once)")
	}

	// same observer, so should not yet get them.
	add(oas, a1, a4)
	add(oas, a2, a4)
	add(oas, a3, a4)
	if !addrsMarch(oas.Addrs(), nil) {
		t.Error("addrs should _still_ be empty (same obs)")
	}

	// different observer, but same observer group.
	add(oas, a1, a5)
	add(oas, a2, a5)
	add(oas, a3, a5)
	if !addrsMarch(oas.Addrs(), nil) {
		t.Error("addrs should _still_ be empty (same obs group)")
	}

	add(oas, a1, b1)
	add(oas, a1, b2)
	add(oas, a1, b3)
	if !addrsMarch(oas.Addrs(), []ma.Multiaddr{a1}) {
		t.Error("addrs should only have a1")
	}

	add(oas, a2, a5)
	add(oas, a1, a5)
	add(oas, a1, a5)
	add(oas, a2, b1)
	add(oas, a1, b1)
	add(oas, a1, b1)
	add(oas, a2, b2)
	add(oas, a1, b2)
	add(oas, a1, b2)
	add(oas, a2, b4)
	add(oas, a2, b5)
	if !addrsMarch(oas.Addrs(), []ma.Multiaddr{a1, a2}) {
		t.Error("addrs should only have a1, a2")
	}

	// change the timeout constant so we can time it out.
	oas.SetTTL(time.Millisecond * 200)
	<-time.After(time.Millisecond * 210)
	if !addrsMarch(oas.Addrs(), nil) {
		t.Error("addrs should have timed out")
	}
}

func TestAddAddrsProfile(b *testing.T) {
	m := func(s string) ma.Multiaddr {
		m, err := ma.NewMultiaddr(s)
		if err != nil {
			b.Fatal(err)
		}
		return m
	}
	oas := &ObservedAddrSet{}

	add := func(oas *ObservedAddrSet, observed, observer ma.Multiaddr) {
		dummyLocal := m("/ip4/127.0.0.1/tcp/10086")
		dummyDirection := net.DirOutbound

		oas.Add(observed, dummyLocal, observer, dummyDirection)
	}

	a1 := m("/ip4/1.2.3.4/tcp/1231")
	a2 := m("/ip4/1.2.3.4/tcp/1232")
	a3 := m("/ip4/1.2.3.4/tcp/1233")
	a4 := m("/ip4/1.2.3.4/tcp/1234")
	a5 := m("/ip4/1.2.3.4/tcp/1235")

	b1 := m("/ip4/1.2.3.6/tcp/1236")
	b2 := m("/ip4/1.2.3.7/tcp/1237")
	b3 := m("/ip4/1.2.3.8/tcp/1237")
	b4 := m("/ip4/1.2.3.9/tcp/1237")
	b5 := m("/ip4/1.2.3.10/tcp/1237")

	_ = []ma.Multiaddr{a1, a2, a3, a4, a5, b1, b2, b3, b4, b5}

	var wg sync.WaitGroup
	for i := 0; i < 1000; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for j := 0; j < 10000; j++ {
				add(oas, a1, b1)
			}
		}()
	}

	wg.Wait()
}
<commit_msg>test: disable TestAddAddrsProfile when the race detector is enabled<commit_after>package identify

import (
	"sync"
	"testing"
	"time"

	detectrace "github.com/ipfs/go-detect-race"
	net "github.com/libp2p/go-libp2p-net"
	ma "github.com/multiformats/go-multiaddr"
)

// TestObsAddrSet
func TestObsAddrSet(t *testing.T) {
	m := func(s string) ma.Multiaddr {
		m, err := ma.NewMultiaddr(s)
		if err != nil {
			t.Error(err)
		}
		return m
	}

	addrsMarch := func(a, b []ma.Multiaddr) bool {
		if len(a) != len(b) {
			return false
		}

		for _, aa := range a {
			found := false
			for _, bb := range b {
				if aa.Equal(bb) {
					found = true
					break
				}
			}
			if !found {
				return false
			}
		}
		return true
	}

	a1 := m("/ip4/1.2.3.4/tcp/1231")
	a2 := m("/ip4/1.2.3.4/tcp/1232")
	a3 := m("/ip4/1.2.3.4/tcp/1233")
	a4 := m("/ip4/1.2.3.4/tcp/1234")
	a5 := m("/ip4/1.2.3.4/tcp/1235")

	b1 := m("/ip4/1.2.3.6/tcp/1236")
	b2 := m("/ip4/1.2.3.7/tcp/1237")
	b3 := m("/ip4/1.2.3.8/tcp/1237")
	b4 := m("/ip4/1.2.3.9/tcp/1237")
	b5 := m("/ip4/1.2.3.10/tcp/1237")

	oas := &ObservedAddrSet{}

	if !addrsMarch(oas.Addrs(), nil) {
		t.Error("addrs should be empty")
	}

	add := func(oas *ObservedAddrSet, observed, observer ma.Multiaddr) {
		dummyLocal := m("/ip4/127.0.0.1/tcp/10086")
		dummyDirection := net.DirOutbound

		oas.Add(observed, dummyLocal, observer, dummyDirection)
	}

	add(oas, a1, a4)
	add(oas, a2, a4)
	add(oas, a3, a4)

	// these are all different so we should not yet get them.
	if !addrsMarch(oas.Addrs(), nil) {
		t.Error("addrs should _still_ be empty (once)")
	}

	// same observer, so should not yet get them.
	add(oas, a1, a4)
	add(oas, a2, a4)
	add(oas, a3, a4)
	if !addrsMarch(oas.Addrs(), nil) {
		t.Error("addrs should _still_ be empty (same obs)")
	}

	// different observer, but same observer group.
	add(oas, a1, a5)
	add(oas, a2, a5)
	add(oas, a3, a5)
	if !addrsMarch(oas.Addrs(), nil) {
		t.Error("addrs should _still_ be empty (same obs group)")
	}

	add(oas, a1, b1)
	add(oas, a1, b2)
	add(oas, a1, b3)
	if !addrsMarch(oas.Addrs(), []ma.Multiaddr{a1}) {
		t.Error("addrs should only have a1")
	}

	add(oas, a2, a5)
	add(oas, a1, a5)
	add(oas, a1, a5)
	add(oas, a2, b1)
	add(oas, a1, b1)
	add(oas, a1, b1)
	add(oas, a2, b2)
	add(oas, a1, b2)
	add(oas, a1, b2)
	add(oas, a2, b4)
	add(oas, a2, b5)
	if !addrsMarch(oas.Addrs(), []ma.Multiaddr{a1, a2}) {
		t.Error("addrs should only have a1, a2")
	}

	// change the timeout constant so we can time it out.
	oas.SetTTL(time.Millisecond * 200)
	<-time.After(time.Millisecond * 210)
	if !addrsMarch(oas.Addrs(), nil) {
		t.Error("addrs should have timed out")
	}
}

func TestAddAddrsProfile(b *testing.T) {
	if detectrace.WithRace() {
		b.Skip("test too slow when the race detector is running")
	}
	m := func(s string) ma.Multiaddr {
		m, err := ma.NewMultiaddr(s)
		if err != nil {
			b.Fatal(err)
		}
		return m
	}
	oas := &ObservedAddrSet{}

	add := func(oas *ObservedAddrSet, observed, observer ma.Multiaddr) {
		dummyLocal := m("/ip4/127.0.0.1/tcp/10086")
		dummyDirection := net.DirOutbound

		oas.Add(observed, dummyLocal, observer, dummyDirection)
	}

	a1 := m("/ip4/1.2.3.4/tcp/1231")
	a2 := m("/ip4/1.2.3.4/tcp/1232")
	a3 := m("/ip4/1.2.3.4/tcp/1233")
	a4 := m("/ip4/1.2.3.4/tcp/1234")
	a5 := m("/ip4/1.2.3.4/tcp/1235")

	b1 := m("/ip4/1.2.3.6/tcp/1236")
	b2 := m("/ip4/1.2.3.7/tcp/1237")
	b3 := m("/ip4/1.2.3.8/tcp/1237")
	b4 := m("/ip4/1.2.3.9/tcp/1237")
	b5 := m("/ip4/1.2.3.10/tcp/1237")

	_ = []ma.Multiaddr{a1, a2, a3, a4, a5, b1, b2, b3, b4, b5}

	var wg sync.WaitGroup
	for i := 0; i < 1000; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for j := 0; j < 10000; j++ {
				add(oas, a1, b1)
			}
		}()
	}

	wg.Wait()
}
<|endoftext|>


<commit_before><commit_msg>highlight the deprecated apis<commit_after><|endoftext|>


<commit_before><commit_msg>Fixing CLI backwards compatibility (#5121)<commit_after><|endoftext|>


<commit_before>package torrent

import (
	"fmt"

	"github.com/cenkalti/rain/internal/peerprotocol"
	"github.com/cenkalti/rain/internal/verifier"
)

func (t *torrent) handleVerifyCommand() {
	t.log.Info("verifying")
	t.doVerify = true
	if t.status() == Stopped {
		t.bitfield = nil
		t.start()
	} else {
		t.stop(nil)
	}
}

func (t *torrent) handleVerificationDone(ve *verifier.Verifier) {
	if t.verifier != ve {
		panic("invalid verifier")
	}
	t.verifier = nil

	if ve.Error != nil {
		t.stop(fmt.Errorf("file verification error: %s", ve.Error))
		return
	}

	// Now we have a constructed and verified bitfield.
	t.mBitfield.Lock()
	t.bitfield = ve.Bitfield
	t.mBitfield.Unlock()

	// Save the bitfield to resume db.
	err := t.writeBitfield()
	if err != nil {
		t.stop(err)
		return
	}

	var haveMessages []peerprotocol.HaveMessage

	// Mark downloaded pieces.
	for i := uint32(0); i < t.bitfield.Len(); i++ {
		if t.bitfield.Test(i) {
			t.pieces[i].Done = true
			haveMessages = append(haveMessages, peerprotocol.HaveMessage{Index: i})
		}
	}

	if t.doVerify {
		t.doVerify = false
		t.stop(nil)
		return
	}

	// Tell connected peers that pieces we have.
	for pe := range t.peers {
		for _, msg := range haveMessages {
			pe.SendMessage(msg)
		}
		t.updateInterestedState(pe)
	}

	if t.checkCompletion() && t.stopAfterDownload {
		t.stop(nil)
		return
	}
	t.processQueuedMessages()
	t.addFixedPeers()
	t.startAcceptor()
	t.startAnnouncers()
	t.startPieceDownloaders()
}
<commit_msg>clear completed status when missing piece detected; fix #8<commit_after>package torrent

import (
	"fmt"

	"github.com/cenkalti/rain/internal/peerprotocol"
	"github.com/cenkalti/rain/internal/verifier"
)

func (t *torrent) handleVerifyCommand() {
	t.log.Info("verifying")
	t.doVerify = true
	if t.status() == Stopped {
		t.bitfield = nil
		t.start()
	} else {
		t.stop(nil)
	}
}

func (t *torrent) handleVerificationDone(ve *verifier.Verifier) {
	if t.verifier != ve {
		panic("invalid verifier")
	}
	t.verifier = nil

	if ve.Error != nil {
		t.stop(fmt.Errorf("file verification error: %s", ve.Error))
		return
	}

	// Now we have a constructed and verified bitfield.
	t.mBitfield.Lock()
	t.bitfield = ve.Bitfield
	t.mBitfield.Unlock()

	// Save the bitfield to resume db.
	err := t.writeBitfield()
	if err != nil {
		t.stop(err)
		return
	}

	var haveMessages []peerprotocol.HaveMessage

	// Mark downloaded pieces.
	for i := uint32(0); i < t.bitfield.Len(); i++ {
		if t.bitfield.Test(i) {
			t.pieces[i].Done = true
			haveMessages = append(haveMessages, peerprotocol.HaveMessage{Index: i})
		}
	}

	// We may detect missing pieces after verification. Then, status must be set from Seeding to Downloading.
	if !t.bitfield.All() {
		t.completed = false
		t.completeC = make(chan struct{})
	}

	if t.doVerify {
		// Stop after manual verification command.
		t.doVerify = false
		t.stop(nil)
		return
	}

	// Tell connected peers that pieces we have.
	for pe := range t.peers {
		for _, msg := range haveMessages {
			pe.SendMessage(msg)
		}
		t.updateInterestedState(pe)
	}

	if t.checkCompletion() && t.stopAfterDownload {
		t.stop(nil)
		return
	}
	t.processQueuedMessages()
	t.addFixedPeers()
	t.startAcceptor()
	t.startAnnouncers()
	t.startPieceDownloaders()
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Govet is a simple checker for static errors in Go source code.
// See doc.go for more information.
package main

import (
	"flag"
	"fmt"
	"io"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"utf8"
)

var verbose = flag.Bool("v", false, "verbose")
var printfuncs = flag.String("printfuncs", "", "comma-separated list of print function names to check")
var exitCode = 0

// setExit sets the value for os.Exit when it is called, later.  It
// remembers the highest value.
func setExit(err int) {
	if err > exitCode {
		exitCode = err
	}
}

// Usage is a replacement usage function for the flags package.
func Usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

// File is a wrapper for the state of a file used in the parser.
// The parse tree walkers are all methods of this type.
type File struct {
	file *token.File
}

func main() {
	flag.Usage = Usage
	flag.Parse()

	if *printfuncs != "" {
		for _, name := range strings.Split(*printfuncs, ",", -1) {
			if len(name) == 0 {
				flag.Usage()
			}
			skip := 0
			if colon := strings.LastIndex(name, ":"); colon > 0 {
				var err os.Error
				skip, err = strconv.Atoi(name[colon+1:])
				if err != nil {
					error(`illegal format for "Func:N" argument %q; %s`, name, err)
				}
				name = name[:colon]
			}
			if name[len(name)-1] == 'f' {
				printfList[name] = skip
			} else {
				printList[name] = skip
			}
		}
	}

	if flag.NArg() == 0 {
		doFile("stdin", os.Stdin)
	} else {
		for _, name := range flag.Args() {
			// Is it a directory?
			if fi, err := os.Stat(name); err == nil && fi.IsDirectory() {
				walkDir(name)
			} else {
				doFile(name, nil)
			}
		}
	}
	os.Exit(exitCode)
}

// doFile analyzes one file.  If the reader is nil, the source code is read from the
// named file.
func doFile(name string, reader io.Reader) {
	fs := token.NewFileSet()
	parsedFile, err := parser.ParseFile(fs, name, reader, 0)
	if err != nil {
		error("%s: %s", name, err)
		return
	}
	file := &File{fs.File(parsedFile.Pos())}
	file.checkFile(name, parsedFile)
}

// Visitor for filepath.Walk - trivial.  Just calls doFile on each file.
// TODO: if govet becomes richer, might want to process
// a directory (package) at a time.
type V struct{}

func (v V) VisitDir(path string, f *os.FileInfo) bool {
	return true
}

func (v V) VisitFile(path string, f *os.FileInfo) {
	if strings.HasSuffix(path, ".go") {
		doFile(path, nil)
	}
}

// walkDir recursively walks the tree looking for .go files.
func walkDir(root string) {
	errors := make(chan os.Error)
	done := make(chan bool)
	go func() {
		for e := range errors {
			error("walk error: %s", e)
		}
		done <- true
	}()
	filepath.Walk(root, V{}, errors)
	close(errors)
	<-done
}

// error formats the error to standard error, adding program
// identification and a newline
func error(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "govet: "+format+"\n", args...)
	setExit(2)
}

// Println is fmt.Println guarded by -v.
func Println(args ...interface{}) {
	if !*verbose {
		return
	}
	fmt.Println(args...)
}

// Printf is fmt.Printf guarded by -v.
func Printf(format string, args ...interface{}) {
	if !*verbose {
		return
	}
	fmt.Printf(format+"\n", args...)
}

// Bad reports an error and sets the exit code..
func (f *File) Bad(pos token.Pos, args ...interface{}) {
	f.Warn(pos, args...)
	setExit(1)
}

// Badf reports a formatted error and sets the exit code.
func (f *File) Badf(pos token.Pos, format string, args ...interface{}) {
	f.Warnf(pos, format, args...)
	setExit(1)
}

// Warn reports an error but does not set the exit code.
func (f *File) Warn(pos token.Pos, args ...interface{}) {
	loc := f.file.Position(pos).String() + ": "
	fmt.Fprint(os.Stderr, loc+fmt.Sprintln(args...))
}

// Warnf reports a formatted error but does not set the exit code.
func (f *File) Warnf(pos token.Pos, format string, args ...interface{}) {
	loc := f.file.Position(pos).String() + ": "
	fmt.Fprintf(os.Stderr, loc+format+"\n", args...)
}

// checkFile checks all the top-level declarations in a file.
func (f *File) checkFile(name string, file *ast.File) {
	Println("Checking file", name)
	ast.Walk(f, file)
}

// Visit implements the ast.Visitor interface.
func (f *File) Visit(node ast.Node) ast.Visitor {
	// TODO: could return nil for nodes that cannot contain a CallExpr -
	// will shortcut traversal.  Worthwhile?
	switch n := node.(type) {
	case *ast.CallExpr:
		f.checkCallExpr(n)
	}
	return f
}

// checkCallExpr checks a call expression.
func (f *File) checkCallExpr(call *ast.CallExpr) {
	switch x := call.Fun.(type) {
	case *ast.Ident:
		f.checkCall(call, x.Name)
	case *ast.SelectorExpr:
		f.checkCall(call, x.Sel.Name)
	}
}

// printfList records the formatted-print functions. The value is the location
// of the format parameter.
var printfList = map[string]int{
	"Errorf":  0,
	"Fatalf":  0,
	"Fprintf": 1,
	"Panicf":  0,
	"Printf":  0,
	"Sprintf": 0,
}

// printList records the unformatted-print functions. The value is the location
// of the first parameter to be printed.
var printList = map[string]int{
	"Error":  0,
	"Fatal":  0,
	"Fprint": 1, "Fprintln": 1,
	"Panic": 0, "Panicln": 0,
	"Print": 0, "Println": 0,
	"Sprint": 0, "Sprintln": 0,
}

// checkCall triggers the print-specific checks if the call invokes a print function.
func (f *File) checkCall(call *ast.CallExpr, name string) {
	if skip, ok := printfList[name]; ok {
		f.checkPrintf(call, name, skip)
		return
	}
	if skip, ok := printList[name]; ok {
		f.checkPrint(call, name, skip)
		return
	}
}

// checkPrintf checks a call to a formatted print routine such as Printf.
// The skip argument records how many arguments to ignore; that is,
// call.Args[skip] is (well, should be) the format argument.
func (f *File) checkPrintf(call *ast.CallExpr, name string, skip int) {
	if len(call.Args) <= skip {
		return
	}
	// Common case: literal is first argument.
	arg := call.Args[skip]
	lit, ok := arg.(*ast.BasicLit)
	if !ok {
		// Too hard to check.
		if *verbose {
			f.Warn(call.Pos(), "can't check args for call to", name)
		}
		return
	}
	if lit.Kind == token.STRING {
		if strings.Contains(lit.Value, "%") {
			if len(call.Args) > skip+1 {
				f.Badf(call.Pos(), "no formatting directive in %s call", name)
			}
			return
		}
	}
	// Hard part: check formats against args.
	// Trivial but useful test: count.
	numArgs := 0
	for i, w := 0, 0; i < len(lit.Value); i += w {
		w = 1
		if lit.Value[i] == '%' {
			nbytes, nargs := parsePrintfVerb(lit.Value[i:])
			w = nbytes
			numArgs += nargs
		}
	}
	expect := len(call.Args) - (skip + 1)
	if numArgs != expect {
		f.Badf(call.Pos(), "wrong number of args in %s call: %d needed but %d args", name, numArgs, expect)
	}
}

// parsePrintfVerb returns the number of bytes and number of arguments
// consumed by the Printf directive that begins s, including its percent sign
// and verb.
func parsePrintfVerb(s string) (nbytes, nargs int) {
	// There's guaranteed a percent sign.
	nbytes = 1
	end := len(s)
	// There may be flags.
FlagLoop:
	for nbytes < end {
		switch s[nbytes] {
		case '#', '0', '+', '-', ' ':
			nbytes++
		default:
			break FlagLoop
		}
	}
	getNum := func() {
		if nbytes < end && s[nbytes] == '*' {
			nbytes++
			nargs++
		} else {
			for nbytes < end && '0' <= s[nbytes] && s[nbytes] <= '9' {
				nbytes++
			}
		}
	}
	// There may be a width.
	getNum()
	// If there's a period, there may be a precision.
	if nbytes < end && s[nbytes] == '.' {
		nbytes++
		getNum()
	}
	// Now a verb.
	c, w := utf8.DecodeRuneInString(s[nbytes:])
	nbytes += w
	if c != '%' {
		nargs++
	}
	return
}

// checkPrint checks a call to an unformatted print routine such as Println.
// The skip argument records how many arguments to ignore; that is,
// call.Args[skip] is the first argument to be printed.
func (f *File) checkPrint(call *ast.CallExpr, name string, skip int) {
	isLn := strings.HasSuffix(name, "ln")
	args := call.Args
	if len(args) <= skip {
		if *verbose && !isLn {
			f.Badf(call.Pos(), "no args in %s call", name)
		}
		return
	}
	arg := args[skip]
	if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING {
		if strings.Contains(lit.Value, "%") {
			f.Badf(call.Pos(), "possible formatting directive in %s call", name)
		}
	}
	if isLn {
		// The last item, if a string, should not have a newline.
		arg = args[len(call.Args)-1]
		if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING {
			if strings.HasSuffix(lit.Value, `\n"`) {
				f.Badf(call.Pos(), "%s call ends with newline", name)
			}
		}
	}
}

// This function never executes, but it serves as a simple test for the program.
// Test with govet -printfuncs="Bad:1,Badf:1,Warn:1,Warnf:1" govet.go
func BadFunctionUsedInTests() {
	fmt.Println()                      // niladic call
	fmt.Println("%s", "hi")            // % in call to Println
	fmt.Printf("%s", "hi", 3)          // wrong # percents
	fmt.Printf("%s%%%d", "hi", 3)      // right # percents
	fmt.Printf("%.*d", 3, 3)           // right # percents, with a *
	fmt.Printf("%.*d", 3, 3, 3)        // wrong # percents, with a *
	Printf("now is the time", "buddy") // no %s
	f := new(File)
	f.Warn(0, "%s", "hello", 3)  // % in call to added function
	f.Warnf(0, "%s", "hello", 3) // wrong # %s in call to added function
}
<commit_msg>govet: fix bug introduced at 4313054<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Govet is a simple checker for static errors in Go source code.
// See doc.go for more information.
package main

import (
	"flag"
	"fmt"
	"io"
	"go/ast"
	"go/parser"
	"go/token"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"utf8"
)

var verbose = flag.Bool("v", false, "verbose")
var printfuncs = flag.String("printfuncs", "", "comma-separated list of print function names to check")
var exitCode = 0

// setExit sets the value for os.Exit when it is called, later.  It
// remembers the highest value.
func setExit(err int) {
	if err > exitCode {
		exitCode = err
	}
}

// Usage is a replacement usage function for the flags package.
func Usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(2)
}

// File is a wrapper for the state of a file used in the parser.
// The parse tree walkers are all methods of this type.
type File struct {
	file *token.File
}

func main() {
	flag.Usage = Usage
	flag.Parse()

	if *printfuncs != "" {
		for _, name := range strings.Split(*printfuncs, ",", -1) {
			if len(name) == 0 {
				flag.Usage()
			}
			skip := 0
			if colon := strings.LastIndex(name, ":"); colon > 0 {
				var err os.Error
				skip, err = strconv.Atoi(name[colon+1:])
				if err != nil {
					error(`illegal format for "Func:N" argument %q; %s`, name, err)
				}
				name = name[:colon]
			}
			if name[len(name)-1] == 'f' {
				printfList[name] = skip
			} else {
				printList[name] = skip
			}
		}
	}

	if flag.NArg() == 0 {
		doFile("stdin", os.Stdin)
	} else {
		for _, name := range flag.Args() {
			// Is it a directory?
			if fi, err := os.Stat(name); err == nil && fi.IsDirectory() {
				walkDir(name)
			} else {
				doFile(name, nil)
			}
		}
	}
	os.Exit(exitCode)
}

// doFile analyzes one file.  If the reader is nil, the source code is read from the
// named file.
func doFile(name string, reader io.Reader) {
	fs := token.NewFileSet()
	parsedFile, err := parser.ParseFile(fs, name, reader, 0)
	if err != nil {
		error("%s: %s", name, err)
		return
	}
	file := &File{fs.File(parsedFile.Pos())}
	file.checkFile(name, parsedFile)
}

// Visitor for filepath.Walk - trivial.  Just calls doFile on each file.
// TODO: if govet becomes richer, might want to process
// a directory (package) at a time.
type V struct{}

func (v V) VisitDir(path string, f *os.FileInfo) bool {
	return true
}

func (v V) VisitFile(path string, f *os.FileInfo) {
	if strings.HasSuffix(path, ".go") {
		doFile(path, nil)
	}
}

// walkDir recursively walks the tree looking for .go files.
func walkDir(root string) {
	errors := make(chan os.Error)
	done := make(chan bool)
	go func() {
		for e := range errors {
			error("walk error: %s", e)
		}
		done <- true
	}()
	filepath.Walk(root, V{}, errors)
	close(errors)
	<-done
}

// error formats the error to standard error, adding program
// identification and a newline
func error(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, "govet: "+format+"\n", args...)
	setExit(2)
}

// Println is fmt.Println guarded by -v.
func Println(args ...interface{}) {
	if !*verbose {
		return
	}
	fmt.Println(args...)
}

// Printf is fmt.Printf guarded by -v.
func Printf(format string, args ...interface{}) {
	if !*verbose {
		return
	}
	fmt.Printf(format+"\n", args...)
}

// Bad reports an error and sets the exit code..
func (f *File) Bad(pos token.Pos, args ...interface{}) {
	f.Warn(pos, args...)
	setExit(1)
}

// Badf reports a formatted error and sets the exit code.
func (f *File) Badf(pos token.Pos, format string, args ...interface{}) {
	f.Warnf(pos, format, args...)
	setExit(1)
}

// Warn reports an error but does not set the exit code.
func (f *File) Warn(pos token.Pos, args ...interface{}) {
	loc := f.file.Position(pos).String() + ": "
	fmt.Fprint(os.Stderr, loc+fmt.Sprintln(args...))
}

// Warnf reports a formatted error but does not set the exit code.
func (f *File) Warnf(pos token.Pos, format string, args ...interface{}) {
	loc := f.file.Position(pos).String() + ": "
	fmt.Fprintf(os.Stderr, loc+format+"\n", args...)
}

// checkFile checks all the top-level declarations in a file.
func (f *File) checkFile(name string, file *ast.File) {
	Println("Checking file", name)
	ast.Walk(f, file)
}

// Visit implements the ast.Visitor interface.
func (f *File) Visit(node ast.Node) ast.Visitor {
	// TODO: could return nil for nodes that cannot contain a CallExpr -
	// will shortcut traversal.  Worthwhile?
	switch n := node.(type) {
	case *ast.CallExpr:
		f.checkCallExpr(n)
	}
	return f
}

// checkCallExpr checks a call expression.
func (f *File) checkCallExpr(call *ast.CallExpr) {
	switch x := call.Fun.(type) {
	case *ast.Ident:
		f.checkCall(call, x.Name)
	case *ast.SelectorExpr:
		f.checkCall(call, x.Sel.Name)
	}
}

// printfList records the formatted-print functions. The value is the location
// of the format parameter.
var printfList = map[string]int{
	"Errorf":  0,
	"Fatalf":  0,
	"Fprintf": 1,
	"Panicf":  0,
	"Printf":  0,
	"Sprintf": 0,
}

// printList records the unformatted-print functions. The value is the location
// of the first parameter to be printed.
var printList = map[string]int{
	"Error":  0,
	"Fatal":  0,
	"Fprint": 1, "Fprintln": 1,
	"Panic": 0, "Panicln": 0,
	"Print": 0, "Println": 0,
	"Sprint": 0, "Sprintln": 0,
}

// checkCall triggers the print-specific checks if the call invokes a print function.
func (f *File) checkCall(call *ast.CallExpr, name string) {
	if skip, ok := printfList[name]; ok {
		f.checkPrintf(call, name, skip)
		return
	}
	if skip, ok := printList[name]; ok {
		f.checkPrint(call, name, skip)
		return
	}
}

// checkPrintf checks a call to a formatted print routine such as Printf.
// The skip argument records how many arguments to ignore; that is,
// call.Args[skip] is (well, should be) the format argument.
func (f *File) checkPrintf(call *ast.CallExpr, name string, skip int) {
	if len(call.Args) <= skip {
		return
	}
	// Common case: literal is first argument.
	arg := call.Args[skip]
	lit, ok := arg.(*ast.BasicLit)
	if !ok {
		// Too hard to check.
		if *verbose {
			f.Warn(call.Pos(), "can't check args for call to", name)
		}
		return
	}
	if lit.Kind == token.STRING {
		if !strings.Contains(lit.Value, "%") {
			if len(call.Args) > skip+1 {
				f.Badf(call.Pos(), "no formatting directive in %s call", name)
			}
			return
		}
	}
	// Hard part: check formats against args.
	// Trivial but useful test: count.
	numArgs := 0
	for i, w := 0, 0; i < len(lit.Value); i += w {
		w = 1
		if lit.Value[i] == '%' {
			nbytes, nargs := parsePrintfVerb(lit.Value[i:])
			w = nbytes
			numArgs += nargs
		}
	}
	expect := len(call.Args) - (skip + 1)
	if numArgs != expect {
		f.Badf(call.Pos(), "wrong number of args in %s call: %d needed but %d args", name, numArgs, expect)
	}
}

// parsePrintfVerb returns the number of bytes and number of arguments
// consumed by the Printf directive that begins s, including its percent sign
// and verb.
func parsePrintfVerb(s string) (nbytes, nargs int) {
	// There's guaranteed a percent sign.
	nbytes = 1
	end := len(s)
	// There may be flags.
FlagLoop:
	for nbytes < end {
		switch s[nbytes] {
		case '#', '0', '+', '-', ' ':
			nbytes++
		default:
			break FlagLoop
		}
	}
	getNum := func() {
		if nbytes < end && s[nbytes] == '*' {
			nbytes++
			nargs++
		} else {
			for nbytes < end && '0' <= s[nbytes] && s[nbytes] <= '9' {
				nbytes++
			}
		}
	}
	// There may be a width.
	getNum()
	// If there's a period, there may be a precision.
	if nbytes < end && s[nbytes] == '.' {
		nbytes++
		getNum()
	}
	// Now a verb.
	c, w := utf8.DecodeRuneInString(s[nbytes:])
	nbytes += w
	if c != '%' {
		nargs++
	}
	return
}

// checkPrint checks a call to an unformatted print routine such as Println.
// The skip argument records how many arguments to ignore; that is,
// call.Args[skip] is the first argument to be printed.
func (f *File) checkPrint(call *ast.CallExpr, name string, skip int) {
	isLn := strings.HasSuffix(name, "ln")
	args := call.Args
	if len(args) <= skip {
		if *verbose && !isLn {
			f.Badf(call.Pos(), "no args in %s call", name)
		}
		return
	}
	arg := args[skip]
	if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING {
		if strings.Contains(lit.Value, "%") {
			f.Badf(call.Pos(), "possible formatting directive in %s call", name)
		}
	}
	if isLn {
		// The last item, if a string, should not have a newline.
		arg = args[len(call.Args)-1]
		if lit, ok := arg.(*ast.BasicLit); ok && lit.Kind == token.STRING {
			if strings.HasSuffix(lit.Value, `\n"`) {
				f.Badf(call.Pos(), "%s call ends with newline", name)
			}
		}
	}
}

// This function never executes, but it serves as a simple test for the program.
// Test with govet -printfuncs="Bad:1,Badf:1,Warn:1,Warnf:1" govet.go
func BadFunctionUsedInTests() {
	fmt.Println()                      // niladic call
	fmt.Println("%s", "hi")            // % in call to Println
	fmt.Printf("%s", "hi", 3)          // wrong # percents
	fmt.Printf("%s%%%d", "hi", 3)      // right # percents
	fmt.Printf("%.*d", 3, 3)           // right # percents, with a *
	fmt.Printf("%.*d", 3, 3, 3)        // wrong # percents, with a *
	Printf("now is the time", "buddy") // no %s
	f := new(File)
	f.Warn(0, "%s", "hello", 3)  // % in call to added function
	f.Warnf(0, "%s", "hello", 3) // wrong # %s in call to added function
}
<|endoftext|>


<commit_before><commit_msg>Always close object storage writer (even after errors)<commit_after><|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"

	"k8s.io/klog"
)

// PackagesBuilder adds miscellaneous OS packages that we need
type PackagesBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &DockerBuilder{}

// Build is responsible for installing packages
func (b *PackagesBuilder) Build(c *fi.ModelBuilderContext) error {
	// kubelet needs:
	//   conntrack  - kops #5671
	//   ebtables - kops #1711
	//   ethtool - kops #1830
	if b.Distribution.IsDebianFamily() {
		// From containerd: https://github.com/containerd/cri/blob/master/contrib/ansible/tasks/bootstrap_ubuntu.yaml
		c.AddTask(&nodetasks.Package{Name: "bridge-utils"})
		c.AddTask(&nodetasks.Package{Name: "conntrack"})
		c.AddTask(&nodetasks.Package{Name: "ebtables"})
		c.AddTask(&nodetasks.Package{Name: "ethtool"})
		c.AddTask(&nodetasks.Package{Name: "iptables"})
		c.AddTask(&nodetasks.Package{Name: "libapparmor1"})
		c.AddTask(&nodetasks.Package{Name: "libseccomp2"})
		c.AddTask(&nodetasks.Package{Name: "libltdl7"})
		c.AddTask(&nodetasks.Package{Name: "pigz"})
		c.AddTask(&nodetasks.Package{Name: "socat"})
		c.AddTask(&nodetasks.Package{Name: "util-linux"})
	} else if b.Distribution.IsRHELFamily() {
		// From containerd: https://github.com/containerd/cri/blob/master/contrib/ansible/tasks/bootstrap_centos.yaml
		c.AddTask(&nodetasks.Package{Name: "conntrack-tools"})
		c.AddTask(&nodetasks.Package{Name: "ebtables"})
		c.AddTask(&nodetasks.Package{Name: "ethtool"})
		c.AddTask(&nodetasks.Package{Name: "iptables"})
		c.AddTask(&nodetasks.Package{Name: "libseccomp"})
		c.AddTask(&nodetasks.Package{Name: "libtool-ltdl"})
		c.AddTask(&nodetasks.Package{Name: "socat"})
		c.AddTask(&nodetasks.Package{Name: "util-linux"})
		// Handle some packages differently for each distro
		switch b.Distribution {
		case distros.DistributionRhel7:
			// Easier to install container-selinux from CentOS than extras
			c.AddTask(&nodetasks.Package{
				Name:   "container-selinux",
				Source: s("http://vault.centos.org/7.6.1810/extras/x86_64/Packages/container-selinux-2.107-1.el7_6.noarch.rpm"),
				Hash:   s("7de4211fa0dfd240d8827b93763e1eb5f0d56411"),
			})
		case distros.DistributionAmazonLinux2:
			// Amazon Linux 2 doesn't have SELinux enabled by default
		default:
			c.AddTask(&nodetasks.Package{Name: "container-selinux"})
			c.AddTask(&nodetasks.Package{Name: "pigz"})
		}
	} else {
		// Hopefully they are already installed
		klog.Warningf("unknown distribution, skipping required packages install: %v", b.Distribution)
	}

	return nil
}
<commit_msg>Add cgroup related deps<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"

	"k8s.io/klog"
)

// PackagesBuilder adds miscellaneous OS packages that we need
type PackagesBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &DockerBuilder{}

// Build is responsible for installing packages
func (b *PackagesBuilder) Build(c *fi.ModelBuilderContext) error {
	// kubelet needs:
	//   conntrack  - kops #5671
	//   ebtables - kops #1711
	//   ethtool - kops #1830
	if b.Distribution.IsDebianFamily() {
		// From containerd: https://github.com/containerd/cri/blob/master/contrib/ansible/tasks/bootstrap_ubuntu.yaml
		c.AddTask(&nodetasks.Package{Name: "bridge-utils"})
		c.AddTask(&nodetasks.Package{Name: "cgroupfs-mount"})
		c.AddTask(&nodetasks.Package{Name: "conntrack"})
		c.AddTask(&nodetasks.Package{Name: "ebtables"})
		c.AddTask(&nodetasks.Package{Name: "ethtool"})
		c.AddTask(&nodetasks.Package{Name: "iptables"})
		c.AddTask(&nodetasks.Package{Name: "libapparmor1"})
		c.AddTask(&nodetasks.Package{Name: "libseccomp2"})
		c.AddTask(&nodetasks.Package{Name: "libltdl7"})
		c.AddTask(&nodetasks.Package{Name: "pigz"})
		c.AddTask(&nodetasks.Package{Name: "socat"})
		c.AddTask(&nodetasks.Package{Name: "util-linux"})
	} else if b.Distribution.IsRHELFamily() {
		// From containerd: https://github.com/containerd/cri/blob/master/contrib/ansible/tasks/bootstrap_centos.yaml
		c.AddTask(&nodetasks.Package{Name: "conntrack-tools"})
		c.AddTask(&nodetasks.Package{Name: "ebtables"})
		c.AddTask(&nodetasks.Package{Name: "ethtool"})
		c.AddTask(&nodetasks.Package{Name: "iptables"})
		c.AddTask(&nodetasks.Package{Name: "libcgroup"})
		c.AddTask(&nodetasks.Package{Name: "libseccomp"})
		c.AddTask(&nodetasks.Package{Name: "libtool-ltdl"})
		c.AddTask(&nodetasks.Package{Name: "socat"})
		c.AddTask(&nodetasks.Package{Name: "util-linux"})
		// Handle some packages differently for each distro
		switch b.Distribution {
		case distros.DistributionRhel7:
			// Easier to install container-selinux from CentOS than extras
			c.AddTask(&nodetasks.Package{
				Name:   "container-selinux",
				Source: s("http://vault.centos.org/7.6.1810/extras/x86_64/Packages/container-selinux-2.107-1.el7_6.noarch.rpm"),
				Hash:   s("7de4211fa0dfd240d8827b93763e1eb5f0d56411"),
			})
		case distros.DistributionAmazonLinux2:
			// Amazon Linux 2 doesn't have SELinux enabled by default
		default:
			c.AddTask(&nodetasks.Package{Name: "container-selinux"})
			c.AddTask(&nodetasks.Package{Name: "pigz"})
		}
	} else {
		// Hopefully they are already installed
		klog.Warningf("unknown distribution, skipping required packages install: %v", b.Distribution)
	}

	return nil
}
<|endoftext|>


<commit_before><commit_msg>close body<commit_after><|endoftext|>


<commit_before>package arn

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestStreamAnime(t *testing.T) {
	allAnime, err := StreamAnime()

	assert.NoError(t, err)
	assert.NotNil(t, allAnime)

	validAnimeStatus := []string{
		"finished",
		"current",
		"tba",
		"upcoming",
		"unreleased",
	}

	for anime := range allAnime {
		assert.NotEmpty(t, anime.ID)
		assert.Contains(t, validAnimeStatus, anime.Status)
	}
}
<commit_msg>Improved test coverage<commit_after>package arn

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestStreamAnime(t *testing.T) {
	allAnime, err := StreamAnime()

	assert.NoError(t, err)
	assert.NotNil(t, allAnime)

	validAnimeStatus := []string{
		"finished",
		"current",
		"tba",
		"upcoming",
		"unreleased",
	}

	for anime := range allAnime {
		assert.NotEmpty(t, anime.ID)
		assert.Contains(t, validAnimeStatus, anime.Status)
		assert.NotEmpty(t, anime.Link())

		anime.Episodes()
		anime.Characters()
		anime.GetMapping("shoboi/anime")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package interpodaffinity

import (
	"fmt"

	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	listersv1 "k8s.io/client-go/listers/core/v1"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/pkg/scheduler/apis/config"
	"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
	"k8s.io/kubernetes/pkg/scheduler/framework"
	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
	"k8s.io/kubernetes/pkg/scheduler/internal/parallelize"
)

const (
	// Name is the name of the plugin used in the plugin registry and configurations.
	Name = "InterPodAffinity"
)

var _ framework.PreFilterPlugin = &InterPodAffinity{}
var _ framework.FilterPlugin = &InterPodAffinity{}
var _ framework.PreScorePlugin = &InterPodAffinity{}
var _ framework.ScorePlugin = &InterPodAffinity{}

// InterPodAffinity is a plugin that checks inter pod affinity
type InterPodAffinity struct {
	parallelizer            parallelize.Parallelizer
	args                    config.InterPodAffinityArgs
	sharedLister            framework.SharedLister
	nsLister                listersv1.NamespaceLister
	enableNamespaceSelector bool
}

// Name returns name of the plugin. It is used in logs, etc.
func (pl *InterPodAffinity) Name() string {
	return Name
}

// New initializes a new plugin and returns it.
func New(plArgs runtime.Object, h framework.Handle, fts feature.Features) (framework.Plugin, error) {
	if h.SnapshotSharedLister() == nil {
		return nil, fmt.Errorf("SnapshotSharedlister is nil")
	}
	args, err := getArgs(plArgs)
	if err != nil {
		return nil, err
	}
	if err := validation.ValidateInterPodAffinityArgs(args); err != nil {
		return nil, err
	}
	pl := &InterPodAffinity{
		parallelizer:            h.Parallelizer(),
		args:                    args,
		sharedLister:            h.SnapshotSharedLister(),
		enableNamespaceSelector: fts.EnablePodAffinityNamespaceSelector,
	}

	if pl.enableNamespaceSelector {
		pl.nsLister = h.SharedInformerFactory().Core().V1().Namespaces().Lister()
	}
	return pl, nil
}

func getArgs(obj runtime.Object) (config.InterPodAffinityArgs, error) {
	ptr, ok := obj.(*config.InterPodAffinityArgs)
	if !ok {
		return config.InterPodAffinityArgs{}, fmt.Errorf("want args to be of type InterPodAffinityArgs, got %T", obj)
	}
	return *ptr, nil
}

// Updates Namespaces with the set of namespaces identified by NamespaceSelector.
// If successful, NamespaceSelector is set to nil.
// The assumption is that the term is for an incoming pod, in which case
// namespaceSelector is either unrolled into Namespaces (and so the selector
// is set to Nothing()) or is Empty(), which means match everything. Therefore,
// there when matching against this term, there is no need to lookup the existing
// pod's namespace labels to match them against term's namespaceSelector explicitly.
func (pl *InterPodAffinity) mergeAffinityTermNamespacesIfNotEmpty(at *framework.AffinityTerm) error {
	if at.NamespaceSelector.Empty() {
		return nil
	}
	ns, err := pl.nsLister.List(at.NamespaceSelector)
	if err != nil {
		return err
	}
	for _, n := range ns {
		at.Namespaces.Insert(n.Name)
	}
	at.NamespaceSelector = labels.Nothing()
	return nil
}

// GetNamespaceLabelsSnapshot returns a snapshot of the labels associated with
// the namespace.
func GetNamespaceLabelsSnapshot(ns string, nsLister listersv1.NamespaceLister) (nsLabels labels.Set) {
	podNS, err := nsLister.Get(ns)
	if err == nil {
		// Create and return snapshot of the labels.
		return labels.Merge(podNS.Labels, nil)
	}
	klog.ErrorS(err, "getting namespace, assuming empty set of namespace labels", "namespace", ns)
	return
}
<commit_msg>implement EnqueueExtensions in interPodAffinity<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package interpodaffinity

import (
	"fmt"

	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	listersv1 "k8s.io/client-go/listers/core/v1"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/pkg/scheduler/apis/config"
	"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
	"k8s.io/kubernetes/pkg/scheduler/framework"
	"k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature"
	"k8s.io/kubernetes/pkg/scheduler/internal/parallelize"
)

const (
	// Name is the name of the plugin used in the plugin registry and configurations.
	Name = "InterPodAffinity"
)

var _ framework.PreFilterPlugin = &InterPodAffinity{}
var _ framework.FilterPlugin = &InterPodAffinity{}
var _ framework.PreScorePlugin = &InterPodAffinity{}
var _ framework.ScorePlugin = &InterPodAffinity{}
var _ framework.EnqueueExtensions = &InterPodAffinity{}

// InterPodAffinity is a plugin that checks inter pod affinity
type InterPodAffinity struct {
	parallelizer            parallelize.Parallelizer
	args                    config.InterPodAffinityArgs
	sharedLister            framework.SharedLister
	nsLister                listersv1.NamespaceLister
	enableNamespaceSelector bool
}

// Name returns name of the plugin. It is used in logs, etc.
func (pl *InterPodAffinity) Name() string {
	return Name
}

// EventsToRegister returns the possible events that may make a failed Pod
// schedulable
func (pl *InterPodAffinity) EventsToRegister() []framework.ClusterEvent {
	return []framework.ClusterEvent{
		// All ActionType includes the following events:
		// - Delete. An unschedulable Pod may fail due to violating an existing Pod's anti-affinity constraints,
		// deleting an existing Pod may make it schedulable.
		// - Update. Updating on an existing Pod's labels (e.g., removal) may make
		// an unschedulable Pod schedulable.
		// - Add. An unschedulable Pod may fail due to violating pod-affinity constraints,
		// adding an assigned Pod may make it schedulable.
		{Resource: framework.Pod, ActionType: framework.All},
		{Resource: framework.Node, ActionType: framework.Add | framework.UpdateNodeLabel},
	}
}

// New initializes a new plugin and returns it.
func New(plArgs runtime.Object, h framework.Handle, fts feature.Features) (framework.Plugin, error) {
	if h.SnapshotSharedLister() == nil {
		return nil, fmt.Errorf("SnapshotSharedlister is nil")
	}
	args, err := getArgs(plArgs)
	if err != nil {
		return nil, err
	}
	if err := validation.ValidateInterPodAffinityArgs(args); err != nil {
		return nil, err
	}
	pl := &InterPodAffinity{
		parallelizer:            h.Parallelizer(),
		args:                    args,
		sharedLister:            h.SnapshotSharedLister(),
		enableNamespaceSelector: fts.EnablePodAffinityNamespaceSelector,
	}

	if pl.enableNamespaceSelector {
		pl.nsLister = h.SharedInformerFactory().Core().V1().Namespaces().Lister()
	}
	return pl, nil
}

func getArgs(obj runtime.Object) (config.InterPodAffinityArgs, error) {
	ptr, ok := obj.(*config.InterPodAffinityArgs)
	if !ok {
		return config.InterPodAffinityArgs{}, fmt.Errorf("want args to be of type InterPodAffinityArgs, got %T", obj)
	}
	return *ptr, nil
}

// Updates Namespaces with the set of namespaces identified by NamespaceSelector.
// If successful, NamespaceSelector is set to nil.
// The assumption is that the term is for an incoming pod, in which case
// namespaceSelector is either unrolled into Namespaces (and so the selector
// is set to Nothing()) or is Empty(), which means match everything. Therefore,
// there when matching against this term, there is no need to lookup the existing
// pod's namespace labels to match them against term's namespaceSelector explicitly.
func (pl *InterPodAffinity) mergeAffinityTermNamespacesIfNotEmpty(at *framework.AffinityTerm) error {
	if at.NamespaceSelector.Empty() {
		return nil
	}
	ns, err := pl.nsLister.List(at.NamespaceSelector)
	if err != nil {
		return err
	}
	for _, n := range ns {
		at.Namespaces.Insert(n.Name)
	}
	at.NamespaceSelector = labels.Nothing()
	return nil
}

// GetNamespaceLabelsSnapshot returns a snapshot of the labels associated with
// the namespace.
func GetNamespaceLabelsSnapshot(ns string, nsLister listersv1.NamespaceLister) (nsLabels labels.Set) {
	podNS, err := nsLister.Get(ns)
	if err == nil {
		// Create and return snapshot of the labels.
		return labels.Merge(podNS.Labels, nil)
	}
	klog.ErrorS(err, "getting namespace, assuming empty set of namespace labels", "namespace", ns)
	return
}
<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"os"
	"time"

	"github.com/syou6162/go-active-learning/lib/example"
)

func FilterLabeledExamples(examples example.Examples) example.Examples {
	var result example.Examples
	for _, e := range examples {
		if e.IsLabeled() {
			result = append(result, e)
		}
	}
	return result
}

func FilterUnlabeledExamples(examples example.Examples) example.Examples {
	result := example.Examples{}

	alreadyLabeledByURL := make(map[string]bool)
	alreadyLabeledByTitle := make(map[string]bool)
	for _, e := range FilterLabeledExamples(examples) {
		alreadyLabeledByURL[e.Url] = true
		alreadyLabeledByURL[e.FinalUrl] = true
		alreadyLabeledByTitle[e.Title] = true
	}

	for _, e := range examples {
		if _, ok := alreadyLabeledByURL[e.Url]; ok {
			continue
		}
		if _, ok := alreadyLabeledByTitle[e.Title]; ok {
			continue
		}
		if !e.IsLabeled() {
			alreadyLabeledByURL[e.Url] = true
			alreadyLabeledByURL[e.FinalUrl] = true
			alreadyLabeledByTitle[e.Title] = true
			result = append(result, e)
		}
	}
	return result
}

func RemoveDuplicate(args []string) []string {
	results := make([]string, 0)
	encountered := map[string]bool{}
	for i := 0; i < len(args); i++ {
		if !encountered[args[i]] {
			encountered[args[i]] = true
			results = append(results, args[i])
		}
	}
	return results
}

func FilterStatusCodeOkExamples(examples example.Examples) example.Examples {
	result := example.Examples{}

	for _, e := range examples {
		if e.StatusCode == 200 {
			result = append(result, e)
		}
	}

	return result
}

func RemoveExample(examples example.Examples, toBeRemoved example.Example) example.Examples {
	result := example.Examples{}

	for _, e := range examples {
		if e.Url != toBeRemoved.Url {
			result = append(result, e)
		}
	}

	return result
}

func RemoveNegativeExamples(examples example.Examples) example.Examples {
	result := example.Examples{}
	for _, e := range examples {
		if e.Label != example.NEGATIVE {
			result = append(result, e)
		}
	}
	return result
}

func SplitTrainAndDev(examples example.Examples) (train example.Examples, dev example.Examples) {
	Shuffle(examples)
	n := int(0.8 * float64(len(examples)))
	return examples[0:n], examples[n:]
}

func GetEnv(key, fallback string) string {
	value, ok := os.LookupEnv(key)
	if !ok {
		value = fallback
	}
	return value
}
<commit_msg>unused<commit_after>package util

import (
	"os"

	"github.com/syou6162/go-active-learning/lib/example"
)

func FilterLabeledExamples(examples example.Examples) example.Examples {
	var result example.Examples
	for _, e := range examples {
		if e.IsLabeled() {
			result = append(result, e)
		}
	}
	return result
}

func FilterUnlabeledExamples(examples example.Examples) example.Examples {
	result := example.Examples{}

	alreadyLabeledByURL := make(map[string]bool)
	alreadyLabeledByTitle := make(map[string]bool)
	for _, e := range FilterLabeledExamples(examples) {
		alreadyLabeledByURL[e.Url] = true
		alreadyLabeledByURL[e.FinalUrl] = true
		alreadyLabeledByTitle[e.Title] = true
	}

	for _, e := range examples {
		if _, ok := alreadyLabeledByURL[e.Url]; ok {
			continue
		}
		if _, ok := alreadyLabeledByTitle[e.Title]; ok {
			continue
		}
		if !e.IsLabeled() {
			alreadyLabeledByURL[e.Url] = true
			alreadyLabeledByURL[e.FinalUrl] = true
			alreadyLabeledByTitle[e.Title] = true
			result = append(result, e)
		}
	}
	return result
}

func RemoveDuplicate(args []string) []string {
	results := make([]string, 0)
	encountered := map[string]bool{}
	for i := 0; i < len(args); i++ {
		if !encountered[args[i]] {
			encountered[args[i]] = true
			results = append(results, args[i])
		}
	}
	return results
}

func FilterStatusCodeOkExamples(examples example.Examples) example.Examples {
	result := example.Examples{}

	for _, e := range examples {
		if e.StatusCode == 200 {
			result = append(result, e)
		}
	}

	return result
}

func RemoveExample(examples example.Examples, toBeRemoved example.Example) example.Examples {
	result := example.Examples{}

	for _, e := range examples {
		if e.Url != toBeRemoved.Url {
			result = append(result, e)
		}
	}

	return result
}

func RemoveNegativeExamples(examples example.Examples) example.Examples {
	result := example.Examples{}
	for _, e := range examples {
		if e.Label != example.NEGATIVE {
			result = append(result, e)
		}
	}
	return result
}

func SplitTrainAndDev(examples example.Examples) (train example.Examples, dev example.Examples) {
	Shuffle(examples)
	n := int(0.8 * float64(len(examples)))
	return examples[0:n], examples[n:]
}

func GetEnv(key, fallback string) string {
	value, ok := os.LookupEnv(key)
	if !ok {
		value = fallback
	}
	return value
}
<|endoftext|>


<commit_before>package hhfrag

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/tabwriter"

	"github.com/BurntSushi/bcbgo/apps/hhsuite"
	"github.com/BurntSushi/bcbgo/io/hhm"
	"github.com/BurntSushi/bcbgo/io/hhr"
	"github.com/BurntSushi/bcbgo/io/pdb"
	"github.com/BurntSushi/bcbgo/seq"
)

type PDBDatabase hhsuite.Database

func (db PDBDatabase) HHsuite() hhsuite.Database {
	resolved := hhsuite.Database(db).Resolve()
	dbName := path.Base(resolved)
	return hhsuite.Database(path.Join(resolved, dbName))
}

func (db PDBDatabase) PDB() string {
	resolved := hhsuite.Database(db).Resolve()
	return path.Join(resolved, "pdb")
}

type Fragments struct {
	Frags      []Fragment
	Start, End int
}

// better returns true if f1 is 'better' than f2. Otherwise false.
func (f1 Fragments) better(f2 Fragments) bool {
	return len(f1.Frags) >= len(f2.Frags)
}

func (frags Fragments) Write(w io.Writer) {
	tabw := tabwriter.NewWriter(w, 0, 4, 4, ' ', 0)
	fmt.Fprintln(tabw, "Hit\tQuery\tTemplate\tProb\tCorrupt")
	for _, frag := range frags.Frags {
		var corruptStr string
		if frag.IsCorrupt() {
			corruptStr = "\tcorrupt"
		}
		fmt.Fprintf(tabw, "%s\t(%d-%d)\t(%d-%d)\t%f%s\n",
			frag.Template.Name,
			frag.Hit.QueryStart, frag.Hit.QueryEnd,
			frag.Hit.TemplateStart, frag.Hit.TemplateEnd,
			frag.Hit.Prob,
			corruptStr)
	}
	tabw.Flush()
}

func FindFragments(pdbDb PDBDatabase, blits bool,
	queryHHM *hhm.HHM, qs seq.Sequence, start, end int) (*Fragments, error) {

	pre := fmt.Sprintf("bcbgo-hhfrag-hhm-%d-%d_", start, end)
	hhmFile, err := ioutil.TempFile("", pre)
	if err != nil {
		return nil, err
	}
	defer os.Remove(hhmFile.Name())
	hhmName := hhmFile.Name()

	if err := hhm.Write(hhmFile, queryHHM.Slice(start, end)); err != nil {
		return nil, err
	}

	var results *hhr.HHR
	if blits {
		results, err = hhsuite.HHBlitsDefault.Run(pdbDb.HHsuite(), hhmName)
	} else {
		results, err = hhsuite.HHSearchDefault.Run(pdbDb.HHsuite(), hhmName)
	}
	if err != nil {
		return nil, err
	}

	frags := make([]Fragment, len(results.Hits))
	for i, hit := range results.Hits {
		hit.QueryStart += start
		hit.QueryEnd += start
		frag, err := NewFragment(pdbDb, qs, hit)
		if err != nil {
			return nil, err
		}
		frags[i] = frag
	}
	return &Fragments{
		Frags: frags,
		Start: start,
		End:   end,
	}, nil
}

// An HHfrag Fragment corresponds to a match between a portion of a query
// HMM and a portion of a template HMM. The former is represented as a slice
// of a regular sequence, where the latter is represented as an hhsuite hit
// and a list of alpha-carbon atoms corresponding to the matched region.
type Fragment struct {
	Query    seq.Sequence
	Template seq.Sequence
	Hit      hhr.Hit
	CaAtoms  pdb.Atoms
}

// IsCorrupt returns true when a particular fragment could not be paired
// with alpha-carbon positions for every residue in the template strand.
// (This problem stems from the fact that we use SEQRES records for sequence
// information, but not all residues in SEQRES have alpha-carbon ATOM records
// associated with them.)
func (frag Fragment) IsCorrupt() bool {
	return frag.CaAtoms == nil
}

// NewFragment constructs a new fragment from a full query sequence and the
// hit from the HHR file.
//
// Since NewFragment requires access to the raw PDB alpha-carbon atoms (and
// the sequence) of the template hit, you'll also need to pass a path to the
// PDB database. (Which is a directory containing a flat list of all
// PDB files used to construct the corresponding hhblits database.) This
// database is usually located inside the 'pdb' directory contained in the
// corresponding hhsuite database. i.e., $HHLIB/data/pdb-select25/pdb
func NewFragment(
	pdbDb PDBDatabase, qs seq.Sequence, hit hhr.Hit) (Fragment, error) {

	pdbName := getTemplatePdbName(hit.Name)
	pdbEntry, err := pdb.New(path.Join(
		pdbDb.PDB(), fmt.Sprintf("%s.pdb", pdbName)))
	if err != nil {
		return Fragment{}, err
	}

	// Load in the sequence from the PDB file using the SEQRES residues.
	ts, te := hit.TemplateStart, hit.TemplateEnd
	chain := pdbEntry.OneChain()
	tseq := seq.Sequence{
		Name:     pdbName,
		Residues: make([]seq.Residue, te-ts+1),
	}

	// We copy here to avoid pinning pdb.Entry objects.
	copy(tseq.Residues, chain.Sequence[ts-1:te])

	frag := Fragment{
		Query:    qs.Slice(hit.QueryStart-1, hit.QueryEnd),
		Template: tseq,
		Hit:      hit,
		CaAtoms:  nil,
	}

	// We designate "corrupt" if the query/template hit regions are of
	// different length. i.e., we don't allow gaps (yet).
	// BUG(burntsushi): Fragments with gaps are marked as corrupt.
	if hit.QueryEnd-hit.QueryStart != hit.TemplateEnd-hit.TemplateStart {
		return frag, nil
	}

	// We also designate "corrupt" if there are any gaps in our alpha-carbon
	// atom list.
	atoms := chain.CaAtomSlice(ts-1, te)
	if atoms == nil {
		return frag, nil
	}

	// One again, we copy to avoid pinning memory.
	frag.CaAtoms = make(pdb.Atoms, len(atoms))
	copy(frag.CaAtoms, atoms)

	return frag, nil
}

func getTemplatePdbName(hitName string) string {
	return strings.SplitN(strings.TrimSpace(hitName), " ", 2)[0]
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<commit_msg>Only use 1 CPU in hhfrag, since we're parallelizing at a higher level.<commit_after>package hhfrag

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"text/tabwriter"

	"github.com/BurntSushi/bcbgo/apps/hhsuite"
	"github.com/BurntSushi/bcbgo/io/hhm"
	"github.com/BurntSushi/bcbgo/io/hhr"
	"github.com/BurntSushi/bcbgo/io/pdb"
	"github.com/BurntSushi/bcbgo/seq"
)

type PDBDatabase hhsuite.Database

func (db PDBDatabase) HHsuite() hhsuite.Database {
	resolved := hhsuite.Database(db).Resolve()
	dbName := path.Base(resolved)
	return hhsuite.Database(path.Join(resolved, dbName))
}

func (db PDBDatabase) PDB() string {
	resolved := hhsuite.Database(db).Resolve()
	return path.Join(resolved, "pdb")
}

type Fragments struct {
	Frags      []Fragment
	Start, End int
}

// better returns true if f1 is 'better' than f2. Otherwise false.
func (f1 Fragments) better(f2 Fragments) bool {
	return len(f1.Frags) >= len(f2.Frags)
}

func (frags Fragments) Write(w io.Writer) {
	tabw := tabwriter.NewWriter(w, 0, 4, 4, ' ', 0)
	fmt.Fprintln(tabw, "Hit\tQuery\tTemplate\tProb\tCorrupt")
	for _, frag := range frags.Frags {
		var corruptStr string
		if frag.IsCorrupt() {
			corruptStr = "\tcorrupt"
		}
		fmt.Fprintf(tabw, "%s\t(%d-%d)\t(%d-%d)\t%f%s\n",
			frag.Template.Name,
			frag.Hit.QueryStart, frag.Hit.QueryEnd,
			frag.Hit.TemplateStart, frag.Hit.TemplateEnd,
			frag.Hit.Prob,
			corruptStr)
	}
	tabw.Flush()
}

func FindFragments(pdbDb PDBDatabase, blits bool,
	queryHHM *hhm.HHM, qs seq.Sequence, start, end int) (*Fragments, error) {

	pre := fmt.Sprintf("bcbgo-hhfrag-hhm-%d-%d_", start, end)
	hhmFile, err := ioutil.TempFile("", pre)
	if err != nil {
		return nil, err
	}
	defer os.Remove(hhmFile.Name())
	hhmName := hhmFile.Name()

	if err := hhm.Write(hhmFile, queryHHM.Slice(start, end)); err != nil {
		return nil, err
	}

	var results *hhr.HHR
	if blits {
		conf := hhsuite.HHBlitsDefault
		conf.CPUs = 1
		results, err = conf.Run(pdbDb.HHsuite(), hhmName)
	} else {
		conf := hhsuite.HHSearchDefault
		conf.CPUs = 1
		results, err = conf.Run(pdbDb.HHsuite(), hhmName)
	}
	if err != nil {
		return nil, err
	}

	frags := make([]Fragment, len(results.Hits))
	for i, hit := range results.Hits {
		hit.QueryStart += start
		hit.QueryEnd += start
		frag, err := NewFragment(pdbDb, qs, hit)
		if err != nil {
			return nil, err
		}
		frags[i] = frag
	}
	return &Fragments{
		Frags: frags,
		Start: start,
		End:   end,
	}, nil
}

// An HHfrag Fragment corresponds to a match between a portion of a query
// HMM and a portion of a template HMM. The former is represented as a slice
// of a regular sequence, where the latter is represented as an hhsuite hit
// and a list of alpha-carbon atoms corresponding to the matched region.
type Fragment struct {
	Query    seq.Sequence
	Template seq.Sequence
	Hit      hhr.Hit
	CaAtoms  pdb.Atoms
}

// IsCorrupt returns true when a particular fragment could not be paired
// with alpha-carbon positions for every residue in the template strand.
// (This problem stems from the fact that we use SEQRES records for sequence
// information, but not all residues in SEQRES have alpha-carbon ATOM records
// associated with them.)
func (frag Fragment) IsCorrupt() bool {
	return frag.CaAtoms == nil
}

// NewFragment constructs a new fragment from a full query sequence and the
// hit from the HHR file.
//
// Since NewFragment requires access to the raw PDB alpha-carbon atoms (and
// the sequence) of the template hit, you'll also need to pass a path to the
// PDB database. (Which is a directory containing a flat list of all
// PDB files used to construct the corresponding hhblits database.) This
// database is usually located inside the 'pdb' directory contained in the
// corresponding hhsuite database. i.e., $HHLIB/data/pdb-select25/pdb
func NewFragment(
	pdbDb PDBDatabase, qs seq.Sequence, hit hhr.Hit) (Fragment, error) {

	pdbName := getTemplatePdbName(hit.Name)
	pdbEntry, err := pdb.New(path.Join(
		pdbDb.PDB(), fmt.Sprintf("%s.pdb", pdbName)))
	if err != nil {
		return Fragment{}, err
	}

	// Load in the sequence from the PDB file using the SEQRES residues.
	ts, te := hit.TemplateStart, hit.TemplateEnd
	chain := pdbEntry.OneChain()
	tseq := seq.Sequence{
		Name:     pdbName,
		Residues: make([]seq.Residue, te-ts+1),
	}

	// We copy here to avoid pinning pdb.Entry objects.
	copy(tseq.Residues, chain.Sequence[ts-1:te])

	frag := Fragment{
		Query:    qs.Slice(hit.QueryStart-1, hit.QueryEnd),
		Template: tseq,
		Hit:      hit,
		CaAtoms:  nil,
	}

	// We designate "corrupt" if the query/template hit regions are of
	// different length. i.e., we don't allow gaps (yet).
	// BUG(burntsushi): Fragments with gaps are marked as corrupt.
	if hit.QueryEnd-hit.QueryStart != hit.TemplateEnd-hit.TemplateStart {
		return frag, nil
	}

	// We also designate "corrupt" if there are any gaps in our alpha-carbon
	// atom list.
	atoms := chain.CaAtomSlice(ts-1, te)
	if atoms == nil {
		return frag, nil
	}

	// One again, we copy to avoid pinning memory.
	frag.CaAtoms = make(pdb.Atoms, len(atoms))
	copy(frag.CaAtoms, atoms)

	return frag, nil
}

func getTemplatePdbName(hitName string) string {
	return strings.SplitN(strings.TrimSpace(hitName), " ", 2)[0]
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package location

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"

	"github.com/shinofara/stand/config"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

type S3 struct {
	cli        *s3.S3
	storageCfg *config.StorageConfig
}

func NewS3(storageCfg *config.StorageConfig) *S3 {

	s3Config := storageCfg.S3Config
	cre := credentials.NewStaticCredentials(
		s3Config.AccessKeyID,
		s3Config.SecretAccessKey,
		"")

	cli := s3.New(session.New(), &aws.Config{
		Credentials: cre,
		Region:      aws.String(s3Config.Region),
	})

	s := &S3{
		cli:        cli,
		storageCfg: storageCfg,
	}

	return s
}

func (s *S3) Save(filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = s.cli.PutObject(&s3.PutObjectInput{
		Bucket: aws.String(s.storageCfg.S3Config.BucketName),
		Key:    aws.String(s.makeUploadPath(file)),
		Body:   file,
	})
	if err != nil {
		return err
	}

	return nil
}

func (s *S3) Clean() error {
	resp, _ := s.findAll()

	pattern := replacePattern(s.storageCfg.Path)

	var keys []string
	for _, obj := range resp.Contents {
		if matched, _ := path.Match(pattern, *obj.Key); matched {
			keys = append(keys, *obj.Key)
		}
	}

	sort.Sort(sort.Reverse(sort.StringSlice(keys)))

	var num int64 = 0
	for _, key := range keys {
		if num > s.storageCfg.LifeCyrcle {
			// delete
			if err := s.delete(key); err != nil {
				return err
			}
		}

		num++
	}
	return nil
}

func (s *S3) findAll() (*s3.ListObjectsOutput, error) {
	return s.cli.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(s.storageCfg.S3Config.BucketName),
	})
}

func (s *S3) delete(key string) error {
	_, err := s.cli.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(s.storageCfg.S3Config.BucketName),
		Key:    aws.String(key),
	})

	if err != nil {
		return err
	}

	return nil
}

func (s *S3) makeUploadPath(file *os.File) string {
	_, name := filepath.Split(file.Name())
	return fmt.Sprintf("%s/%s", s.storageCfg.Path, name)
}

func replacePattern(str string) string {
	rep := regexp.MustCompile(`^/`)
	path := fmt.Sprintf("%s",
		rep.ReplaceAllString(str, ""),
	)

	if regexp.MustCompile(`/$`).MatchString(str) {
		return fmt.Sprintf("%s*", path)
	}

	return fmt.Sprintf("%s/*", path)
}
<commit_msg>ファイルアップロード完了時に対象となったファイルを削除<commit_after>package location

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"

	"github.com/shinofara/stand/config"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

type S3 struct {
	cli        *s3.S3
	storageCfg *config.StorageConfig
}

func NewS3(storageCfg *config.StorageConfig) *S3 {

	s3Config := storageCfg.S3Config
	cre := credentials.NewStaticCredentials(
		s3Config.AccessKeyID,
		s3Config.SecretAccessKey,
		"")

	cli := s3.New(session.New(), &aws.Config{
		Credentials: cre,
		Region:      aws.String(s3Config.Region),
	})

	s := &S3{
		cli:        cli,
		storageCfg: storageCfg,
	}

	return s
}

func (s *S3) Save(filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = s.cli.PutObject(&s3.PutObjectInput{
		Bucket: aws.String(s.storageCfg.S3Config.BucketName),
		Key:    aws.String(s.makeUploadPath(file)),
		Body:   file,
	})
	if err != nil {
		return err
	}

	if err := os.Remove(filename); err != nil {
		return err
	}

	return nil
}

func (s *S3) Clean() error {
	resp, _ := s.findAll()

	pattern := replacePattern(s.storageCfg.Path)

	var keys []string
	for _, obj := range resp.Contents {
		if matched, _ := path.Match(pattern, *obj.Key); matched {
			keys = append(keys, *obj.Key)
		}
	}

	sort.Sort(sort.Reverse(sort.StringSlice(keys)))

	var num int64 = 0
	for _, key := range keys {
		if num > s.storageCfg.LifeCyrcle {
			// delete
			if err := s.delete(key); err != nil {
				return err
			}
		}

		num++
	}
	return nil
}

func (s *S3) findAll() (*s3.ListObjectsOutput, error) {
	return s.cli.ListObjects(&s3.ListObjectsInput{
		Bucket: aws.String(s.storageCfg.S3Config.BucketName),
	})
}

func (s *S3) delete(key string) error {
	_, err := s.cli.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(s.storageCfg.S3Config.BucketName),
		Key:    aws.String(key),
	})

	if err != nil {
		return err
	}

	return nil
}

func (s *S3) makeUploadPath(file *os.File) string {
	_, name := filepath.Split(file.Name())
	return fmt.Sprintf("%s/%s", s.storageCfg.Path, name)
}

func replacePattern(str string) string {
	rep := regexp.MustCompile(`^/`)
	path := fmt.Sprintf("%s",
		rep.ReplaceAllString(str, ""),
	)

	if regexp.MustCompile(`/$`).MatchString(str) {
		return fmt.Sprintf("%s*", path)
	}

	return fmt.Sprintf("%s/*", path)
}
<|endoftext|>


<commit_before>package goftp

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"regexp"
	"strings"
)

// use FEAT to retrieve server features.
func (ftp *FTP) getServerFeatures() (features uint32) {
	features = 0
	code, str := ftp.RawCmd("FEAT")
	if code < 0 || code > 299 {
		return BAD
	}

	lines := strings.Split(str, "\n")

	for _, f := range lines {
		// Only MLSD/MLST is supported
		if (features&MLSD == 0) && strings.Contains(f, "MLST") { // not a bug, see command reference.
			features = features | MLSD

			// Check of NLST and EPLF are disabled since not yet supported

			/*} else if (features&NLST == 0) && strings.Contains(f, "NLST") {
				features = features | NLST
			} else if (features&EPLF == 0) && strings.Contains(f, "EPLF") {
				features = features | EPLF*/
		}
	}

	return features
}

// list the path (or current directory) and parse it.
// Return an array with the files, one with the directories and one with the links
func (ftp *FTP) GetFilesList(path string) (files []string, directories []string, links []string, err error) {
	if ftp.supportedfeatures == 0 {
		ftp.supportedfeatures = ftp.getServerFeatures()
	}
	if len(path) < 2 {
		path = "./"
	}
	if ftp.supportedfeatures&MLSD > 0 {
		//fmt.Println("Using MLSD")
		code, response := ftp.RawPassiveCmd("MLSD " + path)

		if code == 550 {
			return nil, nil, nil, errors.New("550 Requested action not taken. File unavailable.")
		}
		if code < 0 || code > 299 {
			return nil, nil, nil, errors.New(fmt.Sprintf("%v MLSD did not work ", code))
		}
		return ftp.parseMLSD(response, path)
	} else if ftp.supportedfeatures&EPLF > 0 {
		//fmt.Println("Using EPLF")
		code, response := ftp.RawPassiveCmd("EPLF " + path)

		if code == 550 {
			return nil, nil, nil, errors.New("550 Requested action not taken. File unavailable.")
		}
		if code < 0 || code > 299 {
			return nil, nil, nil, errors.New(fmt.Sprintf("%v EPLF did not work ", code))
		}
		return ftp.parseEPLF(response, path)
	} else if ftp.supportedfeatures&NLST > 0 {
		//fmt.Println("Using NLST")
		code, response := ftp.RawPassiveCmd("NLST " + path)

		if code == 550 {
			return nil, nil, nil, errors.New("550 Requested action not taken. File unavailable.")
		}
		if code < 0 || code > 299 {
			return nil, nil, nil, errors.New(fmt.Sprintf("%v NLST did not work ", code))
		}
		return ftp.parseNLST(response, path)
	} else {
		//fmt.Println("Using LIST")
		code, response := ftp.RawPassiveCmd("LIST " + path)
		//fmt.Println(response)
		if code == 550 {
			return nil, nil, nil, errors.New("550 Requested action not taken. File unavailable.")
		}
		if code < 0 || code > 299 {
			return nil, nil, nil, errors.New("LIST did not work")
		}
		return ftp.parseUnixLIST(response, path)
	}
}

// Parse the response of a MLSD
// Return an array with the files, one with the directories and one with the links
func (ftp *FTP) parseMLSD(data []string, basePath string) (files []string, directories []string, links []string, err error) {
	for _, line := range data {
		_, t, subpath := parseLine_MLST(line)

		switch t {
		case "dir":
			if subpath == "." {
			} else if subpath == ".." {
			} else {
				directories = append(directories, basePath+subpath+"/")
			}
		case "file":
			files = append(files, basePath+subpath)
		case "OS.unix=symlink":
			links = append(links, basePath+subpath)
		}

	}
	return files, directories, links, err
}

// Parse a single MLST line
func parseLine_MLST(line string) (perm string, t string, filename string) {
	for _, v := range strings.Split(line, ";") {
		v2 := strings.Split(v, "=")

		switch v2[0] {
		case "perm":
			perm = v2[1]
		case "type":
			t = v2[1]
			if t == "OS.unix" {
				t = t + "=" + v2[2]
				//fmt.Println("Found a link -> " + t)
			}
		default:
			filename = v[1 : len(v)-2]
		}
	}

	return
}

// Parse the response of a EPLF
// Return an array with the files, one with the directories and one with the links
func (ftp *FTP) parseEPLF(data []string, basePath string) (files []string, directories []string, links []string, err error) {
	fmt.Errorf("Not implemented! (EPLF)\n")
	return nil, nil, nil, errors.New("Not implemented! (EPLF)")
}

// Parse the response of a NLST
// Return an array with the files, one with the directories and one with the links
func (ftp *FTP) parseNLST(data []string, basePath string) (files []string, directories []string, links []string, err error) {
	fmt.Errorf("Not implemented! (NLST)\n")
	return nil, nil, nil, errors.New("Not implemented! (NLST)")
}

// Parse the response of a LIST
// Return an array with the files, one with the directories and one with the links
func (ftp *FTP) parseUnixLIST(data []string, basePath string) (files []string, directories []string, links []string, err error) {
	var pattern = regexp.MustCompile(`([-ld])([-rwx]+)\s{2,}\d+\s(\d|\w+)\s{2,}(\(\?\)|\d+|\w+)\s{2,}(\d+)\s*(\w+\s*\d+\s*\d+\:*\d*)\s(\S+)`)
	for _, line := range data {
		match := pattern.FindStringSubmatch(line)
		/*for i, val := range match {
			fmt.Printf("entry %d:%s\n", i, val)
		}*/
		if len(match) > 6 {
			if match[1] == "-" { // a file
				files = append(files, basePath+match[7])
			} else if match[1] == "d" { // a directory
				directories = append(directories, basePath+match[7]+"/")
			} else if match[1] == "l" { // a link
				token := strings.Trim(strings.Split(line, "->")[1], " ")
				links = append(links, token)
			}
		}
	}

	sum := len(files) + len(directories) + len(links)
	if sum > 0 {
		return files, directories, links, nil
	} else { //empty folder
		if ftp.debug {
			log.Printf("Empty folder")
		}
		return nil, nil, nil, nil
	}
}

// extract single lines from a buffered reader
func (ftp *FTP) splitLines(reader *bufio.Reader) (files []string, err error) {
	var line string
	for {
		line, err = reader.ReadString('\n')

		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		files = append(files, string(line))
	}
	return files, nil
}
<commit_msg>fix parseUnixLIST<commit_after>package goftp

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"log"
	"regexp"
	"strings"
)

// use FEAT to retrieve server features.
func (ftp *FTP) getServerFeatures() (features uint32) {
	features = 0
	code, str := ftp.RawCmd("FEAT")
	if code < 0 || code > 299 {
		return BAD
	}

	lines := strings.Split(str, "\n")

	for _, f := range lines {
		// Only MLSD/MLST is supported
		if (features&MLSD == 0) && strings.Contains(f, "MLST") { // not a bug, see command reference.
			features = features | MLSD

			// Check of NLST and EPLF are disabled since not yet supported

			/*} else if (features&NLST == 0) && strings.Contains(f, "NLST") {
				features = features | NLST
			} else if (features&EPLF == 0) && strings.Contains(f, "EPLF") {
				features = features | EPLF*/
		}
	}

	return features
}

// list the path (or current directory) and parse it.
// Return an array with the files, one with the directories and one with the links
func (ftp *FTP) GetFilesList(path string) (files []string, directories []string, links []string, err error) {
	if ftp.supportedfeatures == 0 {
		ftp.supportedfeatures = ftp.getServerFeatures()
	}
	if len(path) < 2 {
		path = "./"
	}
	if ftp.supportedfeatures&MLSD > 0 {
		//fmt.Println("Using MLSD")
		code, response := ftp.RawPassiveCmd("MLSD " + path)

		if code == 550 {
			return nil, nil, nil, errors.New("550 Requested action not taken. File unavailable.")
		}
		if code < 0 || code > 299 {
			return nil, nil, nil, errors.New(fmt.Sprintf("%v MLSD did not work ", code))
		}
		return ftp.parseMLSD(response, path)
	} else if ftp.supportedfeatures&EPLF > 0 {
		//fmt.Println("Using EPLF")
		code, response := ftp.RawPassiveCmd("EPLF " + path)

		if code == 550 {
			return nil, nil, nil, errors.New("550 Requested action not taken. File unavailable.")
		}
		if code < 0 || code > 299 {
			return nil, nil, nil, errors.New(fmt.Sprintf("%v EPLF did not work ", code))
		}
		return ftp.parseEPLF(response, path)
	} else if ftp.supportedfeatures&NLST > 0 {
		//fmt.Println("Using NLST")
		code, response := ftp.RawPassiveCmd("NLST " + path)

		if code == 550 {
			return nil, nil, nil, errors.New("550 Requested action not taken. File unavailable.")
		}
		if code < 0 || code > 299 {
			return nil, nil, nil, errors.New(fmt.Sprintf("%v NLST did not work ", code))
		}
		return ftp.parseNLST(response, path)
	} else {
		//fmt.Println("Using LIST")
		code, response := ftp.RawPassiveCmd("LIST " + path)
		//fmt.Println(response)
		if code == 550 {
			return nil, nil, nil, errors.New("550 Requested action not taken. File unavailable.")
		}
		if code < 0 || code > 299 {
			return nil, nil, nil, errors.New("LIST did not work")
		}
		return ftp.parseUnixLIST(response, path)
	}
}

// Parse the response of a MLSD
// Return an array with the files, one with the directories and one with the links
func (ftp *FTP) parseMLSD(data []string, basePath string) (files []string, directories []string, links []string, err error) {
	for _, line := range data {
		_, t, subpath := parseLine_MLST(line)

		switch t {
		case "dir":
			if subpath == "." {
			} else if subpath == ".." {
			} else {
				directories = append(directories, basePath+subpath+"/")
			}
		case "file":
			files = append(files, basePath+subpath)
		case "OS.unix=symlink":
			links = append(links, basePath+subpath)
		}

	}
	return files, directories, links, err
}

// Parse a single MLST line
func parseLine_MLST(line string) (perm string, t string, filename string) {
	for _, v := range strings.Split(line, ";") {
		v2 := strings.Split(v, "=")

		switch v2[0] {
		case "perm":
			perm = v2[1]
		case "type":
			t = v2[1]
			if t == "OS.unix" {
				t = t + "=" + v2[2]
				//fmt.Println("Found a link -> " + t)
			}
		default:
			filename = v[1 : len(v)-2]
		}
	}

	return
}

// Parse the response of a EPLF
// Return an array with the files, one with the directories and one with the links
func (ftp *FTP) parseEPLF(data []string, basePath string) (files []string, directories []string, links []string, err error) {
	fmt.Errorf("Not implemented! (EPLF)\n")
	return nil, nil, nil, errors.New("Not implemented! (EPLF)")
}

// Parse the response of a NLST
// Return an array with the files, one with the directories and one with the links
func (ftp *FTP) parseNLST(data []string, basePath string) (files []string, directories []string, links []string, err error) {
	fmt.Errorf("Not implemented! (NLST)\n")
	return nil, nil, nil, errors.New("Not implemented! (NLST)")
}

// Parse the response of a LIST
// Return an array with the files, one with the directories and one with the links
func (ftp *FTP) parseUnixLIST(data []string, basePath string) (files []string, directories []string, links []string, err error) {
	var pattern = regexp.MustCompile(`([-ld])([-rwx]+)\s{2,}\d+\s(\d|\w+)\s{2,}(\(\?\)|\d+|[\w\-]+)\s{1,}(\d+)\s*(\w+\s*\d+\s*\d+\:*\d*)\s(\S+)`)
	for _, line := range data {
		match := pattern.FindStringSubmatch(line)
		/*for i, val := range match {
			fmt.Printf("entry %d:%s\n", i, val)
		}*/
		if len(match) > 6 {
			if match[1] == "-" { // a file
				files = append(files, basePath+match[7])
			} else if match[1] == "d" { // a directory
				directories = append(directories, basePath+match[7]+"/")
			} else if match[1] == "l" { // a link
				token := strings.Trim(strings.Split(line, "->")[1], " ")
				links = append(links, token)
			}
		}
	}

	sum := len(files) + len(directories) + len(links)
	if sum > 0 {
		return files, directories, links, nil
	} else { //empty folder
		if ftp.debug {
			log.Printf("Empty folder")
		}
		return nil, nil, nil, nil
	}
}

// extract single lines from a buffered reader
func (ftp *FTP) splitLines(reader *bufio.Reader) (files []string, err error) {
	var line string
	for {
		line, err = reader.ReadString('\n')

		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		files = append(files, string(line))
	}
	return files, nil
}
<|endoftext|>


<commit_before>package function

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type Function struct {
	// Root on disk at which to find/create source and config files.
	Root string

	// Name of the Function.  If not provided, path derivation is attempted when
	// requried (such as for initialization).
	Name string

	// Namespace into which the Function is deployed on supported platforms.
	Namespace string

	// Runtime is the language plus context.  nodejs|go|quarkus|rust etc.
	Runtime string

	// Template for the Function.
	Template string

	// Registry at which to store interstitial containers, in the form
	// [registry]/[user].
	Registry string

	// Optional full OCI image tag in form:
	//   [registry]/[namespace]/[name]:[tag]
	// example:
	//   quay.io/alice/my.function.name
	// Registry is optional and is defaulted to DefaultRegistry
	// example:
	//   alice/my.function.name
	// If Image is provided, it overrides the default of concatenating
	// "Registry+Name:latest" to derive the Image.
	Image string

	// SHA256 hash of the latest image that has been built
	ImageDigest string

	// Builder represents the CNCF Buildpack builder image for a function
	Builder string

	// Map containing known builders.
	// e.g. { "jvm": "docker.io/example/quarkus-jvm-builder" }
	Builders map[string]string

	// Optional list of buildpacks to use when building the function
	Buildpacks []string

	// List of volumes to be mounted to the function
	Volumes Volumes

	// Env variables to be set
	Envs Envs

	// Map containing user-supplied annotations
	// Example: { "division": "finance" }
	Annotations map[string]string

	// Options to be set on deployed function (scaling, etc.)
	Options Options

	// Map of user-supplied labels
	Labels Labels

	// Health endpoints specified by the language pack
	HealthEndpoints map[string]string
}

// NewFunction loads a Function from a path on disk. use .Initialized() to determine if
// the path contained an initialized Function.
// NewFunction creates a Function struct whose attributes are loaded from the
// configuraiton located at path.
func NewFunction(root string) (f Function, err error) {

	// Expand the passed root to its absolute path (default current dir)
	if root, err = filepath.Abs(root); err != nil {
		return
	}

	// Load a Config from the given absolute path
	c, err := newConfig(root)
	if err != nil {
		return
	}

	// Let's set Function name, if it is not already set
	if c.Name == "" {
		pathParts := strings.Split(strings.TrimRight(root, string(os.PathSeparator)), string(os.PathSeparator))
		c.Name = pathParts[len(pathParts)-1]
	}

	// set Function to the value of the config loaded from disk.
	f = fromConfig(c)

	// The only value not included in the config is the effective path on disk
	f.Root = root
	return
}

// WriteConfig writes this Function's configuration to disk.
func (f Function) WriteConfig() (err error) {
	return writeConfig(f)
}

// Initialized returns if the Function has been initialized.
// Any errors are considered failure (invalid or inaccessible root, config file, etc).
func (f Function) Initialized() bool {
	// Load the Function's configuration from disk and check if the (required) value Runtime is populated.
	c, err := newConfig(f.Root)
	if err != nil {
		return false
	}

	return c.Runtime != "" && c.Name != ""
}

// Built indicates the Function has been built.  Does not guarantee the
// image indicated actually exists, just that it _should_ exist based off
// the current state of the Funciton object, in particular the value of
// the Image and ImageDiget fields.
func (f Function) Built() bool {
	// If Image (the override) and ImageDigest (the most recent build stamp) are
	// both empty, the Function is considered unbuilt.
	return f.Image != "" || f.ImageDigest != ""
}

// ImageWithDigest returns the full reference to the image including SHA256 Digest.
// If Digest is empty, image:tag is returned.
func (f Function) ImageWithDigest() string {
	// Return image, if Digest is empty
	if f.ImageDigest == "" {
		return f.Image
	}

	lastSlashIdx := strings.LastIndexAny(f.Image, "/")
	imageAsBytes := []byte(f.Image)

	part1 := string(imageAsBytes[:lastSlashIdx+1])
	part2 := string(imageAsBytes[lastSlashIdx+1:])

	// Remove tag from the image name and append SHA256 hash instead
	return part1 + strings.Split(part2, ":")[0] + "@" + f.ImageDigest
}

// DerivedImage returns the derived image name (OCI container tag) of the
// Function whose source is at root, with the default registry for when
// the image has to be calculated (derived).
// The following are eqivalent due to the use of DefaultRegistry:
// registry:  docker.io/myname
//            myname
// A full image name consists of registry, image name and tag.
// in form [registry]/[image-name]:[tag]
// example docker.io/alice/my.example.func:latest
// Default if not provided is --registry (a required global setting)
// followed by the provided (or derived) image name.
func DerivedImage(root, registry string) (image string, err error) {
	f, err := NewFunction(root)
	if err != nil {
		// an inability to load the Function means it is not yet initialized
		// We could try to be smart here and fall through to the Function name
		// deriviation logic, but that's likely to be confusing.  Better to
		// stay simple and say that derivation of Image depends on first having
		// the Function initialized.
		return
	}

	// If the Function has already had image populated, use this pre-calculated value.
	if f.Image != "" {
		image = f.Image
		return
	}

	// registry is currently required until such time as we support
	// pushing to an implicitly-available in-cluster registry by default.
	if registry == "" {
		err = errors.New("Registry name is required.")
		return
	}

	// If the Function loaded, and there is not yet an Image set, then this is
	// the first build and no explicit image override was specified.  We should
	// therefore derive the image tag from the defined registry and name.
	// form:    [registry]/[user]/[function]:latest
	// example: quay.io/alice/my.function.name:latest
	registry = strings.Trim(registry, "/") // too defensive?
	registryTokens := strings.Split(registry, "/")
	if len(registryTokens) == 1 {
		image = DefaultRegistry + "/" + registry + "/" + f.Name
	} else if len(registryTokens) == 2 {
		image = registry + "/" + f.Name
	} else {
		err = fmt.Errorf("registry should be either 'namespace' or 'registry/namespace'")
	}

	// Explicitly append :latest.  We currently expect source control to drive
	// versioning, rather than rely on Docker Hub tags with explicit version
	// numbers, as is seen in many serverless solutions.  This will be updated
	// to branch name when we add source-driven canary/ bluegreen deployments.
	image = image + ":latest"
	return
}

// assertEmptyRoot ensures that the directory is empty enough to be used for
// initializing a new Function.
func assertEmptyRoot(path string) (err error) {
	// If there exists contentious files (congig files for instance), this Function may have already been initialized.
	files, err := contentiousFilesIn(path)
	if err != nil {
		return
	} else if len(files) > 0 {
		return fmt.Errorf("The chosen directory '%v' contains contentious files: %v.  Has the Service Function already been created?  Try either using a different directory, deleting the Function if it exists, or manually removing the files.", path, files)
	}

	// Ensure there are no non-hidden files, and again none of the aforementioned contentious files.
	empty, err := isEffectivelyEmpty(path)
	if err != nil {
		return
	} else if !empty {
		err = errors.New("The directory must be empty of visible files and recognized config files before it can be initialized.")
		return
	}
	return
}

// contentiousFiles are files which, if extant, preclude the creation of a
// Function rooted in the given directory.
var contentiousFiles = []string{
	ConfigFile,
}

// contentiousFilesIn the given directoy
func contentiousFilesIn(dir string) (contentious []string, err error) {
	files, err := ioutil.ReadDir(dir)
	for _, file := range files {
		for _, name := range contentiousFiles {
			if file.Name() == name {
				contentious = append(contentious, name)
			}
		}
	}
	return
}

// effectivelyEmpty directories are those which have no visible files
func isEffectivelyEmpty(dir string) (bool, error) {
	// Check for any non-hidden files
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return false, err
	}
	for _, file := range files {
		if !strings.HasPrefix(file.Name(), ".") {
			return false, nil
		}
	}
	return true, nil
}
<commit_msg>fix: `build` should honor registry specified in `-r` (#510)<commit_after>package function

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

type Function struct {
	// Root on disk at which to find/create source and config files.
	Root string

	// Name of the Function.  If not provided, path derivation is attempted when
	// requried (such as for initialization).
	Name string

	// Namespace into which the Function is deployed on supported platforms.
	Namespace string

	// Runtime is the language plus context.  nodejs|go|quarkus|rust etc.
	Runtime string

	// Template for the Function.
	Template string

	// Registry at which to store interstitial containers, in the form
	// [registry]/[user].
	Registry string

	// Optional full OCI image tag in form:
	//   [registry]/[namespace]/[name]:[tag]
	// example:
	//   quay.io/alice/my.function.name
	// Registry is optional and is defaulted to DefaultRegistry
	// example:
	//   alice/my.function.name
	// If Image is provided, it overrides the default of concatenating
	// "Registry+Name:latest" to derive the Image.
	Image string

	// SHA256 hash of the latest image that has been built
	ImageDigest string

	// Builder represents the CNCF Buildpack builder image for a function
	Builder string

	// Map containing known builders.
	// e.g. { "jvm": "docker.io/example/quarkus-jvm-builder" }
	Builders map[string]string

	// Optional list of buildpacks to use when building the function
	Buildpacks []string

	// List of volumes to be mounted to the function
	Volumes Volumes

	// Env variables to be set
	Envs Envs

	// Map containing user-supplied annotations
	// Example: { "division": "finance" }
	Annotations map[string]string

	// Options to be set on deployed function (scaling, etc.)
	Options Options

	// Map of user-supplied labels
	Labels Labels

	// Health endpoints specified by the language pack
	HealthEndpoints map[string]string
}

// NewFunction loads a Function from a path on disk. use .Initialized() to determine if
// the path contained an initialized Function.
// NewFunction creates a Function struct whose attributes are loaded from the
// configuraiton located at path.
func NewFunction(root string) (f Function, err error) {

	// Expand the passed root to its absolute path (default current dir)
	if root, err = filepath.Abs(root); err != nil {
		return
	}

	// Load a Config from the given absolute path
	c, err := newConfig(root)
	if err != nil {
		return
	}

	// Let's set Function name, if it is not already set
	if c.Name == "" {
		pathParts := strings.Split(strings.TrimRight(root, string(os.PathSeparator)), string(os.PathSeparator))
		c.Name = pathParts[len(pathParts)-1]
	}

	// set Function to the value of the config loaded from disk.
	f = fromConfig(c)

	// The only value not included in the config is the effective path on disk
	f.Root = root
	return
}

// WriteConfig writes this Function's configuration to disk.
func (f Function) WriteConfig() (err error) {
	return writeConfig(f)
}

// Initialized returns if the Function has been initialized.
// Any errors are considered failure (invalid or inaccessible root, config file, etc).
func (f Function) Initialized() bool {
	// Load the Function's configuration from disk and check if the (required) value Runtime is populated.
	c, err := newConfig(f.Root)
	if err != nil {
		return false
	}

	return c.Runtime != "" && c.Name != ""
}

// Built indicates the Function has been built.  Does not guarantee the
// image indicated actually exists, just that it _should_ exist based off
// the current state of the Funciton object, in particular the value of
// the Image and ImageDiget fields.
func (f Function) Built() bool {
	// If Image (the override) and ImageDigest (the most recent build stamp) are
	// both empty, the Function is considered unbuilt.
	return f.Image != "" || f.ImageDigest != ""
}

// ImageWithDigest returns the full reference to the image including SHA256 Digest.
// If Digest is empty, image:tag is returned.
func (f Function) ImageWithDigest() string {
	// Return image, if Digest is empty
	if f.ImageDigest == "" {
		return f.Image
	}

	lastSlashIdx := strings.LastIndexAny(f.Image, "/")
	imageAsBytes := []byte(f.Image)

	part1 := string(imageAsBytes[:lastSlashIdx+1])
	part2 := string(imageAsBytes[lastSlashIdx+1:])

	// Remove tag from the image name and append SHA256 hash instead
	return part1 + strings.Split(part2, ":")[0] + "@" + f.ImageDigest
}

// DerivedImage returns the derived image name (OCI container tag) of the
// Function whose source is at root, with the default registry for when
// the image has to be calculated (derived).
// The following are eqivalent due to the use of DefaultRegistry:
// registry:  docker.io/myname
//            myname
// A full image name consists of registry, image name and tag.
// in form [registry]/[image-name]:[tag]
// example docker.io/alice/my.example.func:latest
// Default if not provided is --registry (a required global setting)
// followed by the provided (or derived) image name.
func DerivedImage(root, registry string) (image string, err error) {
	f, err := NewFunction(root)
	if err != nil {
		// an inability to load the Function means it is not yet initialized
		// We could try to be smart here and fall through to the Function name
		// deriviation logic, but that's likely to be confusing.  Better to
		// stay simple and say that derivation of Image depends on first having
		// the Function initialized.
		return
	}

	// If the Function has already had image populated
	// and a new registry hasn't been provided, use this pre-calculated value.
	if f.Image != "" && f.Registry == registry {
		image = f.Image
		return
	}

	// registry is currently required until such time as we support
	// pushing to an implicitly-available in-cluster registry by default.
	if registry == "" {
		err = errors.New("Registry name is required.")
		return
	}

	// If the Function loaded, and there is not yet an Image set, then this is
	// the first build and no explicit image override was specified.  We should
	// therefore derive the image tag from the defined registry and name.
	// form:    [registry]/[user]/[function]:latest
	// example: quay.io/alice/my.function.name:latest
	registry = strings.Trim(registry, "/") // too defensive?
	registryTokens := strings.Split(registry, "/")
	if len(registryTokens) == 1 {
		image = DefaultRegistry + "/" + registry + "/" + f.Name
	} else if len(registryTokens) == 2 {
		image = registry + "/" + f.Name
	} else {
		err = fmt.Errorf("registry should be either 'namespace' or 'registry/namespace'")
	}

	// Explicitly append :latest.  We currently expect source control to drive
	// versioning, rather than rely on Docker Hub tags with explicit version
	// numbers, as is seen in many serverless solutions.  This will be updated
	// to branch name when we add source-driven canary/ bluegreen deployments.
	image = image + ":latest"
	return
}

// assertEmptyRoot ensures that the directory is empty enough to be used for
// initializing a new Function.
func assertEmptyRoot(path string) (err error) {
	// If there exists contentious files (congig files for instance), this Function may have already been initialized.
	files, err := contentiousFilesIn(path)
	if err != nil {
		return
	} else if len(files) > 0 {
		return fmt.Errorf("The chosen directory '%v' contains contentious files: %v.  Has the Service Function already been created?  Try either using a different directory, deleting the Function if it exists, or manually removing the files.", path, files)
	}

	// Ensure there are no non-hidden files, and again none of the aforementioned contentious files.
	empty, err := isEffectivelyEmpty(path)
	if err != nil {
		return
	} else if !empty {
		err = errors.New("The directory must be empty of visible files and recognized config files before it can be initialized.")
		return
	}
	return
}

// contentiousFiles are files which, if extant, preclude the creation of a
// Function rooted in the given directory.
var contentiousFiles = []string{
	ConfigFile,
}

// contentiousFilesIn the given directoy
func contentiousFilesIn(dir string) (contentious []string, err error) {
	files, err := ioutil.ReadDir(dir)
	for _, file := range files {
		for _, name := range contentiousFiles {
			if file.Name() == name {
				contentious = append(contentious, name)
			}
		}
	}
	return
}

// effectivelyEmpty directories are those which have no visible files
func isEffectivelyEmpty(dir string) (bool, error) {
	// Check for any non-hidden files
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return false, err
	}
	for _, file := range files {
		if !strings.HasPrefix(file.Name(), ".") {
			return false, nil
		}
	}
	return true, nil
}
<|endoftext|>