<commit_before>package uploads

import (
	"path/filepath"

	"fmt"

	"crypto/md5"

	r "github.com/dancannon/gorethink"
	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

var _ = fmt.Println

// A UploadRequest contains the block to upload and the
// information required to write that block.
type UploadRequest struct {
	*flow.Request
}

type UploadStatus struct {
	FileID string
	Done   bool
}

// UploadService takes care of uploading blocks and constructing the
// file when all blocks have been uploaded.
type UploadService interface {
	Upload(req *UploadRequest) (*UploadStatus, error)
}

// uploadService is an implementation of UploadService.
type uploadService struct {
	tracker     *blockTracker
	files       dai.Files
	uploads     dai.Uploads
	dirs        dai.Dirs
	writer      requestWriter
	requestPath requestPath
	fops        file.Operations
}

// NewUploadService creates a new idService that connects to the database using
// the given session.
func NewUploadService(session *r.Session) *uploadService {
	return &uploadService{
		tracker:     requestBlockTracker,
		files:       dai.NewRFiles(session),
		uploads:     dai.NewRUploads(session),
		dirs:        dai.NewRDirs(session),
		writer:      &blockRequestWriter{},
		requestPath: &mcdirRequestPath{},
		fops:        file.OS,
	}
}

// Upload performs uploading a block and constructing the file
// after all blocks have been uploaded.
func (s *uploadService) Upload(req *UploadRequest) (*UploadStatus, error) {
	dir := s.requestPath.dir(req.Request)
	id := req.UploadID()

	if !s.tracker.idExists(id) {
		return nil, app.ErrInvalid
	}

	if err := s.writeBlock(dir, req); err != nil {
		app.Log.Errorf("Writing block %d for request %s failed: %s", req.FlowChunkNumber, id, err)
		return nil, err
	}

	uploadStatus := &UploadStatus{}

	if s.tracker.done(id) {
		if file, err := s.assemble(req, dir); err != nil {
			app.Log.Errorf("Assembly failed for request %s: %s", req.FlowIdentifier, err)
			// Assembly failed. If file isn't nil then we need to cleanup state.
			if file != nil {
				if err := s.cleanup(req, file.ID); err != nil {
					app.Log.Errorf("Attempted cleanup of failed assembly %s errored with: %s", req.FlowIdentifier, err)
				}
			}
			return nil, err
		} else {
			uploadStatus.FileID = file.ID
			uploadStatus.Done = true
		}

	}
	return uploadStatus, nil
}

// writeBlock will write the request block and update state information
// on the block only if this block hasn't already been written.
func (s *uploadService) writeBlock(dir string, req *UploadRequest) error {
	id := req.UploadID()
	if !s.tracker.isBlockSet(id, int(req.FlowChunkNumber)) {
		if err := s.writer.write(dir, req.Request); err != nil {
			return err
		}
		s.tracker.addToHash(id, req.Chunk)
		s.tracker.setBlock(id, int(req.FlowChunkNumber))
	}
	return nil
}

// assemble moves the upload file to its proper location, creates a database entry
// and take care of all book keeping tasks to make the file accessible.
func (s *uploadService) assemble(req *UploadRequest, dir string) (*schema.File, error) {
	// Look up the upload
	upload, err := s.uploads.ByID(req.FlowIdentifier)
	if err != nil {
		return nil, err
	}

	// Create file entry in database
	file, err := s.createFile(req, upload)
	if err != nil {
		app.Log.Errorf("Assembly failed for request %s, couldn't create file in database: %s", req.FlowIdentifier, err)
		return nil, err
	}

	// Check if this is an upload matching a file that has already been uploaded. If it isn't
	// then copy over the data. If it is, then there isn't any uploaded data to copy over.
	if !upload.IsExisting {
		// Create on disk entry to write chunks to
		if err := s.createDest(file.ID); err != nil {
			app.Log.Errorf("Assembly failed for request %s, couldn't create file on disk: %s", req.FlowIdentifier, err)
			return file, err
		}

		// Move file
		uploadDir := s.requestPath.dir(req.Request)
		s.fops.Rename(filepath.Join(uploadDir, req.UploadID()), app.MCDir.FilePath(file.ID))
	}

	// Finish updating the file state.
	finisher := newFinisher(s.files, s.dirs)
	checksum := s.determineChecksum(req, upload)
	if err := finisher.finish(req, file.ID, checksum, upload); err != nil {
		app.Log.Errorf("Assembly failed for request %s, couldn't finish request: %s", req.FlowIdentifier, err)
		return file, err
	}

	app.Log.Infof("successfully upload fileID %s", file.ID)

	s.cleanupUploadRequest(req.UploadID())
	return file, nil
}

// createFile creates the database file entry.
func (s *uploadService) createFile(req *UploadRequest, upload *schema.Upload) (*schema.File, error) {
	file := schema.NewFile(upload.File.Name, upload.ProjectOwner)
	file.Current = false

	f, err := s.files.Insert(&file, upload.DirectoryID, upload.ProjectID)
	app.Log.Infof("Created file %s, in %s %s", f.ID, upload.DirectoryID, upload.ProjectID)
	if err != nil {
		return nil, err
	}
	return f, nil
}

// createDest creates the destination file and ensures that the directory
// path is also created.
func (s *uploadService) createDest(fileID string) error {
	dir := app.MCDir.FileDir(fileID)
	if err := s.fops.MkdirAll(dir, 0700); err != nil {
		return err
	}
	return nil
}

func (s *uploadService) determineChecksum(req *UploadRequest, upload *schema.Upload) string {
	switch {
	case upload.IsExisting:
		// Existing file so use its checksum, no need to compute.
		return upload.File.Checksum
	case upload.ServerRestarted:
		// Server was restarted, so checksum state in tracker is wrong. Read
		// disk file to get the checksum.
		uploadDir := s.requestPath.dir(req.Request)
		hash, _ := file.HashStr(md5.New(), filepath.Join(uploadDir, req.UploadID()))
		return hash
	default:
		// Checksum in tracker is correct since its state has been properly
		// updated as blocks are uploaded.
		return s.tracker.hash(req.UploadID())
	}
}

// cleanup is called when an error has occurred. It attempts to clean up
// the state in the database for this particular entry.
func (s *uploadService) cleanup(req *UploadRequest, fileID string) error {
	upload, err := s.uploads.ByID(req.FlowIdentifier)
	if err != nil {
		return err
	}
	_, err = s.files.Delete(fileID, upload.DirectoryID, upload.ProjectID)
	return err
}

//cleanupUploadRequest removes the upload request and file chunks.
func (s *uploadService) cleanupUploadRequest(uploadID string) {
	s.tracker.clear(uploadID)
	s.uploads.Delete(uploadID)
	s.fops.RemoveAll(app.MCDir.UploadDir(uploadID))
}
<commit_msg>Change s to service.<commit_after>package uploads

import (
	"path/filepath"

	"fmt"

	"crypto/md5"

	r "github.com/dancannon/gorethink"
	"github.com/materials-commons/gohandy/file"
	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

var _ = fmt.Println

// A UploadRequest contains the block to upload and the
// information required to write that block.
type UploadRequest struct {
	*flow.Request
}

type UploadStatus struct {
	FileID string
	Done   bool
}

// UploadService takes care of uploading blocks and constructing the
// file when all blocks have been uploaded.
type UploadService interface {
	Upload(req *UploadRequest) (*UploadStatus, error)
}

// uploadService is an implementation of UploadService.
type uploadService struct {
	tracker     *blockTracker
	files       dai.Files
	uploads     dai.Uploads
	dirs        dai.Dirs
	writer      requestWriter
	requestPath requestPath
	fops        file.Operations
}

// NewUploadService creates a new idService that connects to the database using
// the given session.
func NewUploadService(session *r.Session) *uploadService {
	return &uploadService{
		tracker:     requestBlockTracker,
		files:       dai.NewRFiles(session),
		uploads:     dai.NewRUploads(session),
		dirs:        dai.NewRDirs(session),
		writer:      &blockRequestWriter{},
		requestPath: &mcdirRequestPath{},
		fops:        file.OS,
	}
}

// Upload performs uploading a block and constructing the file
// after all blocks have been uploaded.
func (service *uploadService) Upload(req *UploadRequest) (*UploadStatus, error) {
	dir := service.requestPath.dir(req.Request)
	id := req.UploadID()

	if !service.tracker.idExists(id) {
		return nil, app.ErrInvalid
	}

	if err := service.writeBlock(dir, req); err != nil {
		app.Log.Errorf("Writing block %d for request %s failed: %s", req.FlowChunkNumber, id, err)
		return nil, err
	}

	uploadStatus := &UploadStatus{}

	if service.tracker.done(id) {
		if file, err := service.assemble(req, dir); err != nil {
			app.Log.Errorf("Assembly failed for request %s: %s", req.FlowIdentifier, err)
			// Assembly failed. If file isn't nil then we need to cleanup state.
			if file != nil {
				if err := service.cleanup(req, file.ID); err != nil {
					app.Log.Errorf("Attempted cleanup of failed assembly %s errored with: %s", req.FlowIdentifier, err)
				}
			}
			return nil, err
		} else {
			uploadStatus.FileID = file.ID
			uploadStatus.Done = true
		}

	}
	return uploadStatus, nil
}

// writeBlock will write the request block and update state information
// on the block only if this block hasn't already been written.
func (service *uploadService) writeBlock(dir string, req *UploadRequest) error {
	id := req.UploadID()
	if !service.tracker.isBlockSet(id, int(req.FlowChunkNumber)) {
		if err := service.writer.write(dir, req.Request); err != nil {
			return err
		}
		service.tracker.addToHash(id, req.Chunk)
		service.tracker.setBlock(id, int(req.FlowChunkNumber))
	}
	return nil
}

// assemble moves the upload file to its proper location, creates a database entry
// and take care of all book keeping tasks to make the file accessible.
func (service *uploadService) assemble(req *UploadRequest, dir string) (*schema.File, error) {
	// Look up the upload
	upload, err := service.uploads.ByID(req.FlowIdentifier)
	if err != nil {
		return nil, err
	}

	// Create file entry in database
	file, err := service.createFile(req, upload)
	if err != nil {
		app.Log.Errorf("Assembly failed for request %s, couldn't create file in database: %s", req.FlowIdentifier, err)
		return nil, err
	}

	// Check if this is an upload matching a file that has already been uploaded. If it isn't
	// then copy over the data. If it is, then there isn't any uploaded data to copy over.
	if !upload.IsExisting {
		// Create on disk entry to write chunks to
		if err := service.createDest(file.ID); err != nil {
			app.Log.Errorf("Assembly failed for request %s, couldn't create file on disk: %s", req.FlowIdentifier, err)
			return file, err
		}

		// Move file
		uploadDir := service.requestPath.dir(req.Request)
		service.fops.Rename(filepath.Join(uploadDir, req.UploadID()), app.MCDir.FilePath(file.ID))
	}

	// Finish updating the file state.
	finisher := newFinisher(service.files, service.dirs)
	checksum := service.determineChecksum(req, upload)
	if err := finisher.finish(req, file.ID, checksum, upload); err != nil {
		app.Log.Errorf("Assembly failed for request %s, couldn't finish request: %s", req.FlowIdentifier, err)
		return file, err
	}

	app.Log.Infof("successfully upload fileID %s", file.ID)

	service.cleanupUploadRequest(req.UploadID())
	return file, nil
}

// createFile creates the database file entry.
func (service *uploadService) createFile(req *UploadRequest, upload *schema.Upload) (*schema.File, error) {
	file := schema.NewFile(upload.File.Name, upload.ProjectOwner)
	file.Current = false

	f, err := service.files.Insert(&file, upload.DirectoryID, upload.ProjectID)
	app.Log.Infof("Created file %s, in %s %s", f.ID, upload.DirectoryID, upload.ProjectID)
	if err != nil {
		return nil, err
	}
	return f, nil
}

// createDest creates the destination file and ensures that the directory
// path is also created.
func (service *uploadService) createDest(fileID string) error {
	dir := app.MCDir.FileDir(fileID)
	if err := service.fops.MkdirAll(dir, 0700); err != nil {
		return err
	}
	return nil
}

func (service *uploadService) determineChecksum(req *UploadRequest, upload *schema.Upload) string {
	switch {
	case upload.IsExisting:
		// Existing file so use its checksum, no need to compute.
		return upload.File.Checksum
	case upload.ServerRestarted:
		// Server was restarted, so checksum state in tracker is wrong. Read
		// disk file to get the checksum.
		uploadDir := service.requestPath.dir(req.Request)
		hash, _ := file.HashStr(md5.New(), filepath.Join(uploadDir, req.UploadID()))
		return hash
	default:
		// Checksum in tracker is correct since its state has been properly
		// updated as blocks are uploaded.
		return service.tracker.hash(req.UploadID())
	}
}

// cleanup is called when an error has occurred. It attempts to clean up
// the state in the database for this particular entry.
func (service *uploadService) cleanup(req *UploadRequest, fileID string) error {
	upload, err := service.uploads.ByID(req.FlowIdentifier)
	if err != nil {
		return err
	}
	_, err = service.files.Delete(fileID, upload.DirectoryID, upload.ProjectID)
	return err
}

//cleanupUploadRequest removes the upload request and file chunks.
func (service *uploadService) cleanupUploadRequest(uploadID string) {
	service.tracker.clear(uploadID)
	service.uploads.Delete(uploadID)
	service.fops.RemoveAll(app.MCDir.UploadDir(uploadID))
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cmd

import (
	"bytes"
	"errors"
	"fmt"
	"go/build"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

func ParseTargets(a string) map[string]struct{} {
	targetsSlice := strings.Fields(a)
	if len(targetsSlice) == 0 {
		if runtime.GOOS == "darwin" {
			targetsSlice = []string{"android", "ios"}
		} else {
			targetsSlice = []string{"android"}
		}
	}
	targets := map[string]struct{}{}
	for _, i := range targetsSlice {
		switch i {
		case "android":
			targets["android"] = struct{}{}
			targets["android/arm"] = struct{}{}
			targets["android/arm64"] = struct{}{}
			// targets["android/386"] = struct{}{}
			// targets["android/amd64"] = struct{}{}
		case "android/arm", "android/arm64", "android/386", "android/amd64":
			targets["android"] = struct{}{}
			targets[i] = struct{}{}
		case "ios":
			targets["ios"] = struct{}{}
			targets["ios/arm"] = struct{}{}
			targets["ios/arm64"] = struct{}{}
			// targets["ios/386"] = struct{}{}
			targets["ios/amd64"] = struct{}{}
		case "ios/arm", "ios/arm64", "ios/386", "ios/amd64":
			targets["ios"] = struct{}{}
			targets[i] = struct{}{}
		}
	}
	return targets
}

func Build(flags *Flags, args []string) error {
	iosDir, err := PackageDir(flags, "gomatcha.io/matcha")
	if err != nil {
		return err
	}
	// flags.BuildBinary = true
	flags.BuildO = iosDir
	return Bind(flags, args)
}

func Bind(flags *Flags, args []string) error {
	targets := ParseTargets(flags.BuildTargets)

	// Validate Go
	err := validateGoInstall(flags)
	if err != nil {
		return err
	}

	// Make $WORK.
	tempdir, err := NewTmpDir(flags, "")
	if err != nil {
		return err
	}
	if !flags.BuildWork {
		defer RemoveAll(flags, tempdir)
	}

	// Get $GOPATH/pkg/matcha.
	matchaPkgPath, err := MatchaPkgPath(flags)
	if err != nil {
		return err
	}

	// Get toolchain version.
	installedVersion, err := ReadFile(flags, filepath.Join(matchaPkgPath, "version"))
	if err != nil {
		return errors.New("toolchain partially installed, run `matcha init`")
	}

	// Get go version.
	goVersion, err := GoVersion(flags)
	if err != nil {
		return err
	}

	// Check toolchain matches go version.
	if !bytes.Equal(installedVersion, goVersion) && flags.ShouldRun() {
		return errors.New("toolchain out of date, run `matcha init`")
	}

	// Get current working directory.
	cwd, err := Getwd(flags)
	if err != nil {
		return err
	}

	// Create a build context.
	ctx := build.Default
	ctx.GOARCH = "arm"
	ctx.GOOS = "darwin"
	ctx.BuildTags = append(ctx.BuildTags, "matcha")

	// Get import paths to be built.
	importPaths := []string{}
	if len(args) == 0 {
		importPaths = append(importPaths, ".")
	} else {
		for _, i := range args {
			i = path.Clean(i)
			importPaths = append(importPaths, i)
		}
	}

	// Get packages to be built
	pkgs, err := ImportAll(flags, &ctx, importPaths, cwd, build.ImportComment)
	if err != nil {
		return err
	}

	// Check if any of the package is main.
	for _, pkg := range pkgs {
		if pkg.Name == "main" {
			return fmt.Errorf("binding 'main' package (%s) is not supported", pkg.ImportComment)
		}
	}

	// Get the supporting files
	bridgePath, err := PackageDir(flags, "gomatcha.io/matcha/bridge")
	if err != nil {
		return err
	}

	// Begin iOS
	if _, ok := targets["ios"]; ok {
		// Validate Xcode installation
		if err := validateXcodeInstall(flags); err != nil {
			return err
		}

		// Build the "matcha/bridge" dir
		gopathDir := filepath.Join(tempdir, "IOS-GOPATH")

		// Make $WORK/matcha-ios
		workOutputDir := filepath.Join(tempdir, "matcha-ios")
		if err := Mkdir(flags, workOutputDir); err != nil {
			return err
		}

		// Make binary output dir
		binaryPath := filepath.Join(workOutputDir, "MatchaBridge", "MatchaBridge", "MatchaBridge.a")
		if err := Mkdir(flags, filepath.Dir(binaryPath)); err != nil {
			return err
		}

		// Create the "main" go package, that references the other go packages
		mainPath := filepath.Join(tempdir, "src", "iosbin", "main.go")
		err = WriteFile(flags, mainPath, strings.NewReader(fmt.Sprintf(BindFile, args[0]))) // TODO(KD): Should this be args[0] or should it use the logic to generate pkgs
		if err != nil {
			return fmt.Errorf("failed to create the binding package for iOS: %v", err)
		}

		// if !flags.BuildBinary {
		// 	// Copy package's ios directory if it imports gomatcha.io/bridge.
		// 	for _, pkg := range pkgs {
		// 		importsBridge := false
		// 		for _, i := range pkg.Imports {
		// 			if i == "gomatcha.io/bridge" {
		// 				importsBridge = true
		// 				break
		// 			}
		// 		}

		// 		if importsBridge {
		// 			files, err := ioutil.ReadDir(pkg.Dir)
		// 			if err != nil {
		// 				continue
		// 			}

		// 			for _, i := range files {
		// 				if i.IsDir() && i.Name() == "ios" {
		// 					// Copy directory
		// 					src := filepath.Join(pkg.Dir, "ios")
		// 					dst := filepath.Join(workOutputDir)
		// 					CopyDirContents(flags, dst, src)
		// 				}
		// 			}
		// 		}
		// 	}
		// }

		// Build platform binaries concurrently.
		envs := [][]string{}
		if _, ok := targets["ios/arm"]; ok {
			env, err := DarwinArmEnv(flags)
			if err != nil {
				return err
			}
			envs = append(envs, env)
		}
		if _, ok := targets["ios/arm64"]; ok {
			env, err := DarwinArm64Env(flags)
			if err != nil {
				return err
			}
			envs = append(envs, env)
		}
		if _, ok := targets["ios/386"]; ok {
			env, err := Darwin386Env(flags)
			if err != nil {
				return err
			}
			envs = append(envs, env)
		}
		if _, ok := targets["ios/amd64"]; ok {
			env, err := DarwinAmd64Env(flags)
			if err != nil {
				return err
			}
			envs = append(envs, env)
		}

		type archPath struct {
			arch string
			path string
			err  error
		}
		archs := []archPath{}
		archChan := make(chan archPath)
		for _, i := range envs {
			go func(env []string) {
				arch := FindEnv(env, "GOARCH")
				env = append(env, "GOPATH="+gopathDir+string(filepath.ListSeparator)+GoEnv(flags, "GOPATH"))
				path := filepath.Join(tempdir, "matcha-"+arch+".a")

				// ios needs to be added as a build tag due to https://github.com/golang/go/commit/29eb7d18ed71c057bbdb69d85953a32252f0ea73
				err := GoBuild(flags, []string{mainPath}, env, []string{"matcha", "ios"}, matchaPkgPath, tempdir, "-buildmode=c-archive", "-o", path)
				archChan <- archPath{arch, path, err}
			}(i)

			if !flags.Threaded {
				arch := <-archChan
				if arch.err != nil {
					return arch.err
				}
				archs = append(archs, arch)
			}
		}
		if flags.Threaded {
			for i := 0; i < len(envs); i++ {
				arch := <-archChan
				if arch.err != nil {
					return arch.err
				}
				archs = append(archs, arch)
			}
		}

		// Lipo to build fat binary.
		cmd := exec.Command("xcrun", "lipo", "-create")
		for _, i := range archs {
			cmd.Args = append(cmd.Args, "-arch", ArchClang(i.arch), i.path)
		}
		cmd.Args = append(cmd.Args, "-o", binaryPath)
		if err := RunCmd(flags, tempdir, cmd); err != nil {
			return err
		}

		// Create output dir
		outputDir := flags.BuildO
		if outputDir == "" {
			outputDir = "Matcha-iOS"
		}

		// if !flags.BuildBinary {
		// 	if err := RemoveAll(flags, outputDir); err != nil {
		// 		return err
		// 	}

		// 	// Copy output directory into place.
		// 	if err := CopyDir(flags, outputDir, workOutputDir); err != nil {
		// 		return err
		// 	}
		// } else {

		// Copy binary into place.
		if err := CopyFile(flags, filepath.Join(outputDir, "ios", "MatchaBridge", "MatchaBridge", "MatchaBridge.a"), binaryPath); err != nil {
			return err
		}

		// }
	}
	if _, ok := targets["android"]; ok {
		// Validate Android installation
		if err := ValidateAndroidInstall(flags); err != nil {
			return err
		}

		// Build the "matcha/bridge" dir
		gopathDir := filepath.Join(tempdir, "ANDROID-GOPATH")

		androidArchs := []string{}
		if _, ok := targets["android/arm"]; ok {
			androidArchs = append(androidArchs, "arm")
		}
		if _, ok := targets["android/arm64"]; ok {
			androidArchs = append(androidArchs, "arm64")
		}
		if _, ok := targets["android/386"]; ok {
			androidArchs = append(androidArchs, "386")
		}
		if _, ok := targets["android/amd64"]; ok {
			androidArchs = append(androidArchs, "amd64")
		}

		androidDir := filepath.Join(tempdir, "android")
		mainPath := filepath.Join(tempdir, "androidlib/main.go")

		err = WriteFile(flags, mainPath, strings.NewReader(fmt.Sprintf(BindFile, args[0]))) // TODO(KD): Should this be args[0] or should it use the logic to generate pkgs
		if err != nil {
			return fmt.Errorf("failed to create the main package for android: %v", err)
		}

		javaDir2 := filepath.Join(androidDir, "src", "main", "java", "io", "gomatcha", "bridge")
		if err := Mkdir(flags, javaDir2); err != nil {
			return err
		}
		if err := CopyFile(flags, filepath.Join(javaDir2, "GoValue.java"), filepath.Join(bridgePath, "java-GoValue.java")); err != nil {
			return err
		}
		if err := CopyFile(flags, filepath.Join(javaDir2, "Bridge.java"), filepath.Join(bridgePath, "java-Bridge.java")); err != nil {
			return err
		}
		if err := CopyFile(flags, filepath.Join(javaDir2, "Tracker.java"), filepath.Join(bridgePath, "java-Tracker.java")); err != nil {
			return err
		}

		// Make $WORK/matcha-android
		workOutputDir := filepath.Join(tempdir, "matcha-android")
		if err := Mkdir(flags, workOutputDir); err != nil {
			return err
		}

		// Make aar output file.
		aarDirPath := filepath.Join(workOutputDir, "MatchaBridge")
		aarPath := filepath.Join(workOutputDir, "MatchaBridge", "matchabridge.aar")
		if err := Mkdir(flags, aarDirPath); err != nil {
			return err
		}

		// Generate binding code and java source code only when processing the first package.
		for _, arch := range androidArchs {
			env, err := AndroidEnv(flags, arch)
			if err != nil {
				return err
			}
			env = append(env, "GOPATH="+gopathDir+string(filepath.ListSeparator)+GoEnv(flags, "GOPATH"))

			err = GoBuild(flags,
				[]string{mainPath},
				env,
				[]string{"matcha"},
				matchaPkgPath,
				tempdir,
				"-buildmode=c-shared",
				"-o="+filepath.Join(androidDir, "src/main/jniLibs/"+GetAndroidABI(arch)+"/libgojni.so"),
			)
			if err != nil {
				return err
			}
		}

		if err := BuildAAR(flags, androidDir, pkgs, androidArchs, tempdir, aarPath); err != nil {
			return err
		}

		// Create output dir
		outputDir := flags.BuildO
		if outputDir == "" {
			outputDir = "Matcha-iOS"
		}

		// Copy binary into place.
		if err := CopyFile(flags, filepath.Join(outputDir, "android", "matchabridge.aar"), aarPath); err != nil {
			return err
		}
	}
	return nil
}

var BindFile = `
package main

import (
    _ "gomatcha.io/matcha/bridge"
    _ "%s"
)

import "C"

func main() {}
`
<commit_msg>Re-add 386 and amd64 targets into the default android build<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cmd

import (
	"bytes"
	"errors"
	"fmt"
	"go/build"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"
)

func ParseTargets(a string) map[string]struct{} {
	targetsSlice := strings.Fields(a)
	if len(targetsSlice) == 0 {
		if runtime.GOOS == "darwin" {
			targetsSlice = []string{"android", "ios"}
		} else {
			targetsSlice = []string{"android"}
		}
	}
	targets := map[string]struct{}{}
	for _, i := range targetsSlice {
		switch i {
		case "android":
			targets["android"] = struct{}{}
			targets["android/arm"] = struct{}{}
			targets["android/arm64"] = struct{}{}
			targets["android/386"] = struct{}{}
			targets["android/amd64"] = struct{}{}
		case "android/arm", "android/arm64", "android/386", "android/amd64":
			targets["android"] = struct{}{}
			targets[i] = struct{}{}
		case "ios":
			targets["ios"] = struct{}{}
			targets["ios/arm"] = struct{}{}
			targets["ios/arm64"] = struct{}{}
			// targets["ios/386"] = struct{}{}
			targets["ios/amd64"] = struct{}{}
		case "ios/arm", "ios/arm64", "ios/386", "ios/amd64":
			targets["ios"] = struct{}{}
			targets[i] = struct{}{}
		}
	}
	return targets
}

func Build(flags *Flags, args []string) error {
	iosDir, err := PackageDir(flags, "gomatcha.io/matcha")
	if err != nil {
		return err
	}
	// flags.BuildBinary = true
	flags.BuildO = iosDir
	return Bind(flags, args)
}

func Bind(flags *Flags, args []string) error {
	targets := ParseTargets(flags.BuildTargets)

	// Validate Go
	err := validateGoInstall(flags)
	if err != nil {
		return err
	}

	// Make $WORK.
	tempdir, err := NewTmpDir(flags, "")
	if err != nil {
		return err
	}
	if !flags.BuildWork {
		defer RemoveAll(flags, tempdir)
	}

	// Get $GOPATH/pkg/matcha.
	matchaPkgPath, err := MatchaPkgPath(flags)
	if err != nil {
		return err
	}

	// Get toolchain version.
	installedVersion, err := ReadFile(flags, filepath.Join(matchaPkgPath, "version"))
	if err != nil {
		return errors.New("toolchain partially installed, run `matcha init`")
	}

	// Get go version.
	goVersion, err := GoVersion(flags)
	if err != nil {
		return err
	}

	// Check toolchain matches go version.
	if !bytes.Equal(installedVersion, goVersion) && flags.ShouldRun() {
		return errors.New("toolchain out of date, run `matcha init`")
	}

	// Get current working directory.
	cwd, err := Getwd(flags)
	if err != nil {
		return err
	}

	// Create a build context.
	ctx := build.Default
	ctx.GOARCH = "arm"
	ctx.GOOS = "darwin"
	ctx.BuildTags = append(ctx.BuildTags, "matcha")

	// Get import paths to be built.
	importPaths := []string{}
	if len(args) == 0 {
		importPaths = append(importPaths, ".")
	} else {
		for _, i := range args {
			i = path.Clean(i)
			importPaths = append(importPaths, i)
		}
	}

	// Get packages to be built
	pkgs, err := ImportAll(flags, &ctx, importPaths, cwd, build.ImportComment)
	if err != nil {
		return err
	}

	// Check if any of the package is main.
	for _, pkg := range pkgs {
		if pkg.Name == "main" {
			return fmt.Errorf("binding 'main' package (%s) is not supported", pkg.ImportComment)
		}
	}

	// Get the supporting files
	bridgePath, err := PackageDir(flags, "gomatcha.io/matcha/bridge")
	if err != nil {
		return err
	}

	// Begin iOS
	if _, ok := targets["ios"]; ok {
		// Validate Xcode installation
		if err := validateXcodeInstall(flags); err != nil {
			return err
		}

		// Build the "matcha/bridge" dir
		gopathDir := filepath.Join(tempdir, "IOS-GOPATH")

		// Make $WORK/matcha-ios
		workOutputDir := filepath.Join(tempdir, "matcha-ios")
		if err := Mkdir(flags, workOutputDir); err != nil {
			return err
		}

		// Make binary output dir
		binaryPath := filepath.Join(workOutputDir, "MatchaBridge", "MatchaBridge", "MatchaBridge.a")
		if err := Mkdir(flags, filepath.Dir(binaryPath)); err != nil {
			return err
		}

		// Create the "main" go package, that references the other go packages
		mainPath := filepath.Join(tempdir, "src", "iosbin", "main.go")
		err = WriteFile(flags, mainPath, strings.NewReader(fmt.Sprintf(BindFile, args[0]))) // TODO(KD): Should this be args[0] or should it use the logic to generate pkgs
		if err != nil {
			return fmt.Errorf("failed to create the binding package for iOS: %v", err)
		}

		// if !flags.BuildBinary {
		// 	// Copy package's ios directory if it imports gomatcha.io/bridge.
		// 	for _, pkg := range pkgs {
		// 		importsBridge := false
		// 		for _, i := range pkg.Imports {
		// 			if i == "gomatcha.io/bridge" {
		// 				importsBridge = true
		// 				break
		// 			}
		// 		}

		// 		if importsBridge {
		// 			files, err := ioutil.ReadDir(pkg.Dir)
		// 			if err != nil {
		// 				continue
		// 			}

		// 			for _, i := range files {
		// 				if i.IsDir() && i.Name() == "ios" {
		// 					// Copy directory
		// 					src := filepath.Join(pkg.Dir, "ios")
		// 					dst := filepath.Join(workOutputDir)
		// 					CopyDirContents(flags, dst, src)
		// 				}
		// 			}
		// 		}
		// 	}
		// }

		// Build platform binaries concurrently.
		envs := [][]string{}
		if _, ok := targets["ios/arm"]; ok {
			env, err := DarwinArmEnv(flags)
			if err != nil {
				return err
			}
			envs = append(envs, env)
		}
		if _, ok := targets["ios/arm64"]; ok {
			env, err := DarwinArm64Env(flags)
			if err != nil {
				return err
			}
			envs = append(envs, env)
		}
		if _, ok := targets["ios/386"]; ok {
			env, err := Darwin386Env(flags)
			if err != nil {
				return err
			}
			envs = append(envs, env)
		}
		if _, ok := targets["ios/amd64"]; ok {
			env, err := DarwinAmd64Env(flags)
			if err != nil {
				return err
			}
			envs = append(envs, env)
		}

		type archPath struct {
			arch string
			path string
			err  error
		}
		archs := []archPath{}
		archChan := make(chan archPath)
		for _, i := range envs {
			go func(env []string) {
				arch := FindEnv(env, "GOARCH")
				env = append(env, "GOPATH="+gopathDir+string(filepath.ListSeparator)+GoEnv(flags, "GOPATH"))
				path := filepath.Join(tempdir, "matcha-"+arch+".a")

				// ios needs to be added as a build tag due to https://github.com/golang/go/commit/29eb7d18ed71c057bbdb69d85953a32252f0ea73
				err := GoBuild(flags, []string{mainPath}, env, []string{"matcha", "ios"}, matchaPkgPath, tempdir, "-buildmode=c-archive", "-o", path)
				archChan <- archPath{arch, path, err}
			}(i)

			if !flags.Threaded {
				arch := <-archChan
				if arch.err != nil {
					return arch.err
				}
				archs = append(archs, arch)
			}
		}
		if flags.Threaded {
			for i := 0; i < len(envs); i++ {
				arch := <-archChan
				if arch.err != nil {
					return arch.err
				}
				archs = append(archs, arch)
			}
		}

		// Lipo to build fat binary.
		cmd := exec.Command("xcrun", "lipo", "-create")
		for _, i := range archs {
			cmd.Args = append(cmd.Args, "-arch", ArchClang(i.arch), i.path)
		}
		cmd.Args = append(cmd.Args, "-o", binaryPath)
		if err := RunCmd(flags, tempdir, cmd); err != nil {
			return err
		}

		// Create output dir
		outputDir := flags.BuildO
		if outputDir == "" {
			outputDir = "Matcha-iOS"
		}

		// if !flags.BuildBinary {
		// 	if err := RemoveAll(flags, outputDir); err != nil {
		// 		return err
		// 	}

		// 	// Copy output directory into place.
		// 	if err := CopyDir(flags, outputDir, workOutputDir); err != nil {
		// 		return err
		// 	}
		// } else {

		// Copy binary into place.
		if err := CopyFile(flags, filepath.Join(outputDir, "ios", "MatchaBridge", "MatchaBridge", "MatchaBridge.a"), binaryPath); err != nil {
			return err
		}

		// }
	}
	if _, ok := targets["android"]; ok {
		// Validate Android installation
		if err := ValidateAndroidInstall(flags); err != nil {
			return err
		}

		// Build the "matcha/bridge" dir
		gopathDir := filepath.Join(tempdir, "ANDROID-GOPATH")

		androidArchs := []string{}
		if _, ok := targets["android/arm"]; ok {
			androidArchs = append(androidArchs, "arm")
		}
		if _, ok := targets["android/arm64"]; ok {
			androidArchs = append(androidArchs, "arm64")
		}
		if _, ok := targets["android/386"]; ok {
			androidArchs = append(androidArchs, "386")
		}
		if _, ok := targets["android/amd64"]; ok {
			androidArchs = append(androidArchs, "amd64")
		}

		androidDir := filepath.Join(tempdir, "android")
		mainPath := filepath.Join(tempdir, "androidlib/main.go")

		err = WriteFile(flags, mainPath, strings.NewReader(fmt.Sprintf(BindFile, args[0]))) // TODO(KD): Should this be args[0] or should it use the logic to generate pkgs
		if err != nil {
			return fmt.Errorf("failed to create the main package for android: %v", err)
		}

		javaDir2 := filepath.Join(androidDir, "src", "main", "java", "io", "gomatcha", "bridge")
		if err := Mkdir(flags, javaDir2); err != nil {
			return err
		}
		if err := CopyFile(flags, filepath.Join(javaDir2, "GoValue.java"), filepath.Join(bridgePath, "java-GoValue.java")); err != nil {
			return err
		}
		if err := CopyFile(flags, filepath.Join(javaDir2, "Bridge.java"), filepath.Join(bridgePath, "java-Bridge.java")); err != nil {
			return err
		}
		if err := CopyFile(flags, filepath.Join(javaDir2, "Tracker.java"), filepath.Join(bridgePath, "java-Tracker.java")); err != nil {
			return err
		}

		// Make $WORK/matcha-android
		workOutputDir := filepath.Join(tempdir, "matcha-android")
		if err := Mkdir(flags, workOutputDir); err != nil {
			return err
		}

		// Make aar output file.
		aarDirPath := filepath.Join(workOutputDir, "MatchaBridge")
		aarPath := filepath.Join(workOutputDir, "MatchaBridge", "matchabridge.aar")
		if err := Mkdir(flags, aarDirPath); err != nil {
			return err
		}

		// Generate binding code and java source code only when processing the first package.
		for _, arch := range androidArchs {
			env, err := AndroidEnv(flags, arch)
			if err != nil {
				return err
			}
			env = append(env, "GOPATH="+gopathDir+string(filepath.ListSeparator)+GoEnv(flags, "GOPATH"))

			err = GoBuild(flags,
				[]string{mainPath},
				env,
				[]string{"matcha"},
				matchaPkgPath,
				tempdir,
				"-buildmode=c-shared",
				"-o="+filepath.Join(androidDir, "src/main/jniLibs/"+GetAndroidABI(arch)+"/libgojni.so"),
			)
			if err != nil {
				return err
			}
		}

		if err := BuildAAR(flags, androidDir, pkgs, androidArchs, tempdir, aarPath); err != nil {
			return err
		}

		// Create output dir
		outputDir := flags.BuildO
		if outputDir == "" {
			outputDir = "Matcha-iOS"
		}

		// Copy binary into place.
		if err := CopyFile(flags, filepath.Join(outputDir, "android", "matchabridge.aar"), aarPath); err != nil {
			return err
		}
	}
	return nil
}

var BindFile = `
package main

import (
    _ "gomatcha.io/matcha/bridge"
    _ "%s"
)

import "C"

func main() {}
`
<|endoftext|>


<commit_before>package vtgate

import (
	"flag"
	"fmt"
	"net"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/youtube/vitess/go/mysqlconn"
	"github.com/youtube/vitess/go/sqldb"
	"github.com/youtube/vitess/go/sqltypes"
	"github.com/youtube/vitess/go/vt/callerid"
	"github.com/youtube/vitess/go/vt/servenv"

	querypb "github.com/youtube/vitess/go/vt/proto/query"
	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"
	vtgatepb "github.com/youtube/vitess/go/vt/proto/vtgate"
)

var (
	mysqlServerPort               = flag.Int("mysql_server_port", 0, "If set, also listen for MySQL binary protocol connections on this port.")
	mysqlAuthServerImpl           = flag.String("mysql_auth_server_impl", "static", "Which auth server implementation to use.")
	mysqlAllowClearTextWithoutTLS = flag.Bool("mysql_allow_clear_text_without_tls", false, "If set, the server will allow the use of a clear text password over non-SSL connections.")
)

// vtgateHandler implements the Listener interface.
// It stores the Session in the ClientData of a Connection, if a transaction
// is in progress.
type vtgateHandler struct {
	vtg *VTGate
}

func newVtgateHandler(vtg *VTGate) *vtgateHandler {
	return &vtgateHandler{
		vtg: vtg,
	}
}

func (vh *vtgateHandler) NewConnection(c *mysqlconn.Conn) {
}

func (vh *vtgateHandler) ConnectionClosed(c *mysqlconn.Conn) {
	// Rollback if there is an ongoing transaction. Ignore error.
	ctx := context.Background()
	session, _ := c.ClientData.(*vtgatepb.Session)
	if session == nil || !session.InTransaction {
		return
	}
	_, _, _ = vh.vtg.Execute(ctx, "rollback", make(map[string]interface{}), "", topodatapb.TabletType_MASTER, session, false, &querypb.ExecuteOptions{})
}

func (vh *vtgateHandler) ComQuery(c *mysqlconn.Conn, query string) (*sqltypes.Result, error) {
	// FIXME(alainjobart): Add some kind of timeout to the context.
	ctx := context.Background()

	// Fill in the ImmediateCallerID with the UserData returned by
	// the AuthServer plugin for that user. If nothing was
	// returned, use the User. This lets the plugin map a MySQL
	// user used for authentication to a Vitess User used for
	// Table ACLs and Vitess authentication in general.
	im := c.UserData.Get()
	ef := callerid.NewEffectiveCallerID(
		c.User,                  /* principal: who */
		c.RemoteAddr().String(), /* component: running client process */
		"VTGate MySQL Connector" /* subcomponent: part of the client */)
	ctx = callerid.NewContext(ctx, ef, im)

	session, _ := c.ClientData.(*vtgatepb.Session)
	session, result, err := vh.vtg.Execute(ctx, query, make(map[string]interface{}), c.SchemaName, topodatapb.TabletType_MASTER, session, false /* notInTransaction */, &querypb.ExecuteOptions{
		IncludedFields: querypb.ExecuteOptions_ALL,
	})
	c.ClientData = session
	return result, sqldb.NewSQLErrorFromError(err)
}

func init() {
	var listener *mysqlconn.Listener

	servenv.OnRun(func() {
		// Flag is not set, just return.
		if *mysqlServerPort == 0 {
			return
		}

		// If no VTGate was created, just return.
		if rpcVTGate == nil {
			return
		}

		// Initialize registered AuthServer implementations (or other plugins)
		for _, initFn := range pluginInitializers {
			initFn()
		}
		authServer := mysqlconn.GetAuthServer(*mysqlAuthServerImpl)

		// Create a Listener.
		var err error
		vh := newVtgateHandler(rpcVTGate)
		listener, err = mysqlconn.NewListener("tcp", net.JoinHostPort("", fmt.Sprintf("%v", *mysqlServerPort)), authServer, vh)
		if err != nil {
			log.Fatalf("mysqlconn.NewListener failed: %v", err)
		}
		listener.AllowClearTextWithoutTLS = *mysqlAllowClearTextWithoutTLS

		// And starts listening.
		go func() {
			listener.Accept()
		}()
	})

	servenv.OnTerm(func() {
		if listener != nil {
			listener.Close()
		}
	})
}

var pluginInitializers []func()

// RegisterPluginInitializer lets plugins register themselves to be init'ed at servenv.OnRun-time
func RegisterPluginInitializer(initializer func()) {
	pluginInitializers = append(pluginInitializers, initializer)
}
<commit_msg>Add flags for configuring ssl in vtgate mysql plugin<commit_after>package vtgate

import (
	"flag"
	"fmt"
	"net"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	"github.com/youtube/vitess/go/mysqlconn"
	"github.com/youtube/vitess/go/sqldb"
	"github.com/youtube/vitess/go/sqltypes"
	"github.com/youtube/vitess/go/vt/callerid"
	"github.com/youtube/vitess/go/vt/servenv"

	querypb "github.com/youtube/vitess/go/vt/proto/query"
	topodatapb "github.com/youtube/vitess/go/vt/proto/topodata"
	vtgatepb "github.com/youtube/vitess/go/vt/proto/vtgate"
	"github.com/youtube/vitess/go/vt/servenv/grpcutils"
)

var (
	mysqlServerPort               = flag.Int("mysql_server_port", 0, "If set, also listen for MySQL binary protocol connections on this port.")
	mysqlAuthServerImpl           = flag.String("mysql_auth_server_impl", "static", "Which auth server implementation to use.")
	mysqlAllowClearTextWithoutTLS = flag.Bool("mysql_allow_clear_text_without_tls", false, "If set, the server will allow the use of a clear text password over non-SSL connections.")

	mysqlSslCert = flag.String("mysql_server_ssl_cert", "", "Path to the ssl cert for mysql server plugin SSL")
	mysqlSslKey  = flag.String("mysql_server_ssl_key", "", "Path to ssl key for mysql server plugin SSL")
	mysqlSslCa   = flag.String("mysql_server_ssl_ca", "", "Path to ssl CA for mysql server plugin SSL. If specified, server will require and validate client certs.")
)

// vtgateHandler implements the Listener interface.
// It stores the Session in the ClientData of a Connection, if a transaction
// is in progress.
type vtgateHandler struct {
	vtg *VTGate
}

func newVtgateHandler(vtg *VTGate) *vtgateHandler {
	return &vtgateHandler{
		vtg: vtg,
	}
}

func (vh *vtgateHandler) NewConnection(c *mysqlconn.Conn) {
}

func (vh *vtgateHandler) ConnectionClosed(c *mysqlconn.Conn) {
	// Rollback if there is an ongoing transaction. Ignore error.
	ctx := context.Background()
	session, _ := c.ClientData.(*vtgatepb.Session)
	if session == nil || !session.InTransaction {
		return
	}
	_, _, _ = vh.vtg.Execute(ctx, "rollback", make(map[string]interface{}), "", topodatapb.TabletType_MASTER, session, false, &querypb.ExecuteOptions{})
}

func (vh *vtgateHandler) ComQuery(c *mysqlconn.Conn, query string) (*sqltypes.Result, error) {
	// FIXME(alainjobart): Add some kind of timeout to the context.
	ctx := context.Background()

	// Fill in the ImmediateCallerID with the UserData returned by
	// the AuthServer plugin for that user. If nothing was
	// returned, use the User. This lets the plugin map a MySQL
	// user used for authentication to a Vitess User used for
	// Table ACLs and Vitess authentication in general.
	im := c.UserData.Get()
	ef := callerid.NewEffectiveCallerID(
		c.User,                  /* principal: who */
		c.RemoteAddr().String(), /* component: running client process */
		"VTGate MySQL Connector" /* subcomponent: part of the client */)
	ctx = callerid.NewContext(ctx, ef, im)

	session, _ := c.ClientData.(*vtgatepb.Session)
	session, result, err := vh.vtg.Execute(ctx, query, make(map[string]interface{}), c.SchemaName, topodatapb.TabletType_MASTER, session, false /* notInTransaction */, &querypb.ExecuteOptions{
		IncludedFields: querypb.ExecuteOptions_ALL,
	})
	c.ClientData = session
	return result, sqldb.NewSQLErrorFromError(err)
}

func init() {
	var listener *mysqlconn.Listener

	servenv.OnRun(func() {
		// Flag is not set, just return.
		if *mysqlServerPort == 0 {
			return
		}

		// If no VTGate was created, just return.
		if rpcVTGate == nil {
			return
		}

		// Initialize registered AuthServer implementations (or other plugins)
		for _, initFn := range pluginInitializers {
			initFn()
		}
		authServer := mysqlconn.GetAuthServer(*mysqlAuthServerImpl)

		// Create a Listener.
		var err error
		vh := newVtgateHandler(rpcVTGate)
		listener, err = mysqlconn.NewListener("tcp", net.JoinHostPort("", fmt.Sprintf("%v", *mysqlServerPort)), authServer, vh)
		if err != nil {
			log.Fatalf("mysqlconn.NewListener failed: %v", err)
		}
		if *mysqlSslCert != "" && *mysqlSslKey != "" {
			listener.TLSConfig, err = grpcutils.TLSServerConfig(*mysqlSslCert, *mysqlSslKey, *mysqlSslCa)
			if err != nil {
				log.Fatalf("grpcutils.TLSServerConfig failed: %v", err)
				return
			}
		}
		listener.AllowClearTextWithoutTLS = *mysqlAllowClearTextWithoutTLS

		// And starts listening.
		go func() {
			listener.Accept()
		}()
	})

	servenv.OnTerm(func() {
		if listener != nil {
			listener.Close()
		}
	})
}

var pluginInitializers []func()

// RegisterPluginInitializer lets plugins register themselves to be init'ed at servenv.OnRun-time
func RegisterPluginInitializer(initializer func()) {
	pluginInitializers = append(pluginInitializers, initializer)
}
<|endoftext|>


<commit_before>package cmd

import (
	"os"

	"bytes"
	"context"
	"fmt"
	"sort"
	"strings"

	"github.com/kr/pretty"
	"github.com/spf13/cobra"
	"github.com/zrepl/zrepl/zfs"
	"log"
)

var testCmd = &cobra.Command{
	Use:   "test",
	Short: "test configuration",
}

var testCmdGlobal struct {
	log  Logger
	conf *Config
}

var testConfigSyntaxCmd = &cobra.Command{
	Use:   "config",
	Short: "parse config file and dump parsed datastructure",
	Run:   doTestConfig,
}

var testDatasetMapFilter = &cobra.Command{
	Use:     "pattern jobname test/zfs/dataset/path",
	Short:   "test dataset mapping / filter specified in config",
	Example: ` zrepl test pattern prune.clean_backups tank/backups/legacyscript/foo`,
	Run:     doTestDatasetMapFilter,
}

var testPrunePolicyArgs struct {
	side        PrunePolicySide
	showKept    bool
	showRemoved bool
}

var testPrunePolicyCmd = &cobra.Command{
	Use:   "prune jobname",
	Short: "do a dry-run of the pruning part of a job",
	Run:   doTestPrunePolicy,
}

func init() {
	cobra.OnInitialize(testCmdGlobalInit)
	RootCmd.AddCommand(testCmd)
	testCmd.AddCommand(testConfigSyntaxCmd)
	testCmd.AddCommand(testDatasetMapFilter)

	testPrunePolicyCmd.Flags().VarP(&testPrunePolicyArgs.side, "side", "s", "prune_lhs (left) or prune_rhs (right)")
	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showKept, "kept", false, "show kept snapshots")
	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showRemoved, "removed", true, "show removed snapshots")
	testCmd.AddCommand(testPrunePolicyCmd)
}

func testCmdGlobalInit() {

	testCmdGlobal.log = log.New(os.Stdout, "", 0)

	ctx := context.WithValue(context.Background(), contextKeyLog, testCmdGlobal.log)

	var err error
	if testCmdGlobal.conf, err = ParseConfig(ctx, rootArgs.configFile); err != nil {
		testCmdGlobal.log.Printf("error parsing config file: %s", err)
		os.Exit(1)
	}

}

func doTestConfig(cmd *cobra.Command, args []string) {

	log, conf := testCmdGlobal.log, testCmdGlobal.conf

	log.Printf("config ok")
	log.Printf("%# v", pretty.Formatter(conf))
	return
}

func doTestDatasetMapFilter(cmd *cobra.Command, args []string) {

	log, conf := testCmdGlobal.log, testCmdGlobal.conf

	if len(args) != 2 {
		log.Printf("specify job name as first postitional argument, test input as second")
		log.Printf(cmd.UsageString())
		os.Exit(1)
	}
	n, i := args[0], args[1]

	jobi, err := conf.LookupJob(n)
	if err != nil {
		log.Printf("%s", err)
		os.Exit(1)
	}

	var mf *DatasetMapFilter
	switch j := jobi.(type) {
	case *PullJob:
		mf = j.Mapping
	case *SourceJob:
		mf = j.Datasets
	case *LocalJob:
		mf = j.Mapping
	default:
		panic("incomplete implementation")
	}

	ip, err := zfs.NewDatasetPath(i)
	if err != nil {
		log.Printf("cannot parse test input as ZFS dataset path: %s", err)
		os.Exit(1)
	}

	if mf.filterMode {
		pass, err := mf.Filter(ip)
		if err != nil {
			log.Printf("error evaluating filter: %s", err)
			os.Exit(1)
		}
		log.Printf("filter result: %v", pass)
	} else {
		res, err := mf.Map(ip)
		if err != nil {
			log.Printf("error evaluating mapping: %s", err)
			os.Exit(1)
		}
		toStr := "NO MAPPING"
		if res != nil {
			toStr = res.ToString()
		}
		log.Printf("%s => %s", ip.ToString(), toStr)

	}

}

func doTestPrunePolicy(cmd *cobra.Command, args []string) {

	log, conf := testCmdGlobal.log, testCmdGlobal.conf

	if cmd.Flags().NArg() != 1 {
		log.Printf("specify job name as first positional argument")
		log.Printf(cmd.UsageString())
		os.Exit(1)
	}

	jobname := cmd.Flags().Arg(0)
	jobi, err := conf.LookupJob(jobname)
	if err != nil {
		log.Printf("%s", err)
		os.Exit(1)
	}

	jobp, ok := jobi.(PruningJob)
	if !ok {
		log.Printf("job doesn't do any prunes")
		os.Exit(0)
	}

	log.Printf("job dump:\n%s", pretty.Sprint(jobp))

	pruner, err := jobp.Pruner(testPrunePolicyArgs.side, true)
	if err != nil {
		log.Printf("cannot create test pruner: %s", err)
		os.Exit(1)
	}

	log.Printf("start pruning")

	ctx := context.WithValue(context.Background(), contextKeyLog, log)
	result, err := pruner.Run(ctx)
	if err != nil {
		log.Printf("error running pruner: %s", err)
		os.Exit(1)
	}

	sort.Slice(result, func(i, j int) bool {
		return strings.Compare(result[i].Filesystem.ToString(), result[j].Filesystem.ToString()) == -1
	})

	var b bytes.Buffer
	for _, r := range result {
		fmt.Fprintf(&b, "%s\n", r.Filesystem.ToString())

		if testPrunePolicyArgs.showKept {
			fmt.Fprintf(&b, "\tkept:\n")
			for _, v := range r.Keep {
				fmt.Fprintf(&b, "\t- %s\n", v.Name)
			}
		}

		if testPrunePolicyArgs.showRemoved {
			fmt.Fprintf(&b, "\tremoved:\n")
			for _, v := range r.Remove {
				fmt.Fprintf(&b, "\t- %s\n", v.Name)
			}
		}

	}

	log.Printf("pruning result:\n%s", b.String())

}
<commit_msg>cmd: test: would always run testCmdGlobalInit<commit_after>package cmd

import (
	"os"

	"bytes"
	"context"
	"fmt"
	"sort"
	"strings"

	"github.com/kr/pretty"
	"github.com/spf13/cobra"
	"github.com/zrepl/zrepl/zfs"
	"log"
)

var testCmd = &cobra.Command{
	Use:              "test",
	Short:            "test configuration",
	PersistentPreRun: testCmdGlobalInit,
}

var testCmdGlobal struct {
	log  Logger
	conf *Config
}

var testConfigSyntaxCmd = &cobra.Command{
	Use:   "config",
	Short: "parse config file and dump parsed datastructure",
	Run:   doTestConfig,
}

var testDatasetMapFilter = &cobra.Command{
	Use:     "pattern jobname test/zfs/dataset/path",
	Short:   "test dataset mapping / filter specified in config",
	Example: ` zrepl test pattern prune.clean_backups tank/backups/legacyscript/foo`,
	Run:     doTestDatasetMapFilter,
}

var testPrunePolicyArgs struct {
	side        PrunePolicySide
	showKept    bool
	showRemoved bool
}

var testPrunePolicyCmd = &cobra.Command{
	Use:   "prune jobname",
	Short: "do a dry-run of the pruning part of a job",
	Run:   doTestPrunePolicy,
}

func init() {
	RootCmd.AddCommand(testCmd)
	testCmd.AddCommand(testConfigSyntaxCmd)
	testCmd.AddCommand(testDatasetMapFilter)

	testPrunePolicyCmd.Flags().VarP(&testPrunePolicyArgs.side, "side", "s", "prune_lhs (left) or prune_rhs (right)")
	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showKept, "kept", false, "show kept snapshots")
	testPrunePolicyCmd.Flags().BoolVar(&testPrunePolicyArgs.showRemoved, "removed", true, "show removed snapshots")
	testCmd.AddCommand(testPrunePolicyCmd)
}

func testCmdGlobalInit(cmd *cobra.Command, args []string) {

	testCmdGlobal.log = log.New(os.Stdout, "", 0)

	ctx := context.WithValue(context.Background(), contextKeyLog, testCmdGlobal.log)

	var err error
	if testCmdGlobal.conf, err = ParseConfig(ctx, rootArgs.configFile); err != nil {
		testCmdGlobal.log.Printf("error parsing config file: %s", err)
		os.Exit(1)
	}

}

func doTestConfig(cmd *cobra.Command, args []string) {

	log, conf := testCmdGlobal.log, testCmdGlobal.conf

	log.Printf("config ok")
	log.Printf("%# v", pretty.Formatter(conf))
	return
}

func doTestDatasetMapFilter(cmd *cobra.Command, args []string) {

	log, conf := testCmdGlobal.log, testCmdGlobal.conf

	if len(args) != 2 {
		log.Printf("specify job name as first postitional argument, test input as second")
		log.Printf(cmd.UsageString())
		os.Exit(1)
	}
	n, i := args[0], args[1]

	jobi, err := conf.LookupJob(n)
	if err != nil {
		log.Printf("%s", err)
		os.Exit(1)
	}

	var mf *DatasetMapFilter
	switch j := jobi.(type) {
	case *PullJob:
		mf = j.Mapping
	case *SourceJob:
		mf = j.Datasets
	case *LocalJob:
		mf = j.Mapping
	default:
		panic("incomplete implementation")
	}

	ip, err := zfs.NewDatasetPath(i)
	if err != nil {
		log.Printf("cannot parse test input as ZFS dataset path: %s", err)
		os.Exit(1)
	}

	if mf.filterMode {
		pass, err := mf.Filter(ip)
		if err != nil {
			log.Printf("error evaluating filter: %s", err)
			os.Exit(1)
		}
		log.Printf("filter result: %v", pass)
	} else {
		res, err := mf.Map(ip)
		if err != nil {
			log.Printf("error evaluating mapping: %s", err)
			os.Exit(1)
		}
		toStr := "NO MAPPING"
		if res != nil {
			toStr = res.ToString()
		}
		log.Printf("%s => %s", ip.ToString(), toStr)

	}

}

func doTestPrunePolicy(cmd *cobra.Command, args []string) {

	log, conf := testCmdGlobal.log, testCmdGlobal.conf

	if cmd.Flags().NArg() != 1 {
		log.Printf("specify job name as first positional argument")
		log.Printf(cmd.UsageString())
		os.Exit(1)
	}

	jobname := cmd.Flags().Arg(0)
	jobi, err := conf.LookupJob(jobname)
	if err != nil {
		log.Printf("%s", err)
		os.Exit(1)
	}

	jobp, ok := jobi.(PruningJob)
	if !ok {
		log.Printf("job doesn't do any prunes")
		os.Exit(0)
	}

	log.Printf("job dump:\n%s", pretty.Sprint(jobp))

	pruner, err := jobp.Pruner(testPrunePolicyArgs.side, true)
	if err != nil {
		log.Printf("cannot create test pruner: %s", err)
		os.Exit(1)
	}

	log.Printf("start pruning")

	ctx := context.WithValue(context.Background(), contextKeyLog, log)
	result, err := pruner.Run(ctx)
	if err != nil {
		log.Printf("error running pruner: %s", err)
		os.Exit(1)
	}

	sort.Slice(result, func(i, j int) bool {
		return strings.Compare(result[i].Filesystem.ToString(), result[j].Filesystem.ToString()) == -1
	})

	var b bytes.Buffer
	for _, r := range result {
		fmt.Fprintf(&b, "%s\n", r.Filesystem.ToString())

		if testPrunePolicyArgs.showKept {
			fmt.Fprintf(&b, "\tkept:\n")
			for _, v := range r.Keep {
				fmt.Fprintf(&b, "\t- %s\n", v.Name)
			}
		}

		if testPrunePolicyArgs.showRemoved {
			fmt.Fprintf(&b, "\tremoved:\n")
			for _, v := range r.Remove {
				fmt.Fprintf(&b, "\t- %s\n", v.Name)
			}
		}

	}

	log.Printf("pruning result:\n%s", b.String())

}
<|endoftext|>


<commit_before>package cmd

import (
	"io/ioutil"
	"os"
	"os/exec"
)

func openLynx(html string) {
	execCmd(html, "lynx", "/tmp/sreq.html")
}

func openEditor(body string, editor string) {
	execCmd(body, editor, "/tmp/sreq.txt")
}

func execCmd(body string, cmdName string, file string) {
	text := []byte(body)
	ioutil.WriteFile(file, text, os.ModePerm)
	cmd := exec.Command(cmdName, file)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Run()
}
<commit_msg>Remove unneccesary file<commit_after><|endoftext|>


<commit_before>package http_util

import (
	"bytes"
	"crypto/tls"
	"github.com/bysir-zl/bygo/util"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

func Get(url string, params util.OrderKV, header map[string]string) (code int, response string, err error) {
	up := params.EncodeString()
	if up != "" {
		if strings.Contains(url, "?") {
			url = url + "&" + up
		} else {
			url = url + "?" + up
		}
	}
	code, bs, err := request(url, "GET", nil, header, nil)
	response = util.B2S(bs)
	return
}

func Post(url string, params util.OrderKV, header map[string]string) (code int, response string, err error) {
	code, bs, err := request(url, "POST", params.Encode(), header, nil)
	response = util.B2S(bs)
	return
}

func PostByte(url string, post []byte, header map[string]string) (code int, response string, err error) {
	code, bs, err := request(url, "POST", post, header, nil)
	response = util.B2S(bs)
	return
}

func PostWithCookie(url string, params util.OrderKV, cookie map[string]string) (code int, response string, err error) {
	code, bs, err := request(url, "POST", params.Encode(), nil, cookie)
	response = util.B2S(bs)
	return
}

func GetWithCookie(url string, params util.OrderKV, cookie map[string]string) (code int, response string, err error) {
	code, bs, err := request(url, "GET", params.Encode(), nil, cookie)
	response = util.B2S(bs)
	return
}

func request(url string, method string, post []byte, header map[string]string, cookie map[string]string) (code int, result []byte, err error) {
	var response *http.Response

	// 忽略https证书验证
	transport := &http.Transport{
		TLSClientConfig:    &tls.Config{InsecureSkipVerify: true},
		DisableCompression: true,
	}
	client := &http.Client{Transport: transport}
	var req *http.Request
	if method == "GET" {
		req, _ = http.NewRequest("GET", url, nil)
	} else if method == "POST" {
		req, _ = http.NewRequest("POST", url, bytes.NewReader(post))
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}

	if header != nil&&len(header) != 0 {
		for key, value := range header {
			req.Header.Add(key, value)
		}
	}
	if cookie != nil&&len(cookie) != 0 {
		for key, value := range cookie {
			req.AddCookie(&http.Cookie{Name:key, Value:value})
		}
	}
	response, err = client.Do(req)
	if err != nil {
		return
	}

	code = response.StatusCode
	body, _ := ioutil.ReadAll(response.Body)
	result = body
	return

}

func BuildQuery(key []string, value []string) string {
	var bf bytes.Buffer
	for i, k := range key {
		if bf.Len() == 0 {
			bf.WriteByte('&')
		}
		bf.WriteString(k + "=" + url.QueryEscape(value[i]))
	}
	return bf.String()
}

func BuildQueryWithOutEmptyValue(key []string, value []string) string {
	var bf bytes.Buffer
	for i, k := range key {
		if v := value[i]; v != "" {
			if bf.Len() == 0 {
				bf.WriteByte('&')
			}
			bf.WriteString(k + "=" + url.QueryEscape(v))
		}
	}
	return bf.String()
}

func QueryString2Map(que string) (set map[string]string) {
	set = map[string]string{}
	if !strings.Contains(que, "&") {
		return
	}
	for _, kv := range strings.Split(que, "&") {
		kAv := strings.Split(kv, "=")
		if len(kAv) == 2 {
			k, err := url.QueryUnescape(kAv[0])
			v, err2 := url.QueryUnescape(kAv[1])
			if err == nil && err2 == nil {
				set[k] = v
			}
		}
	}
	return
}

// like php - rawurlencode
// rawurlencode and urlencode is different form the ' ' will encode to '%20', is not '+'
func RawUrlEncode(origin string) string {
	x := url.QueryEscape(origin)
	x = strings.Replace(x, "+", "%20", -1)
	return x
}
<commit_msg>add http util timeout<commit_after>package http_util

import (
	"bytes"
	"crypto/tls"
	"github.com/bysir-zl/bygo/util"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

func Get(url string, params util.OrderKV, header map[string]string) (code int, response string, err error) {
	up := params.EncodeString()
	if up != "" {
		if strings.Contains(url, "?") {
			url = url + "&" + up
		} else {
			url = url + "?" + up
		}
	}
	code, bs, err := request(url, "GET", nil, header, nil)
	response = util.B2S(bs)
	return
}

func Post(url string, params util.OrderKV, header map[string]string) (code int, response string, err error) {
	code, bs, err := request(url, "POST", params.Encode(), header, nil)
	response = util.B2S(bs)
	return
}

func PostByte(url string, post []byte, header map[string]string) (code int, response string, err error) {
	code, bs, err := request(url, "POST", post, header, nil)
	response = util.B2S(bs)
	return
}

func PostWithCookie(url string, params util.OrderKV, cookie map[string]string) (code int, response string, err error) {
	code, bs, err := request(url, "POST", params.Encode(), nil, cookie)
	response = util.B2S(bs)
	return
}

func GetWithCookie(url string, params util.OrderKV, cookie map[string]string) (code int, response string, err error) {
	code, bs, err := request(url, "GET", params.Encode(), nil, cookie)
	response = util.B2S(bs)
	return
}

func request(url string, method string, post []byte, header map[string]string, cookie map[string]string) (code int, result []byte, err error) {
	var response *http.Response

	// 忽略https证书验证
	transport := &http.Transport{
		TLSClientConfig:    &tls.Config{InsecureSkipVerify: true},
		DisableCompression: true,
	}
	client := &http.Client{Transport: transport, Timeout: 5 * time.Second}
	var req *http.Request
	if method == "GET" {
		req, _ = http.NewRequest("GET", url, nil)
	} else if method == "POST" {
		req, _ = http.NewRequest("POST", url, bytes.NewReader(post))
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}

	if header != nil && len(header) != 0 {
		for key, value := range header {
			req.Header.Add(key, value)
		}
	}
	if cookie != nil && len(cookie) != 0 {
		for key, value := range cookie {
			req.AddCookie(&http.Cookie{Name: key, Value: value})
		}
	}
	response, err = client.Do(req)
	if err != nil {
		return
	}

	code = response.StatusCode
	body, _ := ioutil.ReadAll(response.Body)
	result = body
	return

}

func BuildQuery(key []string, value []string) string {
	var bf bytes.Buffer
	for i, k := range key {
		if bf.Len() == 0 {
			bf.WriteByte('&')
		}
		bf.WriteString(k + "=" + url.QueryEscape(value[i]))
	}
	return bf.String()
}

func BuildQueryWithOutEmptyValue(key []string, value []string) string {
	var bf bytes.Buffer
	for i, k := range key {
		if v := value[i]; v != "" {
			if bf.Len() == 0 {
				bf.WriteByte('&')
			}
			bf.WriteString(k + "=" + url.QueryEscape(v))
		}
	}
	return bf.String()
}

func QueryString2Map(que string) (set map[string]string) {
	set = map[string]string{}
	if !strings.Contains(que, "&") {
		return
	}
	for _, kv := range strings.Split(que, "&") {
		kAv := strings.Split(kv, "=")
		if len(kAv) == 2 {
			k, err := url.QueryUnescape(kAv[0])
			v, err2 := url.QueryUnescape(kAv[1])
			if err == nil && err2 == nil {
				set[k] = v
			}
		}
	}
	return
}

// like php - rawurlencode
// rawurlencode and urlencode is different form the ' ' will encode to '%20', is not '+'
func RawUrlEncode(origin string) string {
	x := url.QueryEscape(origin)
	x = strings.Replace(x, "+", "%20", -1)
	return x
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/timakin/ts/loader"
	"github.com/codegangsta/cli"
)

var Commands = []cli.Command{
	commandAll,
//	commandBiz,
	commandHack,
}

var commandAll = cli.Command{
	Name:  "pop",
	Usage: "",
	Description: "Show today's news from major tech news sites, HN, PH, and subreddit of /programming.",
	Action: doAll,
}

//var commandBiz = cli.Command{
//	Name:  "biz",
//	Usage: "",
//	Description: `
//`,
//	Action: doBiz,
//}
var commandHack = cli.Command{
	Name:  "test",
	Usage: "",
	Description: `
`,
	Action: doHack,
}

func pp(str string) {
  fmt.Printf(str)
}

func doAll(c *cli.Context) {
		hn := make(chan loader.ResultData)
		ph := make(chan loader.ResultData)
		re := make(chan loader.ResultData)
		go loader.GetHNFeed(hn)
		go loader.GetPHFeed(ph)
		go loader.GetRedditFeed(re)
		hnres := <- hn
		phres := <- ph
		reres := <- re
		var HNData loader.Feed = &hnres
		var PHData loader.Feed = &phres
		var REData loader.Feed = &reres
		HNData.Display()
		PHData.Display()
		REData.Display()

		var uri string
		pp("[TechCrunch]\n")
		uri = "http://feeds.feedburner.com/TechCrunch/"
		loader.GetRSSFeed(uri)
		pp("[Mashable]\n")
		uri = "http://feeds.mashable.com/Mashable"
		loader.GetRSSFeed(uri)
		pp("[Hatena]\n")
		uri = "http://b.hatena.ne.jp/search/tag?q=%E3%83%97%E3%83%AD%E3%82%B0%E3%83%A9%E3%83%9F%E3%83%B3%E3%82%B0&users=10&mode=rss"
		loader.GetRSSFeed(uri)
}

//func doBiz(c *cli.Context) {
//}
func doHack(c *cli.Context) {
}
<commit_msg>Temporary fix of Ascii Art title display<commit_after>package main

import (
	"fmt"
	"github.com/timakin/ts/loader"
	"github.com/codegangsta/cli"
)

var Commands = []cli.Command{
	commandAll,
//	commandBiz,
	commandHack,
}

var commandAll = cli.Command{
	Name:  "pop",
	Usage: "",
	Description: "Show today's news from major tech news sites, HN, PH, and subreddit of /programming.",
	Action: doAll,
}

//var commandBiz = cli.Command{
//	Name:  "biz",
//	Usage: "",
//	Description: `
//`,
//	Action: doBiz,
//}
var commandHack = cli.Command{
	Name:  "test",
	Usage: "",
	Description: `
`,
	Action: doHack,
}

func pp(str string) {
  fmt.Printf(str)
}

func doAll(c *cli.Context) {
		pp("▁ ▂ ▄ ▅ ▆ ▇ █ тecнѕтacĸ █ ▇ ▆ ▅ ▄ ▂ ▁\n")
		hn := make(chan loader.ResultData)
		ph := make(chan loader.ResultData)
		re := make(chan loader.ResultData)
		go loader.GetHNFeed(hn)
		go loader.GetPHFeed(ph)
		go loader.GetRedditFeed(re)
		hnres := <- hn
		phres := <- ph
		reres := <- re
		var HNData loader.Feed = &hnres
		var PHData loader.Feed = &phres
		var REData loader.Feed = &reres
		HNData.Display()
		PHData.Display()
		REData.Display()

		var uri string
		pp("[TechCrunch]\n")
		uri = "http://feeds.feedburner.com/TechCrunch/"
		loader.GetRSSFeed(uri)
		pp("[Mashable]\n")
		uri = "http://feeds.mashable.com/Mashable"
		loader.GetRSSFeed(uri)
		pp("[Hatena]\n")
		uri = "http://b.hatena.ne.jp/search/tag?q=%E3%83%97%E3%83%AD%E3%82%B0%E3%83%A9%E3%83%9F%E3%83%B3%E3%82%B0&users=10&mode=rss"
		loader.GetRSSFeed(uri)
}

//func doBiz(c *cli.Context) {
//}
func doHack(c *cli.Context) {
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"strings"
)

type Page struct {
	Name       string
	Slug       string
	SrcFile    string
	StructName string
}

var Pages = map[string]Page{
	"consumer":  Page{"Consumers", "consumers", "consumer.go", "ConsumerConfig"},
	"depends":   Page{"Dependencies", "dependencies", "dependency_config.go", "DependencyConfig"},
	"downloads": Page{"Downloads", "downloads", "download_config.go", "DownloadConfig"},
	"errands":   Page{"Errands", "errands", "errand.go", "Errand"},
	"templates": Page{"Templates", "templates", "templates/templates.go", "Template"},
	"variables": Page{"Input and Output Variables", "input-and-output-variables", "variables/variable.go", "Variable"},
}

const PageHeader = `---
date: 2017-11-11 00:00:00
title: "%s"
slug: %s
type: "docs"
toc: true
---

%s

Field | Type | Description
------|------|-------------
%s
`

func GetJsonFieldFromTag(tag string) string {
	for _, s := range strings.Split(tag, " ") {
		s = strings.Trim(s, "`")
		if strings.HasPrefix(s, "json:\"") {
			s = s[6 : len(s)-1]
			return s
		}
	}
	return ""
}

func ParseType(expr ast.Expr) string {
	switch t := expr.(type) {
	case *ast.Ident:
		return t.Name
	case *ast.SelectorExpr:
		return ParseType(t.X) + "." + t.Sel.String() // probably wrong
	case *ast.ArrayType:
		return "[" + ParseType(t.Elt) + "]"
	case *ast.StarExpr:
		return ParseType(t.X)
	case *ast.MapType:
		return "{" + ParseType(t.Key) + ":" + ParseType(t.Value) + "}"
	case *ast.InterfaceType:
		return "any"
	default:
		fmt.Printf("%T\n", t)
		panic("type not supported in documentation: ")
	}
	return ""
}

func StructTable(page Page, topLevelDoc string, s *ast.TypeSpec) string {
	structType := s.Type.(*ast.StructType)
	result := ""
	for _, field := range structType.Fields.List {
		tag := GetJsonFieldFromTag(field.Tag.Value)
		typ := ParseType(field.Type)
		result += "|" + tag + "|" + typ + "|"
		doc := strings.TrimSpace(field.Doc.Text())
		if doc != "" {
			for _, line := range strings.Split(doc, "\n") {
				if strings.HasPrefix(line, "#") {
					line = line[1:]
				}
				line = strings.TrimSpace(line)
				if line == "" {
					result += "\n|||"
				} else {
					result += line + " "
				}
			}
		}
		result += "\n"
	}
	return fmt.Sprintf(PageHeader, page.Name, page.Slug, topLevelDoc, result)
}

func GenerateStructDocs(f *ast.File, page Page) string {
	for _, decl := range f.Decls {
		if gen, ok := decl.(*ast.GenDecl); ok && gen.Tok == token.TYPE {
			for _, spec := range gen.Specs {
				if s, ok := spec.(*ast.TypeSpec); ok {
					switch s.Type.(type) {
					case *ast.StructType:
						if s.Name.String() == page.StructName {
							return StructTable(page, gen.Doc.Text(), s)
						}
					}
				}
			}
		}
	}
	return ""
}

func main() {
	os.Mkdir("docs/generated/", 0755)
	for _, page := range Pages {
		fset := token.NewFileSet()
		f, err := parser.ParseFile(fset, page.SrcFile, nil, parser.ParseComments)
		if err != nil {
			panic(err)
		}
		str := GenerateStructDocs(f, page)
		filename := "docs/generated/" + page.Slug + ".md"
		fmt.Println("Writing ", filename)
		ioutil.WriteFile(filename, []byte(str), 0644)
	}
}
<commit_msg>Vanity improvements to generated docs<commit_after>package main

import (
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"strings"
)

type Page struct {
	Name       string
	Slug       string
	SrcFile    string
	StructName string
}

var Pages = map[string]Page{
	"consumer":  Page{"Consumers", "providers-and-consumers", "consumer.go", "ConsumerConfig"},
	"depends":   Page{"Dependencies", "dependencies", "dependency_config.go", "DependencyConfig"},
	"downloads": Page{"Downloads", "downloads", "download_config.go", "DownloadConfig"},
	"errands":   Page{"Errands", "errands", "errand.go", "Errand"},
	"templates": Page{"Templates", "templates", "templates/templates.go", "Template"},
	"variables": Page{"Input and Output Variables", "input-and-output-variables", "variables/variable.go", "Variable"},
}

const PageHeader = `---
date: 2017-11-11 00:00:00
title: "%s"
slug: %s
type: "docs"
toc: true
---

%s

Field | Type | Description
------|------|-------------
%s
`

func GetJsonFieldFromTag(tag string) string {
	for _, s := range strings.Split(tag, " ") {
		s = strings.Trim(s, "`")
		if strings.HasPrefix(s, "json:\"") {
			s = s[6 : len(s)-1]
			return s
		}
	}
	return ""
}

func ParseType(expr ast.Expr) string {
	switch t := expr.(type) {
	case *ast.Ident:
		return t.Name
	case *ast.SelectorExpr:
		return ParseType(t.X) + "." + t.Sel.String() // probably wrong
	case *ast.ArrayType:
		return "[" + ParseType(t.Elt) + "]"
	case *ast.StarExpr:
		return ParseType(t.X)
	case *ast.MapType:
		return "{" + ParseType(t.Key) + ":" + ParseType(t.Value) + "}"
	case *ast.InterfaceType:
		return "any"
	default:
		fmt.Printf("%T\n", t)
		panic("type not supported in documentation: ")
	}
	return ""
}

func StructTable(page Page, topLevelDoc string, s *ast.TypeSpec) string {
	structType := s.Type.(*ast.StructType)
	result := ""
	for _, field := range structType.Fields.List {
		tag := GetJsonFieldFromTag(field.Tag.Value)
		typ := ParseType(field.Type)
		result += "|" + tag + "|`" + typ + "`|"
		doc := strings.TrimSpace(field.Doc.Text())
		if doc != "" {
			for _, line := range strings.Split(doc, "\n") {
				if strings.HasPrefix(line, "#") {
					line = line[1:]
				}
				line = strings.TrimSpace(line)
				if line == "" {
					result += "\n|||"
				} else {
					result += line + " "
				}
			}
		}
		result += "\n"
	}
	return fmt.Sprintf(PageHeader, page.Name, page.Slug, topLevelDoc, result)
}

func GenerateStructDocs(f *ast.File, page Page) string {
	for _, decl := range f.Decls {
		if gen, ok := decl.(*ast.GenDecl); ok && gen.Tok == token.TYPE {
			for _, spec := range gen.Specs {
				if s, ok := spec.(*ast.TypeSpec); ok {
					switch s.Type.(type) {
					case *ast.StructType:
						if s.Name.String() == page.StructName {
							return StructTable(page, gen.Doc.Text(), s)
						}
					}
				}
			}
		}
	}
	return ""
}

func main() {
	os.Mkdir("docs/generated/", 0755)
	for _, page := range Pages {
		fset := token.NewFileSet()
		f, err := parser.ParseFile(fset, page.SrcFile, nil, parser.ParseComments)
		if err != nil {
			panic(err)
		}
		str := GenerateStructDocs(f, page)
		filename := "docs/generated/" + page.Slug + ".md"
		fmt.Println("Writing ", filename)
		ioutil.WriteFile(filename, []byte(str), 0644)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"

	"github.com/knative/pkg/apis"
	"github.com/knative/pkg/apis/duck"
	duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1"
	"github.com/knative/pkg/kmeta"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// Revision is an immutable snapshot of code and configuration.  A revision
// references a container image, and optionally a build that is responsible for
// materializing that container image from source. Revisions are created by
// updates to a Configuration.
//
// See also: https://github.com/knative/serving/blob/master/docs/spec/overview.md#revision
type Revision struct {
	metav1.TypeMeta `json:",inline"`
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`

	// Spec holds the desired state of the Revision (from the client).
	// +optional
	Spec RevisionSpec `json:"spec,omitempty"`

	// Status communicates the observed state of the Revision (from the controller).
	// +optional
	Status RevisionStatus `json:"status,omitempty"`
}

// Check that Revision can be validated, can be defaulted, and has immutable fields.
var _ apis.Validatable = (*Revision)(nil)
var _ apis.Defaultable = (*Revision)(nil)
var _ apis.Immutable = (*Revision)(nil)

// Check that RevisionStatus may have its conditions managed.
var _ duckv1alpha1.ConditionsAccessor = (*RevisionStatus)(nil)

// Check that Revision implements the Conditions duck type.
var _ = duck.VerifyType(&Revision{}, &duckv1alpha1.Conditions{})

// Check that Revision implements the Generation duck type.
var emptyGenRev duckv1alpha1.Generation
var _ = duck.VerifyType(&Revision{}, &emptyGenRev)

// Check that we can create OwnerReferences to a Revision.
var _ kmeta.OwnerRefable = (*Revision)(nil)

// RevisionTemplateSpec describes the data a revision should have when created from a template.
// Based on: https://github.com/kubernetes/api/blob/e771f807/core/v1/types.go#L3179-L3190
type RevisionTemplateSpec struct {
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`
	// +optional
	Spec RevisionSpec `json:"spec,omitempty"`
}

// RevisionServingStateType is an enumeration of the levels of serving readiness of the Revision.
// See also: https://github.com/knative/serving/blob/master/docs/spec/errors.md#error-conditions-and-reporting
type RevisionServingStateType string

const (
	// The revision is ready to serve traffic. It should have Kubernetes
	// resources, and the Istio route should be pointed to the given resources.
	RevisionServingStateActive RevisionServingStateType = "Active"
	// The revision is not currently serving traffic, but could be made to serve
	// traffic quickly. It should have Kubernetes resources, but the Istio route
	// should be pointed to the activator.
	RevisionServingStateReserve RevisionServingStateType = "Reserve"
	// The revision has been decommissioned and is not needed to serve traffic
	// anymore. It should not have any Istio routes or Kubernetes resources.
	// A Revision may be brought out of retirement, but it may take longer than
	// it would from a "Reserve" state.
	// Note: currently not set anywhere. See https://github.com/knative/serving/issues/1203
	RevisionServingStateRetired RevisionServingStateType = "Retired"
)

// RevisionRequestConcurrencyModelType is an enumeration of the
// concurrency models supported by a Revision.
// Deprecated in favor of RevisionContainerConcurrencyType.
type RevisionRequestConcurrencyModelType string

const (
	// RevisionRequestConcurrencyModelSingle guarantees that only one
	// request will be handled at a time (concurrently) per instance
	// of Revision Container.
	RevisionRequestConcurrencyModelSingle RevisionRequestConcurrencyModelType = "Single"
	// RevisionRequestConcurencyModelMulti allows more than one request to
	// be handled at a time (concurrently) per instance of Revision
	// Container.
	RevisionRequestConcurrencyModelMulti RevisionRequestConcurrencyModelType = "Multi"
)

// RevisionContainerConcurrencyType is an integer expressing a number of
// in-flight (concurrent) requests.
type RevisionContainerConcurrencyType int64

const (
	// The maximum configurable container concurrency.
	RevisionContainerConcurrencyMax RevisionContainerConcurrencyType = 1000
)

// RevisionSpec holds the desired state of the Revision (from the client).
type RevisionSpec struct {
	// TODO: Generation does not work correctly with CRD. They are scrubbed
	// by the APIserver (https://github.com/kubernetes/kubernetes/issues/58778)
	// So, we add Generation here. Once that gets fixed, remove this and use
	// ObjectMeta.Generation instead.
	// +optional
	Generation int64 `json:"generation,omitempty"`

	// ServingState holds a value describing the desired state the Kubernetes
	// resources should be in for this Revision.
	// Users must not specify this when creating a revision. It is expected
	// that the system will manipulate this based on routability and load.
	// +optional
	ServingState RevisionServingStateType `json:"servingState,omitempty"`

	// ConcurrencyModel specifies the desired concurrency model
	// (Single or Multi) for the
	// Revision. Defaults to Multi.
	// Deprecated in favor of ContainerConcurrency.
	// +optional
	ConcurrencyModel RevisionRequestConcurrencyModelType `json:"concurrencyModel,omitempty"`

	// ContainerConcurrency specifies the maximum allowed
	// in-flight (concurrent) requests per container of the Revision.
	// Defaults to `0` which means unlimited concurrency.
	// This field replaces ConcurrencyModel. A value of `1`
	// is equivalent to `Single` and `0` is equivalent to `Multi`.
	// +optional
	ContainerConcurrency RevisionContainerConcurrencyType `json:"containerConcurrency,omitempty"`

	// ServiceAccountName holds the name of the Kubernetes service account
	// as which the underlying K8s resources should be run. If unspecified
	// this will default to the "default" service account for the namespace
	// in which the Revision exists.
	// This may be used to provide access to private container images by
	// following: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#add-imagepullsecrets-to-a-service-account
	// TODO(ZhiminXiang): verify the corresponding service account exists.
	// +optional
	ServiceAccountName string `json:"serviceAccountName,omitempty"`

	// BuildName optionally holds the name of the Build responsible for
	// producing the container image for its Revision.
	// DEPRECATED: Use BuildRef instead.
	// +optional
	BuildName string `json:"buildName,omitempty"`

	// BuildRef holds the reference to the build (if there is one) responsible
	// for producing the container image for this Revision. Otherwise, nil
	// +optional
	BuildRef *corev1.ObjectReference

	// Container defines the unit of execution for this Revision.
	// In the context of a Revision, we disallow a number of the fields of
	// this Container, including: name, resources, ports, and volumeMounts.
	// TODO(mattmoor): Link to the runtime contract tracked by:
	// https://github.com/knative/serving/issues/627
	// +optional
	Container corev1.Container `json:"container,omitempty"`
}

const (
	// RevisionConditionReady is set when the revision is starting to materialize
	// runtime resources, and becomes true when those resources are ready.
	RevisionConditionReady = duckv1alpha1.ConditionReady
	// RevisionConditionBuildSucceeded is set when the revision has an associated build
	// and is marked True if/once the Build has completed successfully.
	RevisionConditionBuildSucceeded duckv1alpha1.ConditionType = "BuildSucceeded"
	// RevisionConditionResourcesAvailable is set when underlying
	// Kubernetes resources have been provisioned.
	RevisionConditionResourcesAvailable duckv1alpha1.ConditionType = "ResourcesAvailable"
	// RevisionConditionContainerHealthy is set when the revision readiness check completes.
	RevisionConditionContainerHealthy duckv1alpha1.ConditionType = "ContainerHealthy"
	// RevisionConditionActive is set when the revision is receiving traffic.
	RevisionConditionActive duckv1alpha1.ConditionType = "Active"
)

var revCondSet = duckv1alpha1.NewLivingConditionSet(
	RevisionConditionResourcesAvailable,
	RevisionConditionContainerHealthy,
	RevisionConditionActive,
)

var buildCondSet = duckv1alpha1.NewBatchConditionSet()

// RevisionStatus communicates the observed state of the Revision (from the controller).
type RevisionStatus struct {
	// ServiceName holds the name of a core Kubernetes Service resource that
	// load balances over the pods backing this Revision. When the Revision
	// is Active, this service would be an appropriate ingress target for
	// targeting the revision.
	// +optional
	ServiceName string `json:"serviceName,omitempty"`

	// Conditions communicates information about ongoing/complete
	// reconciliation processes that bring the "spec" inline with the observed
	// state of the world.
	// +optional
	Conditions duckv1alpha1.Conditions `json:"conditions,omitempty"`

	// ObservedGeneration is the 'Generation' of the Configuration that
	// was last processed by the controller. The observed generation is updated
	// even if the controller failed to process the spec and create the Revision.
	// +optional
	ObservedGeneration int64 `json:"observedGeneration,omitempty"`

	// LogURL specifies the generated logging url for this particular revision
	// based on the revision url template specified in the controller's config.
	// +optional
	LogURL string `json:"logUrl,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// RevisionList is a list of Revision resources
type RevisionList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`

	Items []Revision `json:"items"`
}

func (r *Revision) GetGroupVersionKind() schema.GroupVersionKind {
	return SchemeGroupVersion.WithKind("Revision")
}

func (r *Revision) BuildRef() *corev1.ObjectReference {
	if r.Spec.BuildRef != nil {
		return r.Spec.BuildRef
	}

	if r.Spec.BuildName != "" {
		return &corev1.ObjectReference{
			APIVersion: "build.knative.dev/v1alpha1",
			Kind:       "Build",
			Namespace:  r.Namespace,
			Name:       r.Spec.BuildName,
		}
	}

	return nil
}

// IsReady looks at the conditions and if the Status has a condition
// RevisionConditionReady returns true if ConditionStatus is True
func (rs *RevisionStatus) IsReady() bool {
	return revCondSet.Manage(rs).IsHappy()
}

func (rs *RevisionStatus) IsActivationRequired() bool {
	if c := revCondSet.Manage(rs).GetCondition(RevisionConditionActive); c != nil {
		return c.Status != corev1.ConditionTrue
	}
	return false
}

func (rs *RevisionStatus) IsRoutable() bool {
	return rs.IsReady() || rs.IsActivationRequired()
}

func (rs *RevisionStatus) GetCondition(t duckv1alpha1.ConditionType) *duckv1alpha1.Condition {
	return revCondSet.Manage(rs).GetCondition(t)
}

func (rs *RevisionStatus) InitializeConditions() {
	revCondSet.Manage(rs).InitializeConditions()

	// We don't include BuildSucceeded here because it could confuse users if
	// no `buildName` was specified.
}

func (rs *RevisionStatus) InitializeBuildCondition() {
	revCondSet.Manage(rs).InitializeCondition(RevisionConditionBuildSucceeded)
}

func (rs *RevisionStatus) PropagateBuildStatus(bs duckv1alpha1.KResourceStatus) {
	bc := buildCondSet.Manage(&bs).GetCondition(duckv1alpha1.ConditionSucceeded)
	if bc == nil {
		return
	}
	switch {
	case bc.Status == corev1.ConditionUnknown:
		revCondSet.Manage(rs).MarkUnknown(RevisionConditionBuildSucceeded, "Building", bc.Message)
	case bc.Status == corev1.ConditionTrue:
		revCondSet.Manage(rs).MarkTrue(RevisionConditionBuildSucceeded)
	case bc.Status == corev1.ConditionFalse:
		revCondSet.Manage(rs).MarkFalse(RevisionConditionBuildSucceeded, bc.Reason, bc.Message)
	}
}

func (rs *RevisionStatus) MarkDeploying(reason string) {
	revCondSet.Manage(rs).MarkUnknown(RevisionConditionResourcesAvailable, reason, "")
	revCondSet.Manage(rs).MarkUnknown(RevisionConditionContainerHealthy, reason, "")
}

func (rs *RevisionStatus) MarkServiceTimeout() {
	revCondSet.Manage(rs).MarkFalse(RevisionConditionResourcesAvailable, "ServiceTimeout",
		"Timed out waiting for a service endpoint to become ready")
}

func (rs *RevisionStatus) MarkProgressDeadlineExceeded(message string) {
	revCondSet.Manage(rs).MarkFalse(RevisionConditionResourcesAvailable, "ProgressDeadlineExceeded", message)
}

func (rs *RevisionStatus) MarkContainerHealthy() {
	revCondSet.Manage(rs).MarkTrue(RevisionConditionContainerHealthy)
}

func (rs *RevisionStatus) MarkResourcesAvailable() {
	revCondSet.Manage(rs).MarkTrue(RevisionConditionResourcesAvailable)
}

func (rs *RevisionStatus) MarkActive() {
	revCondSet.Manage(rs).MarkTrue(RevisionConditionActive)
}

func (rs *RevisionStatus) MarkActivating(reason, message string) {
	revCondSet.Manage(rs).MarkUnknown(RevisionConditionActive, reason, message)
}

func (rs *RevisionStatus) MarkInactive(reason, message string) {
	revCondSet.Manage(rs).MarkFalse(RevisionConditionActive, reason, message)
}

func (rs *RevisionStatus) MarkContainerMissing(message string) {
	revCondSet.Manage(rs).MarkFalse(RevisionConditionContainerHealthy, "ContainerMissing", message)
}

// GetConditions returns the Conditions array. This enables generic handling of
// conditions by implementing the duckv1alpha1.Conditions interface.
func (rs *RevisionStatus) GetConditions() duckv1alpha1.Conditions {
	return rs.Conditions
}

// SetConditions sets the Conditions array. This enables generic handling of
// conditions by implementing the duckv1alpha1.Conditions interface.
func (rs *RevisionStatus) SetConditions(conditions duckv1alpha1.Conditions) {
	rs.Conditions = conditions
}
<commit_msg>BuildRef was missing a JSON annotation. (#2157)<commit_after>/*
Copyright 2018 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"

	"github.com/knative/pkg/apis"
	"github.com/knative/pkg/apis/duck"
	duckv1alpha1 "github.com/knative/pkg/apis/duck/v1alpha1"
	"github.com/knative/pkg/kmeta"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// Revision is an immutable snapshot of code and configuration.  A revision
// references a container image, and optionally a build that is responsible for
// materializing that container image from source. Revisions are created by
// updates to a Configuration.
//
// See also: https://github.com/knative/serving/blob/master/docs/spec/overview.md#revision
type Revision struct {
	metav1.TypeMeta `json:",inline"`
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`

	// Spec holds the desired state of the Revision (from the client).
	// +optional
	Spec RevisionSpec `json:"spec,omitempty"`

	// Status communicates the observed state of the Revision (from the controller).
	// +optional
	Status RevisionStatus `json:"status,omitempty"`
}

// Check that Revision can be validated, can be defaulted, and has immutable fields.
var _ apis.Validatable = (*Revision)(nil)
var _ apis.Defaultable = (*Revision)(nil)
var _ apis.Immutable = (*Revision)(nil)

// Check that RevisionStatus may have its conditions managed.
var _ duckv1alpha1.ConditionsAccessor = (*RevisionStatus)(nil)

// Check that Revision implements the Conditions duck type.
var _ = duck.VerifyType(&Revision{}, &duckv1alpha1.Conditions{})

// Check that Revision implements the Generation duck type.
var emptyGenRev duckv1alpha1.Generation
var _ = duck.VerifyType(&Revision{}, &emptyGenRev)

// Check that we can create OwnerReferences to a Revision.
var _ kmeta.OwnerRefable = (*Revision)(nil)

// RevisionTemplateSpec describes the data a revision should have when created from a template.
// Based on: https://github.com/kubernetes/api/blob/e771f807/core/v1/types.go#L3179-L3190
type RevisionTemplateSpec struct {
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty"`
	// +optional
	Spec RevisionSpec `json:"spec,omitempty"`
}

// RevisionServingStateType is an enumeration of the levels of serving readiness of the Revision.
// See also: https://github.com/knative/serving/blob/master/docs/spec/errors.md#error-conditions-and-reporting
type RevisionServingStateType string

const (
	// The revision is ready to serve traffic. It should have Kubernetes
	// resources, and the Istio route should be pointed to the given resources.
	RevisionServingStateActive RevisionServingStateType = "Active"
	// The revision is not currently serving traffic, but could be made to serve
	// traffic quickly. It should have Kubernetes resources, but the Istio route
	// should be pointed to the activator.
	RevisionServingStateReserve RevisionServingStateType = "Reserve"
	// The revision has been decommissioned and is not needed to serve traffic
	// anymore. It should not have any Istio routes or Kubernetes resources.
	// A Revision may be brought out of retirement, but it may take longer than
	// it would from a "Reserve" state.
	// Note: currently not set anywhere. See https://github.com/knative/serving/issues/1203
	RevisionServingStateRetired RevisionServingStateType = "Retired"
)

// RevisionRequestConcurrencyModelType is an enumeration of the
// concurrency models supported by a Revision.
// Deprecated in favor of RevisionContainerConcurrencyType.
type RevisionRequestConcurrencyModelType string

const (
	// RevisionRequestConcurrencyModelSingle guarantees that only one
	// request will be handled at a time (concurrently) per instance
	// of Revision Container.
	RevisionRequestConcurrencyModelSingle RevisionRequestConcurrencyModelType = "Single"
	// RevisionRequestConcurencyModelMulti allows more than one request to
	// be handled at a time (concurrently) per instance of Revision
	// Container.
	RevisionRequestConcurrencyModelMulti RevisionRequestConcurrencyModelType = "Multi"
)

// RevisionContainerConcurrencyType is an integer expressing a number of
// in-flight (concurrent) requests.
type RevisionContainerConcurrencyType int64

const (
	// The maximum configurable container concurrency.
	RevisionContainerConcurrencyMax RevisionContainerConcurrencyType = 1000
)

// RevisionSpec holds the desired state of the Revision (from the client).
type RevisionSpec struct {
	// TODO: Generation does not work correctly with CRD. They are scrubbed
	// by the APIserver (https://github.com/kubernetes/kubernetes/issues/58778)
	// So, we add Generation here. Once that gets fixed, remove this and use
	// ObjectMeta.Generation instead.
	// +optional
	Generation int64 `json:"generation,omitempty"`

	// ServingState holds a value describing the desired state the Kubernetes
	// resources should be in for this Revision.
	// Users must not specify this when creating a revision. It is expected
	// that the system will manipulate this based on routability and load.
	// +optional
	ServingState RevisionServingStateType `json:"servingState,omitempty"`

	// ConcurrencyModel specifies the desired concurrency model
	// (Single or Multi) for the
	// Revision. Defaults to Multi.
	// Deprecated in favor of ContainerConcurrency.
	// +optional
	ConcurrencyModel RevisionRequestConcurrencyModelType `json:"concurrencyModel,omitempty"`

	// ContainerConcurrency specifies the maximum allowed
	// in-flight (concurrent) requests per container of the Revision.
	// Defaults to `0` which means unlimited concurrency.
	// This field replaces ConcurrencyModel. A value of `1`
	// is equivalent to `Single` and `0` is equivalent to `Multi`.
	// +optional
	ContainerConcurrency RevisionContainerConcurrencyType `json:"containerConcurrency,omitempty"`

	// ServiceAccountName holds the name of the Kubernetes service account
	// as which the underlying K8s resources should be run. If unspecified
	// this will default to the "default" service account for the namespace
	// in which the Revision exists.
	// This may be used to provide access to private container images by
	// following: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#add-imagepullsecrets-to-a-service-account
	// TODO(ZhiminXiang): verify the corresponding service account exists.
	// +optional
	ServiceAccountName string `json:"serviceAccountName,omitempty"`

	// BuildName optionally holds the name of the Build responsible for
	// producing the container image for its Revision.
	// DEPRECATED: Use BuildRef instead.
	// +optional
	BuildName string `json:"buildName,omitempty"`

	// BuildRef holds the reference to the build (if there is one) responsible
	// for producing the container image for this Revision. Otherwise, nil
	// +optional
	BuildRef *corev1.ObjectReference `json:"buildRef,omitempty"`

	// Container defines the unit of execution for this Revision.
	// In the context of a Revision, we disallow a number of the fields of
	// this Container, including: name, resources, ports, and volumeMounts.
	// TODO(mattmoor): Link to the runtime contract tracked by:
	// https://github.com/knative/serving/issues/627
	// +optional
	Container corev1.Container `json:"container,omitempty"`
}

const (
	// RevisionConditionReady is set when the revision is starting to materialize
	// runtime resources, and becomes true when those resources are ready.
	RevisionConditionReady = duckv1alpha1.ConditionReady
	// RevisionConditionBuildSucceeded is set when the revision has an associated build
	// and is marked True if/once the Build has completed successfully.
	RevisionConditionBuildSucceeded duckv1alpha1.ConditionType = "BuildSucceeded"
	// RevisionConditionResourcesAvailable is set when underlying
	// Kubernetes resources have been provisioned.
	RevisionConditionResourcesAvailable duckv1alpha1.ConditionType = "ResourcesAvailable"
	// RevisionConditionContainerHealthy is set when the revision readiness check completes.
	RevisionConditionContainerHealthy duckv1alpha1.ConditionType = "ContainerHealthy"
	// RevisionConditionActive is set when the revision is receiving traffic.
	RevisionConditionActive duckv1alpha1.ConditionType = "Active"
)

var revCondSet = duckv1alpha1.NewLivingConditionSet(
	RevisionConditionResourcesAvailable,
	RevisionConditionContainerHealthy,
	RevisionConditionActive,
)

var buildCondSet = duckv1alpha1.NewBatchConditionSet()

// RevisionStatus communicates the observed state of the Revision (from the controller).
type RevisionStatus struct {
	// ServiceName holds the name of a core Kubernetes Service resource that
	// load balances over the pods backing this Revision. When the Revision
	// is Active, this service would be an appropriate ingress target for
	// targeting the revision.
	// +optional
	ServiceName string `json:"serviceName,omitempty"`

	// Conditions communicates information about ongoing/complete
	// reconciliation processes that bring the "spec" inline with the observed
	// state of the world.
	// +optional
	Conditions duckv1alpha1.Conditions `json:"conditions,omitempty"`

	// ObservedGeneration is the 'Generation' of the Configuration that
	// was last processed by the controller. The observed generation is updated
	// even if the controller failed to process the spec and create the Revision.
	// +optional
	ObservedGeneration int64 `json:"observedGeneration,omitempty"`

	// LogURL specifies the generated logging url for this particular revision
	// based on the revision url template specified in the controller's config.
	// +optional
	LogURL string `json:"logUrl,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// RevisionList is a list of Revision resources
type RevisionList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`

	Items []Revision `json:"items"`
}

func (r *Revision) GetGroupVersionKind() schema.GroupVersionKind {
	return SchemeGroupVersion.WithKind("Revision")
}

func (r *Revision) BuildRef() *corev1.ObjectReference {
	if r.Spec.BuildRef != nil {
		return r.Spec.BuildRef
	}

	if r.Spec.BuildName != "" {
		return &corev1.ObjectReference{
			APIVersion: "build.knative.dev/v1alpha1",
			Kind:       "Build",
			Namespace:  r.Namespace,
			Name:       r.Spec.BuildName,
		}
	}

	return nil
}

// IsReady looks at the conditions and if the Status has a condition
// RevisionConditionReady returns true if ConditionStatus is True
func (rs *RevisionStatus) IsReady() bool {
	return revCondSet.Manage(rs).IsHappy()
}

func (rs *RevisionStatus) IsActivationRequired() bool {
	if c := revCondSet.Manage(rs).GetCondition(RevisionConditionActive); c != nil {
		return c.Status != corev1.ConditionTrue
	}
	return false
}

func (rs *RevisionStatus) IsRoutable() bool {
	return rs.IsReady() || rs.IsActivationRequired()
}

func (rs *RevisionStatus) GetCondition(t duckv1alpha1.ConditionType) *duckv1alpha1.Condition {
	return revCondSet.Manage(rs).GetCondition(t)
}

func (rs *RevisionStatus) InitializeConditions() {
	revCondSet.Manage(rs).InitializeConditions()

	// We don't include BuildSucceeded here because it could confuse users if
	// no `buildName` was specified.
}

func (rs *RevisionStatus) InitializeBuildCondition() {
	revCondSet.Manage(rs).InitializeCondition(RevisionConditionBuildSucceeded)
}

func (rs *RevisionStatus) PropagateBuildStatus(bs duckv1alpha1.KResourceStatus) {
	bc := buildCondSet.Manage(&bs).GetCondition(duckv1alpha1.ConditionSucceeded)
	if bc == nil {
		return
	}
	switch {
	case bc.Status == corev1.ConditionUnknown:
		revCondSet.Manage(rs).MarkUnknown(RevisionConditionBuildSucceeded, "Building", bc.Message)
	case bc.Status == corev1.ConditionTrue:
		revCondSet.Manage(rs).MarkTrue(RevisionConditionBuildSucceeded)
	case bc.Status == corev1.ConditionFalse:
		revCondSet.Manage(rs).MarkFalse(RevisionConditionBuildSucceeded, bc.Reason, bc.Message)
	}
}

func (rs *RevisionStatus) MarkDeploying(reason string) {
	revCondSet.Manage(rs).MarkUnknown(RevisionConditionResourcesAvailable, reason, "")
	revCondSet.Manage(rs).MarkUnknown(RevisionConditionContainerHealthy, reason, "")
}

func (rs *RevisionStatus) MarkServiceTimeout() {
	revCondSet.Manage(rs).MarkFalse(RevisionConditionResourcesAvailable, "ServiceTimeout",
		"Timed out waiting for a service endpoint to become ready")
}

func (rs *RevisionStatus) MarkProgressDeadlineExceeded(message string) {
	revCondSet.Manage(rs).MarkFalse(RevisionConditionResourcesAvailable, "ProgressDeadlineExceeded", message)
}

func (rs *RevisionStatus) MarkContainerHealthy() {
	revCondSet.Manage(rs).MarkTrue(RevisionConditionContainerHealthy)
}

func (rs *RevisionStatus) MarkResourcesAvailable() {
	revCondSet.Manage(rs).MarkTrue(RevisionConditionResourcesAvailable)
}

func (rs *RevisionStatus) MarkActive() {
	revCondSet.Manage(rs).MarkTrue(RevisionConditionActive)
}

func (rs *RevisionStatus) MarkActivating(reason, message string) {
	revCondSet.Manage(rs).MarkUnknown(RevisionConditionActive, reason, message)
}

func (rs *RevisionStatus) MarkInactive(reason, message string) {
	revCondSet.Manage(rs).MarkFalse(RevisionConditionActive, reason, message)
}

func (rs *RevisionStatus) MarkContainerMissing(message string) {
	revCondSet.Manage(rs).MarkFalse(RevisionConditionContainerHealthy, "ContainerMissing", message)
}

// GetConditions returns the Conditions array. This enables generic handling of
// conditions by implementing the duckv1alpha1.Conditions interface.
func (rs *RevisionStatus) GetConditions() duckv1alpha1.Conditions {
	return rs.Conditions
}

// SetConditions sets the Conditions array. This enables generic handling of
// conditions by implementing the duckv1alpha1.Conditions interface.
func (rs *RevisionStatus) SetConditions(conditions duckv1alpha1.Conditions) {
	rs.Conditions = conditions
}
<|endoftext|>