<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

/*

Package file provides a file-based implementation of the catalog
package.

*/
package file

import (
	"github.com/couchbaselabs/tuqtng/catalog"
	"github.com/couchbaselabs/tuqtng/query"
	"os"
	"path/filepath"
	"strings"
)

// site is the root for the file-based Site.
type site struct {
	path      string
	pools     map[string]*pool
	poolNames []string
}

func (s *site) URL() string {
	return "file://" + s.path
}

func (s *site) PoolNames() ([]string, query.Error) {
	return s.poolNames, nil
}

func (s *site) Pool(name string) (p catalog.Pool, e query.Error) {
	p, ok := s.pools[strings.ToUpper(name)]
	if !ok {
		e = query.NewError(nil, "Pool "+name+" not found.")
	}

	return
}

// NewSite creates a new file-based site for the given filepath.
func NewSite(path string) (s catalog.Site, e query.Error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return nil, query.NewError(err, "")
	}

	fs := &site{path: path}

	e = fs.loadPools()
	if e != nil {
		return
	}

	s = fs
	return
}

func (s *site) loadPools() (e query.Error) {
	f, err := os.Open(s.path)
	if err != nil {
		if f != nil {
			f.Close()
		}

		return query.NewError(err, "")
	}

	dirs, err := f.Readdirnames(0)
	f.Close()

	if err != nil {
		return query.NewError(err, "")
	}

	s.pools = make(map[string]*pool, len(dirs))
	s.poolNames = dirs

	var p *pool
	for _, dir := range dirs {
		diru := strings.ToUpper(dir)
		if _, ok := s.pools[diru]; ok {
			return query.NewError(nil, "Duplicate pool name "+dir)
		}

		p, e = newPool(s, dir)
		if e != nil {
			return
		}

		s.pools[diru] = p
	}

	return
}

// pool represents a file-based Pool.
type pool struct {
	site        *site
	name        string
	buckets     map[string]*bucket
	bucketNames []string
}

func (p *pool) Name() string {
	return p.name
}

func (p *pool) BucketNames() ([]string, query.Error) {
	return p.bucketNames, nil
}

func (p *pool) Bucket(name string) (b catalog.Bucket, e query.Error) {
	b, ok := p.buckets[strings.ToUpper(name)]
	if !ok {
		e = query.NewError(nil, "Bucket "+name+" not found.")
	}

	return
}

func (p *pool) path() string {
	return filepath.Join(p.site.path, p.name)
}

// newPool creates a new pool.
func newPool(s *site, dir string) (p *pool, e query.Error) {
	p = new(pool)
	p.site = s
	p.name = dir

	e = p.loadBuckets()
	return
}

func (p *pool) loadBuckets() (e query.Error) {
	f, err := os.Open(p.path())
	if err != nil {
		if f != nil {
			f.Close()
		}

		return query.NewError(err, "")
	}

	dirs, err := f.Readdirnames(0)
	f.Close()

	if err != nil {
		return query.NewError(err, "")
	}

	p.buckets = make(map[string]*bucket, len(dirs))
	p.bucketNames = dirs

	var b *bucket
	for _, dir := range dirs {
		diru := strings.ToUpper(dir)
		if _, ok := p.buckets[diru]; ok {
			return query.NewError(nil, "Duplicate bucket name "+dir)
		}

		b, e = newBucket(p, dir)
		if e != nil {
			return
		}

		p.buckets[diru] = b
	}

	return
}

// bucket is a file-based bucket.
type bucket struct {
	pool      *pool
	name      string
	filenames []string
	scanners  []catalog.Scanner
}

func (b *bucket) Name() string {
	return b.name
}

func (b *bucket) Count() (int64, query.Error) {
	return int64(len(b.filenames)), nil
}

func (b *bucket) Scanners() ([]catalog.Scanner, query.Error) {
	return b.scanners, nil
}

func (b *bucket) Fetch(id string) (item query.Item, e query.Error) {
	path := filepath.Join(b.path(), id)
	item, e = fetch(path)
	if e != nil {
		item = nil
	}

	return
}

func (b *bucket) path() string {
	return filepath.Join(b.pool.path(), b.name)
}

// newBucket creates a new bucket.
func newBucket(p *pool, dir string) (b *bucket, e query.Error) {
	b = new(bucket)
	b.pool = p
	b.name = dir

	f, err := os.Open(b.path())
	if err != nil {
		if f != nil {
			f.Close()
		}

		return nil, query.NewError(err, "")
	}

	b.filenames, err = f.Readdirnames(0)
	f.Close()
	if err != nil {
		return nil, query.NewError(err, "")
	}

	b.scanners = make([]catalog.Scanner, 1)
	fs := new(fullScanner)
	fs.bucket = b
	b.scanners = append(b.scanners, fs)

	return
}

// fullScanner performs full bucket scans.
type fullScanner struct {
	bucket *bucket
}

func (fs *fullScanner) ScanAll(ch query.ItemChannel, errch query.ErrorChannel) {
	go fs.scanAll(ch, errch)
}

func (fs *fullScanner) scanAll(ch query.ItemChannel, errch query.ErrorChannel) {
	bpath := fs.bucket.path()
	for _, filename := range fs.bucket.filenames {
		item, err := fetch(filepath.Join(bpath, filename))
		if err != nil {
			errch <- err
			break
		}

		ch <- item
	}

	close(ch)
	close(errch)
}

func fetch(path string) (item query.Item, e query.Error) {
	fi, err := os.Stat(path)
	if err != nil {
		return nil, query.NewError(err, "")
	}

	f, err := os.Open(path)
	if err != nil {
		if f != nil {
			f.Close()
		}

		return nil, query.NewError(err, "")
	}

	bytes := make([]byte, fi.Size())
	_, err = f.Read(bytes)
	f.Close()
	if err != nil {
		return nil, query.NewError(err, "")
	}

	return
}
<commit_msg>fixed crash on non-directory entries at pools/buckets level<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

/*

Package file provides a file-based implementation of the catalog
package.

*/
package file

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/couchbaselabs/tuqtng/catalog"
	"github.com/couchbaselabs/tuqtng/query"
)

// site is the root for the file-based Site.
type site struct {
	path      string
	pools     map[string]*pool
	poolNames []string
}

func (s *site) URL() string {
	return "file://" + s.path
}

func (s *site) PoolNames() ([]string, query.Error) {
	return s.poolNames, nil
}

func (s *site) Pool(name string) (p catalog.Pool, e query.Error) {
	p, ok := s.pools[strings.ToUpper(name)]
	if !ok {
		e = query.NewError(nil, "Pool "+name+" not found.")
	}

	return
}

// NewSite creates a new file-based site for the given filepath.
func NewSite(path string) (s catalog.Site, e query.Error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return nil, query.NewError(err, "")
	}

	fs := &site{path: path}

	e = fs.loadPools()
	if e != nil {
		return
	}

	s = fs
	return
}

func (s *site) loadPools() (e query.Error) {
	dirEntries, err := ioutil.ReadDir(s.path)
	if err != nil {
		return query.NewError(err, "")
	}

	s.pools = make(map[string]*pool)
	s.poolNames = make([]string, 0)

	var p *pool
	for _, dirEntry := range dirEntries {
		if dirEntry.IsDir() {
			s.poolNames = append(s.poolNames, dirEntry.Name())
			diru := strings.ToUpper(dirEntry.Name())
			if _, ok := s.pools[diru]; ok {
				return query.NewError(nil, "Duplicate pool name "+dirEntry.Name())
			}

			p, e = newPool(s, dirEntry.Name())
			if e != nil {
				return
			}

			s.pools[diru] = p
		}
	}

	return
}

// pool represents a file-based Pool.
type pool struct {
	site        *site
	name        string
	buckets     map[string]*bucket
	bucketNames []string
}

func (p *pool) Name() string {
	return p.name
}

func (p *pool) BucketNames() ([]string, query.Error) {
	return p.bucketNames, nil
}

func (p *pool) Bucket(name string) (b catalog.Bucket, e query.Error) {
	b, ok := p.buckets[strings.ToUpper(name)]
	if !ok {
		e = query.NewError(nil, "Bucket "+name+" not found.")
	}

	return
}

func (p *pool) path() string {
	return filepath.Join(p.site.path, p.name)
}

// newPool creates a new pool.
func newPool(s *site, dir string) (p *pool, e query.Error) {
	p = new(pool)
	p.site = s
	p.name = dir

	e = p.loadBuckets()
	return
}

func (p *pool) loadBuckets() (e query.Error) {
	dirEntries, err := ioutil.ReadDir(p.path())
	if err != nil {
		return query.NewError(err, "")
	}

	p.buckets = make(map[string]*bucket)
	p.bucketNames = make([]string, 0)

	var b *bucket
	for _, dirEntry := range dirEntries {
		if dirEntry.IsDir() {
			diru := strings.ToUpper(dirEntry.Name())
			if _, ok := p.buckets[diru]; ok {
				return query.NewError(nil, "Duplicate bucket name "+dirEntry.Name())
			}

			b, e = newBucket(p, dirEntry.Name())
			if e != nil {
				return
			}

			p.buckets[diru] = b
		}
	}

	return
}

// bucket is a file-based bucket.
type bucket struct {
	pool      *pool
	name      string
	filenames []string
	scanners  []catalog.Scanner
}

func (b *bucket) Name() string {
	return b.name
}

func (b *bucket) Count() (int64, query.Error) {
	return int64(len(b.filenames)), nil
}

func (b *bucket) Scanners() ([]catalog.Scanner, query.Error) {
	return b.scanners, nil
}

func (b *bucket) Fetch(id string) (item query.Item, e query.Error) {
	path := filepath.Join(b.path(), id)
	item, e = fetch(path)
	if e != nil {
		item = nil
	}

	return
}

func (b *bucket) path() string {
	return filepath.Join(b.pool.path(), b.name)
}

// newBucket creates a new bucket.
func newBucket(p *pool, dir string) (b *bucket, e query.Error) {
	b = new(bucket)
	b.pool = p
	b.name = dir

	f, err := os.Open(b.path())
	if err != nil {
		if f != nil {
			f.Close()
		}

		return nil, query.NewError(err, "")
	}

	b.filenames, err = f.Readdirnames(0)
	f.Close()
	if err != nil {
		return nil, query.NewError(err, "")
	}

	b.scanners = make([]catalog.Scanner, 1)
	fs := new(fullScanner)
	fs.bucket = b
	b.scanners = append(b.scanners, fs)

	return
}

// fullScanner performs full bucket scans.
type fullScanner struct {
	bucket *bucket
}

func (fs *fullScanner) ScanAll(ch query.ItemChannel, errch query.ErrorChannel) {
	go fs.scanAll(ch, errch)
}

func (fs *fullScanner) scanAll(ch query.ItemChannel, errch query.ErrorChannel) {
	bpath := fs.bucket.path()
	for _, filename := range fs.bucket.filenames {
		item, err := fetch(filepath.Join(bpath, filename))
		if err != nil {
			errch <- err
			break
		}

		ch <- item
	}

	close(ch)
	close(errch)
}

func fetch(path string) (item query.Item, e query.Error) {
	fi, err := os.Stat(path)
	if err != nil {
		return nil, query.NewError(err, "")
	}

	f, err := os.Open(path)
	if err != nil {
		if f != nil {
			f.Close()
		}

		return nil, query.NewError(err, "")
	}

	bytes := make([]byte, fi.Size())
	_, err = f.Read(bytes)
	f.Close()
	if err != nil {
		return nil, query.NewError(err, "")
	}

	return
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resources

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/knative/build-pipeline/pkg/apis/pipeline/v1alpha1"
	listers "github.com/knative/build-pipeline/pkg/client/listers/pipeline/v1alpha1"
	buildv1alpha1 "github.com/knative/build/pkg/apis/build/v1alpha1"
	"go.uber.org/zap"
	corev1 "k8s.io/api/core/v1"
)

var (
	outputDir = "/workspace/output/"
)

// AddOutputResources reads the output resources and adds the corresponding container steps
// This function also reads the inputs to check if resources are redeclared in inputs and has any custom
// target directory.
// Steps executed:
//  1. If taskrun has owner reference as pipelinerun then all outputs are copied to parents PVC
// and also runs any custom upload steps (upload to blob store)
//  2.  If taskrun does not have pipelinerun as owner reference then all outputs resources execute their custom
// upload steps (like upload to blob store )
//
// Resource source path determined
// 1. If resource is declared in inputs then target path from input resource is used to identify source path
// 2. If resource is declared in outputs only then the default is /output/resource_name
func AddOutputResources(
	b *buildv1alpha1.Build,
	taskName string,
	taskSpec *v1alpha1.TaskSpec,
	taskRun *v1alpha1.TaskRun,
	pipelineResourceLister listers.PipelineResourceLister,
	logger *zap.SugaredLogger,
) error {

	if taskSpec == nil || taskSpec.Outputs == nil {
		return nil
	}

	pipelineRunpvcName := taskRun.GetPipelineRunPVCName()

	// track resources that are present in input of task cuz these resources will be copied onto PVC
	inputResourceMap := map[string]string{}

	if taskSpec.Inputs != nil {
		for _, input := range taskSpec.Inputs.Resources {
			var targetPath = filepath.Join(workspaceDir, input.Name)
			if input.TargetPath != "" {
				targetPath = filepath.Join(workspaceDir, input.TargetPath)
			}
			inputResourceMap[input.Name] = targetPath
		}
	}

	for _, output := range taskSpec.Outputs.Resources {
		boundResource, err := getBoundResource(output.Name, taskRun.Spec.Outputs.Resources)
		if err != nil {
			return fmt.Errorf("Failed to get bound resource: %s", err)
		}

		resource, err := pipelineResourceLister.PipelineResources(taskRun.Namespace).Get(boundResource.ResourceRef.Name)
		if err != nil {
			return fmt.Errorf("Failed to get output pipeline Resource for task %q: %q", boundResource.ResourceRef.Name, taskName)
		}

		// if resource is declared in input then copy outputs to pvc
		// To build copy step it needs source path(which is targetpath of input resourcemap) from task input source
		sourcePath := inputResourceMap[boundResource.Name]
		if sourcePath == "" {
			sourcePath = filepath.Join(outputDir, boundResource.Name)
		}
		switch resource.Spec.Type {
		case v1alpha1.PipelineResourceTypeStorage:
			storageResource, err := v1alpha1.NewStorageResource(resource)
			if err != nil {
				return fmt.Errorf("task %q invalid git Pipeline Resource: %q",
					taskName,
					boundResource.ResourceRef.Name,
				)
			}
			err = addStoreUploadStep(b, storageResource, sourcePath)
			if err != nil {
				return fmt.Errorf("task %q invalid Pipeline Resource: %q; invalid upload steps err: %v",
					taskName, boundResource.ResourceRef.Name, err)
			}
		}

		// copy to pvc if pvc is present
		if pipelineRunpvcName != "" {
			var newSteps []corev1.Container
			for _, dPath := range boundResource.Paths {
				newSteps = append(newSteps, []corev1.Container{{
					Name:  fmt.Sprintf("source-mkdir-%s", resource.GetName()),
					Image: *bashNoopImage,
					Args: []string{
						"-args", strings.Join([]string{"mkdir", "-p", dPath}, " "),
					},
					VolumeMounts: []corev1.VolumeMount{getPvcMount(pipelineRunpvcName)},
				}, {
					Name:  fmt.Sprintf("source-copy-%s", resource.GetName()),
					Image: *bashNoopImage,
					Args: []string{
						"-args", strings.Join([]string{"cp", "-r", fmt.Sprintf("%s/.", sourcePath), dPath}, " "),
					},
					VolumeMounts: []corev1.VolumeMount{getPvcMount(pipelineRunpvcName)},
				}}...)
			}
			b.Spec.Steps = append(b.Spec.Steps, newSteps...)
		}
	}

	if pipelineRunpvcName == "" {
		return nil
	}

	// attach pvc volume only if it is not already attached
	for _, buildVol := range b.Spec.Volumes {
		if buildVol.Name == pipelineRunpvcName {
			return nil
		}
	}
	b.Spec.Volumes = append(b.Spec.Volumes, GetPVCVolume(pipelineRunpvcName))
	return nil
}

func addStoreUploadStep(build *buildv1alpha1.Build,
	storageResource v1alpha1.PipelineStorageResourceInterface,
	sourcePath string,
) error {
	storageResource.SetDestinationDirectory(sourcePath)
	gcsContainers, err := storageResource.GetUploadContainerSpec()
	if err != nil {
		return err
	}
	mountedSecrets := map[string]string{}

	for _, volume := range build.Spec.Volumes {
		mountedSecrets[volume.Name] = ""
	}
	var buildSteps []corev1.Container
	for _, gcsContainer := range gcsContainers {
		gcsContainer.VolumeMounts = append(gcsContainer.VolumeMounts, corev1.VolumeMount{
			Name:      "workspace",
			MountPath: workspaceDir,
		})
		// Map holds list of secrets that are mounted as volumes
		for _, secretParam := range storageResource.GetSecretParams() {
			volName := fmt.Sprintf("volume-%s-%s", storageResource.GetName(), secretParam.SecretName)

			gcsSecretVolume := corev1.Volume{
				Name: volName,
				VolumeSource: corev1.VolumeSource{
					Secret: &corev1.SecretVolumeSource{
						SecretName: secretParam.SecretName,
					},
				},
			}

			if _, ok := mountedSecrets[volName]; !ok {
				build.Spec.Volumes = append(build.Spec.Volumes, gcsSecretVolume)
				mountedSecrets[volName] = ""
			}
		}
		buildSteps = append(buildSteps, gcsContainer)
	}
	build.Spec.Steps = append(build.Spec.Steps, buildSteps...)
	return nil
}
<commit_msg>Fix invalid error message for storage output resource<commit_after>/*
Copyright 2018 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package resources

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/knative/build-pipeline/pkg/apis/pipeline/v1alpha1"
	listers "github.com/knative/build-pipeline/pkg/client/listers/pipeline/v1alpha1"
	buildv1alpha1 "github.com/knative/build/pkg/apis/build/v1alpha1"
	"go.uber.org/zap"
	corev1 "k8s.io/api/core/v1"
)

var (
	outputDir = "/workspace/output/"
)

// AddOutputResources reads the output resources and adds the corresponding container steps
// This function also reads the inputs to check if resources are redeclared in inputs and has any custom
// target directory.
// Steps executed:
//  1. If taskrun has owner reference as pipelinerun then all outputs are copied to parents PVC
// and also runs any custom upload steps (upload to blob store)
//  2.  If taskrun does not have pipelinerun as owner reference then all outputs resources execute their custom
// upload steps (like upload to blob store )
//
// Resource source path determined
// 1. If resource is declared in inputs then target path from input resource is used to identify source path
// 2. If resource is declared in outputs only then the default is /output/resource_name
func AddOutputResources(
	b *buildv1alpha1.Build,
	taskName string,
	taskSpec *v1alpha1.TaskSpec,
	taskRun *v1alpha1.TaskRun,
	pipelineResourceLister listers.PipelineResourceLister,
	logger *zap.SugaredLogger,
) error {

	if taskSpec == nil || taskSpec.Outputs == nil {
		return nil
	}

	pipelineRunpvcName := taskRun.GetPipelineRunPVCName()

	// track resources that are present in input of task cuz these resources will be copied onto PVC
	inputResourceMap := map[string]string{}

	if taskSpec.Inputs != nil {
		for _, input := range taskSpec.Inputs.Resources {
			var targetPath = filepath.Join(workspaceDir, input.Name)
			if input.TargetPath != "" {
				targetPath = filepath.Join(workspaceDir, input.TargetPath)
			}
			inputResourceMap[input.Name] = targetPath
		}
	}

	for _, output := range taskSpec.Outputs.Resources {
		boundResource, err := getBoundResource(output.Name, taskRun.Spec.Outputs.Resources)
		if err != nil {
			return fmt.Errorf("Failed to get bound resource: %s", err)
		}

		resource, err := pipelineResourceLister.PipelineResources(taskRun.Namespace).Get(boundResource.ResourceRef.Name)
		if err != nil {
			return fmt.Errorf("Failed to get output pipeline Resource for task %q: %q", boundResource.ResourceRef.Name, taskName)
		}

		// if resource is declared in input then copy outputs to pvc
		// To build copy step it needs source path(which is targetpath of input resourcemap) from task input source
		sourcePath := inputResourceMap[boundResource.Name]
		if sourcePath == "" {
			sourcePath = filepath.Join(outputDir, boundResource.Name)
		}
		switch resource.Spec.Type {
		case v1alpha1.PipelineResourceTypeStorage:
			storageResource, err := v1alpha1.NewStorageResource(resource)
			if err != nil {
				return fmt.Errorf("task %q invalid storage Pipeline Resource: %q",
					taskName,
					boundResource.ResourceRef.Name,
				)
			}
			err = addStoreUploadStep(b, storageResource, sourcePath)
			if err != nil {
				return fmt.Errorf("task %q invalid Pipeline Resource: %q; invalid upload steps err: %v",
					taskName, boundResource.ResourceRef.Name, err)
			}
		}

		// copy to pvc if pvc is present
		if pipelineRunpvcName != "" {
			var newSteps []corev1.Container
			for _, dPath := range boundResource.Paths {
				newSteps = append(newSteps, []corev1.Container{{
					Name:  fmt.Sprintf("source-mkdir-%s", resource.GetName()),
					Image: *bashNoopImage,
					Args: []string{
						"-args", strings.Join([]string{"mkdir", "-p", dPath}, " "),
					},
					VolumeMounts: []corev1.VolumeMount{getPvcMount(pipelineRunpvcName)},
				}, {
					Name:  fmt.Sprintf("source-copy-%s", resource.GetName()),
					Image: *bashNoopImage,
					Args: []string{
						"-args", strings.Join([]string{"cp", "-r", fmt.Sprintf("%s/.", sourcePath), dPath}, " "),
					},
					VolumeMounts: []corev1.VolumeMount{getPvcMount(pipelineRunpvcName)},
				}}...)
			}
			b.Spec.Steps = append(b.Spec.Steps, newSteps...)
		}
	}

	if pipelineRunpvcName == "" {
		return nil
	}

	// attach pvc volume only if it is not already attached
	for _, buildVol := range b.Spec.Volumes {
		if buildVol.Name == pipelineRunpvcName {
			return nil
		}
	}
	b.Spec.Volumes = append(b.Spec.Volumes, GetPVCVolume(pipelineRunpvcName))
	return nil
}

func addStoreUploadStep(build *buildv1alpha1.Build,
	storageResource v1alpha1.PipelineStorageResourceInterface,
	sourcePath string,
) error {
	storageResource.SetDestinationDirectory(sourcePath)
	gcsContainers, err := storageResource.GetUploadContainerSpec()
	if err != nil {
		return err
	}
	mountedSecrets := map[string]string{}

	for _, volume := range build.Spec.Volumes {
		mountedSecrets[volume.Name] = ""
	}
	var buildSteps []corev1.Container
	for _, gcsContainer := range gcsContainers {
		gcsContainer.VolumeMounts = append(gcsContainer.VolumeMounts, corev1.VolumeMount{
			Name:      "workspace",
			MountPath: workspaceDir,
		})
		// Map holds list of secrets that are mounted as volumes
		for _, secretParam := range storageResource.GetSecretParams() {
			volName := fmt.Sprintf("volume-%s-%s", storageResource.GetName(), secretParam.SecretName)

			gcsSecretVolume := corev1.Volume{
				Name: volName,
				VolumeSource: corev1.VolumeSource{
					Secret: &corev1.SecretVolumeSource{
						SecretName: secretParam.SecretName,
					},
				},
			}

			if _, ok := mountedSecrets[volName]; !ok {
				build.Spec.Volumes = append(build.Spec.Volumes, gcsSecretVolume)
				mountedSecrets[volName] = ""
			}
		}
		buildSteps = append(buildSteps, gcsContainer)
	}
	build.Spec.Steps = append(build.Spec.Steps, buildSteps...)
	return nil
}
<|endoftext|>


<commit_before>// +build integration

package test

import (
	"encoding/json"
	"errors"
	"fmt"
	"os/exec"
	"strings"
	"testing"
	"time"
)

func TestServerAuth(t *testing.T) {
	trySuite(t, testServerAuth, retryCount)
}

func testServerAuth(t *t) {
	t.Parallel()
	serv := newServer(t)
	serv.launch()
	defer serv.close()

	basicAuthSuite(serv, t)
}

func TestServerAuthJWT(t *testing.T) {
	trySuite(t, testServerAuthJWT, retryCount)
}

func testServerAuthJWT(t *t) {
	t.Parallel()
	serv := newServer(t, options{auth: "jwt"})
	serv.launch()
	defer serv.close()
	basicAuthSuite(serv, t)
}

func basicAuthSuite(serv server, t *t) {
	// Execute first command in read to wait for store service
	// to start up
	try("Calling micro auth list accounts", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "auth", "list", "accounts")
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "admin") {
			return outp, fmt.Errorf("Output should contain admin")
		}
		return outp, nil
	}, 15*time.Second)

	try("Calling micro auth list rules", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "auth", "list", "rules")
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "*:*:*:*") {
			return outp, fmt.Errorf("Output should contain default rule")
		}
		return outp, nil
	}, 8*time.Second)

	try("Calling micro auth list accounts", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "auth", "list", "accounts")
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "admin") {
			return outp, fmt.Errorf("Output should contain default admin account")
		}
		return outp, nil
	}, 8*time.Second)

	accessToken := ""
	try("Try to get token with default account", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "call", "go.micro.auth", "Auth.Token", `{"id":"admin","secret":"Password1"}`)
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		rsp := map[string]interface{}{}
		err = json.Unmarshal(outp, &rsp)
		token, ok := rsp["token"].(map[string]interface{})
		if !ok {
			return outp, errors.New("Can't find token")
		}
		if accessToken, ok = token["access_token"].(string); !ok {
			return outp, fmt.Errorf("Can't find access token")
		}
		return outp, nil
	}, 5*time.Second)

	try("Try to log in with token we got", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "login", "--token", accessToken)
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		return outp, nil
	}, 5*time.Second)
}
<commit_msg>Update micro server tests<commit_after>// +build integration

package test

import (
	"encoding/json"
	"errors"
	"fmt"
	"os/exec"
	"strings"
	"testing"
	"time"
)

func TestServerAuth(t *testing.T) {
	trySuite(t, testServerAuth, retryCount)
}

func testServerAuth(t *t) {
	t.Parallel()
	serv := newServer(t)
	serv.launch()
	defer serv.close()

	basicAuthSuite(serv, t)
}

func TestServerAuthJWT(t *testing.T) {
	trySuite(t, testServerAuthJWT, retryCount)
}

func testServerAuthJWT(t *t) {
	t.Parallel()
	serv := newServer(t, options{auth: "jwt"})
	serv.launch()
	defer serv.close()
	basicAuthSuite(serv, t)
}

func basicAuthSuite(serv server, t *t) {
	// Execute first command in read to wait for store service
	// to start up
	try("Calling micro auth list accounts", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "auth", "list", "accounts")
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "admin") {
			return outp, fmt.Errorf("Output should contain admin")
		}
		return outp, nil
	}, 15*time.Second)

	try("Calling micro auth list rules", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "auth", "list", "rules")
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "default") {
			return outp, fmt.Errorf("Output should contain default rule")
		}
		return outp, nil
	}, 8*time.Second)

	try("Calling micro auth list accounts", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "auth", "list", "accounts")
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "admin") {
			return outp, fmt.Errorf("Output should contain default admin account")
		}
		return outp, nil
	}, 8*time.Second)

	accessToken := ""
	try("Try to get token with default account", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "call", "go.micro.auth", "Auth.Token", `{"id":"admin","secret":"Password1"}`)
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		rsp := map[string]interface{}{}
		err = json.Unmarshal(outp, &rsp)
		token, ok := rsp["token"].(map[string]interface{})
		if !ok {
			return outp, errors.New("Can't find token")
		}
		if accessToken, ok = token["access_token"].(string); !ok {
			return outp, fmt.Errorf("Can't find access token")
		}
		return outp, nil
	}, 5*time.Second)

	try("Try to log in with token we got", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "login", "--token", accessToken)
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		return outp, nil
	}, 5*time.Second)
}
<|endoftext|>


<commit_before>// Package secureheader adds some HTTP header fields widely
// considered to improve safety of HTTP requests. These fields
// are documented as follows:
//
//   Strict Transport Security: https://tools.ietf.org/html/rfc6797
//   Frame Options:             https://tools.ietf.org/html/draft-ietf-websec-x-frame-options-00
//   Cross Site Scripting:      http://msdn.microsoft.com/en-us/library/dd565647%28v=vs.85%29.aspx
//   Content Type Options:      http://msdn.microsoft.com/en-us/library/ie/gg622941%28v=vs.85%29.aspx
//
// The easiest way to use this package:
//
//   http.ListenAndServe(addr, secureheader.DefaultConfig)
//
// DefaultConfig is initialized with conservative (safer and more
// restrictive) behavior. If you want to change that, set its
// fields to different values before calling ListenAndServe. See
// the example code below.
//
// This package was inspired by Twitter's secureheaders Ruby
// library. See https://github.com/twitter/secureheaders.
package secureheader

// TODO(kr): figure out how to add this one:
//   Content Security Policy:   https://dvcs.w3.org/hg/content-security-policy/raw-file/tip/csp-specification.dev.html
// See https://github.com/kr/secureheader/issues/1.

import (
	"net"
	"net/http"
	"strconv"
	"time"
)

// DefaultConfig is initialized with conservative (safer and more
// restrictive) behavior.
var DefaultConfig = &Config{
	HTTPSRedirect:          true,
	HTTPSUseForwardedProto: ShouldUseForwardedProto(),

	PermitClearLoopback: false,

	ContentTypeOptions: true,

	HSTS:                  true,
	HSTSMaxAge:            300 * 24 * time.Hour,
	HSTSIncludeSubdomains: true,

	FrameOptions:       true,
	FrameOptionsPolicy: Deny,

	XSSProtection:      true,
	XSSProtectionBlock: false,
}

type Config struct {
	// If true, redirects any request with scheme http to the
	// equivalent https URL.
	HTTPSRedirect          bool
	HTTPSUseForwardedProto bool

	// Allow cleartext (non-HTTPS) HTTP connections to a loopback
	// address, even if HTTPSRedirect is true.
	PermitClearLoopback bool

	// If true, sets X-Content-Type-Options to "nosniff".
	ContentTypeOptions bool

	// If true, sets the HTTP Strict Transport Security header
	// field, which instructs browsers to send future requests
	// over HTTPS, even if the URL uses the unencrypted http
	// scheme.
	HSTS                  bool
	HSTSMaxAge            time.Duration
	HSTSIncludeSubdomains bool

	// If true, sets X-Frame-Options, to control when the request
	// should be displayed inside an HTML frame.
	FrameOptions       bool
	FrameOptionsPolicy FramePolicy

	// If true, sets X-XSS-Protection to "1", optionally with
	// "mode=block". See the official documentation, linked above,
	// for the meaning of these values.
	XSSProtection      bool
	XSSProtectionBlock bool

	// Used by ServeHTTP, after setting any extra headers, to
	// reply to the request. Next is typically nil, in which case
	// http.DefaultServeMux is used instead.
	Next http.Handler
}

// ServeHTTP sets header fields on w according to the options in
// c, then either replies directly or runs c.Next to reply.
// Typically c.Next is nil, in which case http.DefaultServeMux is
// used instead.
func (c *Config) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if c.HTTPSRedirect && !c.isHTTPS(r) && !c.okloopback(r) {
		url := *r.URL
		url.Scheme = "https"
		url.Host = r.Host
		http.Redirect(w, r, url.String(), http.StatusMovedPermanently)
		return
	}
	if c.ContentTypeOptions {
		w.Header().Set("X-Content-Type-Options", "nosniff")
	}
	if c.HSTS && c.isHTTPS(r) {
		v := "max-age=" + strconv.FormatInt(int64(c.HSTSMaxAge/time.Second), 10)
		if c.HSTSIncludeSubdomains {
			v += "; includeSubDomains"
		}
		w.Header().Set("Strict-Transport-Security", v)
	}
	if c.FrameOptions {
		w.Header().Set("X-Frame-Options", string(c.FrameOptionsPolicy))
	}
	if c.XSSProtection {
		v := "1"
		if c.XSSProtectionBlock {
			v += "; mode=block"
		}
		w.Header().Set("X-XSS-Protection", v)
	}
	next := c.Next
	if next == nil {
		next = http.DefaultServeMux
	}
	next.ServeHTTP(w, r)
}

// Given that r is cleartext (not HTTPS), okloopback returns
// whether r is on a permitted loopback connection.
func (c *Config) okloopback(r *http.Request) bool {
	return c.PermitClearLoopback && isLoopback(r)
}

func (c *Config) isHTTPS(r *http.Request) bool {
	if c.HTTPSUseForwardedProto {
		return r.Header.Get("X-Forwarded-Proto") == "https"
	}
	return r.TLS != nil
}

// FramePolicy tells the browser under what circumstances to allow
// the response to be displayed inside an HTML frame. There are
// three options:
//
//   Deny            do not permit display in a frame
//   SameOrigin      permit display in a frame from the same origin
//   AllowFrom(url)  permit display in a frame from the given url
type FramePolicy string

const (
	Deny       FramePolicy = "DENY"
	SameOrigin FramePolicy = "SAMEORIGIN"
)

// AllowFrom returns a FramePolicy specifying that the requested
// resource should be included in a frame from only the given url.
func AllowFrom(url string) FramePolicy {
	return FramePolicy("ALLOW-FROM: " + url)
}

// ShouldUseForwardedProto returns whether to trust the
// X-Forwarded-Proto header field.
// DefaultConfig.HTTPSUseForwardedProto is initialized to this
// value.
//
// This value depends on the particular environment where the
// package is built. It is currently true iff build constraint
// "heroku" is satisfied.
func ShouldUseForwardedProto() bool {
	return defaultUseForwardedProto
}

func isLoopback(r *http.Request) bool {
	a, err := net.ResolveTCPAddr("tcp", r.RemoteAddr)
	return err == nil && a.IP.IsLoopback()
}
<commit_msg>add ServeHTTPContext<commit_after>// Package secureheader adds some HTTP header fields widely
// considered to improve safety of HTTP requests. These fields
// are documented as follows:
//
//   Strict Transport Security: https://tools.ietf.org/html/rfc6797
//   Frame Options:             https://tools.ietf.org/html/draft-ietf-websec-x-frame-options-00
//   Cross Site Scripting:      http://msdn.microsoft.com/en-us/library/dd565647%28v=vs.85%29.aspx
//   Content Type Options:      http://msdn.microsoft.com/en-us/library/ie/gg622941%28v=vs.85%29.aspx
//
// The easiest way to use this package:
//
//   http.ListenAndServe(addr, secureheader.DefaultConfig)
//
// DefaultConfig is initialized with conservative (safer and more
// restrictive) behavior. If you want to change that, set its
// fields to different values before calling ListenAndServe. See
// the example code below.
//
// This package was inspired by Twitter's secureheaders Ruby
// library. See https://github.com/twitter/secureheaders.
package secureheader

// TODO(kr): figure out how to add this one:
//   Content Security Policy:   https://dvcs.w3.org/hg/content-security-policy/raw-file/tip/csp-specification.dev.html
// See https://github.com/kr/secureheader/issues/1.

import (
	"net"
	"net/http"
	"strconv"
	"time"

	"golang.org/x/net/context"
)

// DefaultConfig is initialized with conservative (safer and more
// restrictive) behavior.
var DefaultConfig = &Config{
	HTTPSRedirect:          true,
	HTTPSUseForwardedProto: ShouldUseForwardedProto(),

	PermitClearLoopback: false,

	ContentTypeOptions: true,

	HSTS:                  true,
	HSTSMaxAge:            300 * 24 * time.Hour,
	HSTSIncludeSubdomains: true,

	FrameOptions:       true,
	FrameOptionsPolicy: Deny,

	XSSProtection:      true,
	XSSProtectionBlock: false,
}

type Config struct {
	// If true, redirects any request with scheme http to the
	// equivalent https URL.
	HTTPSRedirect          bool
	HTTPSUseForwardedProto bool

	// Allow cleartext (non-HTTPS) HTTP connections to a loopback
	// address, even if HTTPSRedirect is true.
	PermitClearLoopback bool

	// If true, sets X-Content-Type-Options to "nosniff".
	ContentTypeOptions bool

	// If true, sets the HTTP Strict Transport Security header
	// field, which instructs browsers to send future requests
	// over HTTPS, even if the URL uses the unencrypted http
	// scheme.
	HSTS                  bool
	HSTSMaxAge            time.Duration
	HSTSIncludeSubdomains bool

	// If true, sets X-Frame-Options, to control when the request
	// should be displayed inside an HTML frame.
	FrameOptions       bool
	FrameOptionsPolicy FramePolicy

	// If true, sets X-XSS-Protection to "1", optionally with
	// "mode=block". See the official documentation, linked above,
	// for the meaning of these values.
	XSSProtection      bool
	XSSProtectionBlock bool

	// Used by ServeHTTP, after setting any extra headers, to
	// reply to the request. Next is typically nil, in which case
	// http.DefaultServeMux is used instead.
	Next http.Handler
}

// ServeHTTP calls ServeHTTPContext with context.Background().
func (c *Config) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx := context.Background()
	c.ServeHTTPContext(ctx, w, r)
}

// ServeHTTPContext sets header fields on w according to the options in
// c, then either replies directly or runs c.Next to reply.
// Typically c.Next is nil, in which case http.DefaultServeMux is
// used instead.
//
// If c.Next implements ContextHandler, then ServeHTTPContext will
// provide ctx to c.Next.
func (c *Config) ServeHTTPContext(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	if c.HTTPSRedirect && !c.isHTTPS(r) && !c.okloopback(r) {
		url := *r.URL
		url.Scheme = "https"
		url.Host = r.Host
		http.Redirect(w, r, url.String(), http.StatusMovedPermanently)
		return
	}
	if c.ContentTypeOptions {
		w.Header().Set("X-Content-Type-Options", "nosniff")
	}
	if c.HSTS && c.isHTTPS(r) {
		v := "max-age=" + strconv.FormatInt(int64(c.HSTSMaxAge/time.Second), 10)
		if c.HSTSIncludeSubdomains {
			v += "; includeSubDomains"
		}
		w.Header().Set("Strict-Transport-Security", v)
	}
	if c.FrameOptions {
		w.Header().Set("X-Frame-Options", string(c.FrameOptionsPolicy))
	}
	if c.XSSProtection {
		v := "1"
		if c.XSSProtectionBlock {
			v += "; mode=block"
		}
		w.Header().Set("X-XSS-Protection", v)
	}
	next := c.Next
	if next == nil {
		next = http.DefaultServeMux
	}
	if ctxHandler, ok := next.(ContextHandler); ok {
		ctxHandler.ServeHTTPContext(ctx, w, r)
	} else {
		next.ServeHTTP(w, r)
	}
}

// ContextHandler can be used by Config.Next to accept a context.Context.
// See comment for Config.ServeHTTPContext.
type ContextHandler interface {
	ServeHTTPContext(context.Context, http.ResponseWriter, *http.Request)
}

// Given that r is cleartext (not HTTPS), okloopback returns
// whether r is on a permitted loopback connection.
func (c *Config) okloopback(r *http.Request) bool {
	return c.PermitClearLoopback && isLoopback(r)
}

func (c *Config) isHTTPS(r *http.Request) bool {
	if c.HTTPSUseForwardedProto {
		return r.Header.Get("X-Forwarded-Proto") == "https"
	}
	return r.TLS != nil
}

// FramePolicy tells the browser under what circumstances to allow
// the response to be displayed inside an HTML frame. There are
// three options:
//
//   Deny            do not permit display in a frame
//   SameOrigin      permit display in a frame from the same origin
//   AllowFrom(url)  permit display in a frame from the given url
type FramePolicy string

const (
	Deny       FramePolicy = "DENY"
	SameOrigin FramePolicy = "SAMEORIGIN"
)

// AllowFrom returns a FramePolicy specifying that the requested
// resource should be included in a frame from only the given url.
func AllowFrom(url string) FramePolicy {
	return FramePolicy("ALLOW-FROM: " + url)
}

// ShouldUseForwardedProto returns whether to trust the
// X-Forwarded-Proto header field.
// DefaultConfig.HTTPSUseForwardedProto is initialized to this
// value.
//
// This value depends on the particular environment where the
// package is built. It is currently true iff build constraint
// "heroku" is satisfied.
func ShouldUseForwardedProto() bool {
	return defaultUseForwardedProto
}

func isLoopback(r *http.Request) bool {
	a, err := net.ResolveTCPAddr("tcp", r.RemoteAddr)
	return err == nil && a.IP.IsLoopback()
}
<|endoftext|>


<commit_before>// Package category supports post categories
package category

import (
	"errors"
	"github.com/writeas/slug"
)

var (
	ErrNotFound = errors.New("category doesn't exist")
)

// Category represents a post tag with additional metadata, like a title and slug.
type Category struct {
	ID      int64  `json:"-"`
	Hashtag string `json:"hashtag"`
	Slug    string `json:"slug"`
	Title   string `json:"title"`
}

// NewCategory creates a Category you can insert into the database, based on a hashtag. It automatically breaks up the
// hashtag by words, based on capitalization, for both the title and a URL-friendly slug.
func NewCategory(hashtag string) *Category {
	title := titleFromHashtag(hashtag)
	return &Category{
		Hashtag: hashtag,
		Slug:    slug.Make(title),
		Title:   title,
	}
}
<commit_msg>Add NewCategoryFromPartial func<commit_after>// Package category supports post categories
package category

import (
	"errors"
	"github.com/writeas/slug"
)

var (
	ErrNotFound = errors.New("category doesn't exist")
)

// Category represents a post tag with additional metadata, like a title and slug.
type Category struct {
	ID      int64  `json:"-"`
	Hashtag string `json:"hashtag"`
	Slug    string `json:"slug"`
	Title   string `json:"title"`
}

// NewCategory creates a Category you can insert into the database, based on a hashtag. It automatically breaks up the
// hashtag by words, based on capitalization, for both the title and a URL-friendly slug.
func NewCategory(hashtag string) *Category {
	title := titleFromHashtag(hashtag)
	return &Category{
		Hashtag: hashtag,
		Slug:    slug.Make(title),
		Title:   title,
	}
}

// NewCategoryFromPartial creates a Category from a partially-populated Category, such as when a user initially creates
// one.
func NewCategoryFromPartial(cat *Category) *Category {
	newCat := &Category{
		Hashtag: cat.Hashtag,
	}
	// Create title from hashtag, if none supplied
	if cat.Title == "" {
		newCat.Title = titleFromHashtag(cat.Hashtag)
	} else {
		newCat.Title = cat.Title
	}
	// Create slug from title, if none supplied; otherwise ensure slug is valid
	if cat.Slug == "" {
		newCat.Slug = slug.Make(newCat.Title)
	} else {
		newCat.Slug = slug.Make(cat.Slug)
	}
	return newCat
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"sync"
)

var uploadWg = sync.WaitGroup{}

var uploadFlags = flag.NewFlagSet("upload", flag.ExitOnError)
var uploadVerbose = uploadFlags.Bool("v", false, "Verbose")
var uploadDelete = uploadFlags.Bool("delete", false,
	"Delete locally missing items")

type uploadOpType uint8

const (
	uploadFileOp = uploadOpType(iota)
	removeFileOp
	removeRecurseOp
)

type uploadReq struct {
	src        string
	dest       string
	op         uploadOpType
	remoteHash string
}

func recognizeTypeByName(n, def string) string {
	byname := mime.TypeByExtension(n)
	switch {
	case byname != "":
		return byname
	case strings.HasSuffix(n, ".js"):
		return "application/javascript"
	}
	return def
}

func uploadFile(src, dest string) error {
	if *uploadVerbose {
		log.Printf("Uploading %v -> %v", src, dest)
	}

	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	someBytes := make([]byte, 512)
	n, err := f.Read(someBytes)
	if err != nil && err != io.EOF {
		return err
	}
	someBytes = someBytes[:n]
	_, err = f.Seek(0, 0)
	if err != nil {
		return err
	}

	preq, err := http.NewRequest("PUT", dest, f)
	if err != nil {
		return err
	}
	preq.Header.Set("X-CBFS-KeepRevs", strconv.Itoa(*revs))

	ctype := http.DetectContentType(someBytes)
	if strings.HasPrefix(ctype, "text/plain") ||
		strings.HasPrefix(ctype, "application/octet-stream") {
		ctype = recognizeTypeByName(src, ctype)
	}

	preq.Header.Set("Content-Type", ctype)

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("HTTP Error:  %v", resp.Status)
	}
	return nil
}

// This is very similar to rm's version, but uses different channel
// signaling.
func uploadRmDashR(baseUrl string, ch chan uploadReq) error {
	for strings.HasSuffix(baseUrl, "/") {
		baseUrl = baseUrl[:len(baseUrl)-1]
	}

	listing, err := listStuff(baseUrl)
	for err != nil {
		return err
	}
	for fn := range listing.Files {
		ch <- uploadReq{"", baseUrl + "/" + fn, removeFileOp, ""}
	}
	for dn := range listing.Dirs {
		return uploadRmDashR(baseUrl+"/"+dn, ch)
	}
	return nil
}

func localHash(fn string) string {
	f, err := os.Open(fn)
	if err != nil {
		return "unknown"
	}
	defer f.Close()

	h := sha1.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return "unknown"
	}

	return hex.EncodeToString(h.Sum([]byte{}))
}

func uploadWorker(ch chan uploadReq) {
	defer uploadWg.Done()
	for req := range ch {
		retries := 0
		done := false
		for !done {
			var err error
			switch req.op {
			case uploadFileOp:
				if req.remoteHash == "" {
					err = uploadFile(req.src, req.dest)
				} else {
					if localHash(req.src) != req.remoteHash {
						if *uploadVerbose {
							log.Printf("%v has changed, reupping",
								req.src)
						}
						err = uploadFile(req.src, req.dest)
					}
				}
			case removeFileOp:
				if *uploadVerbose {
					log.Printf("Removing file %v", req.dest)
				}
				err = rmFile(req.dest)
			case removeRecurseOp:
				err = uploadRmDashR(req.dest, ch)
			default:
				log.Fatalf("Unhandled case")
			}
			if err != nil {
				if retries < 3 {
					retries++
					log.Printf("Error uploading file: %v... retrying",
						err)
					time.Sleep(time.Duration(retries) * time.Second)
				} else {
					log.Fatalf("Error uploading file: %v", err)
				}
			} else {
				done = true
			}
		}
	}
}

func syncPath(path, dest string, info os.FileInfo, ch chan<- uploadReq) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	children, err := f.Readdir(0)
	if err != nil {
		return err
	}

	retries := 3
	serverListing, err := listStuff(dest)
	for err != nil && retries > 0 {
		serverListing, err = listStuff(dest)
		retries--
	}
	if err != nil {
		return err
	}

	localNames := map[string]os.FileInfo{}
	for _, c := range children {
		switch c.Mode() & os.ModeType {
		case os.ModeCharDevice, os.ModeDevice,
			os.ModeNamedPipe, os.ModeSocket, os.ModeSymlink:
			if *uploadVerbose {
				log.Printf("Ignoring special file: %v", path)
			}
		default:
			localNames[c.Name()] = c
		}
	}

	remoteNames := map[string]bool{}
	for n := range serverListing.Files {
		remoteNames[n] = true
	}
	for n := range serverListing.Dirs {
		remoteNames[n] = true
	}

	missingUpstream := []string{}
	for n, fi := range localNames {
		if !(fi.IsDir() || remoteNames[n]) {
			missingUpstream = append(missingUpstream, n)
		} else if !fi.IsDir() {
			if ri, ok := serverListing.Files[n]; ok {
				ch <- uploadReq{filepath.Join(path, n),
					dest + "/" + n, uploadFileOp, ri.OID}
			}
		}
	}

	toRm := []string{}
	for n := range remoteNames {
		if _, ok := localNames[n]; !ok {
			toRm = append(toRm, n)
		}
	}

	if len(missingUpstream) > 0 {
		for _, m := range missingUpstream {
			ch <- uploadReq{filepath.Join(path, m),
				dest + "/" + m, uploadFileOp, ""}
		}
	}

	if *uploadDelete && len(toRm) > 0 {
		for _, m := range toRm {
			ch <- uploadReq{"", dest + "/" + m, removeFileOp, ""}
			ch <- uploadReq{"", dest + "/" + m, removeRecurseOp, ""}
		}
	}

	return nil
}

func syncUp(src, u string, ch chan<- uploadReq) {
	for strings.HasSuffix(u, "/") {
		u = u[:len(u)-1]
	}
	for strings.HasSuffix(src, "/") {
		src = src[:len(src)-1]
	}

	err := filepath.Walk(src,
		func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				shortPath := path[len(src):]
				err = syncPath(path, u+shortPath, info, ch)
			}
			return err
		})
	if err != nil {
		log.Fatalf("Traversal error: %v", err)
	}
}

func uploadCommand(args []string) {
	uploadFlags.Parse(args)

	if uploadFlags.NArg() < 2 {
		log.Fatalf("src and dest required")
	}

	fi, err := os.Stat(uploadFlags.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	if fi.IsDir() {
		ch := make(chan uploadReq, 1000)

		for i := 0; i < *workers; i++ {
			uploadWg.Add(1)
			go uploadWorker(ch)
		}

		start := time.Now()
		syncUp(uploadFlags.Arg(0), uploadFlags.Arg(1), ch)

		close(ch)
		log.Printf("Finished traversal in %v", time.Since(start))
		uploadWg.Wait()
		log.Printf("Finished sync in %v", time.Since(start))
	} else {
		err = uploadFile(uploadFlags.Arg(0), uploadFlags.Arg(1))
		if err != nil {
			log.Fatalf("Error uploading file: %v", err)
		}
	}
}
<commit_msg>When I ignore something, just ignore it.<commit_after>package main

import (
	"crypto/sha1"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"log"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"sync"
)

var uploadWg = sync.WaitGroup{}

var uploadFlags = flag.NewFlagSet("upload", flag.ExitOnError)
var uploadVerbose = uploadFlags.Bool("v", false, "Verbose")
var uploadDelete = uploadFlags.Bool("delete", false,
	"Delete locally missing items")

type uploadOpType uint8

const (
	uploadFileOp = uploadOpType(iota)
	removeFileOp
	removeRecurseOp
)

type uploadReq struct {
	src        string
	dest       string
	op         uploadOpType
	remoteHash string
}

func recognizeTypeByName(n, def string) string {
	byname := mime.TypeByExtension(n)
	switch {
	case byname != "":
		return byname
	case strings.HasSuffix(n, ".js"):
		return "application/javascript"
	}
	return def
}

func uploadFile(src, dest string) error {
	if *uploadVerbose {
		log.Printf("Uploading %v -> %v", src, dest)
	}

	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	someBytes := make([]byte, 512)
	n, err := f.Read(someBytes)
	if err != nil && err != io.EOF {
		return err
	}
	someBytes = someBytes[:n]
	_, err = f.Seek(0, 0)
	if err != nil {
		return err
	}

	preq, err := http.NewRequest("PUT", dest, f)
	if err != nil {
		return err
	}
	preq.Header.Set("X-CBFS-KeepRevs", strconv.Itoa(*revs))

	ctype := http.DetectContentType(someBytes)
	if strings.HasPrefix(ctype, "text/plain") ||
		strings.HasPrefix(ctype, "application/octet-stream") {
		ctype = recognizeTypeByName(src, ctype)
	}

	preq.Header.Set("Content-Type", ctype)

	resp, err := http.DefaultClient.Do(preq)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != 201 {
		return fmt.Errorf("HTTP Error:  %v", resp.Status)
	}
	return nil
}

// This is very similar to rm's version, but uses different channel
// signaling.
func uploadRmDashR(baseUrl string, ch chan uploadReq) error {
	for strings.HasSuffix(baseUrl, "/") {
		baseUrl = baseUrl[:len(baseUrl)-1]
	}

	listing, err := listStuff(baseUrl)
	for err != nil {
		return err
	}
	for fn := range listing.Files {
		ch <- uploadReq{"", baseUrl + "/" + fn, removeFileOp, ""}
	}
	for dn := range listing.Dirs {
		return uploadRmDashR(baseUrl+"/"+dn, ch)
	}
	return nil
}

func localHash(fn string) string {
	f, err := os.Open(fn)
	if err != nil {
		return "unknown"
	}
	defer f.Close()

	h := sha1.New()
	_, err = io.Copy(h, f)
	if err != nil {
		return "unknown"
	}

	return hex.EncodeToString(h.Sum([]byte{}))
}

func uploadWorker(ch chan uploadReq) {
	defer uploadWg.Done()
	for req := range ch {
		retries := 0
		done := false
		for !done {
			var err error
			switch req.op {
			case uploadFileOp:
				if req.remoteHash == "" {
					err = uploadFile(req.src, req.dest)
				} else {
					if localHash(req.src) != req.remoteHash {
						if *uploadVerbose {
							log.Printf("%v has changed, reupping",
								req.src)
						}
						err = uploadFile(req.src, req.dest)
					}
				}
			case removeFileOp:
				if *uploadVerbose {
					log.Printf("Removing file %v", req.dest)
				}
				err = rmFile(req.dest)
			case removeRecurseOp:
				err = uploadRmDashR(req.dest, ch)
			default:
				log.Fatalf("Unhandled case")
			}
			if err != nil {
				if retries < 3 {
					retries++
					log.Printf("Error uploading file: %v... retrying",
						err)
					time.Sleep(time.Duration(retries) * time.Second)
				} else {
					log.Fatalf("Error uploading file: %v", err)
				}
			} else {
				done = true
			}
		}
	}
}

func syncPath(path, dest string, info os.FileInfo, ch chan<- uploadReq) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	children, err := f.Readdir(0)
	if err != nil {
		return err
	}

	retries := 3
	serverListing, err := listStuff(dest)
	for err != nil && retries > 0 {
		serverListing, err = listStuff(dest)
		retries--
	}
	if err != nil {
		return err
	}

	localNames := map[string]os.FileInfo{}
	for _, c := range children {
		switch c.Mode() & os.ModeType {
		case os.ModeCharDevice, os.ModeDevice,
			os.ModeNamedPipe, os.ModeSocket, os.ModeSymlink:
			if *uploadVerbose {
				log.Printf("Ignoring special file: %v", path)
			}
			return nil
		default:
			localNames[c.Name()] = c
		}
	}

	remoteNames := map[string]bool{}
	for n := range serverListing.Files {
		remoteNames[n] = true
	}
	for n := range serverListing.Dirs {
		remoteNames[n] = true
	}

	missingUpstream := []string{}
	for n, fi := range localNames {
		if !(fi.IsDir() || remoteNames[n]) {
			missingUpstream = append(missingUpstream, n)
		} else if !fi.IsDir() {
			if ri, ok := serverListing.Files[n]; ok {
				ch <- uploadReq{filepath.Join(path, n),
					dest + "/" + n, uploadFileOp, ri.OID}
			}
		}
	}

	toRm := []string{}
	for n := range remoteNames {
		if _, ok := localNames[n]; !ok {
			toRm = append(toRm, n)
		}
	}

	if len(missingUpstream) > 0 {
		for _, m := range missingUpstream {
			ch <- uploadReq{filepath.Join(path, m),
				dest + "/" + m, uploadFileOp, ""}
		}
	}

	if *uploadDelete && len(toRm) > 0 {
		for _, m := range toRm {
			ch <- uploadReq{"", dest + "/" + m, removeFileOp, ""}
			ch <- uploadReq{"", dest + "/" + m, removeRecurseOp, ""}
		}
	}

	return nil
}

func syncUp(src, u string, ch chan<- uploadReq) {
	for strings.HasSuffix(u, "/") {
		u = u[:len(u)-1]
	}
	for strings.HasSuffix(src, "/") {
		src = src[:len(src)-1]
	}

	err := filepath.Walk(src,
		func(path string, info os.FileInfo, err error) error {
			if err == nil && info.IsDir() {
				shortPath := path[len(src):]
				err = syncPath(path, u+shortPath, info, ch)
			}
			return err
		})
	if err != nil {
		log.Fatalf("Traversal error: %v", err)
	}
}

func uploadCommand(args []string) {
	uploadFlags.Parse(args)

	if uploadFlags.NArg() < 2 {
		log.Fatalf("src and dest required")
	}

	fi, err := os.Stat(uploadFlags.Arg(0))
	if err != nil {
		log.Fatal(err)
	}

	if fi.IsDir() {
		ch := make(chan uploadReq, 1000)

		for i := 0; i < *workers; i++ {
			uploadWg.Add(1)
			go uploadWorker(ch)
		}

		start := time.Now()
		syncUp(uploadFlags.Arg(0), uploadFlags.Arg(1), ch)

		close(ch)
		log.Printf("Finished traversal in %v", time.Since(start))
		uploadWg.Wait()
		log.Printf("Finished sync in %v", time.Since(start))
	} else {
		err = uploadFile(uploadFlags.Arg(0), uploadFlags.Arg(1))
		if err != nil {
			log.Fatalf("Error uploading file: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Verify that unbuffered channels act as pure fifos.

package main

const N = 10

func AsynchFifo() {
	ch := new(chan int, N);
	for i := 0; i < N; i++ {
		ch -< i
	}
	for i := 0; i < N; i++ {
		if <-ch != i {
			print "bad receive\n";
			sys.exit(1);
		}
	}
}

func Chain(ch *chan<- int, val int, in *chan<- int, out *chan-< int) {
	<-in;
	if <-ch != val {
		panic val
	}
	out -< 1
}

// thread together a daisy chain to read the elements in sequence
func SynchFifo() {
	ch := new(chan int);
	in := new(chan int);
	start := in;
	for i := 0; i < N; i++ {
		out := new(chan int);
		go Chain(ch, i, in, out);
		in = out;
	}
	start -< 0;
	for i := 0; i < N; i++ {
		ch -< i
	}
}

func main() {
	AsynchFifo();
	SynchFifo();
}

<commit_msg>fix bug - need to read final message in daisy chain to avoid hang<commit_after>// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Verify that unbuffered channels act as pure fifos.

package main

const N = 10

func AsynchFifo() {
	ch := new(chan int, N);
	for i := 0; i < N; i++ {
		ch -< i
	}
	for i := 0; i < N; i++ {
		if <-ch != i {
			print "bad receive\n";
			sys.exit(1);
		}
	}
}

func Chain(ch *chan<- int, val int, in *chan<- int, out *chan-< int) {
	<-in;
	if <-ch != val {
		panic val
	}
	out -< 1
}

// thread together a daisy chain to read the elements in sequence
func SynchFifo() {
	ch := new(chan int);
	in := new(chan int);
	start := in;
	for i := 0; i < N; i++ {
		out := new(chan int);
		go Chain(ch, i, in, out);
		in = out;
	}
	start -< 0;
	for i := 0; i < N; i++ {
		ch -< i
	}
	<-in
}

func main() {
	AsynchFifo();
	SynchFifo();
}

<|endoftext|>


<commit_before>package channel

import (
	"bytes"
	"strings"
	"time"

	"github.com/scrapli/scrapligo/logging"
)

func (c *Channel) getPrompt() *channelResult {
	err := c.SendReturn()
	if err != nil {
		return &channelResult{
			result: make([]byte, 0),
			error:  err,
		}
	}

	b := make([]byte, 100)

	for {
		chunk, readErr := c.Read()
		b = append(b, chunk...)

		if readErr != nil {
			return &channelResult{
				result: make([]byte, 0),
				error:  readErr,
			}
		}

		channelMatch := c.CommsPromptPattern.Match(b)
		if channelMatch {
			return &channelResult{
				result: b,
				error:  nil,
			}
		}
	}
}

// GetPrompt fetch the current prompt.
func (c *Channel) GetPrompt() (string, error) {
	_c := make(chan *channelResult, 1)

	go func() {
		r := c.getPrompt()
		_c <- r
		close(_c)
	}()

	timer := time.NewTimer(c.DetermineOperationTimeout(*c.TimeoutOps))

	select {
	case r := <-_c:
		return strings.TrimSpace(string(bytes.Trim(r.result, "\x00"))), r.error
	case <-timer.C:
		logging.LogError(c.FormatLogMessage("error", "timed out sending getting prompt"))

		return "", ErrChannelTimeout
	}
}
<commit_msg>added capacity to prompt slice<commit_after>package channel

import (
	"bytes"
	"strings"
	"time"

	"github.com/scrapli/scrapligo/logging"
)

func (c *Channel) getPrompt() *channelResult {
	err := c.SendReturn()
	if err != nil {
		return &channelResult{
			result: make([]byte, 0),
			error:  err,
		}
	}

	b := make([]byte, 0, 100)

	for {
		chunk, readErr := c.Read()
		b = append(b, chunk...)

		if readErr != nil {
			return &channelResult{
				result: make([]byte, 0),
				error:  readErr,
			}
		}

		channelMatch := c.CommsPromptPattern.Match(b)
		if channelMatch {
			return &channelResult{
				result: b,
				error:  nil,
			}
		}
	}
}

// GetPrompt fetch the current prompt.
func (c *Channel) GetPrompt() (string, error) {
	_c := make(chan *channelResult, 1)

	go func() {
		r := c.getPrompt()
		_c <- r
		close(_c)
	}()

	timer := time.NewTimer(c.DetermineOperationTimeout(*c.TimeoutOps))

	select {
	case r := <-_c:
		return strings.TrimSpace(string(bytes.Trim(r.result, "\x00"))), r.error
	case <-timer.C:
		logging.LogError(c.FormatLogMessage("error", "timed out sending getting prompt"))

		return "", ErrChannelTimeout
	}
}
<|endoftext|>


<commit_before>// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "runtime"

const	N	= 1000;		// sent messages
const	M	= 10;		// receiving goroutines
const	W	= 2;		// channel buffering
var	h	[N]int;		// marking of send/recv

func
r(c chan int, m int) {
	for {
		select {
		case r := <- c:
			if h[r] != 1 {
				panicln("r",
					"m=", m,
					"r=", r,
					"h=", h[r],
				);
			}
			h[r] = 2;
		}
	}
}

func
s(c chan int) {
	for n:=0; n<N; n++ {
		r := n;
		if h[r] != 0 {
			panicln("s");
		}
		h[r] = 1;
		c <- r;
	}
}

func
main() {
	c := make(chan int, W);
	for m:=0; m<M; m++ {
		go r(c, m);
		runtime.Gosched();
	}
	runtime.Gosched();
	runtime.Gosched();
	s(c);
}
<commit_msg>trailing comma's are not accepted with current syntax<commit_after>// $G $D/$F.go && $L $F.$A && ./$A.out

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import "runtime"

const	N	= 1000;		// sent messages
const	M	= 10;		// receiving goroutines
const	W	= 2;		// channel buffering
var	h	[N]int;		// marking of send/recv

func
r(c chan int, m int) {
	for {
		select {
		case r := <- c:
			if h[r] != 1 {
				panicln("r",
					"m=", m,
					"r=", r,
					"h=", h[r]);
			}
			h[r] = 2;
		}
	}
}

func
s(c chan int) {
	for n:=0; n<N; n++ {
		r := n;
		if h[r] != 0 {
			panicln("s");
		}
		h[r] = 1;
		c <- r;
	}
}

func
main() {
	c := make(chan int, W);
	for m:=0; m<M; m++ {
		go r(c, m);
		runtime.Gosched();
	}
	runtime.Gosched();
	runtime.Gosched();
	s(c);
}
<|endoftext|>


<commit_before>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr + "-" + mode
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }

        if mode == "reader" && ! reader {
          log.Println(nntp.name, "we don't support reader on this feed, dropping")
          return
        }
        self.register_outfeed <- nntp
        nntp.runConnection(self, false, stream, reader, mode, c)
        self.deregister_outfeed <- nntp

      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      log.Println("federate", nntp.MessageID())
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          if strings.HasSuffix(feed.name, "-stream") {
            log.Println("send", nntp.MessageID(), "to", feed.name)
            feed.check <- nntp.MessageID()
          }
        }
      }
    case nntp := <- self.ask_for_article:
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          if strings.HasSuffix(feed.name, "-reader") {
            log.Println("asking", feed.name, "for", nntp.MessageID(), "mode", feed.mode)
            feed.article <- nntp.MessageID()
          }
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      //log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      //root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      //self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, "stream", c)
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<commit_msg>logging<commit_after>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr + "-" + mode
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }

        if mode == "reader" && ! reader {
          log.Println(nntp.name, "we don't support reader on this feed, dropping")
          return
        }
        self.register_outfeed <- nntp
        nntp.runConnection(self, false, stream, reader, mode, c)
        self.deregister_outfeed <- nntp

      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      log.Println("federate", nntp.MessageID())
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          if strings.HasSuffix(feed.name, "-stream") {
            log.Println("send", nntp.MessageID(), "to", feed.name)
            feed.check <- nntp.MessageID()
          }
        } else {
          log.Println("not allowed", feed.name)
        }
      }
    case nntp := <- self.ask_for_article:
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          if strings.HasSuffix(feed.name, "-reader") {
            log.Println("asking", feed.name, "for", nntp.MessageID(), "mode", feed.mode)
            feed.article <- nntp.MessageID()
          }
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      //log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      //root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      //self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, "stream", c)
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<|endoftext|>


<commit_before>package mknod

import (
	"errors"
	"flag"
	"strconv"
	"strings"
	"syscall"
)

var (
	flagSet   = flag.NewFlagSet("mknod", flag.PanicOnError)
	majorFlag = flagSet.Int("major", -1, "Major number of the block device")
	minorFlag = flagSet.Int("minor", -1, "Minor number of the block device")
	typeFlag  = flagSet.String("type", "", "Type of the node to create (i.e. socket, link, regular, block, directory, character, fifo)")
	modeFlag  = flagSet.String("mode", "644", "Mode (in octal) to create the node with")
	helpFlag  = flagSet.Bool("help", false, "Show this help")

	typemap = map[string]uint32{
		"socket":    syscall.S_IFSOCK,
		"link":      syscall.S_IFLNK,
		"regular":   syscall.S_IFREG,
		"block":     syscall.S_IFBLK,
		"directory": syscall.S_IFDIR,
		"character": syscall.S_IFCHR,
		"fifo":      syscall.S_IFIFO,
	}
)

func Mknod(call []string) error {
	e := flagSet.Parse(call[1:])
	if e != nil {
		return e
	}

	if flagSet.NArg() != 1 || *helpFlag {
		println("`mknod` [options] <file>")
		flagSet.PrintDefaults()
		return nil
	}

	mode, ok := typemap[strings.ToLower(*typeFlag)]
	if !ok {
		return errors.New("Invalid node type \"" + *typeFlag + "\"")
	}

	if mode == syscall.S_IFBLK && (*majorFlag == -1 || *minorFlag == -1) {
		return errors.New("When creating a block device, both minor and major number have to be given")
	}

	fmode, e := strconv.Btoui64(*modeFlag, 8)
	if e != nil {
		return e
	}
	mode |= uint32(fmode)

	e = syscall.Mknod(flagSet.Arg(0), mode, *majorFlag<<8|*minorFlag)
	return e
}
<commit_msg>Mknod: fixed<commit_after>package mknod

import (
	"errors"
	"flag"
	"strconv"
	"strings"
	"syscall"
)

var (
	flagSet   = flag.NewFlagSet("mknod", flag.PanicOnError)
	majorFlag = flagSet.Int("major", -1, "Major number of the block device")
	minorFlag = flagSet.Int("minor", -1, "Minor number of the block device")
	typeFlag  = flagSet.String("type", "", "Type of the node to create (i.e. socket, link, regular, block, directory, character, fifo)")
	modeFlag  = flagSet.String("mode", "644", "Mode (in octal) to create the node with")
	helpFlag  = flagSet.Bool("help", false, "Show this help")

	typemap = map[string]uint32{
		"socket":    syscall.S_IFSOCK,
		"link":      syscall.S_IFLNK,
		"regular":   syscall.S_IFREG,
		"block":     syscall.S_IFBLK,
		"directory": syscall.S_IFDIR,
		"character": syscall.S_IFCHR,
		"fifo":      syscall.S_IFIFO,
	}
)

func Mknod(call []string) error {
	e := flagSet.Parse(call[1:])
	if e != nil {
		return e
	}

	if flagSet.NArg() != 1 || *helpFlag {
		println("`mknod` [options] <file>")
		flagSet.PrintDefaults()
		return nil
	}

	mode, ok := typemap[strings.ToLower(*typeFlag)]
	if !ok {
		return errors.New("Invalid node type \"" + *typeFlag + "\"")
	}

	if mode == syscall.S_IFBLK && (*majorFlag == -1 || *minorFlag == -1) {
		return errors.New("When creating a block device, both minor and major number have to be given")
	}

	fmode, e := strconv.ParseInt(*modeFlag, 8, 32)
	if e != nil {
		return e
	}
	mode |= uint32(fmode)

	e = syscall.Mknod(flagSet.Arg(0), mode, *majorFlag<<8|*minorFlag)
	return e
}
<|endoftext|>


<commit_before>package peerwriter

import (
	"bytes"
	"encoding/binary"
	"net"
	"time"

	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/torrent/internal/peerprotocol"
	"github.com/cenkalti/rain/torrent/internal/pieceio"
)

const keepAlivePeriod = 2 * time.Minute

type PeerWriter struct {
	conn       net.Conn
	queueC     chan peerprotocol.Message
	writeQueue []peerprotocol.Message
	writeC     chan peerprotocol.Message
	log        logger.Logger
	stopC      chan struct{}
	doneC      chan struct{}
}

func New(conn net.Conn, l logger.Logger) *PeerWriter {
	return &PeerWriter{
		conn:       conn,
		queueC:     make(chan peerprotocol.Message),
		writeQueue: make([]peerprotocol.Message, 0),
		writeC:     make(chan peerprotocol.Message),
		log:        l,
		stopC:      make(chan struct{}),
		doneC:      make(chan struct{}),
	}
}

func (p *PeerWriter) SendMessage(msg peerprotocol.Message) {
	select {
	case p.queueC <- msg:
	case <-p.doneC:
	}
}

func (p *PeerWriter) SendPiece(msg peerprotocol.RequestMessage, pi *pieceio.Piece) {
	m := Piece{Piece: pi, Begin: msg.Begin, Length: msg.Length}
	select {
	case p.queueC <- m:
	case <-p.doneC:
	}
}

func (p *PeerWriter) Stop() {
	close(p.stopC)
}

func (p *PeerWriter) Done() chan struct{} {
	return p.doneC
}

func (p *PeerWriter) Run() {
	defer close(p.doneC)

	go p.messageWriter()

	for {
		if len(p.writeQueue) == 0 {
			select {
			case msg := <-p.queueC:
				p.queueMessage(msg)
			case <-p.stopC:
				return
			}
		}
		msg := p.writeQueue[0]
		select {
		case msg = <-p.queueC:
			p.queueMessage(msg)
		case p.writeC <- msg:
			// TODO peer write queue array grows indefinitely. Try using linked list.
			p.writeQueue = p.writeQueue[1:]
		case <-p.stopC:
			return
		}
	}
}

func (p *PeerWriter) queueMessage(msg peerprotocol.Message) {
	// TODO cancel pending requests on choke
	p.writeQueue = append(p.writeQueue, msg)
}

func (p *PeerWriter) messageWriter() {
	keepAliveTicker := time.NewTicker(keepAlivePeriod)
	defer keepAliveTicker.Stop()
	for {
		select {
		case msg := <-p.writeC:
			p.log.Debugf("writing message of type: %q", msg.ID())
			payload, err := msg.MarshalBinary()
			if err != nil {
				p.log.Errorf("cannot marshal message [%v]: %s", msg.ID(), err.Error())
				p.conn.Close()
				return
			}
			buf := bytes.NewBuffer(make([]byte, 0, 4+1+len(payload)))
			var header = struct {
				Length uint32
				ID     peerprotocol.MessageID
			}{
				uint32(1 + len(payload)),
				msg.ID(),
			}
			_ = binary.Write(buf, binary.BigEndian, &header)
			buf.Write(payload)
			_, err = p.conn.Write(buf.Bytes())
			if err != nil {
				p.log.Errorf("cannot write message [%v]: %s", msg.ID(), err.Error())
				p.conn.Close()
				return
			}
		case <-keepAliveTicker.C:
			_, err := p.conn.Write([]byte{0, 0, 0, 0})
			if err != nil {
				p.log.Errorf("cannot write keepalive message: %s", err.Error())
				p.conn.Close()
				return
			}
		case <-p.stopC:
			return
		}
	}
}
<commit_msg>send more frequent keepalive packets<commit_after>package peerwriter

import (
	"bytes"
	"encoding/binary"
	"net"
	"time"

	"github.com/cenkalti/rain/internal/logger"
	"github.com/cenkalti/rain/torrent/internal/peerprotocol"
	"github.com/cenkalti/rain/torrent/internal/pieceio"
)

const keepAlivePeriod = 2 * time.Minute

type PeerWriter struct {
	conn       net.Conn
	queueC     chan peerprotocol.Message
	writeQueue []peerprotocol.Message
	writeC     chan peerprotocol.Message
	log        logger.Logger
	stopC      chan struct{}
	doneC      chan struct{}
}

func New(conn net.Conn, l logger.Logger) *PeerWriter {
	return &PeerWriter{
		conn:       conn,
		queueC:     make(chan peerprotocol.Message),
		writeQueue: make([]peerprotocol.Message, 0),
		writeC:     make(chan peerprotocol.Message),
		log:        l,
		stopC:      make(chan struct{}),
		doneC:      make(chan struct{}),
	}
}

func (p *PeerWriter) SendMessage(msg peerprotocol.Message) {
	select {
	case p.queueC <- msg:
	case <-p.doneC:
	}
}

func (p *PeerWriter) SendPiece(msg peerprotocol.RequestMessage, pi *pieceio.Piece) {
	m := Piece{Piece: pi, Begin: msg.Begin, Length: msg.Length}
	select {
	case p.queueC <- m:
	case <-p.doneC:
	}
}

func (p *PeerWriter) Stop() {
	close(p.stopC)
}

func (p *PeerWriter) Done() chan struct{} {
	return p.doneC
}

func (p *PeerWriter) Run() {
	defer close(p.doneC)

	go p.messageWriter()

	for {
		if len(p.writeQueue) == 0 {
			select {
			case msg := <-p.queueC:
				p.queueMessage(msg)
			case <-p.stopC:
				return
			}
		}
		msg := p.writeQueue[0]
		select {
		case msg = <-p.queueC:
			p.queueMessage(msg)
		case p.writeC <- msg:
			// TODO peer write queue array grows indefinitely. Try using linked list.
			p.writeQueue = p.writeQueue[1:]
		case <-p.stopC:
			return
		}
	}
}

func (p *PeerWriter) queueMessage(msg peerprotocol.Message) {
	// TODO cancel pending requests on choke
	p.writeQueue = append(p.writeQueue, msg)
}

func (p *PeerWriter) messageWriter() {
	keepAliveTicker := time.NewTicker(keepAlivePeriod / 2)
	defer keepAliveTicker.Stop()
	for {
		select {
		case msg := <-p.writeC:
			p.log.Debugf("writing message of type: %q", msg.ID())
			payload, err := msg.MarshalBinary()
			if err != nil {
				p.log.Errorf("cannot marshal message [%v]: %s", msg.ID(), err.Error())
				p.conn.Close()
				return
			}
			buf := bytes.NewBuffer(make([]byte, 0, 4+1+len(payload)))
			var header = struct {
				Length uint32
				ID     peerprotocol.MessageID
			}{
				uint32(1 + len(payload)),
				msg.ID(),
			}
			_ = binary.Write(buf, binary.BigEndian, &header)
			buf.Write(payload)
			_, err = p.conn.Write(buf.Bytes())
			if err != nil {
				p.log.Errorf("cannot write message [%v]: %s", msg.ID(), err.Error())
				p.conn.Close()
				return
			}
		case <-keepAliveTicker.C:
			_, err := p.conn.Write([]byte{0, 0, 0, 0})
			if err != nil {
				p.log.Errorf("cannot write keepalive message: %s", err.Error())
				p.conn.Close()
				return
			}
		case <-p.stopC:
			return
		}
	}
}
<|endoftext|>