<commit_before>package commands

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	cmds "github.com/jbenet/go-ipfs/commands"
	files "github.com/jbenet/go-ipfs/commands/files"
	core "github.com/jbenet/go-ipfs/core"
	importer "github.com/jbenet/go-ipfs/importer"
	"github.com/jbenet/go-ipfs/importer/chunk"
	dag "github.com/jbenet/go-ipfs/merkledag"
	pinning "github.com/jbenet/go-ipfs/pin"
	ft "github.com/jbenet/go-ipfs/unixfs"
	u "github.com/jbenet/go-ipfs/util"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

// how many bytes of progress to wait before sending a progress update message
const progressReaderIncrement = 1024 * 256

const progressOptionName = "progress"

type AddedObject struct {
	Name  string
	Hash  string `json:",omitempty"`
	Bytes int64  `json:",omitempty"`
}

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add an object to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG. A smarter partial add with a staging area (like git)
remains to be implemented.
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to IPFS").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption("quiet", "q", "Write minimal output"),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data"),
	},
	PreRun: func(req cmds.Request) error {
		req.SetOption(progressOptionName, true)

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			return nil
		}

		size, err := sizeFile.Size()
		if err != nil {
			// see comment above
			return nil
		}
		log.Debugf("Total size of file being added: %v\n", size)
		req.Values()["size"] = size

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.Context().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		progress, _, _ := req.Option(progressOptionName).Bool()

		outChan := make(chan interface{})
		res.SetOutput((<-chan interface{})(outChan))

		go func() {
			defer close(outChan)

			for {
				file, err := req.Files().NextFile()
				if (err != nil && err != io.EOF) || file == nil {
					return
				}

				_, err = addFile(n, file, outChan, progress)
				if err != nil {
					return
				}
			}
		}()
	},
	PostRun: func(res cmds.Response) {
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}
		res.SetOutput(nil)

		quiet, _, err := res.Request().Option("quiet").Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		size := int64(0)
		s, found := res.Request().Values()["size"]
		if found {
			size = s.(int64)
		}
		showProgressBar := size >= progressBarMinSize

		var bar *pb.ProgressBar
		var terminalWidth int
		if showProgressBar {
			bar = pb.New64(size).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.Start()

			// the progress bar lib doesn't give us a way to get the width of the output,
			// so as a hack we just use a callback to measure the output, then git rid of it
			terminalWidth = 0
			bar.Callback = func(line string) {
				terminalWidth = len(line)
				bar.Callback = nil
				bar.Output = os.Stderr
				log.Infof("terminal width: %v\n", terminalWidth)
			}
			bar.Update()
		}

		lastFile := ""
		var totalProgress, prevFiles, lastBytes int64

		for out := range outChan {
			output := out.(*AddedObject)
			if len(output.Hash) > 0 {
				if showProgressBar {
					// clear progress bar line before we print "added x" output
					fmt.Fprintf(os.Stderr, "\r%s\r", strings.Repeat(" ", terminalWidth))
				}
				if quiet {
					fmt.Printf("%s\n", output.Hash)
				} else {
					fmt.Printf("added %s %s\n", output.Hash, output.Name)
				}

			} else {
				log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

				if !showProgressBar {
					continue
				}

				if len(lastFile) == 0 {
					lastFile = output.Name
				}
				if output.Name != lastFile || output.Bytes < lastBytes {
					prevFiles += lastBytes
					lastFile = output.Name
				}
				lastBytes = output.Bytes
				delta := prevFiles + lastBytes - totalProgress
				totalProgress = bar.Add64(delta)
			}

			if showProgressBar {
				bar.Update()
			}
		}
	},
	Type: AddedObject{},
}

func add(n *core.IpfsNode, readers []io.Reader) ([]*dag.Node, error) {
	mp, ok := n.Pinning.(pinning.ManualPinner)
	if !ok {
		return nil, errors.New("invalid pinner type! expected manual pinner")
	}

	dagnodes := make([]*dag.Node, 0)

	for _, reader := range readers {
		node, err := importer.BuildDagFromReader(reader, n.DAG, mp, chunk.DefaultSplitter)
		if err != nil {
			return nil, err
		}
		dagnodes = append(dagnodes, node)
	}

	err := n.Pinning.Flush()
	if err != nil {
		return nil, err
	}

	return dagnodes, nil
}

func addNode(n *core.IpfsNode, node *dag.Node) error {
	err := n.DAG.AddRecursive(node) // add the file to the graph + local storage
	if err != nil {
		return err
	}

	err = n.Pinning.Pin(node, true) // ensure we keep it
	if err != nil {
		return err
	}

	return nil
}

func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool) (*dag.Node, error) {
	if file.IsDirectory() {
		return addDir(n, file, out, progress)
	}

	// if the progress flag was specified, wrap the file so that we can send
	// progress updates to the client (over the output channel)
	var reader io.Reader = file
	if progress {
		reader = &progressReader{file: file, out: out}
	}

	dns, err := add(n, []io.Reader{reader})
	if err != nil {
		return nil, err
	}

	log.Infof("adding file: %s", file.FileName())
	if err := outputDagnode(out, file.FileName(), dns[len(dns)-1]); err != nil {
		return nil, err
	}
	return dns[len(dns)-1], nil // last dag node is the file.
}

func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool) (*dag.Node, error) {
	log.Infof("adding directory: %s", dir.FileName())

	tree := &dag.Node{Data: ft.FolderPBData()}

	for {
		file, err := dir.NextFile()
		if err != nil && err != io.EOF {
			return nil, err
		}
		if file == nil {
			break
		}

		node, err := addFile(n, file, out, progress)
		if err != nil {
			return nil, err
		}

		_, name := path.Split(file.FileName())

		err = tree.AddNodeLink(name, node)
		if err != nil {
			return nil, err
		}
	}

	err := outputDagnode(out, dir.FileName(), tree)
	if err != nil {
		return nil, err
	}

	err = addNode(n, tree)
	if err != nil {
		return nil, err
	}

	return tree, nil
}

// outputDagnode sends dagnode info over the output channel
func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
	o, err := getOutput(dn)
	if err != nil {
		return err
	}

	out <- &AddedObject{
		Hash: o.Hash,
		Name: name,
	}

	return nil
}

type progressReader struct {
	file         files.File
	out          chan interface{}
	bytes        int64
	lastProgress int64
}

func (i *progressReader) Read(p []byte) (int, error) {
	n, err := i.file.Read(p)

	i.bytes += int64(n)
	if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF {
		i.lastProgress = i.bytes
		i.out <- &AddedObject{
			Name:  i.file.FileName(),
			Bytes: i.bytes,
		}
	}

	return n, err
}
<commit_msg>core/commands: add: Don't show progress bar when using --quiet option<commit_after>package commands

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	cmds "github.com/jbenet/go-ipfs/commands"
	files "github.com/jbenet/go-ipfs/commands/files"
	core "github.com/jbenet/go-ipfs/core"
	importer "github.com/jbenet/go-ipfs/importer"
	"github.com/jbenet/go-ipfs/importer/chunk"
	dag "github.com/jbenet/go-ipfs/merkledag"
	pinning "github.com/jbenet/go-ipfs/pin"
	ft "github.com/jbenet/go-ipfs/unixfs"
	u "github.com/jbenet/go-ipfs/util"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

// how many bytes of progress to wait before sending a progress update message
const progressReaderIncrement = 1024 * 256

const progressOptionName = "progress"

type AddedObject struct {
	Name  string
	Hash  string `json:",omitempty"`
	Bytes int64  `json:",omitempty"`
}

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add an object to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG. A smarter partial add with a staging area (like git)
remains to be implemented.
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to IPFS").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption("quiet", "q", "Write minimal output"),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data"),
	},
	PreRun: func(req cmds.Request) error {
		if quiet, _, _ := req.Option("quiet").Bool(); quiet {
			return nil
		}

		req.SetOption(progressOptionName, true)

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			return nil
		}

		size, err := sizeFile.Size()
		if err != nil {
			// see comment above
			return nil
		}
		log.Debugf("Total size of file being added: %v\n", size)
		req.Values()["size"] = size

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.Context().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		progress, _, _ := req.Option(progressOptionName).Bool()

		outChan := make(chan interface{})
		res.SetOutput((<-chan interface{})(outChan))

		go func() {
			defer close(outChan)

			for {
				file, err := req.Files().NextFile()
				if (err != nil && err != io.EOF) || file == nil {
					return
				}

				_, err = addFile(n, file, outChan, progress)
				if err != nil {
					return
				}
			}
		}()
	},
	PostRun: func(res cmds.Response) {
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}
		res.SetOutput(nil)

		quiet, _, err := res.Request().Option("quiet").Bool()
		if err != nil {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		size := int64(0)
		s, found := res.Request().Values()["size"]
		if found {
			size = s.(int64)
		}
		showProgressBar := !quiet && size >= progressBarMinSize

		var bar *pb.ProgressBar
		var terminalWidth int
		if showProgressBar {
			bar = pb.New64(size).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.Start()

			// the progress bar lib doesn't give us a way to get the width of the output,
			// so as a hack we just use a callback to measure the output, then git rid of it
			terminalWidth = 0
			bar.Callback = func(line string) {
				terminalWidth = len(line)
				bar.Callback = nil
				bar.Output = os.Stderr
				log.Infof("terminal width: %v\n", terminalWidth)
			}
			bar.Update()
		}

		lastFile := ""
		var totalProgress, prevFiles, lastBytes int64

		for out := range outChan {
			output := out.(*AddedObject)
			if len(output.Hash) > 0 {
				if showProgressBar {
					// clear progress bar line before we print "added x" output
					fmt.Fprintf(os.Stderr, "\r%s\r", strings.Repeat(" ", terminalWidth))
				}
				if quiet {
					fmt.Printf("%s\n", output.Hash)
				} else {
					fmt.Printf("added %s %s\n", output.Hash, output.Name)
				}

			} else {
				log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

				if !showProgressBar {
					continue
				}

				if len(lastFile) == 0 {
					lastFile = output.Name
				}
				if output.Name != lastFile || output.Bytes < lastBytes {
					prevFiles += lastBytes
					lastFile = output.Name
				}
				lastBytes = output.Bytes
				delta := prevFiles + lastBytes - totalProgress
				totalProgress = bar.Add64(delta)
			}

			if showProgressBar {
				bar.Update()
			}
		}
	},
	Type: AddedObject{},
}

func add(n *core.IpfsNode, readers []io.Reader) ([]*dag.Node, error) {
	mp, ok := n.Pinning.(pinning.ManualPinner)
	if !ok {
		return nil, errors.New("invalid pinner type! expected manual pinner")
	}

	dagnodes := make([]*dag.Node, 0)

	for _, reader := range readers {
		node, err := importer.BuildDagFromReader(reader, n.DAG, mp, chunk.DefaultSplitter)
		if err != nil {
			return nil, err
		}
		dagnodes = append(dagnodes, node)
	}

	err := n.Pinning.Flush()
	if err != nil {
		return nil, err
	}

	return dagnodes, nil
}

func addNode(n *core.IpfsNode, node *dag.Node) error {
	err := n.DAG.AddRecursive(node) // add the file to the graph + local storage
	if err != nil {
		return err
	}

	err = n.Pinning.Pin(node, true) // ensure we keep it
	if err != nil {
		return err
	}

	return nil
}

func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool) (*dag.Node, error) {
	if file.IsDirectory() {
		return addDir(n, file, out, progress)
	}

	// if the progress flag was specified, wrap the file so that we can send
	// progress updates to the client (over the output channel)
	var reader io.Reader = file
	if progress {
		reader = &progressReader{file: file, out: out}
	}

	dns, err := add(n, []io.Reader{reader})
	if err != nil {
		return nil, err
	}

	log.Infof("adding file: %s", file.FileName())
	if err := outputDagnode(out, file.FileName(), dns[len(dns)-1]); err != nil {
		return nil, err
	}
	return dns[len(dns)-1], nil // last dag node is the file.
}

func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool) (*dag.Node, error) {
	log.Infof("adding directory: %s", dir.FileName())

	tree := &dag.Node{Data: ft.FolderPBData()}

	for {
		file, err := dir.NextFile()
		if err != nil && err != io.EOF {
			return nil, err
		}
		if file == nil {
			break
		}

		node, err := addFile(n, file, out, progress)
		if err != nil {
			return nil, err
		}

		_, name := path.Split(file.FileName())

		err = tree.AddNodeLink(name, node)
		if err != nil {
			return nil, err
		}
	}

	err := outputDagnode(out, dir.FileName(), tree)
	if err != nil {
		return nil, err
	}

	err = addNode(n, tree)
	if err != nil {
		return nil, err
	}

	return tree, nil
}

// outputDagnode sends dagnode info over the output channel
func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
	o, err := getOutput(dn)
	if err != nil {
		return err
	}

	out <- &AddedObject{
		Hash: o.Hash,
		Name: name,
	}

	return nil
}

type progressReader struct {
	file         files.File
	out          chan interface{}
	bytes        int64
	lastProgress int64
}

func (i *progressReader) Read(p []byte) (int, error) {
	n, err := i.file.Read(p)

	i.bytes += int64(n)
	if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF {
		i.lastProgress = i.bytes
		i.out <- &AddedObject{
			Name:  i.file.FileName(),
			Bytes: i.bytes,
		}
	}

	return n, err
}
<|endoftext|>


<commit_before>package container

import (
	"testing"

	"github.com/docker/docker/api/types"
)

func TestCalculateBlockIO(t *testing.T) {
	blkio := types.BlkioStats{
		IoServiceBytesRecursive: []types.BlkioStatEntry{{8, 0, "read", 1234}, {8, 1, "read", 4567}, {8, 0, "write", 123}, {8, 1, "write", 456}},
	}
	blkRead, blkWrite := calculateBlockIO(blkio)
	if blkRead != 5801 {
		t.Fatalf("blkRead = %d, want 5801", blkRead)
	}
	if blkWrite != 579 {
		t.Fatalf("blkWrite = %d, want 579", blkWrite)
	}
}
<commit_msg>fixed:go vetting warning unkeyed fields<commit_after>package container

import (
	"testing"

	"github.com/docker/docker/api/types"
)

func TestCalculateBlockIO(t *testing.T) {
	blkio := types.BlkioStats{
		IoServiceBytesRecursive: []types.BlkioStatEntry{{Major: 8, Minor: 0, Op: "read", Value: 1234}, {Major: 8, Minor: 1, Op: "read", Value: 4567}, {Major: 8, Minor: 0, Op: "write", Value: 123}, {Major: 8, Minor: 1, Op: "write", Value: 456}},
	}
	blkRead, blkWrite := calculateBlockIO(blkio)
	if blkRead != 5801 {
		t.Fatalf("blkRead = %d, want 5801", blkRead)
	}
	if blkWrite != 579 {
		t.Fatalf("blkWrite = %d, want 579", blkWrite)
	}
}
<|endoftext|>


<commit_before>package commandevaluators

import (
	"errors"
	"log"
	"strings"

	"github.com/byuoitav/av-api/base"
	"github.com/byuoitav/av-api/dbo"
	"github.com/byuoitav/configuration-database-microservice/accessors"
	"github.com/byuoitav/event-router-microservice/eventinfrastructure"
)

// Standby is struct that implements the CommandEvaluation struct
type StandbyDefault struct {
}

// Evaluate fulfills the CommmandEvaluation evaluate requirement.
func (s *StandbyDefault) Evaluate(room base.PublicRoom) (actions []base.ActionStructure, err error) {

	log.Printf("Evaluating for Standby Command.")

	var devices []accessors.Device
	eventInfo := eventinfrastructure.EventInfo{
		Type:           eventinfrastructure.CORESTATE,
		EventCause:     eventinfrastructure.USERINPUT,
		EventInfoKey:   "power",
		EventInfoValue: "standby",
	}

	if strings.EqualFold(room.Power, "standby") {

		log.Printf("Room-wide power set. Retrieving all devices.")
		devices, err = dbo.GetDevicesByRoom(room.Building, room.Room)
		if err != nil {
			return
		}

		log.Printf("Setting power to 'standby' state for all output devices.")
		for _, device := range devices {

			containsStandby := false
			for _, ps := range device.PowerStates {
				if ps == "Standby" {
					containsStandby = true
					break
				}
			}
			if containsStandby {
				log.Printf("Adding device %+v", device.Name)
				eventInfo.Device = device.Name
				actions = append(actions, base.ActionStructure{
					Action:              "Standby",
					Device:              device,
					GeneratingEvaluator: "StandbyDefault",
					DeviceSpecific:      false,
					EventLog:            []eventinfrastructure.EventInfo{eventInfo},
				})
			}
			/*
				if device.Output {

					log.Printf("Adding device %+v", device.Name)
					eventInfo.Device = device.Name
					actions = append(actions, base.ActionStructure{
						Action:              "Standby",
						Device:              device,
						GeneratingEvaluator: "StandbyDefault",
						DeviceSpecific:      false,
						EventLog:            []eventinfrastructure.EventInfo{eventInfo},
					})
				}
			*/
		}
	}

	// now we go through and check if power 'standby' was set for any other device.
	for _, device := range room.Displays {
		log.Printf("Evaluating displays for command power standby. ")
		actions, err = s.evaluateDevice(device.Device, actions, devices, room.Room, room.Building, eventInfo)
		if err != nil {
			return
		}
	}

	for _, device := range room.AudioDevices {
		log.Printf("Evaluating audio devices for command power on. ")
		actions, err = s.evaluateDevice(device.Device, actions, devices, room.Room, room.Building, eventInfo)
		if err != nil {
			return
		}
	}
	log.Printf("%v actions generated.", len(actions))
	log.Printf("Evaluation complete.")

	return
}

// Validate fulfills the Fulfill requirement on the command interface
func (s *StandbyDefault) Validate(action base.ActionStructure) (err error) {
	log.Printf("Validating action for command Standby.")

	ok, _ := CheckCommands(action.Device.Commands, "Standby")
	if !ok || !strings.EqualFold(action.Action, "Standby") {
		log.Printf("ERROR. %s is an invalid command for %s", action.Action, action.Device.GetFullName())
		return errors.New(action.Action + " is an invalid command for" + action.Device.GetFullName())
	}

	log.Printf("Done.")
	return
}

// GetIncompatibleCommands keeps track of actions that are incompatable (on the same device)
func (s *StandbyDefault) GetIncompatibleCommands() (incompatableActions []string) {
	incompatableActions = []string{
		"PowerOn",
	}

	return
}

// Evaluate devices just pulls out the process we do with the audio-devices and displays into one function.
func (s *StandbyDefault) evaluateDevice(device base.Device, actions []base.ActionStructure, devices []accessors.Device, room string, building string, eventInfo eventinfrastructure.EventInfo) ([]base.ActionStructure, error) {
	// Check if we even need to start anything
	if strings.EqualFold(device.Power, "standby") {
		// check if we already added it
		index := checkActionListForDevice(actions, device.Name, room, building)
		if index == -1 {

			// Get the device, check the list of already retreived devices first, if not there,
			// hit the DB up for it.
			dev, err := getDevice(devices, device.Name, room, building)
			if err != nil {
				return actions, err
			}

			eventInfo.Device = device.Name
			actions = append(actions, base.ActionStructure{
				Action:              "Standby",
				Device:              dev,
				GeneratingEvaluator: "StandbyDefault",
				DeviceSpecific:      true,
				EventLog:            []eventinfrastructure.EventInfo{eventInfo},
			})
		}
	}
	return actions, nil
}
<commit_msg>equal fold, removed comments<commit_after>package commandevaluators

import (
	"errors"
	"log"
	"strings"

	"github.com/byuoitav/av-api/base"
	"github.com/byuoitav/av-api/dbo"
	"github.com/byuoitav/configuration-database-microservice/accessors"
	"github.com/byuoitav/event-router-microservice/eventinfrastructure"
)

// Standby is struct that implements the CommandEvaluation struct
type StandbyDefault struct {
}

// Evaluate fulfills the CommmandEvaluation evaluate requirement.
func (s *StandbyDefault) Evaluate(room base.PublicRoom) (actions []base.ActionStructure, err error) {

	log.Printf("Evaluating for Standby Command.")

	var devices []accessors.Device
	eventInfo := eventinfrastructure.EventInfo{
		Type:           eventinfrastructure.CORESTATE,
		EventCause:     eventinfrastructure.USERINPUT,
		EventInfoKey:   "power",
		EventInfoValue: "standby",
	}

	if strings.EqualFold(room.Power, "standby") {

		log.Printf("Room-wide power set. Retrieving all devices.")
		devices, err = dbo.GetDevicesByRoom(room.Building, room.Room)
		if err != nil {
			return
		}

		log.Printf("Setting power to 'standby' state for all devices with a 'standby' power state.")
		for _, device := range devices {
			containsStandby := false
			for _, ps := range device.PowerStates {
				if strings.EqualFold(ps, "Standby") {
					containsStandby = true
					break
				}
			}

			if containsStandby {
				log.Printf("Adding device %+v", device.Name)
				eventInfo.Device = device.Name
				actions = append(actions, base.ActionStructure{
					Action:              "Standby",
					Device:              device,
					GeneratingEvaluator: "StandbyDefault",
					DeviceSpecific:      false,
					EventLog:            []eventinfrastructure.EventInfo{eventInfo},
				})
			}
		}
	}

	// now we go through and check if power 'standby' was set for any other device.
	for _, device := range room.Displays {
		log.Printf("Evaluating displays for command power standby. ")
		actions, err = s.evaluateDevice(device.Device, actions, devices, room.Room, room.Building, eventInfo)
		if err != nil {
			return
		}
	}

	for _, device := range room.AudioDevices {
		log.Printf("Evaluating audio devices for command power on. ")
		actions, err = s.evaluateDevice(device.Device, actions, devices, room.Room, room.Building, eventInfo)
		if err != nil {
			return
		}
	}
	log.Printf("%v actions generated.", len(actions))
	log.Printf("Evaluation complete.")

	return
}

// Validate fulfills the Fulfill requirement on the command interface
func (s *StandbyDefault) Validate(action base.ActionStructure) (err error) {
	log.Printf("Validating action for command Standby.")

	ok, _ := CheckCommands(action.Device.Commands, "Standby")
	if !ok || !strings.EqualFold(action.Action, "Standby") {
		log.Printf("ERROR. %s is an invalid command for %s", action.Action, action.Device.GetFullName())
		return errors.New(action.Action + " is an invalid command for" + action.Device.GetFullName())
	}

	log.Printf("Done.")
	return
}

// GetIncompatibleCommands keeps track of actions that are incompatable (on the same device)
func (s *StandbyDefault) GetIncompatibleCommands() (incompatableActions []string) {
	incompatableActions = []string{
		"PowerOn",
	}

	return
}

// Evaluate devices just pulls out the process we do with the audio-devices and displays into one function.
func (s *StandbyDefault) evaluateDevice(device base.Device, actions []base.ActionStructure, devices []accessors.Device, room string, building string, eventInfo eventinfrastructure.EventInfo) ([]base.ActionStructure, error) {
	// Check if we even need to start anything
	if strings.EqualFold(device.Power, "standby") {
		// check if we already added it
		index := checkActionListForDevice(actions, device.Name, room, building)
		if index == -1 {

			// Get the device, check the list of already retreived devices first, if not there,
			// hit the DB up for it.
			dev, err := getDevice(devices, device.Name, room, building)
			if err != nil {
				return actions, err
			}

			eventInfo.Device = device.Name
			actions = append(actions, base.ActionStructure{
				Action:              "Standby",
				Device:              dev,
				GeneratingEvaluator: "StandbyDefault",
				DeviceSpecific:      true,
				EventLog:            []eventinfrastructure.EventInfo{eventInfo},
			})
		}
	}
	return actions, nil
}
<|endoftext|>


<commit_before>package coreapi

import (
	context "context"
	fmt "fmt"
	gopath "path"

	core "github.com/ipfs/go-ipfs/core"
	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	namesys "github.com/ipfs/go-ipfs/namesys"
	ipfspath "github.com/ipfs/go-ipfs/path"
	resolver "github.com/ipfs/go-ipfs/path/resolver"
	uio "github.com/ipfs/go-ipfs/unixfs/io"

	ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
	cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
)

// path implements coreiface.Path
type path struct {
	path ipfspath.Path
}

// resolvedPath implements coreiface.resolvedPath
type resolvedPath struct {
	path
	cid       *cid.Cid
	root      *cid.Cid
	remainder string
}

// IpfsPath parses the path from `c`, reruns the parsed path.
func (api *CoreAPI) IpfsPath(c *cid.Cid) coreiface.ResolvedPath {
	return &resolvedPath{
		path:      path{ipfspath.Path("/ipfs/" + c.String())},
		cid:       c,
		root:      c,
		remainder: "",
	}
}

// IpldPath parses the path from `c`, reruns the parsed path.
func (api *CoreAPI) IpldPath(c *cid.Cid) coreiface.ResolvedPath {
	return &resolvedPath{
		path:      path{ipfspath.Path("/ipld/" + c.String())},
		cid:       c,
		root:      c,
		remainder: "",
	}
}

// ResolveNode resolves the path `p` using Unixfs resolver, gets and returns the
// resolved Node.
func (api *CoreAPI) ResolveNode(ctx context.Context, p coreiface.Path) (ipld.Node, error) {
	return resolveNode(ctx, api.node.DAG, api.node.Namesys, p)
}

// ResolvePath resolves the path `p` using Unixfs resolver, returns the
// resolved path.
func (api *CoreAPI) ResolvePath(ctx context.Context, p coreiface.Path) (coreiface.ResolvedPath, error) {
	return resolvePath(ctx, api.node.DAG, api.node.Namesys, p)
}

func resolveNode(ctx context.Context, ng ipld.NodeGetter, nsys namesys.NameSystem, p coreiface.Path) (ipld.Node, error) {
	rp, err := resolvePath(ctx, ng, nsys, p)
	if err != nil {
		return nil, err
	}

	node, err := ng.Get(ctx, rp.Cid())
	if err != nil {
		return nil, err
	}
	return node, nil
}

func resolvePath(ctx context.Context, ng ipld.NodeGetter, nsys namesys.NameSystem, p coreiface.Path) (coreiface.ResolvedPath, error) {
	if _, ok := p.(coreiface.ResolvedPath); ok {
		return p.(coreiface.ResolvedPath), nil
	}

	ipath := p.(*path).path
	ipath, err := core.ResolveIPNS(ctx, nsys, ipath)
	if err == core.ErrNoNamesys {
		return nil, coreiface.ErrOffline
	} else if err != nil {
		return nil, err
	}

	var resolveOnce resolver.ResolveOnce

	switch p.Namespace() {
	case "ipfs":
		resolveOnce = uio.ResolveUnixfsOnce
	case "ipld":
		resolveOnce = resolver.ResolveSingle
	default:
		return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace())
	}

	r := &resolver.Resolver{
		DAG:         ng,
		ResolveOnce: resolveOnce,
	}

	node, rest, err := r.ResolveToLastNode(ctx, ipath)
	if err != nil {
		return nil, err
	}

	var root *cid.Cid
	if ipath.IsJustAKey() {
		root = node.Cid()
	}

	return &resolvedPath{
		path:      path{ipath},
		cid:       node.Cid(),
		root:      root,
		remainder: gopath.Join(rest...),
	}, nil
}

// ParsePath parses path `p` using ipfspath parser, returns the parsed path.
func (api *CoreAPI) ParsePath(p string) (coreiface.Path, error) {
	pp, err := ipfspath.ParsePath(p)
	if err != nil {
		return nil, err
	}

	return &path{path: pp}, nil
}

func (p *path) String() string {
	return p.path.String()
}

func (p *path) Namespace() string {
	if len(p.path.Segments()) < 1 {
		panic("path without namespace") //this shouldn't happen under any scenario
	}
	return p.path.Segments()[0]
}

func (p *path) Mutable() bool {
	//TODO: MFS: check for /local
	return p.Namespace() == "ipns"
}

func (p *resolvedPath) Cid() *cid.Cid {
	return p.cid
}

func (p *resolvedPath) Root() *cid.Cid {
	return p.root
}

func (p *resolvedPath) Remainder() string {
	return p.remainder
}
<commit_msg>coreapi: fix ipns path resolving<commit_after>package coreapi

import (
	context "context"
	fmt "fmt"
	gopath "path"

	core "github.com/ipfs/go-ipfs/core"
	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	namesys "github.com/ipfs/go-ipfs/namesys"
	ipfspath "github.com/ipfs/go-ipfs/path"
	resolver "github.com/ipfs/go-ipfs/path/resolver"
	uio "github.com/ipfs/go-ipfs/unixfs/io"

	ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
	cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
)

// path implements coreiface.Path
type path struct {
	path ipfspath.Path
}

// resolvedPath implements coreiface.resolvedPath
type resolvedPath struct {
	path
	cid       *cid.Cid
	root      *cid.Cid
	remainder string
}

// IpfsPath parses the path from `c`, reruns the parsed path.
func (api *CoreAPI) IpfsPath(c *cid.Cid) coreiface.ResolvedPath {
	return &resolvedPath{
		path:      path{ipfspath.Path("/ipfs/" + c.String())},
		cid:       c,
		root:      c,
		remainder: "",
	}
}

// IpldPath parses the path from `c`, reruns the parsed path.
func (api *CoreAPI) IpldPath(c *cid.Cid) coreiface.ResolvedPath {
	return &resolvedPath{
		path:      path{ipfspath.Path("/ipld/" + c.String())},
		cid:       c,
		root:      c,
		remainder: "",
	}
}

// ResolveNode resolves the path `p` using Unixfs resolver, gets and returns the
// resolved Node.
func (api *CoreAPI) ResolveNode(ctx context.Context, p coreiface.Path) (ipld.Node, error) {
	return resolveNode(ctx, api.node.DAG, api.node.Namesys, p)
}

// ResolvePath resolves the path `p` using Unixfs resolver, returns the
// resolved path.
func (api *CoreAPI) ResolvePath(ctx context.Context, p coreiface.Path) (coreiface.ResolvedPath, error) {
	return resolvePath(ctx, api.node.DAG, api.node.Namesys, p)
}

func resolveNode(ctx context.Context, ng ipld.NodeGetter, nsys namesys.NameSystem, p coreiface.Path) (ipld.Node, error) {
	rp, err := resolvePath(ctx, ng, nsys, p)
	if err != nil {
		return nil, err
	}

	node, err := ng.Get(ctx, rp.Cid())
	if err != nil {
		return nil, err
	}
	return node, nil
}

func resolvePath(ctx context.Context, ng ipld.NodeGetter, nsys namesys.NameSystem, p coreiface.Path) (coreiface.ResolvedPath, error) {
	if _, ok := p.(coreiface.ResolvedPath); ok {
		return p.(coreiface.ResolvedPath), nil
	}

	ipath := p.(*path).path
	ipath, err := core.ResolveIPNS(ctx, nsys, ipath)
	if err == core.ErrNoNamesys {
		return nil, coreiface.ErrOffline
	} else if err != nil {
		return nil, err
	}

	var resolveOnce resolver.ResolveOnce

	switch ipath.Segments()[0] {
	case "ipfs":
		resolveOnce = uio.ResolveUnixfsOnce
	case "ipld":
		resolveOnce = resolver.ResolveSingle
	default:
		return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace())
	}

	r := &resolver.Resolver{
		DAG:         ng,
		ResolveOnce: resolveOnce,
	}

	node, rest, err := r.ResolveToLastNode(ctx, ipath)
	if err != nil {
		return nil, err
	}

	var root *cid.Cid
	if ipath.IsJustAKey() {
		root = node.Cid()
	}

	return &resolvedPath{
		path:      path{ipath},
		cid:       node.Cid(),
		root:      root,
		remainder: gopath.Join(rest...),
	}, nil
}

// ParsePath parses path `p` using ipfspath parser, returns the parsed path.
func (api *CoreAPI) ParsePath(p string) (coreiface.Path, error) {
	pp, err := ipfspath.ParsePath(p)
	if err != nil {
		return nil, err
	}

	return &path{path: pp}, nil
}

func (p *path) String() string {
	return p.path.String()
}

func (p *path) Namespace() string {
	if len(p.path.Segments()) < 1 {
		panic("path without namespace") //this shouldn't happen under any scenario
	}
	return p.path.Segments()[0]
}

func (p *path) Mutable() bool {
	//TODO: MFS: check for /local
	return p.Namespace() == "ipns"
}

func (p *resolvedPath) Cid() *cid.Cid {
	return p.cid
}

func (p *resolvedPath) Root() *cid.Cid {
	return p.root
}

func (p *resolvedPath) Remainder() string {
	return p.remainder
}
<|endoftext|>


<commit_before>package core

import (
	"database/sql"
	"errors"
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/mdlayher/wavepipe/data"

	"github.com/wtolson/go-taglib"
)

// fsFileSource represents a file source which indexes files in the local filesystem
type fsFileSource struct{}

// MediaScan scans for media files in the local filesystem
func (fsFileSource) MediaScan(mediaFolder string, walkCancelChan chan struct{}) error {
	// Halt walk if needed
	var mutex sync.RWMutex
	haltWalk := false
	go func() {
		// Wait for signal
		<-walkCancelChan

		// Halt!
		mutex.Lock()
		haltWalk = true
		mutex.Unlock()
	}()

	// Track metrics about the walk
	artistCount := 0
	albumCount := 0
	songCount := 0
	folderCount := 0
	startTime := time.Now()

	// Invoke a recursive file walk on the given media folder, passing closure variables into
	// walkFunc to enable additional functionality
	log.Println("fs: beginning media scan:", mediaFolder)
	err := filepath.Walk(mediaFolder, func(currPath string, info os.FileInfo, err error) error {
		// Stop walking immediately if needed
		mutex.RLock()
		if haltWalk {
			return errors.New("media scan: halted by channel")
		}
		mutex.RUnlock()

		// Make sure path is actually valid
		if info == nil {
			return errors.New("media scan: invalid path: " + currPath)
		}

		// Check for an existing folder for this item
		folder := new(data.Folder)
		if info.IsDir() {
			// If directory, use this path
			folder.Path = currPath
		} else {
			// If file, use the directory path
			folder.Path = path.Dir(currPath)
		}

		// Attempt to load folder
		if err := folder.Load(); err != nil && err == sql.ErrNoRows {
			// Ensure this is actually a folder
			if !info.IsDir() {
				return nil
			}

			// Set short title
			folder.Title = path.Base(folder.Path)

			// Check for a parent folder
			pFolder := new(data.Folder)
			pFolder.Path = path.Dir(currPath)
			if err := pFolder.Load(); err != nil && err != sql.ErrNoRows {
				return err
			}

			// Copy parent folder's ID
			folder.ParentID = pFolder.ID

			// Save new folder
			if err := folder.Save(); err != nil {
				return err
			}

			// Continue traversal
			folderCount++
			return nil
		}

		// Check for a valid media extension
		if !validSet.Has(path.Ext(currPath)) {
			return nil
		}

		// Attempt to scan media file with taglib
		file, err := taglib.Read(currPath)
		if err != nil {
			return fmt.Errorf("%s: %s", currPath, err.Error())
		}
		defer file.Close()

		// Generate a song model from the TagLib file
		song, err := data.SongFromFile(file)
		if err != nil {
			return err
		}

		// Populate filesystem-related struct fields using OS info
		song.FileName = currPath
		song.FileSize = info.Size()

		// Use this folder's ID
		song.FolderID = folder.ID

		// Extract type from the extension, capitalize it, drop the dot
		song.FileType = strings.ToUpper(path.Ext(info.Name()))[1:]
		song.LastModified = info.ModTime().Unix()

		// Generate an artist model from this song's metadata
		artist := data.ArtistFromSong(song)

		// Check for existing artist
		// Note: if the artist exists, this operation also loads necessary scanning information
		// such as their artist ID, for use in album and song generation
		if err := artist.Load(); err == sql.ErrNoRows {
			// Save new artist
			if err := artist.Save(); err != nil {
				log.Println(err)
			} else if err == nil {
				log.Printf("Artist: [%04d] %s", artist.ID, artist.Title)
				artistCount++
			}
		}

		// Generate the album model from this song's metadata
		album := data.AlbumFromSong(song)
		album.ArtistID = artist.ID

		// Check for existing album
		// Note: if the album exists, this operation also loads necessary scanning information
		// such as the album ID, for use in song generation
		if err := album.Load(); err == sql.ErrNoRows {
			// Save album
			if err := album.Save(); err != nil {
				log.Println(err)
			} else if err == nil {
				log.Printf("  - Album: [%04d] %s - %d - %s", album.ID, album.Artist, album.Year, album.Title)
				albumCount++
			}
		}

		// Add ID fields to song
		song.ArtistID = artist.ID
		song.AlbumID = album.ID

		// Check for existing song
		if err := song.Load(); err == sql.ErrNoRows {
			// Save song (don't log these because they really slow things down)
			if err := song.Save(); err != nil {
				log.Println(err)
			} else if err == nil {
				songCount++
			}
		}

		// Successful media scan
		return nil
	})

	// Check for filesystem walk errors
	if err != nil {
		return err
	}

	// Print metrics
	log.Printf("fs: media scan complete [time: %s]", time.Since(startTime).String())
	log.Printf("fs: added: [artists: %d] [albums: %d] [songs: %d] [folders: %d]", artistCount, albumCount, songCount, folderCount)

	// No errors
	return nil
}

// OrphanScan scans for missing "orphaned" media files in the local filesystem
func (fsFileSource) OrphanScan(baseFolder string, subFolder string, orphanCancelChan chan struct{}) error {
	// Halt scan if needed
	var mutex sync.RWMutex
	haltOrphanScan := false
	go func() {
		// Wait for signal
		<-orphanCancelChan

		// Halt!
		mutex.Lock()
		haltOrphanScan = true
		mutex.Unlock()
	}()

	// Track metrics about the scan
	songCount := 0
	startTime := time.Now()

	log.Println("fs: beginning orphan scan")

	// Check if a baseFolder is set, meaning remove ANYTHING not under this base
	if baseFolder != "" {
		log.Println("fs: orphan scanning base folder:", baseFolder)

		// Scan for all songs NOT under the base folder
		songs, err := data.DB.SongsNotInPath(baseFolder)
		if err != nil {
			return err
		}

		// Remove all songs which are not in this path
		for _, s := range songs {
			// Remove song from database
			if err := s.Delete(); err != nil {
				return err
			}

			songCount++
		}
	}

	// If no subfolder set, use the base folder to check file existence
	if subFolder == "" {
		subFolder = baseFolder
	}

	// Scan for all songs in subfolder
	log.Println("fs: orphan scanning subfolder:", subFolder)
	songs, err := data.DB.SongsInPath(subFolder)
	if err != nil {
		return err
	}

	// Iterate all songs in this path
	for _, s := range songs {
		// Check that the song still exists in this place
		if _, err := os.Stat(s.FileName); os.IsNotExist(err) {
			// Remove song from database
			if err := s.Delete(); err != nil {
				return err
			}

			songCount++
		}
	}

	// Now that songs have been purged, check for albums
	albumCount, err := data.DB.PurgeOrphanAlbums()
	if err != nil {
		return err
	}

	// Finally, check for artists
	artistCount, err := data.DB.PurgeOrphanArtists()
	if err != nil {
		return err
	}

	// Print metrics
	log.Printf("fs: orphan scan complete [time: %s]", time.Since(startTime).String())
	log.Printf("fs: removed: [artists: %d] [albums: %d] [songs: %d]", artistCount, albumCount, songCount)
	return nil
}
<commit_msg>Fix folder scanning, add orphan scanning for folders<commit_after>package core

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/mdlayher/wavepipe/data"

	"github.com/wtolson/go-taglib"
)

// fsFileSource represents a file source which indexes files in the local filesystem
type fsFileSource struct{}

// MediaScan scans for media files in the local filesystem
func (fsFileSource) MediaScan(mediaFolder string, walkCancelChan chan struct{}) error {
	// Halt walk if needed
	var mutex sync.RWMutex
	haltWalk := false
	go func() {
		// Wait for signal
		<-walkCancelChan

		// Halt!
		mutex.Lock()
		haltWalk = true
		mutex.Unlock()
	}()

	// Track metrics about the walk
	artistCount := 0
	albumCount := 0
	songCount := 0
	folderCount := 0
	startTime := time.Now()

	// Invoke a recursive file walk on the given media folder, passing closure variables into
	// walkFunc to enable additional functionality
	log.Println("fs: beginning media scan:", mediaFolder)
	err := filepath.Walk(mediaFolder, func(currPath string, info os.FileInfo, err error) error {
		// Stop walking immediately if needed
		mutex.RLock()
		if haltWalk {
			return errors.New("media scan: halted by channel")
		}
		mutex.RUnlock()

		// Make sure path is actually valid
		if info == nil {
			return errors.New("media scan: invalid path: " + currPath)
		}

		// Check for an existing folder for this item
		folder := new(data.Folder)
		if info.IsDir() {
			// If directory, use this path
			folder.Path = currPath
		} else {
			// If file, use the directory path
			folder.Path = path.Dir(currPath)
		}

		// Attempt to load folder
		if err := folder.Load(); err != nil && err == sql.ErrNoRows {
			// Make sure items actually exist at this path
			files, err := ioutil.ReadDir(folder.Path)
			if err != nil {
				return err
			}

			// No items, skip it
			if len(files) == 0 {
				return nil
			}

			// Set short title
			folder.Title = path.Base(folder.Path)

			// Check for a parent folder
			pFolder := new(data.Folder)
			pFolder.Path = path.Dir(currPath)
			if err := pFolder.Load(); err != nil && err != sql.ErrNoRows {
				return err
			}

			// Copy parent folder's ID
			folder.ParentID = pFolder.ID

			// Save new folder
			if err := folder.Save(); err != nil {
				return err
			}

			// Continue traversal
			folderCount++
		}

		// Check for a valid media extension
		if !validSet.Has(path.Ext(currPath)) {
			return nil
		}

		// Attempt to scan media file with taglib
		file, err := taglib.Read(currPath)
		if err != nil {
			return fmt.Errorf("%s: %s", currPath, err.Error())
		}
		defer file.Close()

		// Generate a song model from the TagLib file
		song, err := data.SongFromFile(file)
		if err != nil {
			return err
		}

		// Populate filesystem-related struct fields using OS info
		song.FileName = currPath
		song.FileSize = info.Size()

		// Use this folder's ID
		song.FolderID = folder.ID

		// Extract type from the extension, capitalize it, drop the dot
		song.FileType = strings.ToUpper(path.Ext(info.Name()))[1:]
		song.LastModified = info.ModTime().Unix()

		// Generate an artist model from this song's metadata
		artist := data.ArtistFromSong(song)

		// Check for existing artist
		// Note: if the artist exists, this operation also loads necessary scanning information
		// such as their artist ID, for use in album and song generation
		if err := artist.Load(); err == sql.ErrNoRows {
			// Save new artist
			if err := artist.Save(); err != nil {
				log.Println(err)
			} else if err == nil {
				log.Printf("Artist: [%04d] %s", artist.ID, artist.Title)
				artistCount++
			}
		}

		// Generate the album model from this song's metadata
		album := data.AlbumFromSong(song)
		album.ArtistID = artist.ID

		// Check for existing album
		// Note: if the album exists, this operation also loads necessary scanning information
		// such as the album ID, for use in song generation
		if err := album.Load(); err == sql.ErrNoRows {
			// Save album
			if err := album.Save(); err != nil {
				log.Println(err)
			} else if err == nil {
				log.Printf("  - Album: [%04d] %s - %d - %s", album.ID, album.Artist, album.Year, album.Title)
				albumCount++
			}
		}

		// Add ID fields to song
		song.ArtistID = artist.ID
		song.AlbumID = album.ID

		// Check for existing song
		if err := song.Load(); err == sql.ErrNoRows {
			// Save song (don't log these because they really slow things down)
			if err := song.Save(); err != nil {
				log.Println(err)
			} else if err == nil {
				songCount++
			}
		}

		// Successful media scan
		return nil
	})

	// Check for filesystem walk errors
	if err != nil {
		return err
	}

	// Print metrics
	log.Printf("fs: media scan complete [time: %s]", time.Since(startTime).String())
	log.Printf("fs: added: [artists: %d] [albums: %d] [songs: %d] [folders: %d]", artistCount, albumCount, songCount, folderCount)

	// No errors
	return nil
}

// OrphanScan scans for missing "orphaned" media files in the local filesystem
func (fsFileSource) OrphanScan(baseFolder string, subFolder string, orphanCancelChan chan struct{}) error {
	// Halt scan if needed
	var mutex sync.RWMutex
	haltOrphanScan := false
	go func() {
		// Wait for signal
		<-orphanCancelChan

		// Halt!
		mutex.Lock()
		haltOrphanScan = true
		mutex.Unlock()
	}()

	// Track metrics about the scan
	songCount := 0
	startTime := time.Now()

	log.Println("fs: beginning orphan scan")

	// Check if a baseFolder is set, meaning remove ANYTHING not under this base
	if baseFolder != "" {
		log.Println("fs: orphan scanning base folder:", baseFolder)

		// Scan for all songs NOT under the base folder
		songs, err := data.DB.SongsNotInPath(baseFolder)
		if err != nil {
			return err
		}

		// Remove all songs which are not in this path
		for _, s := range songs {
			// Remove song from database
			if err := s.Delete(); err != nil {
				return err
			}

			songCount++
		}
	}

	// If no subfolder set, use the base folder to check file existence
	if subFolder == "" {
		subFolder = baseFolder
	}

	// Scan for all songs in subfolder
	log.Println("fs: orphan scanning subfolder:", subFolder)
	songs, err := data.DB.SongsInPath(subFolder)
	if err != nil {
		return err
	}

	// Iterate all songs in this path
	for _, s := range songs {
		// Check that the song still exists in this place
		if _, err := os.Stat(s.FileName); os.IsNotExist(err) {
			// Remove song from database
			if err := s.Delete(); err != nil {
				return err
			}

			songCount++
		}
	}

	// Now that songs have been purged, check for albums
	albumCount, err := data.DB.PurgeOrphanAlbums()
	if err != nil {
		return err
	}

	// Check for artists
	artistCount, err := data.DB.PurgeOrphanArtists()
	if err != nil {
		return err
	}

	// Check for folders which contain no subfolders or songs
	folderCount, err := data.DB.PurgeOrphanFolders()
	if err != nil {
		return err
	}

	// Print metrics
	log.Printf("fs: orphan scan complete [time: %s]", time.Since(startTime).String())
	log.Printf("fs: removed: [artists: %d] [albums: %d] [songs: %d] [folders: %d]", artistCount, albumCount, songCount, folderCount)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import "github.com/kubeflow/pipelines/backend/src/apiserver/common"

// Resource reference table models the relationship between resources in a loosely coupled way.
type ResourceReference struct {
	// ID of the resource object
	ResourceUUID string `gorm:"column:ResourceUUID; not null; primary_key"`

	// The type of the resource object
	ResourceType common.ResourceType `gorm:"column:ResourceType; not null; primary_key"`

	// The ID of the resource that been referenced to.
	ReferenceUUID string `gorm:"column:ReferenceUUID; not null; "`

	// The name of the resource that been referenced to.
	ReferenceName string `gorm:"column:ReferenceName; not null; "`

	// The type of the resource that been referenced to.
	ReferenceType common.ResourceType `gorm:"column:ReferenceType; not null; primary_key"`

	// The relationship between the resource object and the resource that been referenced to.
	Relationship common.Relationship `gorm:"column:Relationship; not null; "`

	// The json formatted blob of the resource reference.
	Payload string `gorm:"column:Payload; not null; size:65535 "`
}
<commit_msg>Create composite indexes for [ResourceType, ReferenceUUID, ReferenceType] (#1836)<commit_after>// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import "github.com/kubeflow/pipelines/backend/src/apiserver/common"

// Resource reference table models the relationship between resources in a loosely coupled way.
type ResourceReference struct {
	// ID of the resource object
	ResourceUUID string `gorm:"column:ResourceUUID; not null; primary_key"`

	// The type of the resource object
	ResourceType common.ResourceType `gorm:"column:ResourceType; not null; primary_key; index:referencefilter"`

	// The ID of the resource that been referenced to.
	ReferenceUUID string `gorm:"column:ReferenceUUID; not null; index:referencefilter"`

	// The name of the resource that been referenced to.
	ReferenceName string `gorm:"column:ReferenceName; not null; "`

	// The type of the resource that been referenced to.
	ReferenceType common.ResourceType `gorm:"column:ReferenceType; not null; primary_key; index:referencefilter"`

	// The relationship between the resource object and the resource that been referenced to.
	Relationship common.Relationship `gorm:"column:Relationship; not null; "`

	// The json formatted blob of the resource reference.
	Payload string `gorm:"column:Payload; not null; size:65535 "`
}
<|endoftext|>


<commit_before>package runner

import (
	"io"
	"log"
	"os"
	"os/exec"
	"regexp"
)

type Runner interface {
	Run()
}
type subProcess struct {
	port string
	name string
	args []string
}

func New(port string, name string, args []string) Runner {
	return &subProcess{port, name, args}
}

func (sp *subProcess) Run() {

	cmd := exec.Command(sp.name, sp.args...)

	cmd.Env = envForSubProcess()
	stdout, err := cmd.StdoutPipe()
	checkError(err)
	stderr, err := cmd.StderrPipe()
	checkError(err)
	stdin, err := cmd.StderrPipe()
	checkError(err)

	err = cmd.Start()
	checkError(err)

	defer cmd.Wait()

	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	go io.Copy(os.Stdin, stdin)

}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func envForSubProcess() []string {
	var newEnv []string
	for _, v := range os.Environ() {
		if match, _ := regexp.MatchString("PORT", v); match == true {
			newEnv = append(newEnv, v)
		}
	}
	return newEnv
}
<commit_msg>feat: add new env for subprocess<commit_after>package runner

import (
	"io"
	"log"
	"os"
	"os/exec"
	"regexp"
)

type Runner interface {
	Run()
}
type subProcess struct {
	port string
	name string
	args []string
}

func New(port string, name string, args []string) Runner {
	return &subProcess{port, name, args}
}

func (sp *subProcess) Run() {

	cmd := exec.Command(sp.name, sp.args...)

	cmd.Env = sp.envForSubProcess()
	stdout, err := cmd.StdoutPipe()
	sp.checkError(err)
	stderr, err := cmd.StderrPipe()
	sp.checkError(err)
	stdin, err := cmd.StderrPipe()
	sp.checkError(err)

	err = cmd.Start()
	sp.checkError(err)

	defer cmd.Wait()

	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	go io.Copy(os.Stdin, stdin)

}

func (sp *subProcess) checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func (sp *subProcess) envForSubProcess() []string {
	return sp.addNewPort(sp.removeOldPortFromEnv())
}

func (sp *subProcess) addNewPort(env []string) []string {
	return append(env, "PORT="+sp.port)
}

func (sp *subProcess) removeOldPortFromEnv() []string {
	var newEnv []string
	for _, v := range os.Environ() {
		if match, _ := regexp.MatchString("PORT", v); match == false {
			newEnv = append(newEnv, v)
		}
	}
	return newEnv
}
<|endoftext|>


<commit_before>//
// store.go
//

package srnd

import (
  "github.com/majestrate/srndv2/src/nacl"
  "bufio"
  "bytes"
  "crypto/sha512"
  "errors"
  "io"
  "log"
  "mime"
  "mime/multipart"
  "net/mail"
  "os"
  "os/exec"
  "path/filepath"
  "strings"
)

type ArticleStore interface {
  MessageReader
  MessageWriter

  // get the filepath for an attachment
  AttachmentFilepath(fname string) string
  // get the filepath for an attachment's thumbnail
  ThumbnailFilepath(fname string) string
  // do we have this article?
  HasArticle(msgid string) bool
  // create a file for a message
  CreateFile(msgid string) io.WriteCloser
  // create a file for a temp message, returns nil if it's already open
  CreateTempFile(msgid string) io.WriteCloser
  // get the filename of a message
  GetFilename(msgid string) string
  // get the filename of a temp message
  GetTempFilename(msgid string) string
  // Get a message given its messageid
  GetMessage(msgid string) NNTPMessage
  // get a temp message given its messageid
  // temp message is deleted once read
  ReadTempMessage(msgid string) NNTPMessage
  // store a post
  StorePost(nntp NNTPMessage) error
  // get article headers only
  GetHeaders(msgid string) ArticleHeaders
  // get our temp directory for articles
  TempDir() string
  // get a list of all the attachments we have
  GetAllAttachments() ([]string, error)
  // generate a thumbnail
  GenerateThumbnail(fname string) error
}
type articleStore struct {
  directory string
  temp string
  attachments string
  thumbs string
  database Database
  convert_path string
  ffmpeg_path string
}

func createArticleStore(config map[string]string, database Database) ArticleStore {
  store := articleStore{
    directory: config["store_dir"],
    temp: config["incoming_dir"],
    attachments: config["attachments_dir"],
    thumbs: config["thumbs_dir"],
    convert_path: config["convert_bin"],
    ffmpeg_path: config["ffmpegthumbnailer_bin"],
    database: database,
  }
  store.Init()
  return store
}

func (self articleStore) TempDir() string {
  return self.temp
}

// initialize article store
func (self articleStore) Init() {
  EnsureDir(self.directory)
  EnsureDir(self.temp)
  EnsureDir(self.attachments)
  EnsureDir(self.thumbs)
  if ! CheckFile(self.convert_path) {
    log.Fatal("cannot find executable for convert:", self.convert_path, "not found")
  }
  if ! CheckFile(self.ffmpeg_path) {
    log.Fatal("connt find executable for ffmpegthumbnailer", self.ffmpeg_path, "not found")
  }
}

func (self articleStore) GenerateThumbnail(fname string) error {
  outfname := self.ThumbnailFilepath(fname)
  infname := self.AttachmentFilepath(fname)
  var cmd *exec.Cmd
  if strings.HasPrefix(strings.ToLower(fname), ".gif") {
    cmd = exec.Command(self.convert_path, "-thumbnail", "200", infname, outfname)
  } else {
    cmd = exec.Command(self.ffmpeg_path, "-i", infname, "-o", outfname, "-s", "200")
  }
  exec_out, err := cmd.CombinedOutput()
  if err != nil {
    log.Println("error generating thumbnail", string(exec_out))
  }
  return err
}

func (self articleStore) GetAllAttachments() (names []string, err error) {
  var f *os.File
  f, err = os.Open(self.attachments)
  if err == nil {
    names, err = f.Readdirnames(0)
  }
  return
}

func (self articleStore) ReadMessage(r io.Reader) (NNTPMessage, error) {
  return read_message(r)
}

func (self articleStore) StorePost(nntp NNTPMessage) (err error) {

  f := self.CreateFile(nntp.MessageID())
  if f != nil {
    err = self.WriteMessage(nntp, f)
    f.Close()
  }

  nntp_inner := nntp.Signed()
  if nntp_inner == nil {
    // no inner article
    // store the data in the article
    self.database.RegisterArticle(nntp)
    for _, att := range nntp.Attachments() {
      // save attachments
      go self.saveAttachment(att)
    }
  } else {
    // record a tripcode
    self.database.RegisterSigned(nntp.MessageID(), nntp.Pubkey())
    // we have inner data
    // store the signed data
    self.database.RegisterArticle(nntp_inner)
    for _, att := range nntp_inner.Attachments() {
      go self.saveAttachment(att)
    }
  }
  return
}

// save an attachment
func (self articleStore) saveAttachment(att NNTPAttachment) {
  var err error
  var f io.WriteCloser
  fpath := att.Filepath()
  upload := self.AttachmentFilepath(fpath)
  thumb := self.ThumbnailFilepath(fpath)
  if CheckFile(upload) {
    log.Println("already have file", fpath)
    if ! CheckFile(thumb) && att.NeedsThumbnail() {
      log.Println("create thumbnail for", fpath)
      err = self.GenerateThumbnail(fpath)
      if err != nil {
        log.Println("failed to generate thumbnail", err)
      }
    }
    return
  }
  // save attachment
  log.Println("save attachment", att.Filename(), "to", upload)
  f, err = os.Create(upload)
  if err == nil {
    err = att.WriteTo(f)
    f.Close()
  }
  if err != nil {
    log.Println("did not save attachment", err)
    return
  }

  // generate thumbanils
  if att.NeedsThumbnail() {
    log.Println("create thumbnail for", fpath)
    err = self.GenerateThumbnail(fpath)
    if err != nil {
      log.Println("failed to generate thumbnail", err)
    }
  }
}

// eh this isn't really needed is it?
func (self articleStore) WriteMessage(nntp NNTPMessage, wr io.Writer) (err error) {
  return nntp.WriteTo(wr, "\n")
}

// get the filepath for an attachment
func (self articleStore) AttachmentFilepath(fname string) string {
  return filepath.Join(self.attachments, fname)
}

// get the filepath for a thumbanil
func (self articleStore) ThumbnailFilepath(fname string) string {
  // all thumbnails are jpegs now
  return filepath.Join(self.thumbs, fname + ".jpg")
}

// create a file for this article
func (self articleStore) CreateFile(messageID string) io.WriteCloser {
  fname := self.GetFilename(messageID)
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// create a temp file for inboud articles
func (self articleStore) CreateTempFile(messageID string) io.WriteCloser {
  fname := self.GetTempFilename(messageID)
  if CheckFile(fname) {
    log.Println(fname, "already open")
    return nil
  }
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// return true if we have an article
func (self articleStore) HasArticle(messageID string) bool {
  return CheckFile(self.GetFilename(messageID))
}

// get the filename for this article
func (self articleStore) GetFilename(messageID string) string {
  if ! ValidMessageID(messageID) {
    log.Println("!!! bug: tried to open invalid message", messageID, "!!!")
    return ""
  }
  return filepath.Join(self.directory, messageID)
}

// get the filename for this article
func (self articleStore) GetTempFilename(messageID string) string {
  if ! ValidMessageID(messageID) {
    log.Println("!!! bug: tried to open invalid temp message", messageID, "!!!")
    return ""
  }
  return filepath.Join(self.temp, messageID)
}

// loads temp message and deletes old article
func (self articleStore) ReadTempMessage(messageID string) NNTPMessage {
  fname := self.GetTempFilename(messageID)
  nntp := self.readfile(fname)
  DelFile(fname)
  return nntp
}

// read a file give filepath
func (self articleStore) readfile(fname string) NNTPMessage {

  file, err := os.Open(fname)
  if err != nil {
    log.Println("store cannot open file",fname)
    return nil
  }
  message, err := self.ReadMessage(file)
  file.Close()
  if err == nil {
    return message
  }

  log.Println("failed to load file", fname)
  return nil
}

// get the replies for a thread
func (self articleStore) GetThreadReplies(messageID string, last int) []NNTPMessage {
  var repls []NNTPMessage
  if self.database.ThreadHasReplies(messageID) {
    rpls := self.database.GetThreadReplies(messageID, last)
    if rpls == nil {
      return repls
    }
    for _, rpl := range rpls {
      msg := self.GetMessage(rpl)
      if msg == nil {
        log.Println("cannot get message", rpl)
      } else {
        repls = append(repls, msg)
      }
    }
  }
  return repls
}

// load an article
// return nil on failure
func (self articleStore) GetMessage(messageID string) NNTPMessage {
  return self.readfile(self.GetFilename(messageID))
}

// get article with headers only
func (self articleStore) GetHeaders(messageID string) ArticleHeaders {
  // TODO: don't load the entire body
  nntp := self.readfile(self.GetFilename(messageID))
  if nntp == nil {
    return nil
  }
  return nntp.Headers()
}

func read_message(r io.Reader) (NNTPMessage, error) {

  msg, err := mail.ReadMessage(r)
  var nntp nntpArticle

  if err == nil {
    nntp.headers = ArticleHeaders(msg.Header)
    content_type := nntp.ContentType()
    media_type, params, err := mime.ParseMediaType(content_type)
    if err != nil {
      log.Println("failed to parse media type", err, "for mime", content_type)
      return nil, err
    }
    boundary, ok := params["boundary"]
    if ok {
      partReader := multipart.NewReader(msg.Body, boundary)
      for {
        part, err := partReader.NextPart()
        if err == io.EOF {
          return nntp, nil
        } else if err == nil {
          hdr := part.Header
          // get content type of part
          part_type := hdr.Get("Content-Type")
          // parse content type
          media_type, _, err = mime.ParseMediaType(part_type)
          if err == nil {
            if media_type == "text/plain" {
              att := readAttachmentFromMimePart(part)
              if att != nil {
                nntp.message = att.(nntpAttachment)
                nntp.message.header.Set("Content-Type", part_type)
              }
            } else {
              // non plaintext gets added to attachments
              att := readAttachmentFromMimePart(part)
              if att != nil {
                nntp = nntp.Attach(att).(nntpArticle)
              }
            }
          } else {
            log.Println("part has no content type", err)
          }
          part.Close()
        } else {
          log.Println("failed to load part! ", err)
          return nil, err
        }
      }

    } else if media_type == "message/rfc822" {
      // tripcoded message
      sig := nntp.headers.Get("X-Signature-Ed25519-Sha512", "")
      pk := nntp.Pubkey()
      if pk == "" || sig == "" {
        log.Println("invalid sig or pubkey", sig, pk)
        return nil, errors.New("invalid headers")
      }
      log.Printf("got signed message from %s", pk)
      pk_bytes := unhex(pk)
      sig_bytes := unhex(sig)
      r := bufio.NewReader(msg.Body)
      crlf := []byte{13,10}
      for {
        line, err := r.ReadBytes('\n')
        if err == io.EOF {
          break
        }
        nntp.signedPart.body.Write(line[:len(line)-1])
        nntp.signedPart.body.Write(crlf)
      }
      if nntp.signedPart.body.Len() < 2 {
        log.Println("signed body is too small")
      } else {
        body := nntp.signedPart.body.Bytes()[:nntp.signedPart.body.Len()-2]
        body_hash := sha512.Sum512(body)
        log.Printf("hash=%s", hexify(body_hash[:]))
        if nacl.CryptoVerifyFucky(body_hash[:], sig_bytes, pk_bytes) {
          log.Println("signature is valid :^)")
          return nntp, nil
        } else {
          log.Println("!!!signature is invalid!!!")
        }
      }
    } else {
      // plaintext attachment
      var buff bytes.Buffer
      _, err = io.Copy(&buff, msg.Body)
      nntp.message = createPlaintextAttachment(buff.String())
      return nntp, err
    }
  } else {
    log.Println("failed to read message", err)
    return nil, err
  }
  return nntp, err
}

<commit_msg>add stereograph thumbnailing of audio<commit_after>//
// store.go
//

package srnd

import (
  "github.com/majestrate/srndv2/src/nacl"
  "bufio"
  "bytes"
  "crypto/sha512"
  "errors"
  "io"
  "log"
  "mime"
  "mime/multipart"
  "net/mail"
  "os"
  "os/exec"
  "path/filepath"
  "strings"
)

type ArticleStore interface {
  MessageReader
  MessageWriter

  // get the filepath for an attachment
  AttachmentFilepath(fname string) string
  // get the filepath for an attachment's thumbnail
  ThumbnailFilepath(fname string) string
  // do we have this article?
  HasArticle(msgid string) bool
  // create a file for a message
  CreateFile(msgid string) io.WriteCloser
  // create a file for a temp message, returns nil if it's already open
  CreateTempFile(msgid string) io.WriteCloser
  // get the filename of a message
  GetFilename(msgid string) string
  // get the filename of a temp message
  GetTempFilename(msgid string) string
  // Get a message given its messageid
  GetMessage(msgid string) NNTPMessage
  // get a temp message given its messageid
  // temp message is deleted once read
  ReadTempMessage(msgid string) NNTPMessage
  // store a post
  StorePost(nntp NNTPMessage) error
  // get article headers only
  GetHeaders(msgid string) ArticleHeaders
  // get our temp directory for articles
  TempDir() string
  // get a list of all the attachments we have
  GetAllAttachments() ([]string, error)
  // generate a thumbnail
  GenerateThumbnail(fname string) error
}
type articleStore struct {
  directory string
  temp string
  attachments string
  thumbs string
  database Database
  convert_path string
  ffmpeg_path string
  sox_path string
}

func createArticleStore(config map[string]string, database Database) ArticleStore {
  store := articleStore{
    directory: config["store_dir"],
    temp: config["incoming_dir"],
    attachments: config["attachments_dir"],
    thumbs: config["thumbs_dir"],
    convert_path: config["convert_bin"],
    ffmpeg_path: config["ffmpegthumbnailer_bin"],
    sox_path: config["sox_bin"],
    database: database,
  }
  store.Init()
  return store
}

func (self articleStore) TempDir() string {
  return self.temp
}

// initialize article store
func (self articleStore) Init() {
  EnsureDir(self.directory)
  EnsureDir(self.temp)
  EnsureDir(self.attachments)
  EnsureDir(self.thumbs)
  if ! CheckFile(self.convert_path) {
    log.Fatal("cannot find executable for convert: ", self.convert_path, " not found")
  }
  if ! CheckFile(self.ffmpeg_path) {
    log.Fatal("connt find executable for ffmpegthumbnailer: ", self.ffmpeg_path, " not found")
  }
  if ! CheckFile(self.sox_path) {
    log.Fatal("connt find executable for sox: ", self.sox_path, " not found")
  }
}

func (self articleStore) isAudio(fname string) bool {
  for _, ext := range []string{".mp3", ".ogg", ".oga", ".opus", ".flac"} {
    if strings.HasPrefix(fname, ext) {
      return true
    }
  }
  return false
}

func (self articleStore) GenerateThumbnail(fname string) error {
  outfname := self.ThumbnailFilepath(fname)
  infname := self.AttachmentFilepath(fname)
  var cmd *exec.Cmd
  if strings.HasPrefix(strings.ToLower(fname), ".gif") {
    cmd = exec.Command(self.convert_path, "-thumbnail", "200", infname, outfname)
  } else if self.isAudio(fname) {
    cmd = exec.Command(self.sox_path, infname, "-n", "spectrogram", "-a", "-d", "0:30", "-r", "-p", "6", "-x", "200", "-y", "150", "-o", outfname)
  } else {
    cmd = exec.Command(self.ffmpeg_path, "-i", infname, "-o", outfname, "-s", "200")
  }
  exec_out, err := cmd.CombinedOutput()
  if err != nil {
    log.Println("error generating thumbnail", string(exec_out))
  }
  return err
}

func (self articleStore) GetAllAttachments() (names []string, err error) {
  var f *os.File
  f, err = os.Open(self.attachments)
  if err == nil {
    names, err = f.Readdirnames(0)
  }
  return
}

func (self articleStore) ReadMessage(r io.Reader) (NNTPMessage, error) {
  return read_message(r)
}

func (self articleStore) StorePost(nntp NNTPMessage) (err error) {

  f := self.CreateFile(nntp.MessageID())
  if f != nil {
    err = self.WriteMessage(nntp, f)
    f.Close()
  }

  nntp_inner := nntp.Signed()
  if nntp_inner == nil {
    // no inner article
    // store the data in the article
    self.database.RegisterArticle(nntp)
    for _, att := range nntp.Attachments() {
      // save attachments
      go self.saveAttachment(att)
    }
  } else {
    // record a tripcode
    self.database.RegisterSigned(nntp.MessageID(), nntp.Pubkey())
    // we have inner data
    // store the signed data
    self.database.RegisterArticle(nntp_inner)
    for _, att := range nntp_inner.Attachments() {
      go self.saveAttachment(att)
    }
  }
  return
}

// save an attachment
func (self articleStore) saveAttachment(att NNTPAttachment) {
  var err error
  var f io.WriteCloser
  fpath := att.Filepath()
  upload := self.AttachmentFilepath(fpath)
  thumb := self.ThumbnailFilepath(fpath)
  if CheckFile(upload) {
    log.Println("already have file", fpath)
    if ! CheckFile(thumb) && att.NeedsThumbnail() {
      log.Println("create thumbnail for", fpath)
      err = self.GenerateThumbnail(fpath)
      if err != nil {
        log.Println("failed to generate thumbnail", err)
      }
    }
    return
  }
  // save attachment
  log.Println("save attachment", att.Filename(), "to", upload)
  f, err = os.Create(upload)
  if err == nil {
    err = att.WriteTo(f)
    f.Close()
  }
  if err != nil {
    log.Println("did not save attachment", err)
    return
  }

  // generate thumbanils
  if att.NeedsThumbnail() {
    log.Println("create thumbnail for", fpath)
    err = self.GenerateThumbnail(fpath)
    if err != nil {
      log.Println("failed to generate thumbnail", err)
    }
  }
}

// eh this isn't really needed is it?
func (self articleStore) WriteMessage(nntp NNTPMessage, wr io.Writer) (err error) {
  return nntp.WriteTo(wr, "\n")
}

// get the filepath for an attachment
func (self articleStore) AttachmentFilepath(fname string) string {
  return filepath.Join(self.attachments, fname)
}

// get the filepath for a thumbanil
func (self articleStore) ThumbnailFilepath(fname string) string {
  // all thumbnails are jpegs now
  return filepath.Join(self.thumbs, fname + ".jpg")
}

// create a file for this article
func (self articleStore) CreateFile(messageID string) io.WriteCloser {
  fname := self.GetFilename(messageID)
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// create a temp file for inboud articles
func (self articleStore) CreateTempFile(messageID string) io.WriteCloser {
  fname := self.GetTempFilename(messageID)
  if CheckFile(fname) {
    log.Println(fname, "already open")
    return nil
  }
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// return true if we have an article
func (self articleStore) HasArticle(messageID string) bool {
  return CheckFile(self.GetFilename(messageID))
}

// get the filename for this article
func (self articleStore) GetFilename(messageID string) string {
  if ! ValidMessageID(messageID) {
    log.Println("!!! bug: tried to open invalid message", messageID, "!!!")
    return ""
  }
  return filepath.Join(self.directory, messageID)
}

// get the filename for this article
func (self articleStore) GetTempFilename(messageID string) string {
  if ! ValidMessageID(messageID) {
    log.Println("!!! bug: tried to open invalid temp message", messageID, "!!!")
    return ""
  }
  return filepath.Join(self.temp, messageID)
}

// loads temp message and deletes old article
func (self articleStore) ReadTempMessage(messageID string) NNTPMessage {
  fname := self.GetTempFilename(messageID)
  nntp := self.readfile(fname)
  DelFile(fname)
  return nntp
}

// read a file give filepath
func (self articleStore) readfile(fname string) NNTPMessage {

  file, err := os.Open(fname)
  if err != nil {
    log.Println("store cannot open file",fname)
    return nil
  }
  message, err := self.ReadMessage(file)
  file.Close()
  if err == nil {
    return message
  }

  log.Println("failed to load file", fname)
  return nil
}

// get the replies for a thread
func (self articleStore) GetThreadReplies(messageID string, last int) []NNTPMessage {
  var repls []NNTPMessage
  if self.database.ThreadHasReplies(messageID) {
    rpls := self.database.GetThreadReplies(messageID, last)
    if rpls == nil {
      return repls
    }
    for _, rpl := range rpls {
      msg := self.GetMessage(rpl)
      if msg == nil {
        log.Println("cannot get message", rpl)
      } else {
        repls = append(repls, msg)
      }
    }
  }
  return repls
}

// load an article
// return nil on failure
func (self articleStore) GetMessage(messageID string) NNTPMessage {
  return self.readfile(self.GetFilename(messageID))
}

// get article with headers only
func (self articleStore) GetHeaders(messageID string) ArticleHeaders {
  // TODO: don't load the entire body
  nntp := self.readfile(self.GetFilename(messageID))
  if nntp == nil {
    return nil
  }
  return nntp.Headers()
}

func read_message(r io.Reader) (NNTPMessage, error) {

  msg, err := mail.ReadMessage(r)
  var nntp nntpArticle

  if err == nil {
    nntp.headers = ArticleHeaders(msg.Header)
    content_type := nntp.ContentType()
    media_type, params, err := mime.ParseMediaType(content_type)
    if err != nil {
      log.Println("failed to parse media type", err, "for mime", content_type)
      return nil, err
    }
    boundary, ok := params["boundary"]
    if ok {
      partReader := multipart.NewReader(msg.Body, boundary)
      for {
        part, err := partReader.NextPart()
        if err == io.EOF {
          return nntp, nil
        } else if err == nil {
          hdr := part.Header
          // get content type of part
          part_type := hdr.Get("Content-Type")
          // parse content type
          media_type, _, err = mime.ParseMediaType(part_type)
          if err == nil {
            if media_type == "text/plain" {
              att := readAttachmentFromMimePart(part)
              if att != nil {
                nntp.message = att.(nntpAttachment)
                nntp.message.header.Set("Content-Type", part_type)
              }
            } else {
              // non plaintext gets added to attachments
              att := readAttachmentFromMimePart(part)
              if att != nil {
                nntp = nntp.Attach(att).(nntpArticle)
              }
            }
          } else {
            log.Println("part has no content type", err)
          }
          part.Close()
        } else {
          log.Println("failed to load part! ", err)
          return nil, err
        }
      }

    } else if media_type == "message/rfc822" {
      // tripcoded message
      sig := nntp.headers.Get("X-Signature-Ed25519-Sha512", "")
      pk := nntp.Pubkey()
      if pk == "" || sig == "" {
        log.Println("invalid sig or pubkey", sig, pk)
        return nil, errors.New("invalid headers")
      }
      log.Printf("got signed message from %s", pk)
      pk_bytes := unhex(pk)
      sig_bytes := unhex(sig)
      r := bufio.NewReader(msg.Body)
      crlf := []byte{13,10}
      for {
        line, err := r.ReadBytes('\n')
        if err == io.EOF {
          break
        }
        nntp.signedPart.body.Write(line[:len(line)-1])
        nntp.signedPart.body.Write(crlf)
      }
      if nntp.signedPart.body.Len() < 2 {
        log.Println("signed body is too small")
      } else {
        body := nntp.signedPart.body.Bytes()[:nntp.signedPart.body.Len()-2]
        body_hash := sha512.Sum512(body)
        log.Printf("hash=%s", hexify(body_hash[:]))
        if nacl.CryptoVerifyFucky(body_hash[:], sig_bytes, pk_bytes) {
          log.Println("signature is valid :^)")
          return nntp, nil
        } else {
          log.Println("!!!signature is invalid!!!")
        }
      }
    } else {
      // plaintext attachment
      var buff bytes.Buffer
      _, err = io.Copy(&buff, msg.Body)
      nntp.message = createPlaintextAttachment(buff.String())
      return nntp, err
    }
  } else {
    log.Println("failed to read message", err)
    return nil, err
  }
  return nntp, err
}

<|endoftext|>


<commit_before>//
// store.go
//

package srnd

import (

  "github.com/majestrate/srndv2/src/nacl"
  "github.com/gographics/imagick/imagick"
  "bufio"
  "bytes"
  "crypto/sha512"
  "encoding/base32"
  "encoding/base64"
  "errors"
  "fmt"
  "io"
  "log"
  "mime"
  "mime/multipart"
  "net/mail"
  "net/textproto"
  "os"
  "path/filepath"
  "strings"
)

type ArticleStore interface {
  MessageReader
  MessageWriter

  // get the filepath for an attachment
  AttachmentFilepath(fname string) string
  // get the filepath for an attachment's thumbnail
  ThumbnailFilepath(fname string) string
  // do we have this article?
  HasArticle(msgid string) bool
  // create a file for a message
  CreateFile(msgid string) io.WriteCloser
  // create a file for a temp message
  CreateTempFile(msgid string) io.WriteCloser
  // get the filename of a message
  GetFilename(msgid string) string
  // get the filename of a temp message
  GetTempFilename(msgid string) string
  // Get a message given its messageid
  GetMessage(msgid string) NNTPMessage
  // get a temp message given its messageid
  // temp message is deleted once read
  ReadTempMessage(msgid string) NNTPMessage
  // store a post
  StorePost(nntp NNTPMessage) error
  // get article headers only
  GetHeaders(msgid string) ArticleHeaders

  // get our temp directory for articles
  TempDir() string
}
type articleStore struct {
  directory string
  temp string
  attachments string
  thumbs string
  database Database
}

func createArticleStore(config map[string]string, database Database) ArticleStore {
  store := articleStore{
    directory: config["store_dir"],
    temp: config["incoming_dir"],
    attachments: config["attachments_dir"],
    thumbs: config["thumbs_dir"],
    database: database,
  }
  store.Init()
  return store
}

func (self articleStore) TempDir() string {
  return self.temp
}

// initialize article store
func (self articleStore) Init() {
  EnsureDir(self.directory)
  EnsureDir(self.temp)
  EnsureDir(self.attachments)
  EnsureDir(self.thumbs)
}

func (self articleStore) GenerateThumbnail(infname string) (err error) {
  wand := imagick.NewMagickWand()
  defer wand.Destroy()
  // read image source
  err = wand.ReadImage(self.AttachmentFilepath(infname))
  if err == nil {
    // get size
    h := wand.GetImageHeight()
    w := wand.GetImageWidth()

    // calculate scale parameters
    var scale, th, tw float64
    scale = 180
    modifier := scale / float64(w)
    th = modifier * float64(h)
    tw = modifier * float64(w)
    // scale it
    err = wand.ScaleImage(uint(tw), uint(th))
    if err == nil {
      err = wand.WriteImage(self.ThumbnailFilepath(infname))
    }
  }
  return
}

func (self articleStore) ReadMessage(r io.Reader) (NNTPMessage, error) {
  return read_message(r)
}

func (self articleStore) StorePost(nntp NNTPMessage) (err error) {
  f := self.CreateFile(nntp.MessageID())
  if f != nil {
    err = self.WriteMessage(nntp, f)
    f.Close()
  }
  for _, att := range nntp.Attachments() {
    // save attachments in parallel
    go self.saveAttachment(att)
  }
  return
}

// save an attachment
func (self articleStore) saveAttachment(att NNTPAttachment) {
  fpath := att.Filepath()
  upload := self.AttachmentFilepath(fpath)
  // save attachment
  log.Println("save attachment", att.Filename(), "to", upload)
  f, err := os.Create(upload)
  if err == nil {
    err = att.WriteTo(f)
    f.Close()
  }
  if err != nil {
    return
  }

  // generate thumbanils
  if att.NeedsThumbnail() {
    err = self.GenerateThumbnail(fpath)
    if err != nil {
      log.Println("failed to generate thumbnail", err)
      return
    }
  }
}

func (self articleStore) WriteMessage(nntp NNTPMessage, wr io.Writer) (err error) {
  // write headers
  for hdr, hdr_vals := range(nntp.Headers()) {
    for _ , hdr_val := range hdr_vals {
      _, err = io.WriteString(wr, fmt.Sprintf("%s: %s\n", hdr, hdr_val))
      if err != nil {
        return
      }
    }
  }
  // done headers
  _, err = io.WriteString(wr, "\n")
  if err != nil {
    return
  }
  // write body
  err = nntp.WriteBody(wr)
  return
}

// get the filepath for an attachment
func (self articleStore) AttachmentFilepath(fname string) string {
  return filepath.Join(self.attachments, fname)
}

// get the filepath for a thumbanil
func (self articleStore) ThumbnailFilepath(fname string) string {
  return filepath.Join(self.thumbs, fname)
}

// create a file for this article
func (self articleStore) CreateFile(messageID string) io.WriteCloser {
  fname := self.GetFilename(messageID)
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// create a temp file for inboud articles
func (self articleStore) CreateTempFile(messageID string) io.WriteCloser {
  fname := self.GetTempFilename(messageID)
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// return true if we have an article
func (self articleStore) HasArticle(messageID string) bool {
  return self.database.HasArticle(messageID)
}

// get the filename for this article
func (self articleStore) GetFilename(messageID string) string {
  return filepath.Join(self.directory, messageID)
}

// get the filename for this article
func (self articleStore) GetTempFilename(messageID string) string {
  return filepath.Join(self.temp, messageID)
}

// loads temp message and deletes old article
func (self articleStore) ReadTempMessage(messageID string) NNTPMessage {
  fname := self.GetTempFilename(messageID)
  nntp := self.readfile(fname)
  DelFile(fname)
  return nntp
}

// read a file give filepath
func (self articleStore) readfile(fname string) NNTPMessage {

  file, err := os.Open(fname)
  if err != nil {
    log.Println("store cannot open file",fname)
    return nil
  }
  message, err := self.ReadMessage(file)
  file.Close()
  if err == nil {
    return message
  }

  log.Println("failed to load file", fname)
  return nil
}

// get the replies for a thread
func (self articleStore) GetThreadReplies(messageID string, last int) []NNTPMessage {
  var repls []NNTPMessage
  if self.database.ThreadHasReplies(messageID) {
    rpls := self.database.GetThreadReplies(messageID, last)
    if rpls == nil {
      return repls
    }
    for _, rpl := range rpls {
      msg := self.GetMessage(rpl)
      if msg == nil {
        log.Println("cannot get message", rpl)
      } else {
        repls = append(repls, msg)
      }
    }
  }
  return repls
}

// load an article
// return nil on failure
func (self articleStore) GetMessage(messageID string) NNTPMessage {
  return self.readfile(self.GetFilename(messageID))
}

// get article with headers only
func (self articleStore) GetHeaders(messageID string) ArticleHeaders {
  // TODO: don't load the entire body
  nntp := self.readfile(self.GetFilename(messageID))
  if nntp == nil {
    return nil
  }
  return nntp.Headers()
}

func read_message(r io.Reader) (NNTPMessage, error) {

  msg, err := mail.ReadMessage(r)
  nntp := nntpArticle{}

  if err == nil {
    nntp.headers = ArticleHeaders(msg.Header)
    content_type := msg.Header.Get("Content-Type")
    media_type, params, err := mime.ParseMediaType(content_type)
    if err != nil {
      log.Println("failed to parse media type", err)
      return nil, err
    }
    boundary, ok := params["boundary"]
    if ok {
      partReader := multipart.NewReader(msg.Body, boundary)
      for {
        part, err := partReader.NextPart()
        if err == io.EOF {
          return nntp, nil
        } else if err == nil {
          hdr := part.Header
          // get content type of part
          part_type := hdr.Get("Content-Type")
          log.Println("part has content type", part_type)
          // parse content type
          media_type, _, err = mime.ParseMediaType(part_type)
          if err == nil {
            if media_type == "text/plain" {
              att := readAttachmentFromMimePart(part)
              if att != nil {
                att.WriteTo(&nntp.message.body)
              }
              if nntp.message.header == nil {
                nntp.message.header = make(textproto.MIMEHeader)
              }
              nntp.message.header.Set("Content-Type", part_type)

            } else {
              // non plaintext gets added to attachments
              att := readAttachmentFromMimePart(part)
              if att != nil {
                nntp = nntp.Attach(att).(nntpArticle)
              }
            }
          } else {
            log.Println("part has no content type", err)
          }
          part.Close()
        } else {
          log.Println("failed to load part! ", err)
          return nntp, err
        }
      }

    } else if media_type == "message/rfc822" {
      // tripcoded message
      sig := nntp.headers.Get("X-Signature-Ed25519-Sha512", "")
      pk := nntp.headers.Get("X-Pubkey-Ed25519", "")
      if pk == "" || sig == "" {
        log.Println("invalid sig or pubkey", sig, pk)
        return nil, errors.New("invalid headers")
      }
      log.Printf("got signed message from %s", pk)
      pk_bytes := unhex(pk)
      sig_bytes := unhex(sig)
      r := bufio.NewReader(msg.Body)
      crlf := []byte{13,10}
      for {
        line, err := r.ReadBytes('\n')
        if err == io.EOF {
          break
        }
        nntp.signedPart.body.Write(line[:len(line)-1])
        nntp.signedPart.body.Write(crlf)
      }
      if nntp.signedPart.body.Len() < 2 {
        log.Println("signed body is too small")
      } else {
        body := nntp.signedPart.body.Bytes()[:nntp.signedPart.body.Len()-2]
        body_hash := sha512.Sum512(body)
        if nacl.CryptoVerifyFucky(body_hash[:], sig_bytes, pk_bytes) {
          log.Println("signature is valid :^)")
        } else {
          log.Println("!!!signature is invalid!!!")
        }
      }
    } else {
      _, err = nntp.message.body.ReadFrom(msg.Body)
    }
  } else {
    log.Println("failed to read message", err)
  }
  return nntp, err
}

func readAttachmentFromMimePart(part *multipart.Part) NNTPAttachment {
  var buff bytes.Buffer
  content_type := part.Header.Get("Content-Type")
  media_type, _ , err := mime.ParseMediaType(content_type)

  fname := part.FileName()
  idx := strings.LastIndex(fname, ".")
  ext := ".txt"
  if idx > 0 {
    ext = fname[idx:]
  }

  transfer_encoding := part.Header.Get("Content-Transfer-Encoding")

  if transfer_encoding == "base64" {
    reader := base64.NewDecoder(base64.StdEncoding, part)
    _, err = io.Copy(&buff, reader)
  } else {
    _, err = io.Copy(&buff, part)
  }
  if err != nil {
    log.Println("failed to read attachment from mimepart", err)
    return nil
  }
  sha := sha512.Sum512(buff.Bytes())
  hashstr := base32.StdEncoding.EncodeToString(sha[:])
  fpath := hashstr+ext
  hdr := part.Header
  return nntpAttachment{
    header: hdr,
    body: buff,
    mime: media_type,
    filename: fname,
    filepath: fpath,
    ext: ext,
    hash: sha[:],
  }
}
<commit_msg>try returning article on validated explitly<commit_after>//
// store.go
//

package srnd

import (

  "github.com/majestrate/srndv2/src/nacl"
  "github.com/gographics/imagick/imagick"
  "bufio"
  "bytes"
  "crypto/sha512"
  "encoding/base32"
  "encoding/base64"
  "errors"
  "fmt"
  "io"
  "log"
  "mime"
  "mime/multipart"
  "net/mail"
  "net/textproto"
  "os"
  "path/filepath"
  "strings"
)

type ArticleStore interface {
  MessageReader
  MessageWriter

  // get the filepath for an attachment
  AttachmentFilepath(fname string) string
  // get the filepath for an attachment's thumbnail
  ThumbnailFilepath(fname string) string
  // do we have this article?
  HasArticle(msgid string) bool
  // create a file for a message
  CreateFile(msgid string) io.WriteCloser
  // create a file for a temp message
  CreateTempFile(msgid string) io.WriteCloser
  // get the filename of a message
  GetFilename(msgid string) string
  // get the filename of a temp message
  GetTempFilename(msgid string) string
  // Get a message given its messageid
  GetMessage(msgid string) NNTPMessage
  // get a temp message given its messageid
  // temp message is deleted once read
  ReadTempMessage(msgid string) NNTPMessage
  // store a post
  StorePost(nntp NNTPMessage) error
  // get article headers only
  GetHeaders(msgid string) ArticleHeaders

  // get our temp directory for articles
  TempDir() string
}
type articleStore struct {
  directory string
  temp string
  attachments string
  thumbs string
  database Database
}

func createArticleStore(config map[string]string, database Database) ArticleStore {
  store := articleStore{
    directory: config["store_dir"],
    temp: config["incoming_dir"],
    attachments: config["attachments_dir"],
    thumbs: config["thumbs_dir"],
    database: database,
  }
  store.Init()
  return store
}

func (self articleStore) TempDir() string {
  return self.temp
}

// initialize article store
func (self articleStore) Init() {
  EnsureDir(self.directory)
  EnsureDir(self.temp)
  EnsureDir(self.attachments)
  EnsureDir(self.thumbs)
}

func (self articleStore) GenerateThumbnail(infname string) (err error) {
  wand := imagick.NewMagickWand()
  defer wand.Destroy()
  // read image source
  err = wand.ReadImage(self.AttachmentFilepath(infname))
  if err == nil {
    // get size
    h := wand.GetImageHeight()
    w := wand.GetImageWidth()

    // calculate scale parameters
    var scale, th, tw float64
    scale = 180
    modifier := scale / float64(w)
    th = modifier * float64(h)
    tw = modifier * float64(w)
    // scale it
    err = wand.ScaleImage(uint(tw), uint(th))
    if err == nil {
      err = wand.WriteImage(self.ThumbnailFilepath(infname))
    }
  }
  return
}

func (self articleStore) ReadMessage(r io.Reader) (NNTPMessage, error) {
  return read_message(r)
}

func (self articleStore) StorePost(nntp NNTPMessage) (err error) {
  f := self.CreateFile(nntp.MessageID())
  if f != nil {
    err = self.WriteMessage(nntp, f)
    f.Close()
  }
  for _, att := range nntp.Attachments() {
    // save attachments in parallel
    go self.saveAttachment(att)
  }
  return
}

// save an attachment
func (self articleStore) saveAttachment(att NNTPAttachment) {
  fpath := att.Filepath()
  upload := self.AttachmentFilepath(fpath)
  // save attachment
  log.Println("save attachment", att.Filename(), "to", upload)
  f, err := os.Create(upload)
  if err == nil {
    err = att.WriteTo(f)
    f.Close()
  }
  if err != nil {
    return
  }

  // generate thumbanils
  if att.NeedsThumbnail() {
    err = self.GenerateThumbnail(fpath)
    if err != nil {
      log.Println("failed to generate thumbnail", err)
      return
    }
  }
}

func (self articleStore) WriteMessage(nntp NNTPMessage, wr io.Writer) (err error) {
  // write headers
  for hdr, hdr_vals := range(nntp.Headers()) {
    for _ , hdr_val := range hdr_vals {
      _, err = io.WriteString(wr, fmt.Sprintf("%s: %s\n", hdr, hdr_val))
      if err != nil {
        return
      }
    }
  }
  // done headers
  _, err = io.WriteString(wr, "\n")
  if err != nil {
    return
  }
  // write body
  err = nntp.WriteBody(wr)
  return
}

// get the filepath for an attachment
func (self articleStore) AttachmentFilepath(fname string) string {
  return filepath.Join(self.attachments, fname)
}

// get the filepath for a thumbanil
func (self articleStore) ThumbnailFilepath(fname string) string {
  return filepath.Join(self.thumbs, fname)
}

// create a file for this article
func (self articleStore) CreateFile(messageID string) io.WriteCloser {
  fname := self.GetFilename(messageID)
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// create a temp file for inboud articles
func (self articleStore) CreateTempFile(messageID string) io.WriteCloser {
  fname := self.GetTempFilename(messageID)
  file, err := os.Create(fname)
  if err != nil {
    log.Println("cannot open file", fname)
    return nil
  }
  return file
}

// return true if we have an article
func (self articleStore) HasArticle(messageID string) bool {
  return self.database.HasArticle(messageID)
}

// get the filename for this article
func (self articleStore) GetFilename(messageID string) string {
  return filepath.Join(self.directory, messageID)
}

// get the filename for this article
func (self articleStore) GetTempFilename(messageID string) string {
  return filepath.Join(self.temp, messageID)
}

// loads temp message and deletes old article
func (self articleStore) ReadTempMessage(messageID string) NNTPMessage {
  fname := self.GetTempFilename(messageID)
  nntp := self.readfile(fname)
  DelFile(fname)
  return nntp
}

// read a file give filepath
func (self articleStore) readfile(fname string) NNTPMessage {

  file, err := os.Open(fname)
  if err != nil {
    log.Println("store cannot open file",fname)
    return nil
  }
  message, err := self.ReadMessage(file)
  file.Close()
  if err == nil {
    return message
  }

  log.Println("failed to load file", fname)
  return nil
}

// get the replies for a thread
func (self articleStore) GetThreadReplies(messageID string, last int) []NNTPMessage {
  var repls []NNTPMessage
  if self.database.ThreadHasReplies(messageID) {
    rpls := self.database.GetThreadReplies(messageID, last)
    if rpls == nil {
      return repls
    }
    for _, rpl := range rpls {
      msg := self.GetMessage(rpl)
      if msg == nil {
        log.Println("cannot get message", rpl)
      } else {
        repls = append(repls, msg)
      }
    }
  }
  return repls
}

// load an article
// return nil on failure
func (self articleStore) GetMessage(messageID string) NNTPMessage {
  return self.readfile(self.GetFilename(messageID))
}

// get article with headers only
func (self articleStore) GetHeaders(messageID string) ArticleHeaders {
  // TODO: don't load the entire body
  nntp := self.readfile(self.GetFilename(messageID))
  if nntp == nil {
    return nil
  }
  return nntp.Headers()
}

func read_message(r io.Reader) (NNTPMessage, error) {

  msg, err := mail.ReadMessage(r)
  nntp := nntpArticle{}

  if err == nil {
    nntp.headers = ArticleHeaders(msg.Header)
    content_type := msg.Header.Get("Content-Type")
    media_type, params, err := mime.ParseMediaType(content_type)
    if err != nil {
      log.Println("failed to parse media type", err)
      return nil, err
    }
    boundary, ok := params["boundary"]
    if ok {
      partReader := multipart.NewReader(msg.Body, boundary)
      for {
        part, err := partReader.NextPart()
        if err == io.EOF {
          return nntp, nil
        } else if err == nil {
          hdr := part.Header
          // get content type of part
          part_type := hdr.Get("Content-Type")
          log.Println("part has content type", part_type)
          // parse content type
          media_type, _, err = mime.ParseMediaType(part_type)
          if err == nil {
            if media_type == "text/plain" {
              att := readAttachmentFromMimePart(part)
              if att != nil {
                att.WriteTo(&nntp.message.body)
              }
              if nntp.message.header == nil {
                nntp.message.header = make(textproto.MIMEHeader)
              }
              nntp.message.header.Set("Content-Type", part_type)

            } else {
              // non plaintext gets added to attachments
              att := readAttachmentFromMimePart(part)
              if att != nil {
                nntp = nntp.Attach(att).(nntpArticle)
              }
            }
          } else {
            log.Println("part has no content type", err)
          }
          part.Close()
        } else {
          log.Println("failed to load part! ", err)
          return nntp, err
        }
      }

    } else if media_type == "message/rfc822" {
      // tripcoded message
      sig := nntp.headers.Get("X-Signature-Ed25519-Sha512", "")
      pk := nntp.headers.Get("X-Pubkey-Ed25519", "")
      if pk == "" || sig == "" {
        log.Println("invalid sig or pubkey", sig, pk)
        return nil, errors.New("invalid headers")
      }
      log.Printf("got signed message from %s", pk)
      pk_bytes := unhex(pk)
      sig_bytes := unhex(sig)
      r := bufio.NewReader(msg.Body)
      crlf := []byte{13,10}
      for {
        line, err := r.ReadBytes('\n')
        if err == io.EOF {
          break
        }
        nntp.signedPart.body.Write(line[:len(line)-1])
        nntp.signedPart.body.Write(crlf)
      }
      if nntp.signedPart.body.Len() < 2 {
        log.Println("signed body is too small")
      } else {
        body := nntp.signedPart.body.Bytes()[:nntp.signedPart.body.Len()-2]
        body_hash := sha512.Sum512(body)
        if nacl.CryptoVerifyFucky(body_hash[:], sig_bytes, pk_bytes) {
          log.Println("signature is valid :^)")
          return nntp, nil
        } else {
          log.Println("!!!signature is invalid!!!")
        }
      }
    } else {
      _, err = nntp.message.body.ReadFrom(msg.Body)
    }
  } else {
    log.Println("failed to read message", err)
  }
  return nntp, err
}

func readAttachmentFromMimePart(part *multipart.Part) NNTPAttachment {
  var buff bytes.Buffer
  content_type := part.Header.Get("Content-Type")
  media_type, _ , err := mime.ParseMediaType(content_type)

  fname := part.FileName()
  idx := strings.LastIndex(fname, ".")
  ext := ".txt"
  if idx > 0 {
    ext = fname[idx:]
  }

  transfer_encoding := part.Header.Get("Content-Transfer-Encoding")

  if transfer_encoding == "base64" {
    reader := base64.NewDecoder(base64.StdEncoding, part)
    _, err = io.Copy(&buff, reader)
  } else {
    _, err = io.Copy(&buff, part)
  }
  if err != nil {
    log.Println("failed to read attachment from mimepart", err)
    return nil
  }
  sha := sha512.Sum512(buff.Bytes())
  hashstr := base32.StdEncoding.EncodeToString(sha[:])
  fpath := hashstr+ext
  hdr := part.Header
  return nntpAttachment{
    header: hdr,
    body: buff,
    mime: media_type,
    filename: fname,
    filepath: fpath,
    ext: ext,
    hash: sha[:],
  }
}
<|endoftext|>