<commit_before>// Copyright 2017 Vector Creations Ltd
// Copyright 2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package input

import (
	"bytes"
	"context"
	"fmt"

	"github.com/matrix-org/dendrite/internal/eventutil"
	"github.com/matrix-org/dendrite/roomserver/api"
	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
	"github.com/matrix-org/dendrite/roomserver/state"
	"github.com/matrix-org/dendrite/roomserver/types"
	"github.com/matrix-org/gomatrixserverlib"
	"github.com/matrix-org/util"
	"github.com/sirupsen/logrus"
)

// processRoomEvent can only be called once at a time
//
// TODO(#375): This should be rewritten to allow concurrent calls. The
// difficulty is in ensuring that we correctly annotate events with the correct
// state deltas when sending to kafka streams
// TODO: Break up function - we should probably do transaction ID checks before calling this.
// nolint:gocyclo
func (r *Inputer) processRoomEvent(
	ctx context.Context,
	input *api.InputRoomEvent,
) (eventID string, err error) {
	// Parse and validate the event JSON
	headered := input.Event
	event := headered.Unwrap()

	// if we have already got this event then do not process it again, if the input kind is an outlier.
	// Outliers contain no extra information which may warrant a re-processing.
	if input.Kind == api.KindOutlier {
		evs, err := r.DB.EventsFromIDs(ctx, []string{event.EventID()})
		if err == nil && len(evs) == 1 {
			// check hash matches if we're on early room versions where the event ID was a random string
			idFormat, err := headered.RoomVersion.EventIDFormat()
			if err == nil {
				switch idFormat {
				case gomatrixserverlib.EventIDFormatV1:
					if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) {
						util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
						return event.EventID(), nil
					}
				default:
					util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
					return event.EventID(), nil
				}
			}
		}
	}

	// Check that the event passes authentication checks and work out
	// the numeric IDs for the auth events.
	isRejected := false
	authEventNIDs, rejectionErr := helpers.CheckAuthEvents(ctx, r.DB, headered, input.AuthEventIDs)
	if rejectionErr != nil {
		logrus.WithError(rejectionErr).WithField("event_id", event.EventID()).WithField("auth_event_ids", input.AuthEventIDs).Error("helpers.CheckAuthEvents failed for event, rejecting event")
		isRejected = true
	}

	var softfail bool
	if input.Kind == api.KindNew {
		// Check that the event passes authentication checks based on the
		// current room state.
		softfail, err = helpers.CheckForSoftFail(ctx, r.DB, headered, input.StateEventIDs)
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"event_id": event.EventID(),
				"type":     event.Type(),
				"room":     event.RoomID(),
			}).WithError(err).Info("Error authing soft-failed event")
		}
	}

	// If we don't have a transaction ID then get one.
	if input.TransactionID != nil {
		tdID := input.TransactionID
		eventID, err = r.DB.GetTransactionEventID(
			ctx, tdID.TransactionID, tdID.SessionID, event.Sender(),
		)
		// On error OR event with the transaction already processed/processesing
		if err != nil || eventID != "" {
			return
		}
	}

	// Store the event.
	_, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, input.TransactionID, authEventNIDs, isRejected)
	if err != nil {
		return "", fmt.Errorf("r.DB.StoreEvent: %w", err)
	}

	// if storing this event results in it being redacted then do so.
	if !isRejected && redactedEventID == event.EventID() {
		r, rerr := eventutil.RedactEvent(redactionEvent, &event)
		if rerr != nil {
			return "", fmt.Errorf("eventutil.RedactEvent: %w", rerr)
		}
		event = *r
	}

	// For outliers we can stop after we've stored the event itself as it
	// doesn't have any associated state to store and we don't need to
	// notify anyone about it.
	if input.Kind == api.KindOutlier {
		logrus.WithFields(logrus.Fields{
			"event_id": event.EventID(),
			"type":     event.Type(),
			"room":     event.RoomID(),
			"sender":   event.Sender(),
		}).Debug("Stored outlier")
		return event.EventID(), nil
	}

	roomInfo, err := r.DB.RoomInfo(ctx, event.RoomID())
	if err != nil {
		return "", fmt.Errorf("r.DB.RoomInfo: %w", err)
	}
	if roomInfo == nil {
		return "", fmt.Errorf("r.DB.RoomInfo missing for room %s", event.RoomID())
	}

	if stateAtEvent.BeforeStateSnapshotNID == 0 {
		// We haven't calculated a state for this event yet.
		// Lets calculate one.
		err = r.calculateAndSetState(ctx, input, *roomInfo, &stateAtEvent, event, isRejected)
		if err != nil && input.Kind != api.KindOld {
			return "", fmt.Errorf("r.calculateAndSetState: %w", err)
		}
	}

	// We stop here if the event is rejected: We've stored it but won't update forward extremities or notify anyone about it.
	if isRejected || softfail {
		logrus.WithFields(logrus.Fields{
			"event_id":  event.EventID(),
			"type":      event.Type(),
			"room":      event.RoomID(),
			"soft_fail": softfail,
			"sender":    event.Sender(),
		}).Debug("Stored rejected event")
		return event.EventID(), rejectionErr
	}

	switch input.Kind {
	case api.KindNew:
		if err = r.updateLatestEvents(
			ctx,                 // context
			roomInfo,            // room info for the room being updated
			stateAtEvent,        // state at event (below)
			event,               // event
			input.SendAsServer,  // send as server
			input.TransactionID, // transaction ID
			input.HasState,      // rewrites state?
		); err != nil {
			return "", fmt.Errorf("r.updateLatestEvents: %w", err)
		}
	case api.KindOld:
		err = r.WriteOutputEvents(event.RoomID(), []api.OutputEvent{
			{
				Type: api.OutputTypeOldRoomEvent,
				OldRoomEvent: &api.OutputOldRoomEvent{
					Event: headered,
				},
			},
		})
		if err != nil {
			return "", fmt.Errorf("r.WriteOutputEvents (old): %w", err)
		}
	}

	// processing this event resulted in an event (which may not be the one we're processing)
	// being redacted. We are guaranteed to have both sides (the redaction/redacted event),
	// so notify downstream components to redact this event - they should have it if they've
	// been tracking our output log.
	if redactedEventID != "" {
		err = r.WriteOutputEvents(event.RoomID(), []api.OutputEvent{
			{
				Type: api.OutputTypeRedactedEvent,
				RedactedEvent: &api.OutputRedactedEvent{
					RedactedEventID: redactedEventID,
					RedactedBecause: redactionEvent.Headered(headered.RoomVersion),
				},
			},
		})
		if err != nil {
			return "", fmt.Errorf("r.WriteOutputEvents (redactions): %w", err)
		}
	}

	// Update the extremities of the event graph for the room
	return event.EventID(), nil
}

func (r *Inputer) calculateAndSetState(
	ctx context.Context,
	input *api.InputRoomEvent,
	roomInfo types.RoomInfo,
	stateAtEvent *types.StateAtEvent,
	event gomatrixserverlib.Event,
	isRejected bool,
) error {
	var err error
	roomState := state.NewStateResolution(r.DB, roomInfo)

	if input.HasState && !isRejected {
		// Check here if we think we're in the room already.
		stateAtEvent.Overwrite = true
		var joinEventNIDs []types.EventNID
		// Request join memberships only for local users only.
		if joinEventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, roomInfo.RoomNID, true, true); err == nil {
			// If we have no local users that are joined to the room then any state about
			// the room that we have is quite possibly out of date. Therefore in that case
			// we should overwrite it rather than merge it.
			stateAtEvent.Overwrite = len(joinEventNIDs) == 0
		}

		// We've been told what the state at the event is so we don't need to calculate it.
		// Check that those state events are in the database and store the state.
		var entries []types.StateEntry
		if entries, err = r.DB.StateEntriesForEventIDs(ctx, input.StateEventIDs); err != nil {
			return fmt.Errorf("r.DB.StateEntriesForEventIDs: %w", err)
		}
		entries = types.DeduplicateStateEntries(entries)

		if stateAtEvent.BeforeStateSnapshotNID, err = r.DB.AddState(ctx, roomInfo.RoomNID, nil, entries); err != nil {
			return fmt.Errorf("r.DB.AddState: %w", err)
		}
	} else {
		stateAtEvent.Overwrite = false

		// We haven't been told what the state at the event is so we need to calculate it from the prev_events
		if stateAtEvent.BeforeStateSnapshotNID, err = roomState.CalculateAndStoreStateBeforeEvent(ctx, event, isRejected); err != nil {
			return fmt.Errorf("roomState.CalculateAndStoreStateBeforeEvent: %w", err)
		}
	}

	err = r.DB.SetState(ctx, stateAtEvent.EventNID, stateAtEvent.BeforeStateSnapshotNID)
	if err != nil {
		return fmt.Errorf("r.DB.SetState: %w", err)
	}
	return nil
}
<commit_msg>Linting<commit_after>// Copyright 2017 Vector Creations Ltd
// Copyright 2018 New Vector Ltd
// Copyright 2019-2020 The Matrix.org Foundation C.I.C.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package input

import (
	"bytes"
	"context"
	"fmt"

	"github.com/matrix-org/dendrite/internal/eventutil"
	"github.com/matrix-org/dendrite/roomserver/api"
	"github.com/matrix-org/dendrite/roomserver/internal/helpers"
	"github.com/matrix-org/dendrite/roomserver/state"
	"github.com/matrix-org/dendrite/roomserver/types"
	"github.com/matrix-org/gomatrixserverlib"
	"github.com/matrix-org/util"
	"github.com/sirupsen/logrus"
)

// processRoomEvent can only be called once at a time
//
// TODO(#375): This should be rewritten to allow concurrent calls. The
// difficulty is in ensuring that we correctly annotate events with the correct
// state deltas when sending to kafka streams
// TODO: Break up function - we should probably do transaction ID checks before calling this.
// nolint:gocyclo
func (r *Inputer) processRoomEvent(
	ctx context.Context,
	input *api.InputRoomEvent,
) (eventID string, err error) {
	// Parse and validate the event JSON
	headered := input.Event
	event := headered.Unwrap()

	// if we have already got this event then do not process it again, if the input kind is an outlier.
	// Outliers contain no extra information which may warrant a re-processing.
	if input.Kind == api.KindOutlier {
		evs, err2 := r.DB.EventsFromIDs(ctx, []string{event.EventID()})
		if err2 == nil && len(evs) == 1 {
			// check hash matches if we're on early room versions where the event ID was a random string
			idFormat, err2 := headered.RoomVersion.EventIDFormat()
			if err2 == nil {
				switch idFormat {
				case gomatrixserverlib.EventIDFormatV1:
					if bytes.Equal(event.EventReference().EventSHA256, evs[0].EventReference().EventSHA256) {
						util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
						return event.EventID(), nil
					}
				default:
					util.GetLogger(ctx).WithField("event_id", event.EventID()).Infof("Already processed event; ignoring")
					return event.EventID(), nil
				}
			}
		}
	}

	// Check that the event passes authentication checks and work out
	// the numeric IDs for the auth events.
	isRejected := false
	authEventNIDs, rejectionErr := helpers.CheckAuthEvents(ctx, r.DB, headered, input.AuthEventIDs)
	if rejectionErr != nil {
		logrus.WithError(rejectionErr).WithField("event_id", event.EventID()).WithField("auth_event_ids", input.AuthEventIDs).Error("helpers.CheckAuthEvents failed for event, rejecting event")
		isRejected = true
	}

	var softfail bool
	if input.Kind == api.KindNew {
		// Check that the event passes authentication checks based on the
		// current room state.
		softfail, err = helpers.CheckForSoftFail(ctx, r.DB, headered, input.StateEventIDs)
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"event_id": event.EventID(),
				"type":     event.Type(),
				"room":     event.RoomID(),
			}).WithError(err).Info("Error authing soft-failed event")
		}
	}

	// If we don't have a transaction ID then get one.
	if input.TransactionID != nil {
		tdID := input.TransactionID
		eventID, err = r.DB.GetTransactionEventID(
			ctx, tdID.TransactionID, tdID.SessionID, event.Sender(),
		)
		// On error OR event with the transaction already processed/processesing
		if err != nil || eventID != "" {
			return
		}
	}

	// Store the event.
	_, stateAtEvent, redactionEvent, redactedEventID, err := r.DB.StoreEvent(ctx, event, input.TransactionID, authEventNIDs, isRejected)
	if err != nil {
		return "", fmt.Errorf("r.DB.StoreEvent: %w", err)
	}

	// if storing this event results in it being redacted then do so.
	if !isRejected && redactedEventID == event.EventID() {
		r, rerr := eventutil.RedactEvent(redactionEvent, &event)
		if rerr != nil {
			return "", fmt.Errorf("eventutil.RedactEvent: %w", rerr)
		}
		event = *r
	}

	// For outliers we can stop after we've stored the event itself as it
	// doesn't have any associated state to store and we don't need to
	// notify anyone about it.
	if input.Kind == api.KindOutlier {
		logrus.WithFields(logrus.Fields{
			"event_id": event.EventID(),
			"type":     event.Type(),
			"room":     event.RoomID(),
			"sender":   event.Sender(),
		}).Debug("Stored outlier")
		return event.EventID(), nil
	}

	roomInfo, err := r.DB.RoomInfo(ctx, event.RoomID())
	if err != nil {
		return "", fmt.Errorf("r.DB.RoomInfo: %w", err)
	}
	if roomInfo == nil {
		return "", fmt.Errorf("r.DB.RoomInfo missing for room %s", event.RoomID())
	}

	if stateAtEvent.BeforeStateSnapshotNID == 0 {
		// We haven't calculated a state for this event yet.
		// Lets calculate one.
		err = r.calculateAndSetState(ctx, input, *roomInfo, &stateAtEvent, event, isRejected)
		if err != nil && input.Kind != api.KindOld {
			return "", fmt.Errorf("r.calculateAndSetState: %w", err)
		}
	}

	// We stop here if the event is rejected: We've stored it but won't update forward extremities or notify anyone about it.
	if isRejected || softfail {
		logrus.WithFields(logrus.Fields{
			"event_id":  event.EventID(),
			"type":      event.Type(),
			"room":      event.RoomID(),
			"soft_fail": softfail,
			"sender":    event.Sender(),
		}).Debug("Stored rejected event")
		return event.EventID(), rejectionErr
	}

	switch input.Kind {
	case api.KindNew:
		if err = r.updateLatestEvents(
			ctx,                 // context
			roomInfo,            // room info for the room being updated
			stateAtEvent,        // state at event (below)
			event,               // event
			input.SendAsServer,  // send as server
			input.TransactionID, // transaction ID
			input.HasState,      // rewrites state?
		); err != nil {
			return "", fmt.Errorf("r.updateLatestEvents: %w", err)
		}
	case api.KindOld:
		err = r.WriteOutputEvents(event.RoomID(), []api.OutputEvent{
			{
				Type: api.OutputTypeOldRoomEvent,
				OldRoomEvent: &api.OutputOldRoomEvent{
					Event: headered,
				},
			},
		})
		if err != nil {
			return "", fmt.Errorf("r.WriteOutputEvents (old): %w", err)
		}
	}

	// processing this event resulted in an event (which may not be the one we're processing)
	// being redacted. We are guaranteed to have both sides (the redaction/redacted event),
	// so notify downstream components to redact this event - they should have it if they've
	// been tracking our output log.
	if redactedEventID != "" {
		err = r.WriteOutputEvents(event.RoomID(), []api.OutputEvent{
			{
				Type: api.OutputTypeRedactedEvent,
				RedactedEvent: &api.OutputRedactedEvent{
					RedactedEventID: redactedEventID,
					RedactedBecause: redactionEvent.Headered(headered.RoomVersion),
				},
			},
		})
		if err != nil {
			return "", fmt.Errorf("r.WriteOutputEvents (redactions): %w", err)
		}
	}

	// Update the extremities of the event graph for the room
	return event.EventID(), nil
}

func (r *Inputer) calculateAndSetState(
	ctx context.Context,
	input *api.InputRoomEvent,
	roomInfo types.RoomInfo,
	stateAtEvent *types.StateAtEvent,
	event gomatrixserverlib.Event,
	isRejected bool,
) error {
	var err error
	roomState := state.NewStateResolution(r.DB, roomInfo)

	if input.HasState && !isRejected {
		// Check here if we think we're in the room already.
		stateAtEvent.Overwrite = true
		var joinEventNIDs []types.EventNID
		// Request join memberships only for local users only.
		if joinEventNIDs, err = r.DB.GetMembershipEventNIDsForRoom(ctx, roomInfo.RoomNID, true, true); err == nil {
			// If we have no local users that are joined to the room then any state about
			// the room that we have is quite possibly out of date. Therefore in that case
			// we should overwrite it rather than merge it.
			stateAtEvent.Overwrite = len(joinEventNIDs) == 0
		}

		// We've been told what the state at the event is so we don't need to calculate it.
		// Check that those state events are in the database and store the state.
		var entries []types.StateEntry
		if entries, err = r.DB.StateEntriesForEventIDs(ctx, input.StateEventIDs); err != nil {
			return fmt.Errorf("r.DB.StateEntriesForEventIDs: %w", err)
		}
		entries = types.DeduplicateStateEntries(entries)

		if stateAtEvent.BeforeStateSnapshotNID, err = r.DB.AddState(ctx, roomInfo.RoomNID, nil, entries); err != nil {
			return fmt.Errorf("r.DB.AddState: %w", err)
		}
	} else {
		stateAtEvent.Overwrite = false

		// We haven't been told what the state at the event is so we need to calculate it from the prev_events
		if stateAtEvent.BeforeStateSnapshotNID, err = roomState.CalculateAndStoreStateBeforeEvent(ctx, event, isRejected); err != nil {
			return fmt.Errorf("roomState.CalculateAndStoreStateBeforeEvent: %w", err)
		}
	}

	err = r.DB.SetState(ctx, stateAtEvent.EventNID, stateAtEvent.BeforeStateSnapshotNID)
	if err != nil {
		return fmt.Errorf("r.DB.SetState: %w", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

var GUIDest string

func GUIInit() {

}

func GUIEventLoop() {

}

func GUIRender() {

}
<commit_msg>Importing gogobject<commit_after>package main

import (
	"os"

	"github.com/bytbox/gogobject/gtk-3.0"
)

var GUIDest string

func GUIInit() {
	gtk.Init(os.Args)
}

func GUIEventLoop() {

}

func GUIRender() {

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"runtime"

	"image"
	"image/png"

	_ "code.google.com/p/go.image/tiff"

	"github.com/shurcooL/trayhost"
)

// TODO: Load from config. Have ability to set config.
var hostFlag = flag.String("host", "", "Target server host.")
var debugFlag = flag.Bool("debug", false, "Adds menu items for debugging purposes.")

func instantShareHandler() {
	fmt.Println("grab content, content-type of clipboard")

	img, err := trayhost.GetClipboardImage()
	if err != nil {
		log.Println(err)
		return
	}

	// Convert image to desired destination format (currently, always PNG).
	var imageData []byte
	switch img.Kind {
	case trayhost.ImageKindPng:
		imageData = img.Bytes
	case trayhost.ImageKindTiff:
		m, _, err := image.Decode(bytes.NewReader(img.Bytes))
		if err != nil {
			log.Panicln("image.Decode:", err)
		}

		var buf bytes.Buffer
		err = png.Encode(&buf, m)
		if err != nil {
			log.Panicln("png.Encode:", err)
		}
		imageData = buf.Bytes()
	default:
		log.Println("unsupported source image kind:", img.Kind)
		return
	}

	fmt.Println("request URL")

	resp, err := http.Get(*hostFlag + "/api/getfilename?ext=png")
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	filename, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		return
	}

	fmt.Println("display/put URL in clipboard")

	url := *hostFlag + "/" + string(filename)
	trayhost.SetClipboardString(url)
	// TODO: Notification? Or not?

	fmt.Println("upload image in background of size", len(imageData))

	go func() {
		req, err := http.NewRequest("PUT", url, bytes.NewReader(imageData))
		if err != nil {
			log.Println(err)
			return
		}
		req.Header.Set("Content-Type", "application/octet-stream")
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			log.Println(err)
			return
		}
		_ = resp.Body.Close()
		fmt.Println("done")
	}()
}

func main() {
	flag.Parse()
	runtime.LockOSThread()

	menuItems := []trayhost.MenuItem{
		trayhost.MenuItem{
			Title:   "Instant Share",
			Handler: instantShareHandler,
		},
		trayhost.SeparatorMenuItem(),
		trayhost.MenuItem{
			Title:   "Quit",
			Handler: trayhost.Exit,
		},
	}
	if *debugFlag {
		menuItems = append(menuItems,
			trayhost.SeparatorMenuItem(),
			trayhost.MenuItem{
				Title: "Debug: Get Clipboard String",
				Handler: func() {
					str, err := trayhost.GetClipboardString()
					fmt.Printf("GetClipboardString(): %q %v\n", str, err)
				},
			},
			trayhost.MenuItem{
				Title: "Debug: Get Clipboard Image",
				Handler: func() {
					img, err := trayhost.GetClipboardImage()
					fmt.Printf("GetClipboardImage(): %v len(%v) %v\n", img.Kind, len(img.Bytes), err)
				},
			},
			trayhost.MenuItem{
				Title: "Debug: Set Clipboard",
				Handler: func() {
					trayhost.SetClipboardString("http://www.example.org/image.png")
				},
			},
		)
	}

	// TODO: Create a real icon and bake it into the binary.
	// TODO: Optionally, if non-Retina pixel perfection is required, generate or load 1x image and supply that as a second representation.
	iconData, err := ioutil.ReadFile("./icon@2x.png")
	if err != nil {
		panic(err)
	}

	fmt.Println("Starting.")

	trayhost.Initialize("InstantShare", iconData, menuItems)

	trayhost.EnterLoop()

	fmt.Println("Exiting.")
}
<commit_msg>Add timeout to potentially blocking getfilename API call.<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"runtime"
	"time"

	"image"
	"image/png"

	_ "code.google.com/p/go.image/tiff"

	"github.com/shurcooL/trayhost"
)

// TODO: Load from config. Have ability to set config.
var hostFlag = flag.String("host", "", "Target server host.")
var debugFlag = flag.Bool("debug", false, "Adds menu items for debugging purposes.")

func instantShareHandler() {
	fmt.Println("grab content, content-type of clipboard")

	img, err := trayhost.GetClipboardImage()
	if err != nil {
		log.Println(err)
		return
	}

	// Convert image to desired destination format (currently, always PNG).
	var imageData []byte
	switch img.Kind {
	case trayhost.ImageKindPng:
		imageData = img.Bytes
	case trayhost.ImageKindTiff:
		m, _, err := image.Decode(bytes.NewReader(img.Bytes))
		if err != nil {
			log.Panicln("image.Decode:", err)
		}

		var buf bytes.Buffer
		err = png.Encode(&buf, m)
		if err != nil {
			log.Panicln("png.Encode:", err)
		}
		imageData = buf.Bytes()
	default:
		log.Println("unsupported source image kind:", img.Kind)
		return
	}

	fmt.Println("request URL")

	resp, err := (&http.Client{Timeout: 1 * time.Second}).Get(*hostFlag + "/api/getfilename?ext=png")
	if err != nil {
		// TODO: Failure notification?
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	filename, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		// TODO: Failure notification?
		log.Println(err)
		return
	}

	fmt.Println("display/put URL in clipboard")

	url := *hostFlag + "/" + string(filename)
	trayhost.SetClipboardString(url)
	// TODO: Success notification? Or not?

	fmt.Println("upload image in background of size", len(imageData))

	go func() {
		req, err := http.NewRequest("PUT", url, bytes.NewReader(imageData))
		if err != nil {
			log.Println(err)
			return
		}
		req.Header.Set("Content-Type", "application/octet-stream")
		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			log.Println(err)
			return
		}
		_ = resp.Body.Close()
		fmt.Println("done")
	}()
}

func main() {
	flag.Parse()
	runtime.LockOSThread()

	menuItems := []trayhost.MenuItem{
		trayhost.MenuItem{
			Title:   "Instant Share",
			Handler: instantShareHandler,
		},
		trayhost.SeparatorMenuItem(),
		trayhost.MenuItem{
			Title:   "Quit",
			Handler: trayhost.Exit,
		},
	}
	if *debugFlag {
		menuItems = append(menuItems,
			trayhost.SeparatorMenuItem(),
			trayhost.MenuItem{
				Title: "Debug: Get Clipboard String",
				Handler: func() {
					str, err := trayhost.GetClipboardString()
					fmt.Printf("GetClipboardString(): %q %v\n", str, err)
				},
			},
			trayhost.MenuItem{
				Title: "Debug: Get Clipboard Image",
				Handler: func() {
					img, err := trayhost.GetClipboardImage()
					fmt.Printf("GetClipboardImage(): %v len(%v) %v\n", img.Kind, len(img.Bytes), err)
				},
			},
			trayhost.MenuItem{
				Title: "Debug: Set Clipboard",
				Handler: func() {
					trayhost.SetClipboardString("http://www.example.org/image.png")
				},
			},
		)
	}

	// TODO: Create a real icon and bake it into the binary.
	// TODO: Optionally, if non-Retina pixel perfection is required, generate or load 1x image and supply that as a second representation.
	iconData, err := ioutil.ReadFile("./icon@2x.png")
	if err != nil {
		panic(err)
	}

	fmt.Println("Starting.")

	trayhost.Initialize("InstantShare", iconData, menuItems)

	trayhost.EnterLoop()

	fmt.Println("Exiting.")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"time"

	_ "net/http/pprof"

	uuid "github.com/satori/go.uuid"
	"github.com/sirupsen/logrus"
)

func main() {
	logrus.SetLevel(logrus.ErrorLevel)
	interfaces, err := net.Interfaces()
	if err != nil {
		panic("Error:" + err.Error())
	}
	var prefix string
	for _, inter := range interfaces {
		prefix += fmt.Sprintf("|%s%s|", inter.Name, inter.HardwareAddr)
	}
	prefix = uuid.NewV5(uuid.NewV4(), prefix).String()
	portAddrPtr := flag.String("port", "10000", "http port")
	hostAddrPtr := flag.String("host", ":10001", "tcp addr")
	usersPtr := flag.Int("users", 5000, "user count")
	topicsPtr := flag.Int("topics", 1000, "topic count")
	perPtr := flag.Int("per", 20, "topics per user")
	durationPtr := flag.Int64("duration", 30, "duration in second")
	flag.Parse()
	ch := make(chan struct{}, 4)
	go func() {
		for i := 0; i < *usersPtr; i++ {
			ch <- struct{}{}
			cli := NewClient(*hostAddrPtr, prefix+"|user:"+fmt.Sprintln(i), "robot")
			idx := i
			go func() {
				slot := idx % (*topicsPtr / *perPtr)
				start := (*perPtr) * slot
				for i := start; i < start+*perPtr; i++ {
					cli.Subscribe("room:" + fmt.Sprintln(i))
					<-time.After(time.Millisecond * 5)
				}
				<-ch
				go func() {
					for {
						topicID := cli.RandomTopic()
						cli.Publish(topicID, fmt.Sprintln(time.Now().Unix()))
						<-time.After(time.Duration(int64(time.Second) * *durationPtr))
					}
				}()
			}()
			go func() {
				for {
					cli.Ping()
					<-time.After(time.Second * 30)
				}
			}()
		}
	}()
	log.Print(http.ListenAndServe(":"+*portAddrPtr, nil))
}
<commit_msg>no message<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"net/http"
	"time"

	_ "net/http/pprof"

	uuid "github.com/satori/go.uuid"
	"github.com/sirupsen/logrus"
)

func main() {
	logrus.SetLevel(logrus.ErrorLevel)
	interfaces, err := net.Interfaces()
	if err != nil {
		panic("Error:" + err.Error())
	}
	var prefix string
	for _, inter := range interfaces {
		prefix += fmt.Sprintf("|%s%s|", inter.Name, inter.HardwareAddr)
	}
	prefix = uuid.NewV5(uuid.NewV4(), prefix).String()
	portAddrPtr := flag.String("port", "10000", "http port")
	hostAddrPtr := flag.String("host", ":10001", "tcp addr")
	usersPtr := flag.Int("users", 5000, "user count")
	topicsPtr := flag.Int("topics", 250, "topic count")
	perPtr := flag.Int("per", 20, "topics per user")
	durationPtr := flag.Int64("duration", 30, "duration in second")
	flag.Parse()
	ch := make(chan struct{}, 4)
	go func() {
		for i := 0; i < *usersPtr; i++ {
			ch <- struct{}{}
			cli := NewClient(*hostAddrPtr, prefix+"|user:"+fmt.Sprintln(i), "robot")
			idx := i
			go func() {
				slot := idx % (*topicsPtr / *perPtr)
				start := (*perPtr) * slot
				for i := start; i < start+*perPtr; i++ {
					cli.Subscribe("room:" + fmt.Sprintln(i))
					<-time.After(time.Millisecond * 5)
				}
				<-ch
				go func() {
					for {
						topicID := cli.RandomTopic()
						cli.Publish(topicID, fmt.Sprintln(time.Now().Unix()))
						<-time.After(time.Duration(int64(time.Second) * *durationPtr))
					}
				}()
			}()
			go func() {
				for {
					cli.Ping()
					<-time.After(time.Second * 30)
				}
			}()
		}
	}()
	log.Print(http.ListenAndServe(":"+*portAddrPtr, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"io/ioutil"
	"log"
	"os"
	"flag"
	"time"
	"sync/atomic"
	"runtime"
)

import (
	"crypto"
	_ "crypto/md5"
	_ "crypto/sha1"
	_ "crypto/sha256"
	_ "crypto/sha512"
//	_ "golang.org/x/crypto/blake2b"
//	_ "golang.org/x/crypto/blake2s"
//	_ "golang.org/x/crypto/md4"
//	_ "golang.org/x/crypto/ripemd160"
//	_ "golang.org/x/crypto/sha3"
	"hash"
	"reflect"
)

import "github.com/splace/varbinary"
import "github.com/splace/fsflags"

const timeoutStatusCode = 124

type hashIndex struct{
	varbinary.Uint64
}

func NewHashIndex(i uint64) hashIndex{
	return hashIndex{Uint64:varbinary.Uint64(i)}
}

// return new hashindex whos representation is as the source hashindex but with added byte(s)
func hashIndexAppend(hi hashIndex, b ...byte) (nhi hashIndex) {
	buf,_:=hi.MarshalBinary()
	(&nhi).UnmarshalBinary(append(buf,b...))
	return
}

// return new hashindex whos representation is as the source hashindex but with removed byte(s)
func hashIndexTruncate(hi hashIndex,c int) (nhi hashIndex) {
	buf,_:=hi.MarshalBinary()
	(&nhi).UnmarshalBinary(buf[:len(buf)-c])
	return
}

func main() {
	var leadingBitCount uint
	flag.UintVar(&leadingBitCount, "bits", 1, "Number of leading bits being searched for.")
	var bitState bool
	flag.BoolVar(&bitState, "set", false, "Leading bits set.")
	var hashType string
	flag.StringVar(&hashType, "hash", "SHA1", "hash type. one of \"MD4,MD5,SHA1,SHA224,SHA256,SHA384,SHA512,RIPEMD160,SHA3_224,SHA3_256,SHA3_384,SHA3_512,SHA512_224,SHA512_256\"")
	//flag.StringVar(&hashType, "hash", "SHA1", "hash type. one of MD4,MD5,SHA1,SHA224,SHA256,SHA384,SHA512,RIPEMD160,SHA3_224,SHA3_256,SHA3_384,SHA3_512,SHA512_224,SHA512_256,BLAKE2s_256,BLAKE2b_256,BLAKE2b_384,BLAKE2b_512")
	var startHashIndex uint64
	flag.Uint64Var(&startHashIndex, "start", 0, "Hash index to start search from.(default:#0)")
	var stopHashIndex uint64
	flag.Uint64Var(&stopHashIndex, "stop", 0, "Hash index to stop search at.(default:#0 = unlimited)")
	var logInterval time.Duration
	flag.DurationVar(&logInterval, "interval", time.Second, "time between log status reports.")
	var limit time.Duration
	flag.DurationVar(&limit, "end", 0, "search time limit.")
	var help bool
	flag.BoolVar(&help, "help", false, "display help/usage.")
	flag.BoolVar(&help, "h", false, "display help/usage.")
	var quiet bool
	flag.BoolVar(&quiet, "quiet", false, "no progress logging.")
	flag.BoolVar(&quiet, "q", false, "no progress logging.")
	var source fsflags.FileValue
	flag.Var(&source, "i", "input source bytes.(default:<Stdin>)")
	flag.Var(&source, "input", "input source bytes.(default:<Stdin>)")
	var sink fsflags.CreateFileValue
	flag.Var(&sink, "o", "output file, written with input file + nonce appended.(default:Stdout just written with nonce.)")
	flag.Var(&sink, "output", "output file, written with input file + nonce appended.(default:Stdout just written with nonce.)")
	var logToo fsflags.CreateFileValue
	flag.Var(&logToo, "log", "progress log destination.(default:Stderr)")
	flag.Parse()

	if help {
		flag.Usage()
		os.Exit(0)
	}

	// for optimisation #1.1
	var arrayOfBytePerms = [0x100][]byte{}
	for i:=range(arrayOfBytePerms){
		arrayOfBytePerms[i]=[]byte{byte(i)}
	}

	var baseHasher hash.Hash
	switch hashType {
	case "MD4":
		baseHasher = crypto.MD4.New()
	case "MD5":
		baseHasher = crypto.MD5.New()
	case "SHA1":
		baseHasher = crypto.SHA1.New()
	case "SHA224":
		baseHasher = crypto.SHA224.New()
	case "SHA256":
		baseHasher = crypto.SHA256.New()
	case "SHA384":
		baseHasher = crypto.SHA384.New()
	case "SHA512":
		baseHasher = crypto.SHA512.New()
	case "SHA512_224":
		baseHasher = crypto.SHA512_224.New()
	case "SHA512_256":
		baseHasher = crypto.SHA512_256.New()
//	case "RIPEMD160":
//		baseHasher = crypto.RIPEMD160.New()
//	case "SHA3_224":
//		baseHasher = crypto.SHA3_224.New()
//	case "SHA3_256":
//		baseHasher = crypto.SHA3_256.New()
//	case "SHA3_384":
//		baseHasher = crypto.SHA3_384.New()
//	case "SHA3_512":
//		baseHasher = crypto.SHA3_512.New()
//	case "BLAKE2s_256":
//		baseHasher = crypto.BLAKE2s_256.New()
//	case "BLAKE2b_256":
//		baseHasher = crypto.BLAKE2b_256.New()
//	case "BLAKE2b_384":
//		baseHasher = crypto.BLAKE2b_384.New()
//	case "BLAKE2b_512":
//		baseHasher = crypto.BLAKE2b_512.New()
	default:
		log.Fatalf("Aborting, Unknown Hash Scheme:" + hashType)
	}

	if logToo.File == nil {
		logToo.File = os.Stderr
	}

	var progressLog *log.Logger
	if quiet {
		progressLog=log.New(ioutil.Discard, "", log.LstdFlags)
		}else{
		progressLog=log.New(logToo, "", log.LstdFlags)
	}

	if source.File == nil {
		source.File = os.Stdin
	}

	progressLog.Printf("Loading:%q", &source)
	if sink.File == nil {
		sink.File = os.Stdout
		if _, err := io.Copy(baseHasher, source); err != nil {
			log.Fatal(err)
		}
		}else{
		// if not stdout write the source to the sink now, so later writing of the nonce then is appended.
		if _, err := io.Copy(baseHasher, io.TeeReader(source, sink)); err != nil {
			log.Fatal(err)
		}
	}
	source.Close()

	if leadingBitCount > 128 {
		log.Fatalf("Aborting, leading bits matching over 128 not supported (%v)", leadingBitCount)
	}

	// because of optimisation#1, need to find hash index with one byte removed.
	if startHashIndex>0{
		startHashIndex=uint64(hashIndexTruncate(hashIndex{Uint64:varbinary.Uint64(startHashIndex)},1).Uint64)
	}
	if stopHashIndex>0{
		stopHashIndex=uint64(hashIndexTruncate(hashIndex{Uint64:varbinary.Uint64(stopHashIndex)},1).Uint64)+1
		}else{
		stopHashIndex=1<<64-1
	}

	startTime := time.Now()
	doLog := time.NewTicker(logInterval)
	go func() {
		lhashIndex := startHashIndex
		for t := range doLog.C {
			runningFor := t.Sub(startTime)
			if limit > 0 && runningFor > limit {
				progressLog.Print("Aborting: time limit reached")
				os.Exit(timeoutStatusCode)
			}
			progressLog.Printf("#%d @%v\t%.0f#/s\tMean Match:%v", startHashIndex, runningFor/time.Second*time.Second, float64(startHashIndex-lhashIndex)/logInterval.Seconds(), (logInterval / time.Duration(startHashIndex-lhashIndex) * (1 << leadingBitCount) / time.Second * time.Second))
			lhashIndex = startHashIndex
		}
	}()

	var matchCondition func([]byte) bool
	if bitState{
		matchCondition = leadingSetBits(leadingBitCount)
		}else{
		matchCondition = leadingZeroBits(leadingBitCount)
	}

	if sum := baseHasher.Sum(nil); matchCondition(sum) {
		progressLog.Printf("Match on Source file as-is. Hash(%s):[% x]", hashType, sum)
		os.Exit(0)
	}

	stride := uint64(runtime.NumCPU())

	searchStripe := func(start uint64) {
		progressLog.Printf("Starting thread @ #%d", start)
		var hasher, branchHasher hash.Hash
		var sum []byte
		var n int
		buf := make([]byte,8,8)
		for hi := start; hi<=stopHashIndex ; hi += stride {
			hasher = clone(baseHasher).(hash.Hash)
			n=varbinary.Uint64Put(varbinary.Uint64(hi),buf)
			hasher.Write(buf[:n])
			// optimisation#1: rather than check hash, with existing nonce, copy it and check all possible single bytes added to it. (+20% intel core2)
			for i := range arrayOfBytePerms { // optimisation#1.1: use pre-generated array of []byte for added byte. (+5% intel core2)
				branchHasher = clone(hasher).(hash.Hash)
				branchHasher.Write(arrayOfBytePerms[i])
				sum = branchHasher.Sum(nil)
				if matchCondition(sum) {
					doLog.Stop()
					progressLog.Printf("#%d @%.1fs\tMatch:%q+[%s %x] Saving:%q Hash(%s):[% x]", uint64(hashIndexAppend(hashIndex{Uint64:varbinary.Uint64(hi)},arrayOfBytePerms[i][0]).Uint64), time.Since(startTime).Seconds(), &source, varbinary.Uint64(hi), arrayOfBytePerms[i], &sink, hashType, sum)
					n=varbinary.Uint64Put(varbinary.Uint64(hi),buf)
					sink.Write(buf[:n])
					sink.Write(arrayOfBytePerms[i])
					sink.Close()
					os.Exit(0)
				}
			}
			atomic.AddUint64(&startHashIndex, 0x100) //keep track of number checked, optimisation#1: each byte tested
		}
	}

	// start a thread for each core, each searching from different start indexes, but striding the same so always missing each other.
	for t := 0; t < runtime.NumCPU()-1; t++ {
		go func(s uint64) {
			searchStripe(s)
		}(startHashIndex)
		startHashIndex++
	}
	searchStripe(startHashIndex)
	progressLog.Printf("#%d @%.1fs Stopping Search of:%q", startHashIndex, time.Since(startTime).Seconds(), &source)

}

// copy an interface value using reflect (here for pointers to interfaces), because what we want isn't exposed.
func clone(i interface{}) interface{} {
	indirect := reflect.Indirect(reflect.ValueOf(i))
	newIndirect := reflect.New(indirect.Type())
	newIndirect.Elem().Set(reflect.ValueOf(indirect.Interface()))
	return newIndirect.Interface()
}

<commit_msg>remove unused new hashindex<commit_after>package main

import (
	"io"
	"io/ioutil"
	"log"
	"os"
	"flag"
	"time"
	"sync/atomic"
	"runtime"
)

import (
	"crypto"
	_ "crypto/md5"
	_ "crypto/sha1"
	_ "crypto/sha256"
	_ "crypto/sha512"
//	_ "golang.org/x/crypto/blake2b"
//	_ "golang.org/x/crypto/blake2s"
//	_ "golang.org/x/crypto/md4"
//	_ "golang.org/x/crypto/ripemd160"
//	_ "golang.org/x/crypto/sha3"
	"hash"
	"reflect"
)

import "github.com/splace/varbinary"
import "github.com/splace/fsflags"

const timeoutStatusCode = 124

type hashIndex struct{
	varbinary.Uint64
}

// return new hashindex whose representation is as the source hashindex but with added byte(s)
func hashIndexAppend(hi hashIndex, b ...byte) (nhi hashIndex) {
	buf,_:=hi.MarshalBinary()
	(&nhi).UnmarshalBinary(append(buf,b...))
	return
}

// return new hashindex whose representation is as the source hashindex but with removed byte(s)
func hashIndexTruncate(hi hashIndex,c int) (nhi hashIndex) {
	buf,_:=hi.MarshalBinary()
	(&nhi).UnmarshalBinary(buf[:len(buf)-c])
	return
}

func main() {
	var leadingBitCount uint
	flag.UintVar(&leadingBitCount, "bits", 1, "Number of leading bits being searched for.")
	var bitState bool
	flag.BoolVar(&bitState, "set", false, "Leading bits set.")
	var hashType string
	flag.StringVar(&hashType, "hash", "SHA1", "hash type. one of \"MD4,MD5,SHA1,SHA224,SHA256,SHA384,SHA512,RIPEMD160,SHA3_224,SHA3_256,SHA3_384,SHA3_512,SHA512_224,SHA512_256\"")
	//flag.StringVar(&hashType, "hash", "SHA1", "hash type. one of MD4,MD5,SHA1,SHA224,SHA256,SHA384,SHA512,RIPEMD160,SHA3_224,SHA3_256,SHA3_384,SHA3_512,SHA512_224,SHA512_256,BLAKE2s_256,BLAKE2b_256,BLAKE2b_384,BLAKE2b_512")
	var startHashIndex uint64
	flag.Uint64Var(&startHashIndex, "start", 0, "Hash index to start search from.(default:#0)")
	var stopHashIndex uint64
	flag.Uint64Var(&stopHashIndex, "stop", 0, "Hash index to stop search at.(default:#0 = unlimited)")
	var logInterval time.Duration
	flag.DurationVar(&logInterval, "interval", time.Second, "time between log status reports.")
	var limit time.Duration
	flag.DurationVar(&limit, "end", 0, "search time limit.")
	var help bool
	flag.BoolVar(&help, "help", false, "display help/usage.")
	flag.BoolVar(&help, "h", false, "display help/usage.")
	var quiet bool
	flag.BoolVar(&quiet, "quiet", false, "no progress logging.")
	flag.BoolVar(&quiet, "q", false, "no progress logging.")
	var source fsflags.FileValue
	flag.Var(&source, "i", "input source bytes.(default:<Stdin>)")
	flag.Var(&source, "input", "input source bytes.(default:<Stdin>)")
	var sink fsflags.CreateFileValue
	flag.Var(&sink, "o", "output file, written with input file + nonce appended.(default:Stdout just written with nonce.)")
	flag.Var(&sink, "output", "output file, written with input file + nonce appended.(default:Stdout just written with nonce.)")
	var logToo fsflags.CreateFileValue
	flag.Var(&logToo, "log", "progress log destination.(default:Stderr)")
	flag.Parse()

	if help {
		flag.Usage()
		os.Exit(0)
	}

	// for optimisation #1.1
	var arrayOfBytePerms = [0x100][]byte{}
	for i:=range(arrayOfBytePerms){
		arrayOfBytePerms[i]=[]byte{byte(i)}
	}

	var baseHasher hash.Hash
	switch hashType {
	case "MD4":
		baseHasher = crypto.MD4.New()
	case "MD5":
		baseHasher = crypto.MD5.New()
	case "SHA1":
		baseHasher = crypto.SHA1.New()
	case "SHA224":
		baseHasher = crypto.SHA224.New()
	case "SHA256":
		baseHasher = crypto.SHA256.New()
	case "SHA384":
		baseHasher = crypto.SHA384.New()
	case "SHA512":
		baseHasher = crypto.SHA512.New()
	case "SHA512_224":
		baseHasher = crypto.SHA512_224.New()
	case "SHA512_256":
		baseHasher = crypto.SHA512_256.New()
//	case "RIPEMD160":
//		baseHasher = crypto.RIPEMD160.New()
//	case "SHA3_224":
//		baseHasher = crypto.SHA3_224.New()
//	case "SHA3_256":
//		baseHasher = crypto.SHA3_256.New()
//	case "SHA3_384":
//		baseHasher = crypto.SHA3_384.New()
//	case "SHA3_512":
//		baseHasher = crypto.SHA3_512.New()
//	case "BLAKE2s_256":
//		baseHasher = crypto.BLAKE2s_256.New()
//	case "BLAKE2b_256":
//		baseHasher = crypto.BLAKE2b_256.New()
//	case "BLAKE2b_384":
//		baseHasher = crypto.BLAKE2b_384.New()
//	case "BLAKE2b_512":
//		baseHasher = crypto.BLAKE2b_512.New()
	default:
		log.Fatalf("Aborting, Unknown Hash Scheme:" + hashType)
	}

	if logToo.File == nil {
		logToo.File = os.Stderr
	}

	var progressLog *log.Logger
	if quiet {
		progressLog=log.New(ioutil.Discard, "", log.LstdFlags)
		}else{
		progressLog=log.New(logToo, "", log.LstdFlags)
	}

	if source.File == nil {
		source.File = os.Stdin
	}

	progressLog.Printf("Loading:%q", &source)
	if sink.File == nil {
		sink.File = os.Stdout
		if _, err := io.Copy(baseHasher, source); err != nil {
			log.Fatal(err)
		}
		}else{
		// if not stdout write the source to the sink now, so later writing of the nonce then is appended.
		if _, err := io.Copy(baseHasher, io.TeeReader(source, sink)); err != nil {
			log.Fatal(err)
		}
	}
	source.Close()

	if leadingBitCount > 128 {
		log.Fatalf("Aborting, leading bits matching over 128 not supported (%v)", leadingBitCount)
	}

	// because of optimisation#1, need to find hash index with one byte removed.
	if startHashIndex>0{
		startHashIndex=uint64(hashIndexTruncate(hashIndex{Uint64:varbinary.Uint64(startHashIndex)},1).Uint64)
	}
	if stopHashIndex>0{
		stopHashIndex=uint64(hashIndexTruncate(hashIndex{Uint64:varbinary.Uint64(stopHashIndex)},1).Uint64)+1
		}else{
		stopHashIndex=1<<64-1
	}

	startTime := time.Now()
	doLog := time.NewTicker(logInterval)
	go func() {
		lhashIndex := startHashIndex
		for t := range doLog.C {
			runningFor := t.Sub(startTime)
			if limit > 0 && runningFor > limit {
				progressLog.Print("Aborting: time limit reached")
				os.Exit(timeoutStatusCode)
			}
			progressLog.Printf("#%d @%v\t%.0f#/s\tMean Match:%v", startHashIndex, runningFor/time.Second*time.Second, float64(startHashIndex-lhashIndex)/logInterval.Seconds(), (logInterval / time.Duration(startHashIndex-lhashIndex) * (1 << leadingBitCount) / time.Second * time.Second))
			lhashIndex = startHashIndex
		}
	}()

	var matchCondition func([]byte) bool
	if bitState{
		matchCondition = leadingSetBits(leadingBitCount)
		}else{
		matchCondition = leadingZeroBits(leadingBitCount)
	}

	if sum := baseHasher.Sum(nil); matchCondition(sum) {
		progressLog.Printf("Match on Source file as-is. Hash(%s):[% x]", hashType, sum)
		os.Exit(0)
	}

	stride := uint64(runtime.NumCPU())

	searchStripe := func(start uint64) {
		progressLog.Printf("Starting thread @ #%d", start)
		var hasher, branchHasher hash.Hash
		var sum []byte
		var n int
		buf := make([]byte,8,8)
		for hi := start; hi<=stopHashIndex ; hi += stride {
			hasher = clone(baseHasher).(hash.Hash)
			n=varbinary.Uint64Put(varbinary.Uint64(hi),buf)
			hasher.Write(buf[:n])
			// optimisation#1: rather than check hash, with existing nonce, copy it and check all possible single bytes added to it. (+20% intel core2)
			for i := range arrayOfBytePerms { // optimisation#1.1: use pre-generated array of []byte for added byte. (+5% intel core2)
				branchHasher = clone(hasher).(hash.Hash)
				branchHasher.Write(arrayOfBytePerms[i])
				sum = branchHasher.Sum(nil)
				if matchCondition(sum) {
					doLog.Stop()
					progressLog.Printf("#%d @%.1fs\tMatch:%q+[%s %x] Saving:%q Hash(%s):[% x]", uint64(hashIndexAppend(hashIndex{Uint64:varbinary.Uint64(hi)},arrayOfBytePerms[i][0]).Uint64), time.Since(startTime).Seconds(), &source, varbinary.Uint64(hi), arrayOfBytePerms[i], &sink, hashType, sum)
					n=varbinary.Uint64Put(varbinary.Uint64(hi),buf)
					sink.Write(buf[:n])
					sink.Write(arrayOfBytePerms[i])
					sink.Close()
					os.Exit(0)
				}
			}
			atomic.AddUint64(&startHashIndex, 0x100) //keep track of number checked, optimisation#1: each byte tested
		}
	}

	// start a thread for each core, each searching from different start indexes, but striding the same so always missing each other.
	for t := 0; t < runtime.NumCPU()-1; t++ {
		go func(s uint64) {
			searchStripe(s)
		}(startHashIndex)
		startHashIndex++
	}
	searchStripe(startHashIndex)
	progressLog.Printf("#%d @%.1fs Stopping Search of:%q", startHashIndex, time.Since(startTime).Seconds(), &source)

}

// copy an interface value using reflect (here for pointers to interfaces), because what we want isn't exposed.
func clone(i interface{}) interface{} {
	indirect := reflect.Indirect(reflect.ValueOf(i))
	newIndirect := reflect.New(indirect.Type())
	newIndirect.Elem().Set(reflect.ValueOf(indirect.Interface()))
	return newIndirect.Interface()
}

<|endoftext|>


<commit_before>/**
 *  _               _   _
 * | |             | | (_)
 * | |__   __ _ ___| |_ _  ___
 * | '_ \ / _` / __| __| |/ _ \
 * | | | | (_| \__ \ |_| |  __/
 * |_| |_|\__,_|___/\__|_|\___|
 *
 * Hastie - Static Site Generator
 *
 */

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"text/template"
	"time"
)

const debug = false

type Page struct {
	Content  string
	Title    string
	Category string
	Layout   string
	Pages    PagesSlice
	Recent   PagesSlice
	Date     time.Time
	OutFile  string
}

var config map[string]string

type PagesSlice []Page

func (p PagesSlice) Len() int           { return len(p) }
func (p PagesSlice) Less(i, j int) bool { return p[i].Date.Unix() < p[j].Date.Unix() }
func (p PagesSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p PagesSlice) Sort()              { sort.Sort(p) }

func main() {
	//TODO: add verbose / debug - command-line arguments

	setupConfig()

	// create list of directories and files
	site := &SiteStruct{}
	filepath.Walk(config["TemplateDir"], site, nil)

	/* ******************************************
	 * Loop through directories and build pages
	 * ****************************************** */
	var pages PagesSlice
	for _, dir := range site.Directories {

		fmt.Println("Reading Directory: ", dir)
		directory_file_count := 0
		readglob := fmt.Sprintf("%s/*.md", dir)
		var dirfiles, _ = filepath.Glob(readglob)

		// loop through files in directory
		for _, file := range dirfiles {
			if debug {
				fmt.Println("  File: ", file)
			}
			outfile := filepath.Base(file)
			outfile = strings.Replace(outfile, ".md", ".html", 1)

			// read & parse file for parameters
			page := readParseFile(file)

			// skip file if no content
			if page.Content == "" {
				continue // skip to next file
			}
			pages = append(pages, page)
			directory_file_count += 1
		}
	}

	if debug { // spit out pages structure
		fmt.Println("################################################################################")
		fmt.Printf(" %-50s | %-10s | %s \n", "Title", "Category", "Outfile")
		fmt.Println("--------------------------------------------------------------------------------")
		for _, page := range pages {
			fmt.Printf(" %-50s | %-10s | %s \n", page.Title, page.Category, page.OutFile)
		}
		fmt.Println("################################################################################")
	}

	/* ******************************************
	 * Create any data needed from pages
	 * for example recent file list
	 * category list, etc...
	 * ****************************************** */

	// build recent file list, sorted
	recentList := getRecentList(pages)

	/* ******************************************
	 * Loop through pages and generate templates
	 * ****************************************** */
	for _, page := range pages {

		fmt.Println("  Generating Template: ", page.OutFile)

		/* Assign global data to page object
		 * Note: need better templating duplicating data
		         since no logic in templates to limit to 3 */
		page.Pages = recentList
		if len(recentList) > 3 {
			page.Recent = recentList[0:3]
		} else {
			page.Recent = recentList
		}

		/* Templating - writes page data to buffer
		 * read and parse all template files          */
		buffer := new(bytes.Buffer)
		layoutsglob := fmt.Sprintf("%s/*.html", config["LayoutDir"])
		ts, err := template.ParseGlob(layoutsglob)
		if err != nil {
			fmt.Println("Error Parsing Templates: ", err)
			os.Exit(1)
		}
		// pick layout based on specified in file
		templateFile := ""
		if page.Layout == "" {
			templateFile = "post.html"
		} else {
			templateFile = fmt.Sprintf("%s.html", page.Layout)
		}
		ts.Execute(buffer, templateFile, page)

		// writing out file
		writedir := fmt.Sprintf("%s/%s", config["PublishDir"], page.Category)
		if debug {
			fmt.Println(" Write Directory: ", writedir)
		}
		os.MkdirAll(writedir, 0755) // does nothing if already exists

		outfile := fmt.Sprintf("%s/%s", config["PublishDir"], page.OutFile)
		if debug {
			fmt.Println(" Writing File: ", outfile)
		}
		ioutil.WriteFile(outfile, []byte(buffer.String()), 0644)
	}

}

/* ************************************************
 * Read and Parse File
 * @param filename
 * @return Page object
 * ************************************************ */
func readParseFile(filename string) (page Page) {
	if debug {
		fmt.Println("Parsing File: ", filename)
	}
	epoch, _ := time.Parse("20060102", "19700101")

	// setup default page struct
	page = Page{
		Title:    "",
		Category: "",
		Content:  "",
		Layout:   "",
		Date:     epoch,
		OutFile:  filename}

	// read file
	var data, err = ioutil.ReadFile(filename)
	if err != nil {
		fmt.Println("Error Reading: ", filename)
		return
	}

	// go through content parse from --- to ---
	var lines = strings.Split(string(data), "\n")
	var found = 0
	for i, line := range lines {
		line = strings.TrimSpace(line)

		if found == 1 {
			// parse line for param
			colonIndex := strings.Index(line, ":")
			if colonIndex > 0 {
				key := strings.TrimSpace(line[:colonIndex])
				value := strings.TrimSpace(line[colonIndex+1:])
				switch key {
				case "title":
					page.Title = value
				case "category":
					page.Category = value
				case "layout":
					page.Layout = value
				}
			}

		} else if found >= 2 {
			// params over
			lines = lines[i:]
			break
		}

		if line == "---" {
			found += 1
		}

	}

	// switch directory name to just category

	// chop off first directory, since that is the template dir
	page.OutFile = filename[strings.Index(filename, "/")+1:]
	page.OutFile = strings.Replace(page.OutFile, ".md", ".html", 1)

	// next first directory is category
	if strings.Contains(page.OutFile, "/") {
		page.Category = page.OutFile[0:strings.Index(page.OutFile, "/")]
	}

	// parse date from filename
	base := filepath.Base(page.OutFile)
	if base[0:2] == "20" || base[0:2] == "19" { //HACK: if file starts with 20 or 19 assume date
		page.Date, _ = time.Parse("2006-01-02", base[0:10])
		page.OutFile = strings.Replace(page.OutFile, base[0:11], "", 1) // remove date from final filename
	}

	// convert markdown content
	content := strings.Join(lines, "\n")
	output := blackfriday.MarkdownCommon([]byte(content))
	page.Content = string(output)

	return page
}

/* ************************************************
 * Build Recent File List
 *    - return array sorted most recent first
 *    - array includes real link (no date)
 *    - does not include files without date
 * ************************************************ */
func getRecentList(pages PagesSlice) (list PagesSlice) {
	fmt.Println("Creating Recent File List")
	for _, page := range pages {
		// pages without dates are set to epoch
		if page.Date.Format("2006") != "1970" {
			list = append(list, page)
		}
	}
	list.Sort()

	// reverse
	for i, j := 0, len(list)-1; i < j; i, j = i+1, j-1 {
		list[i], list[j] = list[j], list[i]
	}

	return list
}

/* ************************************************
 *  File / Directory Walker
 * ************************************************ */
type SiteStruct struct {
	Files       []string
	Directories []string
	Categories  []string
}

func (v *SiteStruct) VisitDir(path string, f os.FileInfo) bool {
	v.Categories = append(v.Categories, f.Name())
	v.Directories = append(v.Directories, path)
	/* fmt.Println("Directory: ", f.Name)*/
	return true
}

func (v *SiteStruct) VisitFile(path string, f os.FileInfo) {
	v.Files = append(v.Files, path)
	/* fmt.Printf("File: %s  -- Path: %s \n", f.Name, path)*/
}

/* ************************************************
 * Check if File / Directory Exists
 * ************************************************ */
func exists(path string) bool {
	_, err := os.Stat(path)
	if err != nil {
		if e, ok := err.(*os.PathError); ok && (e.Error == os.ENOENT || e.Error == os.ENOTDIR) {
			return false
		}
		return false
	}
	return true
}

/* ************************************************
 * Read Config file or set defaults
 * ************************************************ */
func setupConfig() {
	file, err := ioutil.Readfile("hastie.json")
	if err != nil {
		// set defaults
		config["TemplateDir"] = "posts"
		config["LayoutDir"] = "layouts"
		config["PublishDir"] = "public"
	} else {
		if err := json.Unmarshal(file, &config); err != nil {
			fmt.Printf("Error parsing config: %s", err)
			os.Exit(1)
		}
	}
}
<commit_msg>New WalkFn implemented, cleanups to be done still<commit_after>/**
 *  _               _   _
 * | |             | | (_)
 * | |__   __ _ ___| |_ _  ___
 * | '_ \ / _` / __| __| |/ _ \
 * | | | | (_| \__ \ |_| |  __/
 * |_| |_|\__,_|___/\__|_|\___|
 *
 * Hastie - Static Site Generator
 *
 */

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"text/template"
	"time"
)

const debug = true

type Page struct {
	Content  string
	Title    string
	Category string
	Layout   string
	Pages    PagesSlice
	Recent   PagesSlice
	Date     time.Time
	OutFile  string
}

var config map[string]string

type PagesSlice []Page

func (p PagesSlice) Len() int           { return len(p) }
func (p PagesSlice) Less(i, j int) bool { return p[i].Date.Unix() < p[j].Date.Unix() }
func (p PagesSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
func (p PagesSlice) Sort()              { sort.Sort(p) }

// create list of directories and files
var site = &SiteStruct{}

func main() {
	//TODO: add verbose / debug - command-line arguments

	setupConfig()

	//filepath.Walk(config["TemplateDir"], site, nil)
	filepath.Walk(config["TemplateDir"], Walker)

	/* ******************************************
	 * Loop through directories and build pages
	 * ****************************************** */
	var pages PagesSlice
	for _, dir := range site.Directories {

		fmt.Println("Reading Directory: ", dir)
		directory_file_count := 0
		readglob := fmt.Sprintf("%s/*.md", dir)
		var dirfiles, _ = filepath.Glob(readglob)
		fmt.Println("---> dirfiles:", dirfiles)

		// loop through files in directory
		for _, file := range dirfiles {
			if debug {
				fmt.Println("  File: ", file)
			}
			outfile := filepath.Base(file)
			outfile = strings.Replace(outfile, ".md", ".html", 1)

			// read & parse file for parameters
			page := readParseFile(file)

			// skip file if no content
			if page.Content == "" {
				continue // skip to next file
			}
			pages = append(pages, page)
			directory_file_count += 1
		}
	}

	if debug { // spit out pages structure
		fmt.Println("################################################################################")
		fmt.Printf(" %-50s | %-10s | %s \n", "Title", "Category", "Outfile")
		fmt.Println("--------------------------------------------------------------------------------")
		for _, page := range pages {
			fmt.Printf(" %-50s | %-10s | %s \n", page.Title, page.Category, page.OutFile)
		}
		fmt.Println("################################################################################")
	}

	/* ******************************************
	 * Create any data needed from pages
	 * for example recent file list
	 * category list, etc...
	 * ****************************************** */

	// build recent file list, sorted
	recentList := getRecentList(pages)

	/* ******************************************
	 * Loop through pages and generate templates
	 * ****************************************** */
	for _, page := range pages {

		fmt.Println("  Generating Template: ", page.OutFile)

		/* Assign global data to page object
		 * Note: need better templating duplicating data
		         since no logic in templates to limit to 3 */
		page.Pages = recentList
		if len(recentList) > 3 {
			page.Recent = recentList[0:3]
		} else {
			page.Recent = recentList
		}

		/* Templating - writes page data to buffer
		 * read and parse all template files          */
		buffer := new(bytes.Buffer)
		layoutsglob := fmt.Sprintf("%s/*.html", config["LayoutDir"])
		ts, err := template.ParseGlob(layoutsglob)
		if err != nil {
			fmt.Println("Error Parsing Templates: ", err)
			os.Exit(1)
		}
		// pick layout based on specified in file
		templateFile := ""
		if page.Layout == "" {
			templateFile = "post.html"
		} else {
			templateFile = fmt.Sprintf("%s.html", page.Layout)
		}
		ts.ExecuteTemplate(buffer, templateFile, page)

		// writing out file
		writedir := fmt.Sprintf("%s/%s", config["PublishDir"], page.Category)
		if debug {
			fmt.Println(" Write Directory: ", writedir)
		}
		os.MkdirAll(writedir, 0755) // does nothing if already exists

		outfile := fmt.Sprintf("%s/%s", config["PublishDir"], page.OutFile)
		if debug {
			fmt.Println(" Writing File: ", outfile)
		}
		ioutil.WriteFile(outfile, []byte(buffer.String()), 0644)
	}

}

/* ************************************************
 * Read and Parse File
 * @param filename
 * @return Page object
 * ************************************************ */
func readParseFile(filename string) (page Page) {
	if debug {
		fmt.Println("Parsing File: ", filename)
	}
	epoch, _ := time.Parse("20060102", "19700101")

	// setup default page struct
	page = Page{
		Title:    "",
		Category: "",
		Content:  "",
		Layout:   "",
		Date:     epoch,
		OutFile:  filename}

	// read file
	var data, err = ioutil.ReadFile(filename)
	if err != nil {
		fmt.Println("Error Reading: ", filename)
		return
	}

	// go through content parse from --- to ---
	var lines = strings.Split(string(data), "\n")
	var found = 0
	for i, line := range lines {
		line = strings.TrimSpace(line)

		if found == 1 {
			// parse line for param
			colonIndex := strings.Index(line, ":")
			if colonIndex > 0 {
				key := strings.TrimSpace(line[:colonIndex])
				value := strings.TrimSpace(line[colonIndex+1:])
				switch key {
				case "title":
					page.Title = value
				case "category":
					page.Category = value
				case "layout":
					page.Layout = value
				}
			}

		} else if found >= 2 {
			// params over
			lines = lines[i:]
			break
		}

		if line == "---" {
			found += 1
		}

	}

	// switch directory name to just category

	// chop off first directory, since that is the template dir
	page.OutFile = filename[strings.Index(filename, "/")+1:]
	page.OutFile = strings.Replace(page.OutFile, ".md", ".html", 1)

	// next first directory is category
	if strings.Contains(page.OutFile, "/") {
		page.Category = page.OutFile[0:strings.Index(page.OutFile, "/")]
	}

	// parse date from filename
	base := filepath.Base(page.OutFile)
	if base[0:2] == "20" || base[0:2] == "19" { //HACK: if file starts with 20 or 19 assume date
		page.Date, _ = time.Parse("2006-01-02", base[0:10])
		page.OutFile = strings.Replace(page.OutFile, base[0:11], "", 1) // remove date from final filename
	}

	// convert markdown content
	content := strings.Join(lines, "\n")
	output := blackfriday.MarkdownCommon([]byte(content))
	page.Content = string(output)

	return page
}

/* ************************************************
 * Build Recent File List
 *    - return array sorted most recent first
 *    - array includes real link (no date)
 *    - does not include files without date
 * ************************************************ */
func getRecentList(pages PagesSlice) (list PagesSlice) {
	fmt.Println("Creating Recent File List")
	for _, page := range pages {
		// pages without dates are set to epoch
		if page.Date.Format("2006") != "1970" {
			list = append(list, page)
		}
	}
	list.Sort()

	// reverse
	for i, j := 0, len(list)-1; i < j; i, j = i+1, j-1 {
		list[i], list[j] = list[j], list[i]
	}

	return list
}

/* ************************************************
 *  File / Directory Walker
 * ************************************************ */
type SiteStruct struct {
	Files       []string
	Directories []string
	Categories  []string
}

func Walker(fn string, fi os.FileInfo, err error) error {
	if err != nil {
		fmt.Println("Walker:", err)
		return nil
	}

	if fi.IsDir() {
		fmt.Println("Found Dir", fi.Name())
		site.Categories = append(site.Categories, fi.Name())
		site.Directories = append(site.Directories, fn)
		return nil
	} else {
		site.Files = append(site.Files, fi.Name())
		return nil
	}
	return nil

}

func (v *SiteStruct) VisitDir(path string, f os.FileInfo) bool {
	v.Categories = append(v.Categories, f.Name())
	v.Directories = append(v.Directories, path)
	/* fmt.Println("Directory: ", f.Name)*/
	return true
}

func (v *SiteStruct) VisitFile(path string, f os.FileInfo) {
	v.Files = append(v.Files, path)
	/* fmt.Printf("File: %s  -- Path: %s \n", f.Name, path)*/
}

/* ************************************************
 * Check if File / Directory Exists
 * ************************************************ */
func exists(path string) bool {
	_, err := os.Stat(path)
	if err != nil {
		return false
	}
	return true
}

/* ************************************************
 * Read Config file or set defaults
 * ************************************************ */
func setupConfig() {
	file, err := ioutil.ReadFile("hastie.json")
	if err != nil {
		// set defaults
		config["TemplateDir"] = "posts"
		config["LayoutDir"] = "layouts"
		config["PublishDir"] = "public"
	} else {
		if err := json.Unmarshal(file, &config); err != nil {
			fmt.Printf("Error parsing config: %s", err)
			os.Exit(1)
		}
	}
}
<|endoftext|>


<commit_before>package client

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"net/http"
)

var _ = Describe("Client", func() {
	It("should create a instance of a Client", func() {
		xclient, err := NewClient("http://localhost.api.snickers/")
		Expect(err).NotTo(HaveOccurred())
		Expect(xclient.Endpoint).To(Equal("http://localhost.api.snickers/"))
	})

	It("should create an instance of APIError", func() {
		apiError := &APIError{
			Status: http.StatusInternalServerError,
			Errors: "Internal Server Error",
		}
		Expect(apiError.Error()).To(Equal(`Error returned by the Snickers API: {"status":500,"errors":"Internal Server Error"}`))
	})

	It("should execute do generic method", func() {
		server := StartFakeServer(http.StatusOK, `[{
				  "name": "mp4_240p",
				  "description": "Test Preset",
				  "container": "mp4",
				  "rateControl": "vbr",
				  "video": {
				      "height": "720",
				      "width": "1280",
				      "codec": "h264",
				      "bitrate": "10000"
				   },
				  "audio": {
				      "codec": "aac",
				      "bitrate": "64000"
				  }
			}]`)
		defer server.Close()
		client, _ := NewClient(server.URL)

		var respObj []Preset
		err := client.do("GET", "/presets", []interface{}{}, &respObj)
		Expect(respObj[0].Name).To(Equal("mp4_240p"))
		Expect(err).NotTo(HaveOccurred())

	})
})
<commit_msg>fixing identation<commit_after>package client

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"net/http"
)

var _ = Describe("Client", func() {
	It("should create a instance of a Client", func() {
		xclient, err := NewClient("http://localhost.api.snickers/")
		Expect(err).NotTo(HaveOccurred())
		Expect(xclient.Endpoint).To(Equal("http://localhost.api.snickers/"))
	})

	It("should create an instance of APIError", func() {
		apiError := &APIError{
			Status: http.StatusInternalServerError,
			Errors: "Internal Server Error",
		}
		Expect(apiError.Error()).To(Equal(`Error returned by the Snickers API: {"status":500,"errors":"Internal Server Error"}`))
	})

	It("should execute do generic method", func() {
		server := StartFakeServer(http.StatusOK, `[{
			  "name": "mp4_240p",
			  "description": "Test Preset",
			  "container": "mp4",
			  "rateControl": "vbr",
			  "video": {
			      "height": "720",
			      "width": "1280",
			      "codec": "h264",
			      "bitrate": "10000"
			   },
			  "audio": {
			      "codec": "aac",
			      "bitrate": "64000"
			}
		}]`)
		defer server.Close()
		client, _ := NewClient(server.URL)

		var respObj []Preset
		err := client.do("GET", "/presets", []interface{}{}, &respObj)
		Expect(respObj[0].Name).To(Equal("mp4_240p"))
		Expect(err).NotTo(HaveOccurred())

	})
})
<|endoftext|>


<commit_before>package onion

import (
	"reflect"
	"strings"
	"time"
)

var (
	durationValue = reflect.ValueOf(time.Second)
)

// GetStruct fill an structure base on the config nested set, this function use reflection, and its not
// good (in my opinion) for frequent call.
// but its best if you need the config to loaded in structure and use that structure after that.
// Deprecated
func (o *Onion) GetStruct(prefix string, s interface{}) {
	iterateConfig(o, reflect.ValueOf(s), prefix)
}

func join(delim string, parts ...string) string {
	res := ""
	for i := range parts {
		if res != "" && parts[i] != "" {
			res += delim
		}
		res += parts[i]
	}

	return res
}

func setField(o *Onion, v reflect.Value, prefix, name string) {

	switch v.Kind() {
	case reflect.Bool:
		if v.CanSet() {
			v.SetBool(o.GetBoolDefault(join(o.GetDelimiter(), prefix, name), v.Bool()))
		}
	case reflect.Int:
		if v.CanSet() {
			v.SetInt(o.GetInt64Default(join(o.GetDelimiter(), prefix, name), v.Int()))
		}
	case reflect.Int64:
		if v.CanSet() {
			if v.Type().String() == durationValue.Type().String() {
				// its a duration
				v.SetInt(int64(o.GetDurationDefault(join(o.GetDelimiter(), prefix, name), time.Duration(v.Int()))))
				return
			}
			v.SetInt(o.GetInt64Default(join(o.GetDelimiter(), prefix, name), v.Int()))
		} else {
			v.SetInt(33)
		}
	case reflect.String:
		if v.CanSet() {
			v.SetString(o.GetStringDefault(join(o.GetDelimiter(), prefix, name), v.String()))
		}
	case reflect.Float64:
		if v.CanSet() {
			v.SetFloat(o.GetFloat64Default(join(o.GetDelimiter(), prefix, name), v.Float()))
		}
	case reflect.Float32:
		if v.CanSet() {
			v.SetFloat(o.GetFloat64Default(join(o.GetDelimiter(), prefix, name), v.Float()))
		}

	case reflect.Struct:
		iterateConfig(o, v.Addr(), join(o.GetDelimiter(), prefix, name))
	}
}

func iterateConfig(o *Onion, v reflect.Value, op string) {
	prefix := op
	typ := v.Type()
	if typ.Kind() == reflect.Ptr {
		typ = typ.Elem()
		v = v.Elem()
	}
	// Only struct are supported
	if typ.Kind() != reflect.Struct {
		return
	}
	// loop through the struct's fields and set the map
	for i := 0; i < typ.NumField(); i++ {
		p := typ.Field(i)
		if !p.Anonymous {
			name := p.Tag.Get("onion")
			if name == "-" {
				// Ignore this key.
				continue
			}
			if name == "" {
				name = strings.ToLower(p.Name)
			}
			setField(o, v.Field(i), prefix, name)
		} else { // Anonymous structures
			name := p.Tag.Get("onion")
			if name == "-" {
				// Ignore this key.
				continue
			}
			if name == "" {
				prefix = op // Reset the prefix to remove the delimiter
			}
			iterateConfig(o, v.Field(i).Addr(), join(o.GetDelimiter(), prefix, name))
		}
	}

}

func searchStringMap(m map[string]interface{}, path ...string) (interface{}, bool) {
	if len(path) == 0 {
		return nil, false
	}
	v, ok := m[path[0]]
	if !ok {
		return nil, false
	}

	if len(path) == 1 {
		return v, true
	}

	switch m := v.(type) {
	case map[string]interface{}:
		return searchStringMap(m, path[1:]...)
	case map[interface{}]interface{}:
		return searchInterfaceMap(m, path[1:]...)
	}
	return nil, false
}

func searchInterfaceMap(m map[interface{}]interface{}, path ...string) (interface{}, bool) {
	if len(path) == 0 {
		return nil, false
	}
	v, ok := m[path[0]]
	if !ok {
		return nil, false
	}

	if len(path) == 1 {
		return v, true
	}

	switch m := v.(type) {
	case map[string]interface{}:
		return searchStringMap(m, path[1:]...)
	case map[interface{}]interface{}:
		return searchInterfaceMap(m, path[1:]...)
	}
	return nil, false
}
<commit_msg>remove unused code<commit_after>package onion

func searchStringMap(m map[string]interface{}, path ...string) (interface{}, bool) {
	if len(path) == 0 {
		return nil, false
	}
	v, ok := m[path[0]]
	if !ok {
		return nil, false
	}

	if len(path) == 1 {
		return v, true
	}

	switch m := v.(type) {
	case map[string]interface{}:
		return searchStringMap(m, path[1:]...)
	case map[interface{}]interface{}:
		return searchInterfaceMap(m, path[1:]...)
	}
	return nil, false
}

func searchInterfaceMap(m map[interface{}]interface{}, path ...string) (interface{}, bool) {
	if len(path) == 0 {
		return nil, false
	}
	v, ok := m[path[0]]
	if !ok {
		return nil, false
	}

	if len(path) == 1 {
		return v, true
	}

	switch m := v.(type) {
	case map[string]interface{}:
		return searchStringMap(m, path[1:]...)
	case map[interface{}]interface{}:
		return searchInterfaceMap(m, path[1:]...)
	}
	return nil, false
}
<|endoftext|>


<commit_before>package main

func main() {}
<commit_msg>Add basic command line parsing + redis link<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/urfave/cli"
	"gopkg.in/redis.v3"
)

func getConnection() *redis.Client {
	client := redis.NewClient(&redis.Options{
		Addr:     "localhost:6379",
		Password: "", // no password set
		DB:       0,  // use default DB
	})

	return client
}

func list(c *cli.Context) error {
	client := getConnection()
	jobs := client.LRange("jobs", 0, -1).Val()
	fmt.Println("[" + strings.Join(jobs, ", ") + "]")
	return nil
}

func add(C *cli.Context) error {
	if !C.Args().Present() {
		fmt.Println("No command given")
		return nil
	}

	client := getConnection()
	client.RPush("jobs", C.Args().First())
	return nil
}

func run(C *cli.Context) error {
	client := getConnection()
	job, err := client.BLPop(0, "jobs").Result()
	if err != nil {
		panic(err)
	}

	fmt.Println("running:", job[1])

	return nil
}

func main() {
	app := cli.NewApp()

	app.EnableBashCompletion = true
	app.Commands = []cli.Command{
		{
			Name:   "list",
			Usage:  "list all pending tasks",
			Action: list,
		},
		{
			Name:   "add",
			Usage:  "add a new task",
			Action: add,
		},
		{
			Name:   "run",
			Usage:  "run the oldest task",
			Action: run,
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	gocontext "context"

	"github.com/docker/containerd/api/execution"
	execEvents "github.com/docker/containerd/execution"
	"github.com/nats-io/go-nats"
	"github.com/urfave/cli"
)

type runConfig struct {
	Image   string `toml:"image"`
	Process struct {
		Args []string `toml:"args"`
		Env  []string `toml:"env"`
		Cwd  string   `toml:"cwd"`
		Uid  int      `toml:"uid"`
		Gid  int      `toml:"gid"`
		Tty  bool     `toml:"tty"`
	} `toml:"process"`
	Network struct {
		Type    string `toml:"type"`
		IP      string `toml:"ip"`
		Gateway string `toml:"gateway"`
	} `toml:"network"`
}

var runCommand = cli.Command{
	Name:  "run",
	Usage: "run a container",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "bundle, b",
			Usage: "path to the container's bundle",
		},
		cli.BoolFlag{
			Name:  "tty, t",
			Usage: "allocate a TTY for the container",
		},
	},
	Action: func(context *cli.Context) error {
		// var config runConfig
		// if _, err := toml.DecodeFile(context.Args().First(), &config); err != nil {
		// 	return err
		// }
		id := context.Args().First()
		if id == "" {
			return fmt.Errorf("container id must be provided")
		}
		executionService, err := getExecutionService(context)
		if err != nil {
			return err
		}

		// setup our event subscriber
		nc, err := nats.Connect(nats.DefaultURL)
		if err != nil {
			return err
		}
		nec, err := nats.NewEncodedConn(nc, nats.JSON_ENCODER)
		if err != nil {
			nc.Close()
			return err
		}
		defer nec.Close()

		evCh := make(chan *execEvents.ContainerExitEvent, 64)
		sub, err := nec.Subscribe(execEvents.ContainersEventsSubjectSubscriber, func(e *execEvents.ContainerExitEvent) {
			evCh <- e
		})
		if err != nil {
			return err
		}
		defer sub.Unsubscribe()

		tmpDir, err := getTempDir(id)
		if err != nil {
			return err
		}
		defer os.RemoveAll(tmpDir)

		crOpts := &execution.CreateContainerRequest{
			ID:         id,
			BundlePath: context.String("bundle"),
			Console:    context.Bool("tty"),
			Stdin:      filepath.Join(tmpDir, "stdin"),
			Stdout:     filepath.Join(tmpDir, "stdout"),
			Stderr:     filepath.Join(tmpDir, "stderr"),
		}

		fwg, err := prepareStdio(crOpts.Stdin, crOpts.Stdout, crOpts.Stderr)
		if err != nil {
			return err
		}

		cr, err := executionService.Create(gocontext.Background(), crOpts)
		if err != nil {
			return err
		}

		if _, err := executionService.Start(gocontext.Background(), &execution.StartContainerRequest{
			ID: cr.Container.ID,
		}); err != nil {
			return err
		}

		var ec uint32
	eventLoop:
		for {
			select {
			case e, more := <-evCh:
				if !more {
					fmt.Println("No More!")
					break eventLoop
				}

				if e.ID == cr.Container.ID && e.PID == cr.InitProcess.ID {
					ec = e.StatusCode
					break eventLoop
				}
			case <-time.After(1 * time.Second):
				if nec.Conn.Status() != nats.CONNECTED {
					break eventLoop
				}
			}
		}

		if _, err := executionService.Delete(gocontext.Background(), &execution.DeleteContainerRequest{
			ID: cr.Container.ID,
		}); err != nil {
			return err
		}

		// Ensure we read all io
		fwg.Wait()

		os.Exit(int(ec))

		return nil
	},
}
<commit_msg>ctr: use full path for bundle in run<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	gocontext "context"

	"github.com/docker/containerd/api/execution"
	execEvents "github.com/docker/containerd/execution"
	"github.com/nats-io/go-nats"
	"github.com/urfave/cli"
)

type runConfig struct {
	Image   string `toml:"image"`
	Process struct {
		Args []string `toml:"args"`
		Env  []string `toml:"env"`
		Cwd  string   `toml:"cwd"`
		Uid  int      `toml:"uid"`
		Gid  int      `toml:"gid"`
		Tty  bool     `toml:"tty"`
	} `toml:"process"`
	Network struct {
		Type    string `toml:"type"`
		IP      string `toml:"ip"`
		Gateway string `toml:"gateway"`
	} `toml:"network"`
}

var runCommand = cli.Command{
	Name:  "run",
	Usage: "run a container",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "bundle, b",
			Usage: "path to the container's bundle",
		},
		cli.BoolFlag{
			Name:  "tty, t",
			Usage: "allocate a TTY for the container",
		},
	},
	Action: func(context *cli.Context) error {
		// var config runConfig
		// if _, err := toml.DecodeFile(context.Args().First(), &config); err != nil {
		// 	return err
		// }
		id := context.Args().First()
		if id == "" {
			return fmt.Errorf("container id must be provided")
		}
		executionService, err := getExecutionService(context)
		if err != nil {
			return err
		}

		// setup our event subscriber
		nc, err := nats.Connect(nats.DefaultURL)
		if err != nil {
			return err
		}
		nec, err := nats.NewEncodedConn(nc, nats.JSON_ENCODER)
		if err != nil {
			nc.Close()
			return err
		}
		defer nec.Close()

		evCh := make(chan *execEvents.ContainerExitEvent, 64)
		sub, err := nec.Subscribe(execEvents.ContainersEventsSubjectSubscriber, func(e *execEvents.ContainerExitEvent) {
			evCh <- e
		})
		if err != nil {
			return err
		}
		defer sub.Unsubscribe()

		tmpDir, err := getTempDir(id)
		if err != nil {
			return err
		}
		defer os.RemoveAll(tmpDir)

		bundle, err := filepath.Abs(context.String("bundle"))
		if err != nil {
			return err
		}
		crOpts := &execution.CreateContainerRequest{
			ID:         id,
			BundlePath: bundle,
			Console:    context.Bool("tty"),
			Stdin:      filepath.Join(tmpDir, "stdin"),
			Stdout:     filepath.Join(tmpDir, "stdout"),
			Stderr:     filepath.Join(tmpDir, "stderr"),
		}

		fwg, err := prepareStdio(crOpts.Stdin, crOpts.Stdout, crOpts.Stderr)
		if err != nil {
			return err
		}

		cr, err := executionService.Create(gocontext.Background(), crOpts)
		if err != nil {
			return err
		}

		if _, err := executionService.Start(gocontext.Background(), &execution.StartContainerRequest{
			ID: cr.Container.ID,
		}); err != nil {
			return err
		}

		var ec uint32
	eventLoop:
		for {
			select {
			case e, more := <-evCh:
				if !more {
					break eventLoop
				}

				if e.ID == cr.Container.ID && e.PID == cr.InitProcess.ID {
					ec = e.StatusCode
					break eventLoop
				}
			case <-time.After(1 * time.Second):
				if nec.Conn.Status() != nats.CONNECTED {
					break eventLoop
				}
			}
		}

		if _, err := executionService.Delete(gocontext.Background(), &execution.DeleteContainerRequest{
			ID: cr.Container.ID,
		}); err != nil {
			return err
		}

		// Ensure we read all io
		fwg.Wait()

		os.Exit(int(ec))

		return nil
	},
}
<|endoftext|>


<commit_before>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"fmt"
	"net/url"
	"strings"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/pkg/console"
)

// du specific flags.
var (
	duFlags = []cli.Flag{
		cli.IntFlag{
			Name:  "depth, d",
			Usage: "print the total for a folder prefix only if it is N or fewer levels below the command line argument",
		},
		cli.BoolFlag{
			Name:  "recursive, r",
			Usage: "recursively print the total for a folder prefix",
		},
		cli.StringFlag{
			Name:  "rewind",
			Usage: "include all object versions no later than specified date",
		},
		cli.BoolFlag{
			Name:  "versions",
			Usage: "include all object versions",
		},
	}
)

// Summarize disk usage.
var duCmd = cli.Command{
	Name:         "du",
	Usage:        "summarize disk usage recursively",
	Action:       mainDu,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        append(append(duFlags, ioFlags...), globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
  MC_ENCRYPT_KEY: list of comma delimited prefix=secret values

EXAMPLES:
  1. Summarize disk usage of 'jazz-songs' bucket recursively.
     {{.Prompt}} {{.HelpName}} s3/jazz-songs

  2. Summarize disk usage of 'louis' prefix in 'jazz-songs' bucket upto two levels.
     {{.Prompt}} {{.HelpName}} --depth=2 s3/jazz-songs/louis/

  3. Summarize disk usage of 'jazz-songs' bucket at a fixed date/time
     {{.Prompt}} {{.HelpName}} --rewind "2020.01.01" s3/jazz-songs/

  4. Summarize disk usage of 'jazz-songs' bucket with all objects versions
     {{.Prompt}} {{.HelpName}} --versions s3/jazz-songs/
`,
}

// Structured message depending on the type of console.
type duMessage struct {
	Prefix string `json:"prefix"`
	Size   int64  `json:"size"`
	Status string `json:"status"`
}

// Colorized message for console printing.
func (r duMessage) String() string {
	humanSize := strings.Join(strings.Fields(humanize.IBytes(uint64(r.Size))), "")

	return fmt.Sprintf("%s\t%s", console.Colorize("Size", humanSize),
		console.Colorize("Prefix", r.Prefix))
}

// JSON'ified message for scripting.
func (r duMessage) JSON() string {
	msgBytes, e := json.MarshalIndent(r, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
	return string(msgBytes)
}

func du(urlStr string, timeRef time.Time, withVersions bool, depth int, encKeyDB map[string][]prefixSSEPair) (int64, error) {
	targetAlias, targetURL, _ := mustExpandAlias(urlStr)
	if !strings.HasSuffix(targetURL, "/") {
		targetURL += "/"
	}

	clnt, pErr := newClientFromAlias(targetAlias, targetURL)
	if pErr != nil {
		errorIf(pErr.Trace(urlStr), "Failed to summarize disk usage `"+urlStr+"`.")
		return 0, exitStatus(globalErrorExitStatus) // End of journey.
	}

	contentCh := clnt.List(globalContext, ListOptions{
		TimeRef:           timeRef,
		WithOlderVersions: withVersions,
		Recursive:         false,
		ShowDir:           DirFirst,
	})
	size := int64(0)
	for content := range contentCh {
		if content.Err != nil {
			switch content.Err.ToGoError().(type) {
			// handle this specifically for filesystem related errors.
			case BrokenSymlink, TooManyLevelsSymlink, PathNotFound, ObjectOnGlacier:
				continue
			case PathInsufficientPermission:
				errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.")
				continue
			}
			errorIf(content.Err.Trace(urlStr), "Failed to find disk usage of `"+urlStr+"` recursively.")
			return 0, exitStatus(globalErrorExitStatus)
		}
		if content.URL.String() == targetURL {
			continue
		}

		if content.Type.IsDir() {
			depth := depth
			if depth > 0 {
				depth--
			}

			subDirAlias := content.URL.Path
			if targetAlias != "" {
				subDirAlias = targetAlias + "/" + content.URL.Path
			}
			used, err := du(subDirAlias, timeRef, withVersions, depth, encKeyDB)
			if err != nil {
				return 0, err
			}
			size += used
		} else {
			size += content.Size
		}
	}

	if depth != 0 {
		u, err := url.Parse(targetURL)
		if err != nil {
			panic(err)
		}

		printMsg(duMessage{
			Prefix: strings.Trim(u.Path, "/"),
			Size:   size,
			Status: "success",
		})
	}

	return size, nil
}

// main for du command.
func mainDu(ctx *cli.Context) error {
	if !ctx.Args().Present() {
		cli.ShowCommandHelpAndExit(ctx, "du", 1)
	}

	// Set colors.
	console.SetColor("Remove", color.New(color.FgGreen, color.Bold))
	console.SetColor("Prefix", color.New(color.FgCyan, color.Bold))
	console.SetColor("Size", color.New(color.FgYellow))

	// Parse encryption keys per command.
	encKeyDB, err := getEncKeys(ctx)
	fatalIf(err, "Unable to parse encryption keys.")

	// du specific flags.
	depth := ctx.Int("depth")
	if depth == 0 {
		if ctx.Bool("recursive") {
			if !ctx.IsSet("depth") {
				depth = -1
			}
		} else {
			depth = 1
		}
	}

	withVersions := ctx.Bool("versions")
	timeRef := parseRewindFlag(ctx.String("rewind"))

	var duErr error
	for _, urlStr := range ctx.Args() {
		if _, err := du(urlStr, timeRef, withVersions, depth, encKeyDB); duErr == nil {
			duErr = err
		}
	}

	return duErr
}
<commit_msg>du: Use recursive listing when no detailed report is asked (#3737)<commit_after>// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"context"
	"fmt"
	"net/url"
	"strings"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/colorjson"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/pkg/console"
)

// du specific flags.
var (
	duFlags = []cli.Flag{
		cli.IntFlag{
			Name:  "depth, d",
			Usage: "print the total for a folder prefix only if it is N or fewer levels below the command line argument",
		},
		cli.BoolFlag{
			Name:  "recursive, r",
			Usage: "recursively print the total for a folder prefix",
		},
		cli.StringFlag{
			Name:  "rewind",
			Usage: "include all object versions no later than specified date",
		},
		cli.BoolFlag{
			Name:  "versions",
			Usage: "include all object versions",
		},
	}
)

// Summarize disk usage.
var duCmd = cli.Command{
	Name:         "du",
	Usage:        "summarize disk usage recursively",
	Action:       mainDu,
	OnUsageError: onUsageError,
	Before:       setGlobalsFromContext,
	Flags:        append(append(duFlags, ioFlags...), globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} [FLAGS] TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
  MC_ENCRYPT_KEY: list of comma delimited prefix=secret values

EXAMPLES:
  1. Summarize disk usage of 'jazz-songs' bucket recursively.
     {{.Prompt}} {{.HelpName}} s3/jazz-songs

  2. Summarize disk usage of 'louis' prefix in 'jazz-songs' bucket upto two levels.
     {{.Prompt}} {{.HelpName}} --depth=2 s3/jazz-songs/louis/

  3. Summarize disk usage of 'jazz-songs' bucket at a fixed date/time
     {{.Prompt}} {{.HelpName}} --rewind "2020.01.01" s3/jazz-songs/

  4. Summarize disk usage of 'jazz-songs' bucket with all objects versions
     {{.Prompt}} {{.HelpName}} --versions s3/jazz-songs/
`,
}

// Structured message depending on the type of console.
type duMessage struct {
	Prefix string `json:"prefix"`
	Size   int64  `json:"size"`
	Status string `json:"status"`
}

// Colorized message for console printing.
func (r duMessage) String() string {
	humanSize := strings.Join(strings.Fields(humanize.IBytes(uint64(r.Size))), "")

	return fmt.Sprintf("%s\t%s", console.Colorize("Size", humanSize),
		console.Colorize("Prefix", r.Prefix))
}

// JSON'ified message for scripting.
func (r duMessage) JSON() string {
	msgBytes, e := json.MarshalIndent(r, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")
	return string(msgBytes)
}

func du(ctx context.Context, urlStr string, timeRef time.Time, withVersions bool, depth int, encKeyDB map[string][]prefixSSEPair) (int64, error) {
	targetAlias, targetURL, _ := mustExpandAlias(urlStr)
	if !strings.HasSuffix(targetURL, "/") {
		targetURL += "/"
	}

	clnt, pErr := newClientFromAlias(targetAlias, targetURL)
	if pErr != nil {
		errorIf(pErr.Trace(urlStr), "Failed to summarize disk usage `"+urlStr+"`.")
		return 0, exitStatus(globalErrorExitStatus) // End of journey.
	}

	// No disk usage details below this level,
	// just do a recursive listing
	recursive := depth == 1

	contentCh := clnt.List(ctx, ListOptions{
		TimeRef:           timeRef,
		WithOlderVersions: withVersions,
		Recursive:         recursive,
		ShowDir:           DirFirst,
	})
	size := int64(0)
	for content := range contentCh {
		if content.Err != nil {
			switch content.Err.ToGoError().(type) {
			// handle this specifically for filesystem related errors.
			case BrokenSymlink, TooManyLevelsSymlink, PathNotFound, ObjectOnGlacier:
				continue
			case PathInsufficientPermission:
				errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.")
				continue
			}
			errorIf(content.Err.Trace(urlStr), "Failed to find disk usage of `"+urlStr+"` recursively.")
			return 0, exitStatus(globalErrorExitStatus)
		}
		if content.URL.String() == targetURL {
			continue
		}

		if content.Type.IsDir() {
			depth := depth
			if depth > 0 {
				depth--
			}

			subDirAlias := content.URL.Path
			if targetAlias != "" {
				subDirAlias = targetAlias + "/" + content.URL.Path
			}
			used, err := du(ctx, subDirAlias, timeRef, withVersions, depth, encKeyDB)
			if err != nil {
				return 0, err
			}
			size += used
		} else {
			size += content.Size
		}
	}

	if depth != 0 {
		u, err := url.Parse(targetURL)
		if err != nil {
			panic(err)
		}

		printMsg(duMessage{
			Prefix: strings.Trim(u.Path, "/"),
			Size:   size,
			Status: "success",
		})
	}

	return size, nil
}

// main for du command.
func mainDu(cliCtx *cli.Context) error {
	if !cliCtx.Args().Present() {
		cli.ShowCommandHelpAndExit(cliCtx, "du", 1)
	}

	// Set colors.
	console.SetColor("Remove", color.New(color.FgGreen, color.Bold))
	console.SetColor("Prefix", color.New(color.FgCyan, color.Bold))
	console.SetColor("Size", color.New(color.FgYellow))

	ctx, cancelRm := context.WithCancel(globalContext)
	defer cancelRm()

	// Parse encryption keys per command.
	encKeyDB, err := getEncKeys(cliCtx)
	fatalIf(err, "Unable to parse encryption keys.")

	// du specific flags.
	depth := cliCtx.Int("depth")
	if depth == 0 {
		if cliCtx.Bool("recursive") {
			if !cliCtx.IsSet("depth") {
				depth = -1
			}
		} else {
			depth = 1
		}
	}

	withVersions := cliCtx.Bool("versions")
	timeRef := parseRewindFlag(cliCtx.String("rewind"))

	var duErr error
	for _, urlStr := range cliCtx.Args() {
		if !isAliasURLDir(ctx, urlStr, nil, time.Time{}) {
			fatalIf(errInvalidArgument().Trace(urlStr), fmt.Sprintf("Source `%s` is not a folder. Only folders are supported by mirror command.", urlStr))
		}

		if _, err := du(ctx, urlStr, timeRef, withVersions, depth, encKeyDB); duErr == nil {
			duErr = err
		}
	}

	return duErr
}
<|endoftext|>