<commit_before>package goinsta

import (
	"encoding/json"
	"errors"
	"fmt"
)

type Users struct {
	inst *Instagram

	// It's a bit confusing have the same structure
	// in the Instagram strucure and in the multiple users
	// calls

	endpoint string

	Status   string `json:"status"`
	BigList  bool   `json:"big_list"`
	Users    []User `json:"users"`
	PageSize int    `json:"page_size"`
	NextID   string `json:"next_max_id"`
}

func newUsers(inst *Instagram) *Users {
	users := &Users{inst: inst}

	return users
}

// SetInstagram sets new instagram to user structure
func (users *Users) SetInstagram(inst *Instagram) {
	users.inst = inst
}

var ErrNoMore = errors.New("User list end reached")

// Next allows to paginate after calling:
// Account.Follow* and User.Follow*
//
// New user list is stored inside Users
//
// returns ErrNoMore when list reach the end.
func (users *Users) Next() error {
	insta := users.inst
	endpoint := users.endpoint

	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: endpoint,
			Query: map[string]string{
				"max_id":             users.NextID,
				"ig_sig_key_version": goInstaSigKeyVersion,
				"rank_token":         insta.rankToken,
			},
		},
	)
	if err == nil {
		usrs := Users{}
		err = json.Unmarshal(body, &usrs)
		if err == nil {
			if !usrs.BigList || usrs.NextID == "" {
				err = ErrNoMore
			}
			*users = usrs
			users.inst = insta
			users.endpoint = endpoint
		}
	}
	return err
}

type userResp struct {
	Status string `json:"status"`
	User   User   `json:"user"`
}

// User is the representation of instagram's user profile
type User struct {
	//Feed *Feed
	//Followers *Followers
	//Following *Following
	//Status *Friendship
	//Story *Story
	//Messages *Messages
	inst *Instagram

	ID                         int64        `json:"pk"`
	Username                   string       `json:"username"`
	FullName                   string       `json:"full_name"`
	Biography                  string       `json:"biography"`
	ProfilePicURL              string       `json:"profile_pic_url"`
	Email                      string       `json:"email"`
	PhoneNumber                string       `json:"phone_number"`
	IsBusiness                 bool         `json:"is_business"`
	Gender                     int          `json:"gender"`
	ProfilePicID               string       `json:"profile_pic_id"`
	HasAnonymousProfilePicture bool         `json:"has_anonymous_profile_picture"`
	IsPrivate                  bool         `json:"is_private"`
	IsVerified                 bool         `json:"is_verified"`
	MediaCount                 int          `json:"media_count"`
	FollowerCount              int          `json:"follower_count"`
	FollowingCount             int          `json:"following_count"`
	GeoMediaCount              int          `json:"geo_media_count"`
	ExternalURL                string       `json:"external_url"`
	HasBiographyTranslation    bool         `json:"has_biography_translation"`
	ExternalLynxURL            string       `json:"external_lynx_url"`
	HdProfilePicURLInfo        PicURLInfo   `json:"hd_profile_pic_url_info"`
	HdProfilePicVersions       []PicURLInfo `json:"hd_profile_pic_versions"`
	UsertagsCount              int          `json:"usertags_count"`
	HasChaining                bool         `json:"has_chaining"`
	IsFavorite                 bool         `json:"is_favorite"`
	ReelAutoArchive            string       `json:"reel_auto_archive"`
	School                     School       `json:"school"`
	PublicEmail                string       `json:"public_email"`
	PublicPhoneNumber          string       `json:"public_phone_number"`
	PublicPhoneCountryCode     string       `json:"public_phone_country_code"`
	ContactPhoneNumber         string       `json:"contact_phone_number"`
	CityID                     int64        `json:"city_id"`
	CityName                   string       `json:"city_name"`
	AddressStreet              string       `json:"address_street"`
	DirectMessaging            string       `json:"direct_messaging"`
	Latitude                   float64      `json:"latitude"`
	Longitude                  float64      `json:"longitude"`
	Category                   string       `json:"category"`
	BusinessContactMethod      string       `json:"business_contact_method"`
	IsCallToActionEnabled      bool         `json:"is_call_to_action_enabled"`
	FbPageCallToActionID       string       `json:"fb_page_call_to_action_id"`
	Zip                        string       `json:"zip"`
	Friendship                 Friendship   `json:"friendship_status"`
}

// Following returns a list of user following.
//
// Users.Next can be used to paginate
func (user *User) Following() (*Users, error) {
	endpoint := fmt.Sprintf(urlFollowing, user.ID)
	body, err := user.inst.sendRequest(
		&reqOptions{
			Endpoint: endpoint,
			Query: map[string]string{
				"max_id":             "",
				"ig_sig_key_version": goInstaSigKeyVersion,
				"rank_token":         user.inst.rankToken,
			},
		},
	)
	if err == nil {
		users := &Users{}
		err = json.Unmarshal(body, users)
		users.inst = user.inst
		users.endpoint = endpoint
		return users, err
	}
	return nil, err
}

// Followers returns a list of user followers.
//
// Users.Next can be used to paginate
func (user *User) Followers() (*Users, error) {
	endpoint := fmt.Sprintf(urlFollowers, user.ID)
	body, err := user.inst.sendRequest(
		&reqOptions{
			Endpoint: endpoint,
			Query: map[string]string{
				"max_id":             "",
				"ig_sig_key_version": goInstaSigKeyVersion,
				"rank_token":         user.inst.rankToken,
			},
		},
	)
	if err == nil {
		users := &Users{}
		err = json.Unmarshal(body, users)
		users.inst = user.inst
		users.endpoint = endpoint
		return users, err
	}
	return nil, err
}

// Block blocks user
//
// This function updates current User.Friendship structure.
func (user *User) Block() error {
	insta := user.inst
	data, err := insta.prepareData(
		map[string]interface{}{
			"user_id": user.ID,
		},
	)
	if err == nil {
		body, err := insta.sendRequest(
			&reqOptions{
				Endpoint: fmt.Sprintf(urlUserBlock, user.ID),
				Query:    generateSignature(data),
				IsPost:   true,
			},
		)
		if err == nil {
			resp := friendResp{}
			err = json.Unmarshal(body, &resp)
			user.Friendship = resp.Friendship
		}
	}
	return err
}

// Unblock unblocks user
//
// This function updates current User structure.
func (user *User) Unblock() error {
	insta := user.inst
	data, err := insta.prepareData(
		map[string]interface{}{
			"user_id": user.ID,
		},
	)
	if err == nil {
		body, err := insta.sendRequest(
			&reqOptions{
				Endpoint: fmt.Sprintf(urlUserUnblock, user.ID),
				Query:    generateSignature(data),
				IsPost:   true,
			},
		)
		if err == nil {
			resp := friendResp{}
			err = json.Unmarshal(body, &resp)
			user.Friendship = resp.Friendship
		}
	}
	return err
}

// Follow started following some user
//
// This function performs a follow call. If user is private
// you have to wait until he/she accepts you.
//
// If the account is public User.Friendship will be updated
func (user *User) Follow() error {
	insta := user.inst
	data, err := insta.prepareData(
		map[string]interface{}{
			"user_id": user.ID,
		},
	)
	if err == nil {
		body, err := insta.sendRequest(
			&reqOptions{
				Endpoint: fmt.Sprintf(urlUserFollow, user.ID),
				Query:    generateSignature(data),
				IsPost:   true,
			},
		)
		if err == nil {
			resp := friendResp{}
			err = json.Unmarshal(body, &resp)
			user.Friendship = resp.Friendship
		}
	}
	return err
}
<commit_msg>Added Unfollow func<commit_after>package goinsta

import (
	"encoding/json"
	"errors"
	"fmt"
)

type Users struct {
	inst *Instagram

	// It's a bit confusing have the same structure
	// in the Instagram strucure and in the multiple users
	// calls

	endpoint string

	Status   string `json:"status"`
	BigList  bool   `json:"big_list"`
	Users    []User `json:"users"`
	PageSize int    `json:"page_size"`
	NextID   string `json:"next_max_id"`
}

func newUsers(inst *Instagram) *Users {
	users := &Users{inst: inst}

	return users
}

// SetInstagram sets new instagram to user structure
func (users *Users) SetInstagram(inst *Instagram) {
	users.inst = inst
}

var ErrNoMore = errors.New("User list end reached")

// Next allows to paginate after calling:
// Account.Follow* and User.Follow*
//
// New user list is stored inside Users
//
// returns ErrNoMore when list reach the end.
func (users *Users) Next() error {
	insta := users.inst
	endpoint := users.endpoint

	body, err := insta.sendRequest(
		&reqOptions{
			Endpoint: endpoint,
			Query: map[string]string{
				"max_id":             users.NextID,
				"ig_sig_key_version": goInstaSigKeyVersion,
				"rank_token":         insta.rankToken,
			},
		},
	)
	if err == nil {
		usrs := Users{}
		err = json.Unmarshal(body, &usrs)
		if err == nil {
			if !usrs.BigList || usrs.NextID == "" {
				err = ErrNoMore
			}
			*users = usrs
			users.inst = insta
			users.endpoint = endpoint
		}
	}
	return err
}

type userResp struct {
	Status string `json:"status"`
	User   User   `json:"user"`
}

// User is the representation of instagram's user profile
type User struct {
	//Feed *Feed
	//Followers *Followers
	//Following *Following
	//Status *Friendship
	//Story *Story
	//Messages *Messages
	inst *Instagram

	ID                         int64        `json:"pk"`
	Username                   string       `json:"username"`
	FullName                   string       `json:"full_name"`
	Biography                  string       `json:"biography"`
	ProfilePicURL              string       `json:"profile_pic_url"`
	Email                      string       `json:"email"`
	PhoneNumber                string       `json:"phone_number"`
	IsBusiness                 bool         `json:"is_business"`
	Gender                     int          `json:"gender"`
	ProfilePicID               string       `json:"profile_pic_id"`
	HasAnonymousProfilePicture bool         `json:"has_anonymous_profile_picture"`
	IsPrivate                  bool         `json:"is_private"`
	IsVerified                 bool         `json:"is_verified"`
	MediaCount                 int          `json:"media_count"`
	FollowerCount              int          `json:"follower_count"`
	FollowingCount             int          `json:"following_count"`
	GeoMediaCount              int          `json:"geo_media_count"`
	ExternalURL                string       `json:"external_url"`
	HasBiographyTranslation    bool         `json:"has_biography_translation"`
	ExternalLynxURL            string       `json:"external_lynx_url"`
	HdProfilePicURLInfo        PicURLInfo   `json:"hd_profile_pic_url_info"`
	HdProfilePicVersions       []PicURLInfo `json:"hd_profile_pic_versions"`
	UsertagsCount              int          `json:"usertags_count"`
	HasChaining                bool         `json:"has_chaining"`
	IsFavorite                 bool         `json:"is_favorite"`
	ReelAutoArchive            string       `json:"reel_auto_archive"`
	School                     School       `json:"school"`
	PublicEmail                string       `json:"public_email"`
	PublicPhoneNumber          string       `json:"public_phone_number"`
	PublicPhoneCountryCode     string       `json:"public_phone_country_code"`
	ContactPhoneNumber         string       `json:"contact_phone_number"`
	CityID                     int64        `json:"city_id"`
	CityName                   string       `json:"city_name"`
	AddressStreet              string       `json:"address_street"`
	DirectMessaging            string       `json:"direct_messaging"`
	Latitude                   float64      `json:"latitude"`
	Longitude                  float64      `json:"longitude"`
	Category                   string       `json:"category"`
	BusinessContactMethod      string       `json:"business_contact_method"`
	IsCallToActionEnabled      bool         `json:"is_call_to_action_enabled"`
	FbPageCallToActionID       string       `json:"fb_page_call_to_action_id"`
	Zip                        string       `json:"zip"`
	Friendship                 Friendship   `json:"friendship_status"`
}

// Following returns a list of user following.
//
// Users.Next can be used to paginate
func (user *User) Following() (*Users, error) {
	endpoint := fmt.Sprintf(urlFollowing, user.ID)
	body, err := user.inst.sendRequest(
		&reqOptions{
			Endpoint: endpoint,
			Query: map[string]string{
				"max_id":             "",
				"ig_sig_key_version": goInstaSigKeyVersion,
				"rank_token":         user.inst.rankToken,
			},
		},
	)
	if err == nil {
		users := &Users{}
		err = json.Unmarshal(body, users)
		users.inst = user.inst
		users.endpoint = endpoint
		return users, err
	}
	return nil, err
}

// Followers returns a list of user followers.
//
// Users.Next can be used to paginate
func (user *User) Followers() (*Users, error) {
	endpoint := fmt.Sprintf(urlFollowers, user.ID)
	body, err := user.inst.sendRequest(
		&reqOptions{
			Endpoint: endpoint,
			Query: map[string]string{
				"max_id":             "",
				"ig_sig_key_version": goInstaSigKeyVersion,
				"rank_token":         user.inst.rankToken,
			},
		},
	)
	if err == nil {
		users := &Users{}
		err = json.Unmarshal(body, users)
		users.inst = user.inst
		users.endpoint = endpoint
		return users, err
	}
	return nil, err
}

// Block blocks user
//
// This function updates current User.Friendship structure.
func (user *User) Block() error {
	insta := user.inst
	data, err := insta.prepareData(
		map[string]interface{}{
			"user_id": user.ID,
		},
	)
	if err == nil {
		body, err := insta.sendRequest(
			&reqOptions{
				Endpoint: fmt.Sprintf(urlUserBlock, user.ID),
				Query:    generateSignature(data),
				IsPost:   true,
			},
		)
		if err == nil {
			resp := friendResp{}
			err = json.Unmarshal(body, &resp)
			user.Friendship = resp.Friendship
		}
	}
	return err
}

// Unblock unblocks user
//
// This function updates current User structure.
func (user *User) Unblock() error {
	insta := user.inst
	data, err := insta.prepareData(
		map[string]interface{}{
			"user_id": user.ID,
		},
	)
	if err == nil {
		body, err := insta.sendRequest(
			&reqOptions{
				Endpoint: fmt.Sprintf(urlUserUnblock, user.ID),
				Query:    generateSignature(data),
				IsPost:   true,
			},
		)
		if err == nil {
			resp := friendResp{}
			err = json.Unmarshal(body, &resp)
			user.Friendship = resp.Friendship
		}
	}
	return err
}

// Follow started following some user
//
// This function performs a follow call. If user is private
// you have to wait until he/she accepts you.
//
// If the account is public User.Friendship will be updated
func (user *User) Follow() error {
	insta := user.inst
	data, err := insta.prepareData(
		map[string]interface{}{
			"user_id": user.ID,
		},
	)
	if err == nil {
		body, err := insta.sendRequest(
			&reqOptions{
				Endpoint: fmt.Sprintf(urlUserFollow, user.ID),
				Query:    generateSignature(data),
				IsPost:   true,
			},
		)
		if err == nil {
			resp := friendResp{}
			err = json.Unmarshal(body, &resp)
			user.Friendship = resp.Friendship
		}
	}
	return err
}

// Unfollow unfollows user
//
// User.Friendship will be updated
func (user *User) Unfollow() error {
	insta := user.inst
	data, err := insta.prepareData(
		map[string]interface{}{
			"user_id": user.ID,
		},
	)
	if err == nil {
		body, err := insta.sendRequest(
			&reqOptions{
				Endpoint: fmt.Sprintf(urlUserUnfollow, user.ID),
				Query:    generateSignature(data),
				IsPost:   true,
			},
		)
		if err == nil {
			resp := friendResp{}
			err = json.Unmarshal(body, &resp)
			user.Friendship = resp.Friendship
		}
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	r "github.com/dancannon/gorethink"
	"github.com/jaracil/ei"
)

type UserData struct {
	User      string                            `gorethink:"id,omitempty"`
	Pass      string                            `gorethink:"pass,omitempty"`
	Salt      string                            `gorethink:"salt,omitempty"`
	Tags      map[string]map[string]interface{} `gorethink:"tags,omitempty"`
	Mask      map[string]map[string]interface{} `gorethink:"mask,omitempty"`
	Templates []string                          `gorethink:"templates,omitempty"`
}

var Nobody *UserData = &UserData{User: "nobody", Tags: map[string]map[string]interface{}{}}

func (nc *NexusConn) handleUserReq(req *JsonRpcReq) {
	switch req.Method {
	case "user.create":
		user, err := ei.N(req.Params).M("user").Lower().F(checkRegexp, _userRegexp).F(checkLen, _userMinLen, _userMaxLen).String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}
		pass, err := ei.N(req.Params).M("pass").F(checkLen, _passwordMinLen, _passwordMaxLen).String()
		if err != nil {
			req.Error(ErrInvalidParams, "pass", nil)
			return
		}
		tags := nc.getTags(user)
		if !(ei.N(tags).M("@"+req.Method).BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		ud := UserData{User: user, Salt: safeId(16), Tags: map[string]map[string]interface{}{}, Templates: []string{}}
		ud.Pass, err = HashPass(pass, ud.Salt)
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		_, err = r.Table("users").Insert(&ud).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			if r.IsConflictErr(err) {
				req.Error(ErrUserExists, "", nil)
			} else {
				req.Error(ErrInternal, "", nil)
			}
			return
		}
		req.Result(map[string]interface{}{"ok": true})
	case "user.delete":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}
		tags := nc.getTags(user)
		if !(ei.N(tags).M("@"+req.Method).BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		res, err := r.Table("users").Get(user).Delete().RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Deleted > 0 {
			req.Result(map[string]interface{}{"ok": true})
		} else {
			req.Error(ErrInvalidUser, "", nil)
		}

	case "user.setTags":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}
		prefix, err := ei.N(req.Params).M("prefix").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "prefix", nil)
			return
		}
		tgs, err := ei.N(req.Params).M("tags").MapStr()
		if err != nil {
			req.Error(ErrInvalidParams, "tags", nil)
			return
		}
		tags := nc.getTags(prefix)
		if !(ei.N(tags).M("@"+req.Method).BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		res, err := r.Table("users").Get(user).Update(map[string]interface{}{"tags": map[string]interface{}{prefix: tgs}}).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Unchanged == 0 && res.Replaced == 0 {
			req.Error(ErrInvalidUser, "", nil)
			return
		}
		req.Result(map[string]interface{}{"ok": true})
	case "user.delTags":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}
		prefix, err := ei.N(req.Params).M("prefix").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "prefix", nil)
			return
		}
		tgs, err := ei.N(req.Params).M("tags").Slice()
		if err != nil {
			req.Error(ErrInvalidParams, "tags", nil)
			return
		}
		tags := nc.getTags(prefix)
		if !(ei.N(tags).M("@"+req.Method).BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		res, err := r.Table("users").Get(user).Update(map[string]interface{}{"tags": map[string]interface{}{prefix: r.Literal(r.Row.Field("tags").Field(prefix).Without(tgs))}}).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Unchanged == 0 && res.Replaced == 0 {
			req.Error(ErrInvalidUser, "", nil)
			return
		}
		req.Result(map[string]interface{}{"ok": true})
	case "user.setPass":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}
		pass, err := ei.N(req.Params).M("pass").F(checkLen, _passwordMinLen, _passwordMaxLen).String()
		if err != nil {
			req.Error(ErrInvalidParams, "pass", nil)
			return
		}
		tags := nc.getTags(user)
		if !(ei.N(tags).M("@"+req.Method).BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		salt := safeId(16)
		hp, err := HashPass(pass, salt)
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		res, err := r.Table("users").Get(user).Update(map[string]interface{}{"salt": salt, "pass": hp}).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Unchanged == 0 && res.Replaced == 0 {
			req.Error(ErrInvalidUser, "", nil)
			return
		}
		req.Result(map[string]interface{}{"ok": true})
	case "user.list":
		prefix := ei.N(req.Params).M("prefix").Lower().StringZ()
		limit, err := ei.N(req.Params).M("limit").Int()
		if err != nil {
			limit = 100
		}
		skip, err := ei.N(req.Params).M("skip").Int()
		if err != nil {
			skip = 0
		}
		tags := nc.getTags(prefix)
		if !(ei.N(tags).M("@user.list").BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		term := r.Table("users").
			Between(prefix, prefix+"\uffff").
			Pluck("id", "tags")

		if skip >= 0 {
			term = term.Skip(skip)
		}

		if limit >= 0 {
			term = term.Limit(limit)
		}

		cur, err := term.Map(func(row r.Term) interface{} {
			return ei.M{"user": row.Field("id"), "tags": row.Field("tags").Default(ei.M{})}
		}).Run(db)
		if err != nil {
			req.Error(ErrInternal, err.Error(), nil)
			return
		}
		var all []interface{}
		cur.All(&all)
		req.Result(all)

	case "user.addTemplate":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}

		template, err := ei.N(req.Params).M("template").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "template", nil)
			return
		}

		userTags := ei.N(nc.getTags(user))
		if !(userTags.M("@"+req.Method).BoolZ() || userTags.M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}

		templateTags := ei.N(nc.getTags(template))
		if !(templateTags.M("@"+req.Method).BoolZ() || templateTags.M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}

		res, err := r.Table("users").Get(user).Update(map[string]interface{}{
			"templates": r.Row.Field("templates").Default([]string{}).SetInsert(template),
		}).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Unchanged == 0 && res.Replaced == 0 {
			req.Error(ErrInvalidUser, "", nil)
			return
		}
		req.Result(map[string]interface{}{"ok": true})

	case "user.delTemplate":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}

		template, err := ei.N(req.Params).M("template").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "template", nil)
			return
		}

		userTags := ei.N(nc.getTags(user))
		if !(userTags.M("@"+req.Method).BoolZ() || userTags.M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}

		templateTags := ei.N(nc.getTags(template))
		if !(templateTags.M("@"+req.Method).BoolZ() || templateTags.M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}

		res, err := r.Table("users").Get(user).Update(map[string]interface{}{
			"templates": r.Row.Field("templates").Default(ei.S{}).SetDifference([]string{template}),
		}).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Unchanged == 0 && res.Replaced == 0 {
			req.Error(ErrInvalidUser, "", nil)
			return
		}
		req.Result(map[string]interface{}{"ok": true})
	default:
		req.Error(ErrMethodNotFound, "", nil)
	}
}
<commit_msg>Added listing of users templates<commit_after>package main

import (
	r "github.com/dancannon/gorethink"
	"github.com/jaracil/ei"
)

type UserData struct {
	User      string                            `gorethink:"id,omitempty"`
	Pass      string                            `gorethink:"pass,omitempty"`
	Salt      string                            `gorethink:"salt,omitempty"`
	Tags      map[string]map[string]interface{} `gorethink:"tags,omitempty"`
	Mask      map[string]map[string]interface{} `gorethink:"mask,omitempty"`
	Templates []string                          `gorethink:"templates,omitempty"`
}

var Nobody *UserData = &UserData{User: "nobody", Tags: map[string]map[string]interface{}{}}

func (nc *NexusConn) handleUserReq(req *JsonRpcReq) {
	switch req.Method {
	case "user.create":
		user, err := ei.N(req.Params).M("user").Lower().F(checkRegexp, _userRegexp).F(checkLen, _userMinLen, _userMaxLen).String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}
		pass, err := ei.N(req.Params).M("pass").F(checkLen, _passwordMinLen, _passwordMaxLen).String()
		if err != nil {
			req.Error(ErrInvalidParams, "pass", nil)
			return
		}
		tags := nc.getTags(user)
		if !(ei.N(tags).M("@"+req.Method).BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		ud := UserData{User: user, Salt: safeId(16), Tags: map[string]map[string]interface{}{}, Templates: []string{}}
		ud.Pass, err = HashPass(pass, ud.Salt)
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		_, err = r.Table("users").Insert(&ud).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			if r.IsConflictErr(err) {
				req.Error(ErrUserExists, "", nil)
			} else {
				req.Error(ErrInternal, "", nil)
			}
			return
		}
		req.Result(map[string]interface{}{"ok": true})
	case "user.delete":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}
		tags := nc.getTags(user)
		if !(ei.N(tags).M("@"+req.Method).BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		res, err := r.Table("users").Get(user).Delete().RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Deleted > 0 {
			req.Result(map[string]interface{}{"ok": true})
		} else {
			req.Error(ErrInvalidUser, "", nil)
		}

	case "user.setTags":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}
		prefix, err := ei.N(req.Params).M("prefix").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "prefix", nil)
			return
		}
		tgs, err := ei.N(req.Params).M("tags").MapStr()
		if err != nil {
			req.Error(ErrInvalidParams, "tags", nil)
			return
		}
		tags := nc.getTags(prefix)
		if !(ei.N(tags).M("@"+req.Method).BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		res, err := r.Table("users").Get(user).Update(map[string]interface{}{"tags": map[string]interface{}{prefix: tgs}}).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Unchanged == 0 && res.Replaced == 0 {
			req.Error(ErrInvalidUser, "", nil)
			return
		}
		req.Result(map[string]interface{}{"ok": true})
	case "user.delTags":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}
		prefix, err := ei.N(req.Params).M("prefix").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "prefix", nil)
			return
		}
		tgs, err := ei.N(req.Params).M("tags").Slice()
		if err != nil {
			req.Error(ErrInvalidParams, "tags", nil)
			return
		}
		tags := nc.getTags(prefix)
		if !(ei.N(tags).M("@"+req.Method).BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		res, err := r.Table("users").Get(user).Update(map[string]interface{}{"tags": map[string]interface{}{prefix: r.Literal(r.Row.Field("tags").Field(prefix).Without(tgs))}}).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Unchanged == 0 && res.Replaced == 0 {
			req.Error(ErrInvalidUser, "", nil)
			return
		}
		req.Result(map[string]interface{}{"ok": true})
	case "user.setPass":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}
		pass, err := ei.N(req.Params).M("pass").F(checkLen, _passwordMinLen, _passwordMaxLen).String()
		if err != nil {
			req.Error(ErrInvalidParams, "pass", nil)
			return
		}
		tags := nc.getTags(user)
		if !(ei.N(tags).M("@"+req.Method).BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		salt := safeId(16)
		hp, err := HashPass(pass, salt)
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		res, err := r.Table("users").Get(user).Update(map[string]interface{}{"salt": salt, "pass": hp}).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Unchanged == 0 && res.Replaced == 0 {
			req.Error(ErrInvalidUser, "", nil)
			return
		}
		req.Result(map[string]interface{}{"ok": true})
	case "user.list":
		prefix := ei.N(req.Params).M("prefix").Lower().StringZ()
		limit, err := ei.N(req.Params).M("limit").Int()
		if err != nil {
			limit = 100
		}
		skip, err := ei.N(req.Params).M("skip").Int()
		if err != nil {
			skip = 0
		}
		tags := nc.getTags(prefix)
		if !(ei.N(tags).M("@user.list").BoolZ() || ei.N(tags).M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}
		term := r.Table("users").
			Between(prefix, prefix+"\uffff").
			Pluck("id", "tags")

		if skip >= 0 {
			term = term.Skip(skip)
		}

		if limit >= 0 {
			term = term.Limit(limit)
		}

		cur, err := term.Map(func(row r.Term) interface{} {
			return ei.M{"user": row.Field("id"), "tags": row.Field("tags").Default(ei.M{})}
		}).Run(db)
		if err != nil {
			req.Error(ErrInternal, err.Error(), nil)
			return
		}
		var all []interface{}
		cur.All(&all)
		req.Result(all)

	case "user.addTemplate":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}

		template, err := ei.N(req.Params).M("template").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "template", nil)
			return
		}

		userTags := ei.N(nc.getTags(user))
		if !(userTags.M("@"+req.Method).BoolZ() || userTags.M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}

		templateTags := ei.N(nc.getTags(template))
		if !(templateTags.M("@"+req.Method).BoolZ() || templateTags.M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}

		res, err := r.Table("users").Get(user).Update(map[string]interface{}{
			"templates": r.Row.Field("templates").Default([]string{}).SetInsert(template),
		}).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Unchanged == 0 && res.Replaced == 0 {
			req.Error(ErrInvalidUser, "", nil)
			return
		}
		req.Result(map[string]interface{}{"ok": true})

	case "user.delTemplate":
		user, err := ei.N(req.Params).M("user").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "user", nil)
			return
		}

		template, err := ei.N(req.Params).M("template").Lower().String()
		if err != nil {
			req.Error(ErrInvalidParams, "template", nil)
			return
		}

		userTags := ei.N(nc.getTags(user))
		if !(userTags.M("@"+req.Method).BoolZ() || userTags.M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}

		templateTags := ei.N(nc.getTags(template))
		if !(templateTags.M("@"+req.Method).BoolZ() || templateTags.M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}

		res, err := r.Table("users").Get(user).Update(map[string]interface{}{
			"templates": r.Row.Field("templates").Default(ei.S{}).SetDifference([]string{template}),
		}).RunWrite(db, r.RunOpts{Durability: "hard"})
		if err != nil {
			req.Error(ErrInternal, "", nil)
			return
		}
		if res.Unchanged == 0 && res.Replaced == 0 {
			req.Error(ErrInvalidUser, "", nil)
			return
		}
		req.Result(map[string]interface{}{"ok": true})

	case "user.listTemplate":
		user, err := ei.N(req.Params).M("user").Lower().String()

		userTags := ei.N(nc.getTags(user))
		if !(userTags.M("@"+req.Method).BoolZ() || userTags.M("@admin").BoolZ()) {
			req.Error(ErrPermissionDenied, "", nil)
			return
		}

		type udt struct {
			Templates []string `gorethink:"templates"`
		}

		res, err := r.Table("users").Get(user).Pluck("templates").Run(db)
		if err != nil {
			req.Error(ErrInvalidUser, "", nil)
			return
		}

		ret := udt{Templates: []string{}}
		if err := res.One(&ret); err != nil && err != r.ErrEmptyResult {
			req.Error(ErrInternal, "", nil)
			return
		}

		if len(ret.Templates) == 0 {
			ret.Templates = []string{}
		}
		req.Result(ret.Templates)

	default:
		req.Error(ErrMethodNotFound, "", nil)
	}
}
<|endoftext|>


<commit_before>package gorbac

import (
	"database/sql"
	"errors"
	"fmt"
	"log"
	"time"
)

// User can be Id(int,string)
type User interface{}

type UserManager interface {
	Assign(role Role, userId User) (int64, error)
	Unassign(role Role, userId User) error
	AllRoles(userId int64) (Roles, error)
	HasRole(role Role, userId User) (bool, error)
	RoleCount(userId User) (int64, error)
	ResetAssignments(ensure bool) error
}

type userManager struct {
	rbac  *rbac
	table string
}

var ErrUserRequired = errors.New("user_id is a required argument")

func newUserManager(r *rbac) UserManager {
	var userManager = new(userManager)
	userManager.table = "user_roles"
	userManager.rbac = r
	return userManager
}

func (u userManager) Assign(role Role, userId User) (int64, error) {
	var err error
	var roleId int64

	if _, ok := userId.(string); ok {
		if userId.(string) == "" {
			return 0, ErrUserRequired
		}
	} else if _, ok := userId.(int64); ok {
		if userId.(int64) == 0 {
			return 0, ErrUserRequired
		}
	}

	if _, ok := role.(int64); ok {
		roleId = role.(int64)
	} else if _, ok := role.(string); ok {
		if role.(string)[:1] == "/" {
			roleId, err = u.rbac.Roles().GetRoleId(role.(string))
			if err != nil {
				return 0, err
			}
		} else {
			roleId, err = u.rbac.Roles().TitleId(role.(string))
			if err != nil {
				return 0, err
			}
		}
	}

	if roleId > 0 {
		var query = fmt.Sprintf("INSERT INTO %s (user_id, role_id, assignment_date) VALUES(?,?,?)", u.getTable())
		res, err := u.rbac.db.Exec(query, userId, roleId, time.Now().Nanosecond())
		if err != nil {
			return 0, err
		}

		insertId, _ := res.LastInsertId()

		return insertId, nil
	}

	return 0, fmt.Errorf("role could not be found")
}

func (u userManager) HasRole(role Role, userId User) (bool, error) {
	if _, ok := userId.(string); ok {
		if userId.(string) == "" {
			return false, ErrUserRequired
		}
	} else if _, ok := userId.(int64); ok {
		if userId.(int64) == 0 {
			return false, ErrUserRequired
		}
	}

	roleId, err := u.rbac.Roles().GetRoleId(role)
	if err != nil {
		return false, err
	}

	query := fmt.Sprintf(`
	SELECT COUNT(*) FROM user_roles AS TUR
	JOIN roles AS TRdirect ON (TRdirect.ID=TUR.role_id)
	JOIN roles AS TR ON (TR.Lft BETWEEN TRdirect.Lft AND TRdirect.Rght)
	WHERE
	TUR.user_id=? AND TR.ID=?`)

	var result int64
	err = u.rbac.db.QueryRow(query, userId, roleId).Scan(&result)
	if err != nil {
		if err != sql.ErrNoRows {
			return false, err
		}
	}

	if result > 0 {
		return true, nil
	}

	return false, nil
}

func (u userManager) Unassign(role Role, userId User) error {
	if _, ok := userId.(string); ok {
		if userId.(string) == "" {
			return ErrUserRequired
		}
	} else if _, ok := userId.(int64); ok {
		if userId.(int64) == 0 {
			return ErrUserRequired
		}
	}

	roleId, err := u.rbac.roles.GetRoleId(role)
	if err != nil {
		return err
	}

	_, err = u.rbac.db.Exec(fmt.Sprintf("DELETE FROM %s WHERE user_id=? AND role_id=?", u.getTable()), userId, roleId)
	if err != nil {
		return err
	}

	return nil
}

func (u userManager) AllRoles(userId int64) (Roles, error) {
	if userId == 0 {
		return nil, ErrUserRequired
	}
	query := fmt.Sprintf(`
		SELECT
			TR.Id, TR.Title, TR.Description
		FROM
			%s AS TRel
		JOIN roles AS TR ON
		(TRel.role_id=TR.ID)
		WHERE TRel.user_id=?`, u.getTable())

	rows, err := u.rbac.db.Query(query, userId)
	if err != nil {
		return nil, err
	}

	var roles Roles
	for rows.Next() {
		var role role
		err := rows.Scan(&role.Id, &role.Title, &role.Description)
		if err != nil {
			return nil, err
		}
		roles = append(roles, role)
	}

	return roles, nil
}

func (u userManager) RoleCount(userId User) (int64, error) {
	if _, ok := userId.(string); ok {
		if userId.(string) == "" {
			return 0, ErrUserRequired
		}
	} else if _, ok := userId.(int64); ok {
		if userId.(int64) == 0 {
			return 0, ErrUserRequired
		}
	}

	var result int64
	err := u.rbac.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) AS Result FROM %s WHERE user_id=?", u.getTable()), userId).Scan(&result)

	if err != nil {
		return 0, err
	}

	return result, err
}

func (u userManager) getTable() string {
	return u.table
}

func (u userManager) ResetAssignments(ensure bool) error {
	if !ensure {
		log.Fatal("You must pass true to this function, otherwise it won't work.")
	}

	var err error
	_, err = u.rbac.db.Exec("DELETE FROM user_roles")
	if err != nil {
		return err
	}
	_, err = u.rbac.db.Exec("ALTER TABLE user_roles AUTO_INCREMENT =1")
	if err != nil {
		return err
	}

	u.Assign("root", u.rbac.rootId())

	return nil
}
<commit_msg> Changes to be committed: 	modified:   users.go<commit_after>package gorbac

import (
	"database/sql"
	"errors"
	"fmt"
	"log"
	"time"
)

// User can be Id(int,string)
type User interface{}

type UserManager interface {
	Assign(role Role, userId User) (int64, error)
	Unassign(role Role, userId User) error
	AllRoles(userId User) (Roles, error)
	HasRole(role Role, userId User) (bool, error)
	RoleCount(userId User) (int64, error)
	ResetAssignments(ensure bool) error
}

type userManager struct {
	rbac  *rbac
	table string
}

var ErrUserRequired = errors.New("user_id is a required argument")

func newUserManager(r *rbac) UserManager {
	var userManager = new(userManager)
	userManager.table = "user_roles"
	userManager.rbac = r
	return userManager
}

// Assigns a role to a user
func (u userManager) Assign(role Role, userId User) (int64, error) {
	var err error
	var roleId int64

	if _, ok := userId.(string); ok {
		if userId.(string) == "" {
			return 0, ErrUserRequired
		}
	} else if _, ok := userId.(int64); ok {
		if userId.(int64) == 0 {
			return 0, ErrUserRequired
		}
	}

	if _, ok := role.(int64); ok {
		roleId = role.(int64)
	} else if _, ok := role.(string); ok {
		if role.(string)[:1] == "/" {
			roleId, err = u.rbac.Roles().GetRoleId(role.(string))
			if err != nil {
				return 0, err
			}
		} else {
			roleId, err = u.rbac.Roles().TitleId(role.(string))
			if err != nil {
				return 0, err
			}
		}
	}

	if roleId > 0 {
		var query = fmt.Sprintf("INSERT INTO %s (user_id, role_id, assignment_date) VALUES(?,?,?)", u.getTable())
		res, err := u.rbac.db.Exec(query, userId, roleId, time.Now().Nanosecond())
		if err != nil {
			return 0, err
		}

		insertId, _ := res.LastInsertId()

		return insertId, nil
	}

	return 0, fmt.Errorf("role could not be found")
}

// Checks to see whether a User has a Role or not.
func (u userManager) HasRole(role Role, userId User) (bool, error) {
	if _, ok := userId.(string); ok {
		if userId.(string) == "" {
			return false, ErrUserRequired
		}
	} else if _, ok := userId.(int64); ok {
		if userId.(int64) == 0 {
			return false, ErrUserRequired
		}
	}

	roleId, err := u.rbac.Roles().GetRoleId(role)
	if err != nil {
		return false, err
	}

	query := fmt.Sprintf(`
	SELECT COUNT(*) FROM user_roles AS TUR
	JOIN roles AS TRdirect ON (TRdirect.ID=TUR.role_id)
	JOIN roles AS TR ON (TR.Lft BETWEEN TRdirect.Lft AND TRdirect.Rght)
	WHERE
	TUR.user_id=? AND TR.ID=?`)

	var result int64
	err = u.rbac.db.QueryRow(query, userId, roleId).Scan(&result)
	if err != nil {
		if err != sql.ErrNoRows {
			return false, err
		}
	}

	if result > 0 {
		return true, nil
	}

	return false, nil
}

// Unassigns a Role from a User.
func (u userManager) Unassign(role Role, userId User) error {
	if _, ok := userId.(string); ok {
		if userId.(string) == "" {
			return ErrUserRequired
		}
	} else if _, ok := userId.(int64); ok {
		if userId.(int64) == 0 {
			return ErrUserRequired
		}
	}

	roleId, err := u.rbac.roles.GetRoleId(role)
	if err != nil {
		return err
	}

	_, err = u.rbac.db.Exec(fmt.Sprintf("DELETE FROM %s WHERE user_id=? AND role_id=?", u.getTable()), userId, roleId)
	if err != nil {
		return err
	}

	return nil
}

// Returns all Roles of a User.
func (u userManager) AllRoles(userId User) (Roles, error) {
	if _, ok := userId.(string); ok {
		if userId.(string) == "" {
			return nil, ErrUserRequired
		}
	} else if _, ok := userId.(int64); ok {
		if userId.(int64) == 0 {
			return nil, ErrUserRequired
		}
	}

	query := fmt.Sprintf(`
		SELECT
			TR.Id, TR.Title, TR.Description
		FROM
			%s AS TRel
		JOIN roles AS TR ON
		(TRel.role_id=TR.ID)
		WHERE TRel.user_id=?`, u.getTable())

	rows, err := u.rbac.db.Query(query, userId)
	if err != nil {
		return nil, err
	}

	var roles Roles
	for rows.Next() {
		var role role
		err := rows.Scan(&role.Id, &role.Title, &role.Description)
		if err != nil {
			return nil, err
		}
		roles = append(roles, role)
	}

	return roles, nil
}

func (u userManager) RoleCount(userId User) (int64, error) {
	if _, ok := userId.(string); ok {
		if userId.(string) == "" {
			return 0, ErrUserRequired
		}
	} else if _, ok := userId.(int64); ok {
		if userId.(int64) == 0 {
			return 0, ErrUserRequired
		}
	}

	var result int64
	err := u.rbac.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) AS Result FROM %s WHERE user_id=?", u.getTable()), userId).Scan(&result)

	if err != nil {
		return 0, err
	}

	return result, err
}

func (u userManager) getTable() string {
	return u.table
}

func (u userManager) ResetAssignments(ensure bool) error {
	if !ensure {
		log.Fatal("You must pass true to this function, otherwise it won't work.")
	}

	var err error
	_, err = u.rbac.db.Exec("DELETE FROM user_roles")
	if err != nil {
		return err
	}
	_, err = u.rbac.db.Exec("ALTER TABLE user_roles AUTO_INCREMENT =1")
	if err != nil {
		return err
	}

	u.Assign("root", u.rbac.rootId())

	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2018 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package compdb contains functionality necessary for extracting from a
// compile_commands.json file.
package compdb // import "kythe.io/kythe/go/extractors/config/runextractor/compdb"

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	"bitbucket.org/creachadair/shell"
	"golang.org/x/sync/semaphore"
)

// A compileCommand holds the decoded arguments of a LLVM compilation database
// JSON command spec.
type compileCommand struct {
	Command   string
	Directory string
}

// ExtractCompilations runs the specified extractor over each compilation record
// found in the compile_commands.json file at path.
func ExtractCompilations(ctx context.Context, extractor, path string) error {
	commands, err := readCommands(path)
	if err != nil {
		return err
	}
	env, err := extractorEnv()
	if err != nil {
		return err
	}

	var failCount uint64
	sem := semaphore.NewWeighted(128) // Limit concurrency.
	var wg sync.WaitGroup
	wg.Add(len(commands))
	for _, entry := range commands {
		go func(entry compileCommand) {
			defer wg.Done()
			if err := sem.Acquire(ctx, 1); err != nil {
				atomic.AddUint64(&failCount, 1)
				log.Print(err)
				return
			}
			defer sem.Release(1)

			if err := extractOne(ctx, extractor, entry, env); err != nil {
				// Log error, but continue processing other compilations.
				atomic.AddUint64(&failCount, 1)
				log.Printf("Error extracting compilation: %v", err)
			}
		}(entry)
	}
	wg.Wait()

	if failCount != 0 {
		return fmt.Errorf("Failed to extract %d compilations", failCount)
	}

	return nil
}

// extractOne invokes the extractor for the given compileCommand.
func extractOne(ctx context.Context, extractor string, cc compileCommand, env []string) error {
	cmd := exec.CommandContext(ctx, extractor, "--with_executable")
	args, ok := shell.Split(cc.Command)
	if !ok {
		return fmt.Errorf("unable to split command line")
	}
	cmd.Args = append(cmd.Args, args...)
	var err error
	cmd.Dir, err = filepath.Abs(cc.Directory)
	if err != nil {
		return fmt.Errorf("unable to resolve cmake directory: %v", err)
	}
	cmd.Env = env
	if _, err := cmd.Output(); err != nil {
		if exit, ok := err.(*exec.ExitError); ok {
			return fmt.Errorf("error running extractor: %v (%s)", exit, exit.Stderr)
		}
		return fmt.Errorf("error running extractor: %v", err)
	}
	return nil
}

// readCommands reads the JSON file at path into a slice of compileCommands.
func readCommands(path string) ([]compileCommand, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	var commands []compileCommand
	if err := json.Unmarshal(data, &commands); err != nil {
		return nil, err
	}
	return commands, nil
}

// extractorEnv copies the existing environment and modifies it to be suitable for an extractor invocation.
func extractorEnv() ([]string, error) {
	var env []string
	outputFound := false
	for _, value := range os.Environ() {
		parts := strings.SplitN(value, "=", 2)
		// Until kzip support comes along, we only support writing to a single directory so strip these options.
		if parts[0] == "KYTHE_INDEX_PACK" || parts[0] == "KYTHE_OUTPUT_FILE" {
			continue
		} else if parts[0] == "KYTHE_OUTPUT_DIRECTORY" {
			// Remap KYTHE_OUTPUT_DIRECTORY to be an absolute path.
			output, err := filepath.Abs(parts[1])
			if err != nil {
				return nil, err
			}
			outputFound = true
			env = append(env, "KYTHE_OUTPUT_DIRECTORY="+output)
		} else {
			// Otherwise, preserve the environment unchanged.
			env = append(env, value)
		}

	}
	if !outputFound {
		return nil, errors.New("missing mandatory environment variable: KYTHE_OUTPUT_DIRECTORY")
	}
	return env, nil
}
<commit_msg>feat(compdb): print out command for failed extractions (#4546)<commit_after>/*
 * Copyright 2018 The Kythe Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package compdb contains functionality necessary for extracting from a
// compile_commands.json file.
package compdb // import "kythe.io/kythe/go/extractors/config/runextractor/compdb"

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"

	"bitbucket.org/creachadair/shell"
	"golang.org/x/sync/semaphore"
)

// A compileCommand holds the decoded arguments of a LLVM compilation database
// JSON command spec.
type compileCommand struct {
	Command   string
	Directory string
}

// ExtractCompilations runs the specified extractor over each compilation record
// found in the compile_commands.json file at path.
func ExtractCompilations(ctx context.Context, extractor, path string) error {
	commands, err := readCommands(path)
	if err != nil {
		return err
	}
	env, err := extractorEnv()
	if err != nil {
		return err
	}

	var failCount uint64
	sem := semaphore.NewWeighted(128) // Limit concurrency.
	var wg sync.WaitGroup
	wg.Add(len(commands))
	for _, entry := range commands {
		go func(entry compileCommand) {
			defer wg.Done()
			if err := sem.Acquire(ctx, 1); err != nil {
				atomic.AddUint64(&failCount, 1)
				log.Print(err)
				return
			}
			defer sem.Release(1)

			if err := extractOne(ctx, extractor, entry, env); err != nil {
				// Log error, but continue processing other compilations.
				atomic.AddUint64(&failCount, 1)
				log.Printf("Error extracting compilation with command '%s': %v", entry.Command, err)
			}
		}(entry)
	}
	wg.Wait()

	if failCount != 0 {
		return fmt.Errorf("Failed to extract %d compilations", failCount)
	}

	return nil
}

// extractOne invokes the extractor for the given compileCommand.
func extractOne(ctx context.Context, extractor string, cc compileCommand, env []string) error {
	cmd := exec.CommandContext(ctx, extractor, "--with_executable")
	args, ok := shell.Split(cc.Command)
	if !ok {
		return fmt.Errorf("unable to split command line")
	}
	cmd.Args = append(cmd.Args, args...)
	var err error
	cmd.Dir, err = filepath.Abs(cc.Directory)
	if err != nil {
		return fmt.Errorf("unable to resolve cmake directory: %v", err)
	}
	cmd.Env = env
	if _, err := cmd.Output(); err != nil {
		if exit, ok := err.(*exec.ExitError); ok {
			return fmt.Errorf("error running extractor: %v (%s)", exit, exit.Stderr)
		}
		return fmt.Errorf("error running extractor: %v", err)
	}
	return nil
}

// readCommands reads the JSON file at path into a slice of compileCommands.
func readCommands(path string) ([]compileCommand, error) {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, err
	}
	var commands []compileCommand
	if err := json.Unmarshal(data, &commands); err != nil {
		return nil, err
	}
	return commands, nil
}

// extractorEnv copies the existing environment and modifies it to be suitable for an extractor invocation.
func extractorEnv() ([]string, error) {
	var env []string
	outputFound := false
	for _, value := range os.Environ() {
		parts := strings.SplitN(value, "=", 2)
		// Until kzip support comes along, we only support writing to a single directory so strip these options.
		if parts[0] == "KYTHE_INDEX_PACK" || parts[0] == "KYTHE_OUTPUT_FILE" {
			continue
		} else if parts[0] == "KYTHE_OUTPUT_DIRECTORY" {
			// Remap KYTHE_OUTPUT_DIRECTORY to be an absolute path.
			output, err := filepath.Abs(parts[1])
			if err != nil {
				return nil, err
			}
			outputFound = true
			env = append(env, "KYTHE_OUTPUT_DIRECTORY="+output)
		} else {
			// Otherwise, preserve the environment unchanged.
			env = append(env, value)
		}

	}
	if !outputFound {
		return nil, errors.New("missing mandatory environment variable: KYTHE_OUTPUT_DIRECTORY")
	}
	return env, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package context

import (
	"net/url"

	"code.gitea.io/gitea/modules/auth"
	"code.gitea.io/gitea/modules/setting"
	"github.com/go-macaron/csrf"
	macaron "gopkg.in/macaron.v1"
)

// ToggleOptions contains required or check options
type ToggleOptions struct {
	SignInRequired  bool
	SignOutRequired bool
	AdminRequired   bool
	DisableCSRF     bool
}

// Toggle returns toggle options as middleware
func Toggle(options *ToggleOptions) macaron.Handler {
	return func(ctx *Context) {
		// Cannot view any page before installation.
		if !setting.InstallLock {
			ctx.Redirect(setting.AppSubURL + "/install")
			return
		}

		// Check prohibit login users.
		if ctx.IsSigned {

			if ctx.User.ProhibitLogin {
				ctx.Data["Title"] = ctx.Tr("auth.prohibit_login")
				ctx.HTML(200, "user/auth/prohibit_login")
				return
			}

			// prevent infinite redirection
			// also make sure that the form cannot be accessed by
			// users who don't need this
			if ctx.Req.URL.Path == setting.AppSubURL+"/user/settings/change_password" {
				if !ctx.User.MustChangePassword {
					ctx.Redirect(setting.AppSubURL + "/")
				}
				return
			}

			if ctx.User.MustChangePassword {
				ctx.Data["Title"] = ctx.Tr("auth.must_change_password")
				ctx.Data["ChangePasscodeLink"] = setting.AppSubURL + "/user/change_password"
				ctx.SetCookie("redirect_to", url.QueryEscape(setting.AppSubURL+ctx.Req.RequestURI), 0, setting.AppSubURL)
				ctx.Redirect(setting.AppSubURL + "/user/settings/change_password")
				return
			}
		}

		// Redirect to dashboard if user tries to visit any non-login page.
		if options.SignOutRequired && ctx.IsSigned && ctx.Req.RequestURI != "/" {
			ctx.Redirect(setting.AppSubURL + "/")
			return
		}

		if !options.SignOutRequired && !options.DisableCSRF && ctx.Req.Method == "POST" && !auth.IsAPIPath(ctx.Req.URL.Path) {
			csrf.Validate(ctx.Context, ctx.csrf)
			if ctx.Written() {
				return
			}
		}

		if options.SignInRequired {
			if !ctx.IsSigned {
				// Restrict API calls with error message.
				if auth.IsAPIPath(ctx.Req.URL.Path) {
					ctx.JSON(403, map[string]string{
						"message": "Only signed in user is allowed to call APIs.",
					})
					return
				}

				ctx.SetCookie("redirect_to", url.QueryEscape(setting.AppSubURL+ctx.Req.RequestURI), 0, setting.AppSubURL)
				ctx.Redirect(setting.AppSubURL + "/user/login")
				return
			} else if !ctx.User.IsActive && setting.Service.RegisterEmailConfirm {
				ctx.Data["Title"] = ctx.Tr("auth.active_your_account")
				ctx.HTML(200, "user/auth/activate")
				return
			}
		}

		// Redirect to log in page if auto-signin info is provided and has not signed in.
		if !options.SignOutRequired && !ctx.IsSigned && !auth.IsAPIPath(ctx.Req.URL.Path) &&
			len(ctx.GetCookie(setting.CookieUserName)) > 0 {
			ctx.SetCookie("redirect_to", url.QueryEscape(setting.AppSubURL+ctx.Req.RequestURI), 0, setting.AppSubURL)
			ctx.Redirect(setting.AppSubURL + "/user/login")
			return
		}

		if options.AdminRequired {
			if !ctx.User.IsAdmin {
				ctx.Error(403)
				return
			}
			ctx.Data["PageIsAdmin"] = true
		}
	}
}
<commit_msg>modules/context/auth.go: fix redirect loop (#5965)<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package context

import (
	"net/url"

	"code.gitea.io/gitea/modules/auth"
	"code.gitea.io/gitea/modules/setting"
	"github.com/go-macaron/csrf"
	macaron "gopkg.in/macaron.v1"
)

// ToggleOptions contains required or check options
type ToggleOptions struct {
	SignInRequired  bool
	SignOutRequired bool
	AdminRequired   bool
	DisableCSRF     bool
}

// Toggle returns toggle options as middleware
func Toggle(options *ToggleOptions) macaron.Handler {
	return func(ctx *Context) {
		// Cannot view any page before installation.
		if !setting.InstallLock {
			ctx.Redirect(setting.AppSubURL + "/install")
			return
		}

		// Check prohibit login users.
		if ctx.IsSigned {

			if ctx.User.ProhibitLogin {
				ctx.Data["Title"] = ctx.Tr("auth.prohibit_login")
				ctx.HTML(200, "user/auth/prohibit_login")
				return
			}

			// prevent infinite redirection
			// also make sure that the form cannot be accessed by
			// users who don't need this
			if ctx.Req.URL.Path == "/user/settings/change_password" {
				if !ctx.User.MustChangePassword {
					ctx.Redirect(setting.AppSubURL + "/")
				}
				return
			}

			if ctx.User.MustChangePassword {
				ctx.Data["Title"] = ctx.Tr("auth.must_change_password")
				ctx.Data["ChangePasscodeLink"] = setting.AppSubURL + "/user/change_password"
				ctx.SetCookie("redirect_to", url.QueryEscape(setting.AppSubURL+ctx.Req.RequestURI), 0, setting.AppSubURL)
				ctx.Redirect(setting.AppSubURL + "/user/settings/change_password")
				return
			}
		}

		// Redirect to dashboard if user tries to visit any non-login page.
		if options.SignOutRequired && ctx.IsSigned && ctx.Req.RequestURI != "/" {
			ctx.Redirect(setting.AppSubURL + "/")
			return
		}

		if !options.SignOutRequired && !options.DisableCSRF && ctx.Req.Method == "POST" && !auth.IsAPIPath(ctx.Req.URL.Path) {
			csrf.Validate(ctx.Context, ctx.csrf)
			if ctx.Written() {
				return
			}
		}

		if options.SignInRequired {
			if !ctx.IsSigned {
				// Restrict API calls with error message.
				if auth.IsAPIPath(ctx.Req.URL.Path) {
					ctx.JSON(403, map[string]string{
						"message": "Only signed in user is allowed to call APIs.",
					})
					return
				}

				ctx.SetCookie("redirect_to", url.QueryEscape(setting.AppSubURL+ctx.Req.RequestURI), 0, setting.AppSubURL)
				ctx.Redirect(setting.AppSubURL + "/user/login")
				return
			} else if !ctx.User.IsActive && setting.Service.RegisterEmailConfirm {
				ctx.Data["Title"] = ctx.Tr("auth.active_your_account")
				ctx.HTML(200, "user/auth/activate")
				return
			}
		}

		// Redirect to log in page if auto-signin info is provided and has not signed in.
		if !options.SignOutRequired && !ctx.IsSigned && !auth.IsAPIPath(ctx.Req.URL.Path) &&
			len(ctx.GetCookie(setting.CookieUserName)) > 0 {
			ctx.SetCookie("redirect_to", url.QueryEscape(setting.AppSubURL+ctx.Req.RequestURI), 0, setting.AppSubURL)
			ctx.Redirect(setting.AppSubURL + "/user/login")
			return
		}

		if options.AdminRequired {
			if !ctx.User.IsAdmin {
				ctx.Error(403)
				return
			}
			ctx.Data["PageIsAdmin"] = true
		}
	}
}
<|endoftext|>


<commit_before>Test file to trigger notifications.
<commit_msg>Delete test.go<commit_after><|endoftext|>


<commit_before>// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.

package tail

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/vektra/tail/ratelimiter"
	"github.com/vektra/tail/util"
	"github.com/vektra/tail/watch"
	"gopkg.in/tomb.v1"
)

var (
	ErrStop = fmt.Errorf("tail should now stop")
)

type Line struct {
	Text   string
	Time   time.Time
	Offset int64
	Err    error // Error from tail
}

// NewLine returns a Line with present time.
func NewLine(text string) *Line {
	return &Line{text, time.Now(), 0, nil}
}

// SeekInfo represents arguments to `os.Seek`
type SeekInfo struct {
	Offset int64
	Whence int // os.SEEK_*
}

// Config is used to specify how a file must be tailed.
type Config struct {
	// File-specifc
	Location    *SeekInfo // Seek to this location before tailing
	ReOpen      bool      // Reopen recreated files (tail -F)
	MustExist   bool      // Fail early if the file does not exist
	Poll        bool      // Poll for file changes instead of using inotify
	RateLimiter *ratelimiter.LeakyBucket

	// Generic IO
	Follow      bool // Continue looking for new lines (tail -f)
	MaxLineSize int  // If non-zero, split longer lines into multiple lines

	// Logger, when nil, is set to tail.DefaultLogger
	// To disable logging: set field to tail.DiscardingLogger
	Logger *log.Logger
}

type Tail struct {
	Filename string
	Lines    chan *Line
	Config

	file    *os.File
	reader  *bufio.Reader
	watcher watch.FileWatcher
	changes *watch.FileChanges

	closeOffset int64

	tomb.Tomb // provides: Done, Kill, Dying
}

var (
	// DefaultLogger is used when Config.Logger == nil
	DefaultLogger = log.New(os.Stderr, "", log.LstdFlags)
	// DiscardingLogger can be used to disable logging output
	DiscardingLogger = log.New(ioutil.Discard, "", 0)
)

// TailFile begins tailing the file. Output stream is made available
// via the `Tail.Lines` channel. To handle errors during tailing,
// invoke the `Wait` or `Err` method after finishing reading from the
// `Lines` channel.
func TailFile(filename string, config Config) (*Tail, error) {
	if config.ReOpen && !config.Follow {
		util.Fatal("cannot set ReOpen without Follow.")
	}

	t := &Tail{
		Filename: filename,
		Lines:    make(chan *Line),
		Config:   config,
	}

	// when Logger was not specified in config, use default logger
	if t.Logger == nil {
		t.Logger = log.New(os.Stderr, "", log.LstdFlags)
	}

	if t.Poll {
		t.watcher = watch.NewPollingFileWatcher(filename)
	} else {
		t.watcher = watch.NewInotifyFileWatcher(filename)
	}

	if t.MustExist {
		var err error
		t.file, err = OpenFile(t.Filename)
		if err != nil {
			return nil, err
		}
	}

	go t.tailFileSync()

	return t, nil
}

// Return the file's current position, like stdio's ftell().
// But this value is not very accurate.
// it may readed one line in the chan(tail.Lines),
// so it may lost one line.
func (tail *Tail) Tell() (offset int64, err error) {
	if tail.file == nil {
		return tail.closeOffset, nil
	}

	offset, err = tail.file.Seek(0, os.SEEK_CUR)
	if err == nil {
		offset -= int64(tail.reader.Buffered())
	}
	return
}

// Stop stops the tailing activity.
func (tail *Tail) Stop() error {
	tail.Kill(nil)
	return tail.Wait()
}

func (tail *Tail) close() {
	pos, err := tail.Tell()

	if err == nil {
		tail.closeOffset = pos
	}

	close(tail.Lines)
	if tail.file != nil {
		tail.file.Close()
		tail.file = nil
	}
}

func (tail *Tail) reopen() error {
	if tail.file != nil {
		tail.file.Close()
	}
	for {
		var err error
		tail.file, err = OpenFile(tail.Filename)
		if err != nil {
			if os.IsNotExist(err) {
				tail.Logger.Printf("Waiting for %s to appear...", tail.Filename)
				if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil {
					if err == tomb.ErrDying {
						return err
					}
					return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err)
				}
				continue
			}
			return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err)
		}
		break
	}
	return nil
}

func (tail *Tail) readLine() (string, int64, error) {
	line, err := tail.reader.ReadString('\n')
	sz := int64(len(line))

	if err != nil {
		// Note ReadString "returns the data read before the error" in
		// case of an error, including EOF, so we return it as is. The
		// caller is expected to process it if err is EOF.
		return line, sz, err
	}

	line = strings.TrimRight(line, "\n")

	return line, sz, err
}

func (tail *Tail) tailFileSync() {
	defer tail.Done()
	defer tail.close()

	if !tail.MustExist {
		// deferred first open.
		err := tail.reopen()
		if err != nil {
			if err != tomb.ErrDying {
				tail.Kill(err)
			}
			return
		}
	}

	// Seek to requested location on first open of the file.
	if tail.Location != nil {
		_, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence)
		tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location)
		if err != nil {
			tail.Killf("Seek error on %s: %s", tail.Filename, err)
			return
		}
	}

	tail.openReader()

	offset, err := tail.file.Seek(0, os.SEEK_CUR)
	if err != nil {
		tail.Kill(err)
		return
	}

	// Read line by line.
	for {
		line, sz, err := tail.readLine()

		// Process `line` even if err is EOF.
		if err == nil || (err == io.EOF && line != "") {
			cooloff := !tail.sendLine(line, offset)
			if cooloff {
				// Wait a second before seeking till the end of
				// file when rate limit is reached.
				msg := fmt.Sprintf(
					"Too much log activity; waiting a second " +
						"before resuming tailing")
				tail.Lines <- &Line{msg, time.Now(), 0, fmt.Errorf(msg)}
				select {
				case <-time.After(time.Second):
				case <-tail.Dying():
					return
				}
				err = tail.seekEnd()
				if err != nil {
					tail.Kill(err)
					return
				}
			}
		} else if err == io.EOF {
			if !tail.Follow {
				return
			}
			// When EOF is reached, wait for more data to become
			// available. Wait strategy is based on the `tail.watcher`
			// implementation (inotify or polling).
			err := tail.waitForChanges()
			if err != nil {
				if err != ErrStop {
					tail.Kill(err)
				}
				return
			}
		} else {
			// non-EOF error
			tail.Killf("Error reading %s: %s", tail.Filename, err)
			return
		}

		select {
		case <-tail.Dying():
			return
		default:
		}

		offset += sz
	}
}

// waitForChanges waits until the file has been appended, deleted,
// moved or truncated. When moved or deleted - the file will be
// reopened if ReOpen is true. Truncated files are always reopened.
func (tail *Tail) waitForChanges() error {
	if tail.changes == nil {
		st, err := tail.file.Stat()
		if err != nil {
			return err
		}
		tail.changes = tail.watcher.ChangeEvents(&tail.Tomb, st)
	}

	select {
	case <-tail.changes.Modified:
		return nil
	case <-tail.changes.Deleted:
		tail.changes = nil
		if tail.ReOpen {
			// XXX: we must not log from a library.
			tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename)
			if err := tail.reopen(); err != nil {
				return err
			}
			tail.Logger.Printf("Successfully reopened %s", tail.Filename)
			tail.openReader()
			return nil
		} else {
			tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename)
			return ErrStop
		}
	case <-tail.changes.Truncated:
		// Always reopen truncated files (Follow is true)
		tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename)
		if err := tail.reopen(); err != nil {
			return err
		}
		tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename)
		tail.openReader()
		return nil
	case <-tail.Dying():
		return ErrStop
	}
	panic("unreachable")
}

func (tail *Tail) openReader() {
	if tail.MaxLineSize > 0 {
		// add 2 to account for newline characters
		tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2)
	} else {
		tail.reader = bufio.NewReader(tail.file)
	}
}

func (tail *Tail) seekEnd() error {
	_, err := tail.file.Seek(0, 2)
	if err != nil {
		return fmt.Errorf("Seek error on %s: %s", tail.Filename, err)
	}
	// Reset the read buffer whenever the file is re-seek'ed
	tail.reader.Reset(tail.file)
	return nil
}

// sendLine sends the line(s) to Lines channel, splitting longer lines
// if necessary. Return false if rate limit is reached.
func (tail *Tail) sendLine(line string, sz int64) bool {
	now := time.Now()

	total := 1

	// Split longer lines
	if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize {
		lines := util.PartitionString(line, tail.MaxLineSize)

		total = len(lines)

		offset := sz

		for _, line := range lines {
			tail.Lines <- &Line{line, now, offset, nil}
			offset += int64(len(line))
		}
	} else {
		tail.Lines <- &Line{line, now, sz, nil}
	}

	if tail.Config.RateLimiter != nil {
		ok := tail.Config.RateLimiter.Pour(uint16(total))
		if !ok {
			tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n",
				tail.Filename)
			return false
		}
	}

	return true
}

// Cleanup removes inotify watches added by the tail package. This function is
// meant to be invoked from a process's exit handler. Linux kernel may not
// automatically remove inotify watches after the process exits.
func Cleanup() {
	watch.Cleanup()
}
<commit_msg>Add Size to line so code can calculate the position of the next line<commit_after>// Copyright (c) 2013 ActiveState Software Inc. All rights reserved.

package tail

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/vektra/tail/ratelimiter"
	"github.com/vektra/tail/util"
	"github.com/vektra/tail/watch"
	"gopkg.in/tomb.v1"
)

var (
	ErrStop = fmt.Errorf("tail should now stop")
)

type Line struct {
	Text   string
	Time   time.Time
	Offset int64
	Size   int64
	Err    error // Error from tail
}

// NewLine returns a Line with present time.
func NewLine(text string) *Line {
	return &Line{text, time.Now(), 0, int64(len(text)), nil}
}

// SeekInfo represents arguments to `os.Seek`
type SeekInfo struct {
	Offset int64
	Whence int // os.SEEK_*
}

// Config is used to specify how a file must be tailed.
type Config struct {
	// File-specifc
	Location    *SeekInfo // Seek to this location before tailing
	ReOpen      bool      // Reopen recreated files (tail -F)
	MustExist   bool      // Fail early if the file does not exist
	Poll        bool      // Poll for file changes instead of using inotify
	RateLimiter *ratelimiter.LeakyBucket

	// Generic IO
	Follow      bool // Continue looking for new lines (tail -f)
	MaxLineSize int  // If non-zero, split longer lines into multiple lines

	// Logger, when nil, is set to tail.DefaultLogger
	// To disable logging: set field to tail.DiscardingLogger
	Logger *log.Logger
}

type Tail struct {
	Filename string
	Lines    chan *Line
	Config

	file    *os.File
	reader  *bufio.Reader
	watcher watch.FileWatcher
	changes *watch.FileChanges

	closeOffset int64

	tomb.Tomb // provides: Done, Kill, Dying
}

var (
	// DefaultLogger is used when Config.Logger == nil
	DefaultLogger = log.New(os.Stderr, "", log.LstdFlags)
	// DiscardingLogger can be used to disable logging output
	DiscardingLogger = log.New(ioutil.Discard, "", 0)
)

// TailFile begins tailing the file. Output stream is made available
// via the `Tail.Lines` channel. To handle errors during tailing,
// invoke the `Wait` or `Err` method after finishing reading from the
// `Lines` channel.
func TailFile(filename string, config Config) (*Tail, error) {
	if config.ReOpen && !config.Follow {
		util.Fatal("cannot set ReOpen without Follow.")
	}

	t := &Tail{
		Filename: filename,
		Lines:    make(chan *Line),
		Config:   config,
	}

	// when Logger was not specified in config, use default logger
	if t.Logger == nil {
		t.Logger = log.New(os.Stderr, "", log.LstdFlags)
	}

	if t.Poll {
		t.watcher = watch.NewPollingFileWatcher(filename)
	} else {
		t.watcher = watch.NewInotifyFileWatcher(filename)
	}

	if t.MustExist {
		var err error
		t.file, err = OpenFile(t.Filename)
		if err != nil {
			return nil, err
		}
	}

	go t.tailFileSync()

	return t, nil
}

// Return the file's current position, like stdio's ftell().
// But this value is not very accurate.
// it may readed one line in the chan(tail.Lines),
// so it may lost one line.
func (tail *Tail) Tell() (offset int64, err error) {
	if tail.file == nil {
		return tail.closeOffset, nil
	}

	return tail.tell()
}

func (tail *Tail) tell() (offset int64, err error) {
	offset, err = tail.file.Seek(0, os.SEEK_CUR)
	if err == nil {
		offset -= int64(tail.reader.Buffered())
	}
	return
}

// Stop stops the tailing activity.
func (tail *Tail) Stop() error {
	tail.Kill(nil)
	return tail.Wait()
}

func (tail *Tail) close() {
	pos, err := tail.tell()

	if err == nil {
		tail.closeOffset = pos
	}

	close(tail.Lines)
	if tail.file != nil {
		tail.file.Close()
		tail.file = nil
	}
}

func (tail *Tail) reopen() error {
	if tail.file != nil {
		tail.file.Close()
	}
	for {
		var err error
		tail.file, err = OpenFile(tail.Filename)
		if err != nil {
			if os.IsNotExist(err) {
				tail.Logger.Printf("Waiting for %s to appear...", tail.Filename)
				if err := tail.watcher.BlockUntilExists(&tail.Tomb); err != nil {
					if err == tomb.ErrDying {
						return err
					}
					return fmt.Errorf("Failed to detect creation of %s: %s", tail.Filename, err)
				}
				continue
			}
			return fmt.Errorf("Unable to open file %s: %s", tail.Filename, err)
		}
		break
	}
	return nil
}

func (tail *Tail) readLine() (string, int64, error) {
	line, err := tail.reader.ReadString('\n')
	sz := int64(len(line))

	if err != nil {
		// Note ReadString "returns the data read before the error" in
		// case of an error, including EOF, so we return it as is. The
		// caller is expected to process it if err is EOF.
		return line, sz, err
	}

	line = strings.TrimRight(line, "\n")

	return line, sz, err
}

func (tail *Tail) tailFileSync() {
	defer tail.Done()
	defer tail.close()

	if !tail.MustExist {
		// deferred first open.
		err := tail.reopen()
		if err != nil {
			if err != tomb.ErrDying {
				tail.Kill(err)
			}
			return
		}
	}

	// Seek to requested location on first open of the file.
	if tail.Location != nil {
		_, err := tail.file.Seek(tail.Location.Offset, tail.Location.Whence)
		tail.Logger.Printf("Seeked %s - %+v\n", tail.Filename, tail.Location)
		if err != nil {
			tail.Killf("Seek error on %s: %s", tail.Filename, err)
			return
		}
	}

	tail.openReader()

	offset, err := tail.file.Seek(0, os.SEEK_CUR)
	if err != nil {
		tail.Kill(err)
		return
	}

	// Read line by line.
	for {
		line, sz, err := tail.readLine()

		// Process `line` even if err is EOF.
		if err == nil || (err == io.EOF && line != "") {
			cooloff := !tail.sendLine(line, offset, sz)
			if cooloff {
				// Wait a second before seeking till the end of
				// file when rate limit is reached.
				msg := fmt.Sprintf(
					"Too much log activity; waiting a second " +
						"before resuming tailing")
				tail.Lines <- &Line{msg, time.Now(), 0, int64(len(msg)), fmt.Errorf(msg)}
				select {
				case <-time.After(time.Second):
				case <-tail.Dying():
					return
				}
				err = tail.seekEnd()
				if err != nil {
					tail.Kill(err)
					return
				}
			}
		} else if err == io.EOF {
			if !tail.Follow {
				return
			}
			// When EOF is reached, wait for more data to become
			// available. Wait strategy is based on the `tail.watcher`
			// implementation (inotify or polling).
			err := tail.waitForChanges()
			if err != nil {
				if err != ErrStop {
					tail.Kill(err)
				}
				return
			}
		} else {
			// non-EOF error
			tail.Killf("Error reading %s: %s", tail.Filename, err)
			return
		}

		select {
		case <-tail.Dying():
			return
		default:
		}

		offset += sz
	}
}

// waitForChanges waits until the file has been appended, deleted,
// moved or truncated. When moved or deleted - the file will be
// reopened if ReOpen is true. Truncated files are always reopened.
func (tail *Tail) waitForChanges() error {
	if tail.changes == nil {
		st, err := tail.file.Stat()
		if err != nil {
			return err
		}
		tail.changes = tail.watcher.ChangeEvents(&tail.Tomb, st)
	}

	select {
	case <-tail.changes.Modified:
		return nil
	case <-tail.changes.Deleted:
		tail.changes = nil
		if tail.ReOpen {
			// XXX: we must not log from a library.
			tail.Logger.Printf("Re-opening moved/deleted file %s ...", tail.Filename)
			if err := tail.reopen(); err != nil {
				return err
			}
			tail.Logger.Printf("Successfully reopened %s", tail.Filename)
			tail.openReader()
			return nil
		} else {
			tail.Logger.Printf("Stopping tail as file no longer exists: %s", tail.Filename)
			return ErrStop
		}
	case <-tail.changes.Truncated:
		// Always reopen truncated files (Follow is true)
		tail.Logger.Printf("Re-opening truncated file %s ...", tail.Filename)
		if err := tail.reopen(); err != nil {
			return err
		}
		tail.Logger.Printf("Successfully reopened truncated %s", tail.Filename)
		tail.openReader()
		return nil
	case <-tail.Dying():
		return ErrStop
	}
	panic("unreachable")
}

func (tail *Tail) openReader() {
	if tail.MaxLineSize > 0 {
		// add 2 to account for newline characters
		tail.reader = bufio.NewReaderSize(tail.file, tail.MaxLineSize+2)
	} else {
		tail.reader = bufio.NewReader(tail.file)
	}
}

func (tail *Tail) seekEnd() error {
	_, err := tail.file.Seek(0, 2)
	if err != nil {
		return fmt.Errorf("Seek error on %s: %s", tail.Filename, err)
	}
	// Reset the read buffer whenever the file is re-seek'ed
	tail.reader.Reset(tail.file)
	return nil
}

// sendLine sends the line(s) to Lines channel, splitting longer lines
// if necessary. Return false if rate limit is reached.
func (tail *Tail) sendLine(line string, offset, sz int64) bool {
	now := time.Now()

	total := 1

	// Split longer lines
	if tail.MaxLineSize > 0 && len(line) > tail.MaxLineSize {
		lines := util.PartitionString(line, tail.MaxLineSize)

		total = len(lines)

		cur := offset

		for idx, line := range lines {
			length := int64(len(line))
			if idx == len(lines)-1 {
				length = int64(sz) - cur
			}

			tail.Lines <- &Line{line, now, cur, length, nil}
			cur += int64(len(line))
		}
	} else {
		tail.Lines <- &Line{line, now, offset, sz, nil}
	}

	if tail.Config.RateLimiter != nil {
		ok := tail.Config.RateLimiter.Pour(uint16(total))
		if !ok {
			tail.Logger.Printf("Leaky bucket full (%v); entering 1s cooloff period.\n",
				tail.Filename)
			return false
		}
	}

	return true
}

// Cleanup removes inotify watches added by the tail package. This function is
// meant to be invoked from a process's exit handler. Linux kernel may not
// automatically remove inotify watches after the process exits.
func Cleanup() {
	watch.Cleanup()
}
<|endoftext|>


<commit_before>package scipipe

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

// Task represents a single static shell command, or go function, to be
// executed, and are scheduled and managed by a corresponding Process
type Task struct {
	Name          string
	Command       string
	CustomExecute func(*Task)
	InIPs         map[string]*FileIP
	OutIPs        map[string]*FileIP
	Params        map[string]string
	Done          chan int
	cores         int
	workflow      *Workflow
	process       *Process
}

// ------------------------------------------------------------------------
// Factory method(s)
// ------------------------------------------------------------------------

// NewTask instantiates and initializes a new Task
func NewTask(workflow *Workflow, process *Process, name string, cmdPat string, inIPs map[string]*FileIP, outPathFuncs map[string]func(*Task) string, outPortsDoStream map[string]bool, params map[string]string, prepend string, customExecute func(*Task), cores int) *Task {
	t := &Task{
		Name:          name,
		InIPs:         inIPs,
		OutIPs:        make(map[string]*FileIP),
		Params:        params,
		Command:       "",
		CustomExecute: customExecute,
		Done:          make(chan int),
		cores:         cores,
		workflow:      workflow,
		process:       process,
	}

	// Create Out-IPs
	for oname, outPathFunc := range outPathFuncs {
		oip := NewFileIP(outPathFunc(t))
		if outPortsDoStream[oname] {
			oip.doStream = true
		}
		t.OutIPs[oname] = oip
	}
	t.Command = formatCommand(cmdPat, inIPs, t.OutIPs, params, prepend)
	return t
}

// formatCommand is a helper function for NewTask, that formats a shell command
// based on concrete file paths and parameter values
func formatCommand(cmd string, inIPs map[string]*FileIP, outIPs map[string]*FileIP, params map[string]string, prepend string) string {
	r := getShellCommandPlaceHolderRegex()
	placeHolderMatches := r.FindAllStringSubmatch(cmd, -1)
	for _, placeHolderMatch := range placeHolderMatches {
		var filePath string

		placeHolderStr := placeHolderMatch[0]
		portType := placeHolderMatch[1]
		portName := placeHolderMatch[2]
		sep := " " // Default

		switch portType {
		case "o":
			if outIPs[portName] == nil {
				Fail("Missing outpath for outport '", portName, "' for command '", cmd, "'")
			}
			filePath = outIPs[portName].TempPath()
		case "os":
			if outIPs[portName] == nil {
				Fail("Missing outpath for outport '", portName, "' for command '", cmd, "'")
			}
			filePath = outIPs[portName].FifoPath()
		case "i":
			if inIPs[portName] == nil {
				Fail("Missing in-IP for inport '", portName, "' for command '", cmd, "'")
			}
			// Identify if the place holder represents a reduce-type in-port
			reduceInputs := false
			if len(placeHolderMatch) > 3 {
				sep = placeHolderMatch[5]
				reduceInputs = true
			}
			if reduceInputs && inIPs[portName].Path() == "" {
				// Merge multiple input paths from a substream on the IP, into one string
				ips := []*FileIP{}
				for ip := range inIPs[portName].SubStream.Chan {
					ips = append(ips, ip)
				}
				paths := []string{}
				for _, ip := range ips {
					paths = append(paths, ip.Path())
				}
				filePath = strings.Join(paths, sep)
			} else {
				if inIPs[portName].Path() == "" {
					Fail("Missing inpath for inport '", portName, "', and no substream, for command '", cmd, "'")
				}
				if inIPs[portName].doStream {
					filePath = inIPs[portName].FifoPath()
				} else {
					filePath = inIPs[portName].Path()
				}
			}
		case "p":
			if params[portName] == "" {
				msg := fmt.Sprint("Missing param value param '", portName, "' for command '", cmd, "'")
				CheckWithMsg(errors.New(msg), msg)
			} else {
				filePath = params[portName]
			}
		default:
			Fail("Replace failed for port ", portName, " for command '", cmd, "'")
		}
		cmd = strings.Replace(cmd, placeHolderStr, filePath, -1)
	}

	// Add prepend string to the command
	if prepend != "" {
		cmd = fmt.Sprintf("%s %s", prepend, cmd)
	}

	return cmd
}

// ------------------------------------------------------------------------
// Main API methods: Accessor methods
// ------------------------------------------------------------------------

// InIP returns an IP for the in-port with name portName
func (t *Task) InIP(portName string) *FileIP {
	if t.InIPs[portName] == nil {
		Failf("No such in-portname (%s) in task (%s)\n", portName, t.Name)
	}
	return t.InIPs[portName]
}

// InPath returns the path name of an input file for the task
func (t *Task) InPath(portName string) string {
	return t.InIP(portName).Path()
}

// OutIP returns an IP for the in-port with name portName
func (t *Task) OutIP(portName string) *FileIP {
	if t.OutIPs[portName] == nil {
		Failf("No such out-portname (%s) in task (%s)\n", portName, t.Name)
	}
	return t.OutIPs[portName]
}

// OutPath returns the path name of an input file for the task
func (t *Task) OutPath(portName string) string {
	return t.OutIP(portName).Path()
}

// Param returns the value of a param, for the task
func (t *Task) Param(portName string) string {
	if param, ok := t.Params[portName]; ok {
		return param
	}
	Failf("No such param port '%s' for task '%s'\n", portName, t.Name)
	return "invalid"
}

// ------------------------------------------------------------------------
// Execute the task
// ------------------------------------------------------------------------

// Execute executes the task (the shell command or go function in CustomExecute)
func (t *Task) Execute() {
	defer close(t.Done)

	// Do some sanity checks
	if t.anyTempfileExists() {
		Failf("Task:%s-12s Existing temp files found so existing. Clean up .tmp files before restarting the workflow!", t.Name)
	}

	if t.anyOutputsExist() {
		Audit.Printf("Task:%s-12s Outputs exist, so not executing", t.Name)
		t.Done <- 1
		return
	}

	// Execute task
	t.workflow.IncConcurrentTasks(t.cores) // Will block if max concurrent tasks is reached
	t.createDirs()
	startTime := time.Now()
	if t.CustomExecute != nil {
		outputsStr := ""
		for oipName, oip := range t.OutIPs {
			outputsStr += " " + oipName + ": " + oip.Path()
		}
		Audit.Printf("Task:%-12s Executing custom Go function, producing output(s):%s\n", t.Name, outputsStr)
		t.CustomExecute(t)
	} else {
		t.executeCommand(t.Command)
	}
	finishTime := time.Now()
	t.writeAuditLogs(startTime, finishTime)
	t.atomizeIPs()
	t.workflow.DecConcurrentTasks(t.cores)

	t.Done <- 1
}

// ------------------------------------------------------------------------
// Helper methods for the Execute method
// ------------------------------------------------------------------------

// anyTempFileExists checks if any temporary workflow files exist and if so, returns true
func (t *Task) anyTempfileExists() (anyTempfileExists bool) {
	anyTempfileExists = false
	for _, oip := range t.OutIPs {
		if !oip.doStream {
			otmpPath := oip.TempPath()
			if _, err := os.Stat(otmpPath); err == nil {
				Warning.Printf("Task:%-12s Temp file already exists: %s (Note: If resuming from a failed run, clean up .tmp files first. Also, make sure that two processes don't produce the same output files!).\n", t.Name, otmpPath)
				anyTempfileExists = true
			}
		}
	}
	return
}

// anyOutputsExist if any output file IP, or temporary file IPs, exist
func (t *Task) anyOutputsExist() (anyFileExists bool) {
	anyFileExists = false
	for _, oip := range t.OutIPs {
		if !oip.doStream {
			opath := oip.Path()
			if _, err := os.Stat(opath); err == nil {
				Info.Printf("Task:%-12s Output file already exists, so skipping: %s\n", t.Name, opath)
				anyFileExists = true
			}
		}
	}
	return
}

// createDirs creates directories for out-IPs of the task
func (t *Task) createDirs() {
	for _, oip := range t.OutIPs {
		oipDir := filepath.Dir(oip.Path())
		err := os.MkdirAll(oipDir, 0777)
		CheckWithMsg(err, "Could not create directory: "+oipDir)
	}

}

// executeCommand executes the shell command cmd via bash
func (t *Task) executeCommand(cmd string) {
	Audit.Printf("Task:%-12s Executing command: %s\n", t.Name, cmd)
	out, err := exec.Command("bash", "-c", cmd).CombinedOutput()
	if err != nil {
		Failf("Command failed!\nCommand:\n%s\n\nOutput:\n%s\n\n", cmd, string(out))
	}
}

func (t *Task) writeAuditLogs(startTime time.Time, finishTime time.Time) {
	// Append audit info for the task to all its output IPs
	auditInfo := NewAuditInfo()
	auditInfo.Command = t.Command
	auditInfo.ProcessName = t.process.Name()
	auditInfo.Params = t.Params
	auditInfo.StartTime = startTime
	auditInfo.FinishTime = finishTime
	auditInfo.ExecTimeMS = finishTime.Sub(startTime) / time.Millisecond
	// Set the audit infos from incoming IPs into the "Upstream" map
	for _, iip := range t.InIPs {
		iipPath := iip.Path()
		iipAuditInfo := iip.AuditInfo()
		auditInfo.Upstream[iipPath] = iipAuditInfo
	}
	// Add the current audit info to output ips and write them to file
	for _, oip := range t.OutIPs {
		oip.SetAuditInfo(auditInfo)
		for _, iip := range t.InIPs {
			oip.AddKeys(iip.Keys())
		}
		oip.WriteAuditLogToFile()
	}
}

// Rename temporary output files to their proper file names
func (t *Task) atomizeIPs() {
	for _, oip := range t.OutIPs {
		if !oip.doStream {
			oip.Atomize()
		}
	}
}
<commit_msg>Write Audit for both shell commands and Go functions in main task.Execute method<commit_after>package scipipe

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

// Task represents a single static shell command, or go function, to be
// executed, and are scheduled and managed by a corresponding Process
type Task struct {
	Name          string
	Command       string
	CustomExecute func(*Task)
	InIPs         map[string]*FileIP
	OutIPs        map[string]*FileIP
	Params        map[string]string
	Done          chan int
	cores         int
	workflow      *Workflow
	process       *Process
}

// ------------------------------------------------------------------------
// Factory method(s)
// ------------------------------------------------------------------------

// NewTask instantiates and initializes a new Task
func NewTask(workflow *Workflow, process *Process, name string, cmdPat string, inIPs map[string]*FileIP, outPathFuncs map[string]func(*Task) string, outPortsDoStream map[string]bool, params map[string]string, prepend string, customExecute func(*Task), cores int) *Task {
	t := &Task{
		Name:          name,
		InIPs:         inIPs,
		OutIPs:        make(map[string]*FileIP),
		Params:        params,
		Command:       "",
		CustomExecute: customExecute,
		Done:          make(chan int),
		cores:         cores,
		workflow:      workflow,
		process:       process,
	}

	// Create Out-IPs
	for oname, outPathFunc := range outPathFuncs {
		oip := NewFileIP(outPathFunc(t))
		if outPortsDoStream[oname] {
			oip.doStream = true
		}
		t.OutIPs[oname] = oip
	}
	t.Command = formatCommand(cmdPat, inIPs, t.OutIPs, params, prepend)
	return t
}

// formatCommand is a helper function for NewTask, that formats a shell command
// based on concrete file paths and parameter values
func formatCommand(cmd string, inIPs map[string]*FileIP, outIPs map[string]*FileIP, params map[string]string, prepend string) string {
	r := getShellCommandPlaceHolderRegex()
	placeHolderMatches := r.FindAllStringSubmatch(cmd, -1)
	for _, placeHolderMatch := range placeHolderMatches {
		var filePath string

		placeHolderStr := placeHolderMatch[0]
		portType := placeHolderMatch[1]
		portName := placeHolderMatch[2]
		sep := " " // Default

		switch portType {
		case "o":
			if outIPs[portName] == nil {
				Fail("Missing outpath for outport '", portName, "' for command '", cmd, "'")
			}
			filePath = outIPs[portName].TempPath()
		case "os":
			if outIPs[portName] == nil {
				Fail("Missing outpath for outport '", portName, "' for command '", cmd, "'")
			}
			filePath = outIPs[portName].FifoPath()
		case "i":
			if inIPs[portName] == nil {
				Fail("Missing in-IP for inport '", portName, "' for command '", cmd, "'")
			}
			// Identify if the place holder represents a reduce-type in-port
			reduceInputs := false
			if len(placeHolderMatch) > 3 {
				sep = placeHolderMatch[5]
				reduceInputs = true
			}
			if reduceInputs && inIPs[portName].Path() == "" {
				// Merge multiple input paths from a substream on the IP, into one string
				ips := []*FileIP{}
				for ip := range inIPs[portName].SubStream.Chan {
					ips = append(ips, ip)
				}
				paths := []string{}
				for _, ip := range ips {
					paths = append(paths, ip.Path())
				}
				filePath = strings.Join(paths, sep)
			} else {
				if inIPs[portName].Path() == "" {
					Fail("Missing inpath for inport '", portName, "', and no substream, for command '", cmd, "'")
				}
				if inIPs[portName].doStream {
					filePath = inIPs[portName].FifoPath()
				} else {
					filePath = inIPs[portName].Path()
				}
			}
		case "p":
			if params[portName] == "" {
				msg := fmt.Sprint("Missing param value param '", portName, "' for command '", cmd, "'")
				CheckWithMsg(errors.New(msg), msg)
			} else {
				filePath = params[portName]
			}
		default:
			Fail("Replace failed for port ", portName, " for command '", cmd, "'")
		}
		cmd = strings.Replace(cmd, placeHolderStr, filePath, -1)
	}

	// Add prepend string to the command
	if prepend != "" {
		cmd = fmt.Sprintf("%s %s", prepend, cmd)
	}

	return cmd
}

// ------------------------------------------------------------------------
// Main API methods: Accessor methods
// ------------------------------------------------------------------------

// InIP returns an IP for the in-port with name portName
func (t *Task) InIP(portName string) *FileIP {
	if t.InIPs[portName] == nil {
		Failf("No such in-portname (%s) in task (%s)\n", portName, t.Name)
	}
	return t.InIPs[portName]
}

// InPath returns the path name of an input file for the task
func (t *Task) InPath(portName string) string {
	return t.InIP(portName).Path()
}

// OutIP returns an IP for the in-port with name portName
func (t *Task) OutIP(portName string) *FileIP {
	if t.OutIPs[portName] == nil {
		Failf("No such out-portname (%s) in task (%s)\n", portName, t.Name)
	}
	return t.OutIPs[portName]
}

// OutPath returns the path name of an input file for the task
func (t *Task) OutPath(portName string) string {
	return t.OutIP(portName).Path()
}

// Param returns the value of a param, for the task
func (t *Task) Param(portName string) string {
	if param, ok := t.Params[portName]; ok {
		return param
	}
	Failf("No such param port '%s' for task '%s'\n", portName, t.Name)
	return "invalid"
}

// ------------------------------------------------------------------------
// Execute the task
// ------------------------------------------------------------------------

// Execute executes the task (the shell command or go function in CustomExecute)
func (t *Task) Execute() {
	defer close(t.Done)

	// Do some sanity checks
	if t.anyTempfileExists() {
		Failf("Task:%s-12s Existing temp files found so existing. Clean up .tmp files before restarting the workflow!", t.Name)
	}

	if t.anyOutputsExist() {
		Audit.Printf("Task:%s-12s Outputs exist, so not executing", t.Name)
		t.Done <- 1
		return
	}

	// Execute task
	t.workflow.IncConcurrentTasks(t.cores) // Will block if max concurrent tasks is reached
	t.createDirs()                         // Create output directories needed for any outputs
	startTime := time.Now()
	if t.CustomExecute != nil {
		outputsStr := ""
		for oipName, oip := range t.OutIPs {
			outputsStr += " " + oipName + ": " + oip.Path()
		}
		Audit.Printf("Task:%-12s Executing custom Go function, producing output(s):%s\n", t.Name, outputsStr)
		t.CustomExecute(t)
	} else {
		Audit.Printf("Task:%-12s Executing command: %s\n", t.Name, t.Command)
		t.executeCommand(t.Command)
	}
	finishTime := time.Now()
	t.writeAuditLogs(startTime, finishTime)
	t.atomizeIPs()
	t.workflow.DecConcurrentTasks(t.cores)

	t.Done <- 1
}

// ------------------------------------------------------------------------
// Helper methods for the Execute method
// ------------------------------------------------------------------------

// anyTempFileExists checks if any temporary workflow files exist and if so, returns true
func (t *Task) anyTempfileExists() (anyTempfileExists bool) {
	anyTempfileExists = false
	for _, oip := range t.OutIPs {
		if !oip.doStream {
			otmpPath := oip.TempPath()
			if _, err := os.Stat(otmpPath); err == nil {
				Warning.Printf("Task:%-12s Temp file already exists: %s (Note: If resuming from a failed run, clean up .tmp files first. Also, make sure that two processes don't produce the same output files!).\n", t.Name, otmpPath)
				anyTempfileExists = true
			}
		}
	}
	return
}

// anyOutputsExist if any output file IP, or temporary file IPs, exist
func (t *Task) anyOutputsExist() (anyFileExists bool) {
	anyFileExists = false
	for _, oip := range t.OutIPs {
		if !oip.doStream {
			opath := oip.Path()
			if _, err := os.Stat(opath); err == nil {
				Info.Printf("Task:%-12s Output file already exists, so skipping: %s\n", t.Name, opath)
				anyFileExists = true
			}
		}
	}
	return
}

// createDirs creates directories for out-IPs of the task
func (t *Task) createDirs() {
	for _, oip := range t.OutIPs {
		oipDir := filepath.Dir(oip.Path())
		err := os.MkdirAll(oipDir, 0777)
		CheckWithMsg(err, "Could not create directory: "+oipDir)
	}

}

// executeCommand executes the shell command cmd via bash
func (t *Task) executeCommand(cmd string) {
	out, err := exec.Command("bash", "-c", cmd).CombinedOutput()
	if err != nil {
		Failf("Command failed!\nCommand:\n%s\n\nOutput:\n%s\n\n", cmd, string(out))
	}
}

func (t *Task) writeAuditLogs(startTime time.Time, finishTime time.Time) {
	// Append audit info for the task to all its output IPs
	auditInfo := NewAuditInfo()
	auditInfo.Command = t.Command
	auditInfo.ProcessName = t.process.Name()
	auditInfo.Params = t.Params
	auditInfo.StartTime = startTime
	auditInfo.FinishTime = finishTime
	auditInfo.ExecTimeMS = finishTime.Sub(startTime) / time.Millisecond
	// Set the audit infos from incoming IPs into the "Upstream" map
	for _, iip := range t.InIPs {
		iipPath := iip.Path()
		iipAuditInfo := iip.AuditInfo()
		auditInfo.Upstream[iipPath] = iipAuditInfo
	}
	// Add the current audit info to output ips and write them to file
	for _, oip := range t.OutIPs {
		oip.SetAuditInfo(auditInfo)
		for _, iip := range t.InIPs {
			oip.AddKeys(iip.Keys())
		}
		oip.WriteAuditLogToFile()
	}
}

// Rename temporary output files to their proper file names
func (t *Task) atomizeIPs() {
	for _, oip := range t.OutIPs {
		if !oip.doStream {
			oip.Atomize()
		}
	}
}
<|endoftext|>