Sorry for the size of this patch, I was busy hacking and plumbing away and it got out of hand! I'm allowing this because there doesn't seem to be anyone hacking away on parts of the code that this would break, since the resource code is fairly stable in this change. In particular, it revisits and refreshes some areas of the code that didn't see anything new or innovative since the project first started. I've gotten rid of a lot of cruft, and in particular cleaned up some things that I didn't know how to do better before! Here's hoping I'll continue to learn and have more to improve upon in the future! (Well let's not hope _too_ hard though!) The logical goal of this patch was to make logical grouping of resources possible. For example, it might be more efficient to group three package installations into a single transaction, instead of having to run three separate transactions. This is because a package installation typically has an initial one-time per run cost which shouldn't need to be repeated. Another future goal would be to group file resources sharing a common base path under a common recursive fanotify watcher. Since this depends on fanotify capabilities first, this hasn't been implemented yet, but could be a useful method of reducing the number of separate watches needed, since there is a finite limit. It's worth mentioning that grouping resources typically _reduces_ the parallel execution capability of a particular graph, but depending on the cost/benefit tradeoff, this might be preferential. I'd submit it's almost universally beneficial for pkg resources. This monster patch includes: * the autogroup feature * the grouping interface * a placeholder algorithm * an extensive test case infrastructure to test grouping algorithms * a move of some base resource methods into pgraph refactoring * some config/compile clean ups to remove code duplication * b64 encoding/decoding improvements * a rename of the yaml "res" entries to "kind" (more logical) * some docs * small fixes * and more!
300 lines
7.5 KiB
Go
300 lines
7.5 KiB
Go
// Mgmt
|
|
// Copyright (C) 2013-2016+ James Shubin and the project contributors
|
|
// Written by James Shubin <james@shubin.ca> and the project contributors
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package main
|
|
|
|
import (
|
|
"github.com/godbus/dbus"
|
|
"path"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// returns the string with the first character capitalized
|
|
func FirstToUpper(str string) string {
|
|
return strings.ToUpper(str[0:1]) + str[1:]
|
|
}
|
|
|
|
// return true if a string exists inside a list, otherwise false
|
|
func StrInList(needle string, haystack []string) bool {
|
|
for _, x := range haystack {
|
|
if needle == x {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// remove any duplicate values in the list
|
|
// possibly sub-optimal, O(n^2)? implementation
|
|
func StrRemoveDuplicatesInList(list []string) []string {
|
|
unique := []string{}
|
|
for _, x := range list {
|
|
if !StrInList(x, unique) {
|
|
unique = append(unique, x)
|
|
}
|
|
}
|
|
return unique
|
|
}
|
|
|
|
// remove any of the elements in filter, if they exist in list
|
|
func StrFilterElementsInList(filter []string, list []string) []string {
|
|
result := []string{}
|
|
for _, x := range list {
|
|
if !StrInList(x, filter) {
|
|
result = append(result, x)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// remove any of the elements in filter, if they don't exist in list
|
|
// this is an in order intersection of two lists
|
|
func StrListIntersection(list1 []string, list2 []string) []string {
|
|
result := []string{}
|
|
for _, x := range list1 {
|
|
if StrInList(x, list2) {
|
|
result = append(result, x)
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// reverse a list of strings
|
|
func ReverseStringList(in []string) []string {
|
|
var out []string // empty list
|
|
l := len(in)
|
|
for i := range in {
|
|
out = append(out, in[l-i-1])
|
|
}
|
|
return out
|
|
}
|
|
|
|
// return the sorted list of string keys in a map with string keys
|
|
// NOTE: i thought it would be nice for this to use: map[string]interface{} but
|
|
// it turns out that's not allowed. I know we don't have generics, but common!
|
|
func StrMapKeys(m map[string]string) []string {
|
|
result := []string{}
|
|
for k, _ := range m {
|
|
result = append(result, k)
|
|
}
|
|
sort.Strings(result) // deterministic order
|
|
return result
|
|
}
|
|
|
|
// return the sorted list of bool values in a map with string values
|
|
func BoolMapValues(m map[string]bool) []bool {
|
|
result := []bool{}
|
|
for _, v := range m {
|
|
result = append(result, v)
|
|
}
|
|
//sort.Bools(result) // TODO: deterministic order
|
|
return result
|
|
}
|
|
|
|
// return the sorted list of string values in a map with string values
|
|
func StrMapValues(m map[string]string) []string {
|
|
result := []string{}
|
|
for _, v := range m {
|
|
result = append(result, v)
|
|
}
|
|
sort.Strings(result) // deterministic order
|
|
return result
|
|
}
|
|
|
|
// return true if everyone is true
|
|
func BoolMapTrue(l []bool) bool {
|
|
for _, b := range l {
|
|
if !b {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// Similar to the GNU dirname command
|
|
func Dirname(p string) string {
|
|
if p == "/" {
|
|
return ""
|
|
}
|
|
d, _ := path.Split(path.Clean(p))
|
|
return d
|
|
}
|
|
|
|
func Basename(p string) string {
|
|
_, b := path.Split(path.Clean(p))
|
|
if p == "" {
|
|
return ""
|
|
}
|
|
if p[len(p)-1:] == "/" { // don't loose the tail slash
|
|
b += "/"
|
|
}
|
|
return b
|
|
}
|
|
|
|
// Split a path into an array of tokens excluding any trailing empty tokens
|
|
func PathSplit(p string) []string {
|
|
if p == "/" { // TODO: can't this all be expressed nicely in one line?
|
|
return []string{""}
|
|
}
|
|
return strings.Split(path.Clean(p), "/")
|
|
}
|
|
|
|
// Does path string contain the given path prefix in it?
|
|
func HasPathPrefix(p, prefix string) bool {
|
|
|
|
patharray := PathSplit(p)
|
|
prefixarray := PathSplit(prefix)
|
|
|
|
if len(prefixarray) > len(patharray) {
|
|
return false
|
|
}
|
|
|
|
for i := 0; i < len(prefixarray); i++ {
|
|
if prefixarray[i] != patharray[i] {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func StrInPathPrefixList(needle string, haystack []string) bool {
|
|
for _, x := range haystack {
|
|
if HasPathPrefix(x, needle) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// remove redundant file path prefixes that are under the tree of other files
|
|
func RemoveCommonFilePrefixes(paths []string) []string {
|
|
var result = make([]string, len(paths))
|
|
for i := 0; i < len(paths); i++ { // copy, b/c append can modify the args!!
|
|
result[i] = paths[i]
|
|
}
|
|
// is there a string path which is common everywhere?
|
|
// if so, remove it, and iterate until nothing common is left
|
|
// return what's left over, that's the most common superset
|
|
loop:
|
|
for {
|
|
if len(result) <= 1 {
|
|
return result
|
|
}
|
|
for i := 0; i < len(result); i++ {
|
|
var copied = make([]string, len(result))
|
|
for j := 0; j < len(result); j++ { // copy, b/c append can modify the args!!
|
|
copied[j] = result[j]
|
|
}
|
|
noi := append(copied[:i], copied[i+1:]...) // rm i
|
|
if StrInPathPrefixList(result[i], noi) {
|
|
// delete the element common to everyone
|
|
result = noi
|
|
continue loop
|
|
}
|
|
}
|
|
break
|
|
}
|
|
return result
|
|
}
|
|
|
|
// Delta of path prefix, tells you how many path tokens different the prefix is
|
|
func PathPrefixDelta(p, prefix string) int {
|
|
|
|
if !HasPathPrefix(p, prefix) {
|
|
return -1
|
|
}
|
|
patharray := PathSplit(p)
|
|
prefixarray := PathSplit(prefix)
|
|
return len(patharray) - len(prefixarray)
|
|
}
|
|
|
|
func PathIsDir(p string) bool {
|
|
return p[len(p)-1:] == "/" // a dir has a trailing slash in this context
|
|
}
|
|
|
|
// return the full list of "dependency" paths for a given path in reverse order
|
|
func PathSplitFullReversed(p string) []string {
|
|
var result []string
|
|
split := PathSplit(p)
|
|
count := len(split)
|
|
var x string
|
|
for i := 0; i < count; i++ {
|
|
x = "/" + path.Join(split[0:i+1]...)
|
|
if i != 0 && !(i+1 == count && !PathIsDir(p)) {
|
|
x += "/" // add trailing slash
|
|
}
|
|
result = append(result, x)
|
|
}
|
|
return ReverseStringList(result)
|
|
}
|
|
|
|
// add trailing slashes to any likely dirs in a package manager fileList
|
|
// if removeDirs is true, instead, don't keep the dirs in our output
|
|
func DirifyFileList(fileList []string, removeDirs bool) []string {
|
|
dirs := []string{}
|
|
for _, file := range fileList {
|
|
dir, _ := path.Split(file) // dir
|
|
dir = path.Clean(dir) // clean so cmp is easier
|
|
if !StrInList(dir, dirs) {
|
|
dirs = append(dirs, dir)
|
|
}
|
|
}
|
|
|
|
result := []string{}
|
|
for _, file := range fileList {
|
|
cleanFile := path.Clean(file)
|
|
if !StrInList(cleanFile, dirs) { // we're not a directory!
|
|
result = append(result, file) // pass through
|
|
} else if !removeDirs {
|
|
result = append(result, cleanFile+"/")
|
|
}
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
// special version of time.After that blocks when given a negative integer
|
|
// when used in a case statement, the timer restarts on each select call to it
|
|
func TimeAfterOrBlock(t int) <-chan time.Time {
|
|
if t < 0 {
|
|
return make(chan time.Time) // blocks forever
|
|
}
|
|
return time.After(time.Duration(t) * time.Second)
|
|
}
|
|
|
|
// making using the private bus usable, should be upstream:
|
|
// TODO: https://github.com/godbus/dbus/issues/15
|
|
func SystemBusPrivateUsable() (conn *dbus.Conn, err error) {
|
|
conn, err = dbus.SystemBusPrivate()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err = conn.Auth(nil); err != nil {
|
|
conn.Close()
|
|
conn = nil
|
|
return
|
|
}
|
|
if err = conn.Hello(); err != nil {
|
|
conn.Close()
|
|
conn = nil
|
|
}
|
|
return conn, nil // success
|
|
}
|