Files
mgmt/misc.go
James Shubin 1b01f908e3 Add resource auto grouping
Sorry for the size of this patch, I was busy hacking and plumbing away
and it got out of hand! I'm allowing this because there doesn't seem to
be anyone hacking away on parts of the code that this would break, since
the resource code is fairly stable in this change. In particular, it
revisits and refreshes some areas of the code that didn't see anything
new or innovative since the project first started. I've gotten rid of a
lot of cruft, and in particular cleaned up some things that I didn't
know how to do better before! Here's hoping I'll continue to learn and
have more to improve upon in the future! (Well let's not hope _too_ hard
though!)

The logical goal of this patch was to make logical grouping of resources
possible. For example, it might be more efficient to group three package
installations into a single transaction, instead of having to run three
separate transactions. This is because a package installation typically
has an initial one-time per run cost which shouldn't need to be
repeated.

Another future goal would be to group file resources sharing a common
base path under a common recursive fanotify watcher. Since this depends
on fanotify capabilities first, this hasn't been implemented yet, but
could be a useful method of reducing the number of separate watches
needed, since there is a finite limit.

It's worth mentioning that grouping resources typically _reduces_ the
parallel execution capability of a particular graph, but depending on
the cost/benefit tradeoff, this might be preferential. I'd submit it's
almost universally beneficial for pkg resources.

This monster patch includes:
* the autogroup feature
* the grouping interface
* a placeholder algorithm
* an extensive test case infrastructure to test grouping algorithms
* a move of some base resource methods into pgraph refactoring
* some config/compile clean ups to remove code duplication
* b64 encoding/decoding improvements
* a rename of the yaml "res" entries to "kind" (more logical)
* some docs
* small fixes
* and more!
2016-03-28 20:54:41 -04:00

300 lines
7.5 KiB
Go

// Mgmt
// Copyright (C) 2013-2016+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"github.com/godbus/dbus"
"path"
"sort"
"strings"
"time"
)
// returns the string with the first character capitalized
func FirstToUpper(str string) string {
return strings.ToUpper(str[0:1]) + str[1:]
}
// return true if a string exists inside a list, otherwise false
func StrInList(needle string, haystack []string) bool {
for _, x := range haystack {
if needle == x {
return true
}
}
return false
}
// remove any duplicate values in the list
// possibly sub-optimal, O(n^2)? implementation
func StrRemoveDuplicatesInList(list []string) []string {
unique := []string{}
for _, x := range list {
if !StrInList(x, unique) {
unique = append(unique, x)
}
}
return unique
}
// remove any of the elements in filter, if they exist in list
func StrFilterElementsInList(filter []string, list []string) []string {
result := []string{}
for _, x := range list {
if !StrInList(x, filter) {
result = append(result, x)
}
}
return result
}
// remove any of the elements in filter, if they don't exist in list
// this is an in order intersection of two lists
func StrListIntersection(list1 []string, list2 []string) []string {
result := []string{}
for _, x := range list1 {
if StrInList(x, list2) {
result = append(result, x)
}
}
return result
}
// reverse a list of strings
func ReverseStringList(in []string) []string {
var out []string // empty list
l := len(in)
for i := range in {
out = append(out, in[l-i-1])
}
return out
}
// return the sorted list of string keys in a map with string keys
// NOTE: i thought it would be nice for this to use: map[string]interface{} but
// it turns out that's not allowed. I know we don't have generics, but common!
func StrMapKeys(m map[string]string) []string {
result := []string{}
for k, _ := range m {
result = append(result, k)
}
sort.Strings(result) // deterministic order
return result
}
// return the sorted list of bool values in a map with string values
func BoolMapValues(m map[string]bool) []bool {
result := []bool{}
for _, v := range m {
result = append(result, v)
}
//sort.Bools(result) // TODO: deterministic order
return result
}
// return the sorted list of string values in a map with string values
func StrMapValues(m map[string]string) []string {
result := []string{}
for _, v := range m {
result = append(result, v)
}
sort.Strings(result) // deterministic order
return result
}
// return true if everyone is true
func BoolMapTrue(l []bool) bool {
for _, b := range l {
if !b {
return false
}
}
return true
}
// Similar to the GNU dirname command
func Dirname(p string) string {
if p == "/" {
return ""
}
d, _ := path.Split(path.Clean(p))
return d
}
func Basename(p string) string {
_, b := path.Split(path.Clean(p))
if p == "" {
return ""
}
if p[len(p)-1:] == "/" { // don't loose the tail slash
b += "/"
}
return b
}
// Split a path into an array of tokens excluding any trailing empty tokens
func PathSplit(p string) []string {
if p == "/" { // TODO: can't this all be expressed nicely in one line?
return []string{""}
}
return strings.Split(path.Clean(p), "/")
}
// Does path string contain the given path prefix in it?
func HasPathPrefix(p, prefix string) bool {
patharray := PathSplit(p)
prefixarray := PathSplit(prefix)
if len(prefixarray) > len(patharray) {
return false
}
for i := 0; i < len(prefixarray); i++ {
if prefixarray[i] != patharray[i] {
return false
}
}
return true
}
func StrInPathPrefixList(needle string, haystack []string) bool {
for _, x := range haystack {
if HasPathPrefix(x, needle) {
return true
}
}
return false
}
// remove redundant file path prefixes that are under the tree of other files
func RemoveCommonFilePrefixes(paths []string) []string {
var result = make([]string, len(paths))
for i := 0; i < len(paths); i++ { // copy, b/c append can modify the args!!
result[i] = paths[i]
}
// is there a string path which is common everywhere?
// if so, remove it, and iterate until nothing common is left
// return what's left over, that's the most common superset
loop:
for {
if len(result) <= 1 {
return result
}
for i := 0; i < len(result); i++ {
var copied = make([]string, len(result))
for j := 0; j < len(result); j++ { // copy, b/c append can modify the args!!
copied[j] = result[j]
}
noi := append(copied[:i], copied[i+1:]...) // rm i
if StrInPathPrefixList(result[i], noi) {
// delete the element common to everyone
result = noi
continue loop
}
}
break
}
return result
}
// Delta of path prefix, tells you how many path tokens different the prefix is
func PathPrefixDelta(p, prefix string) int {
if !HasPathPrefix(p, prefix) {
return -1
}
patharray := PathSplit(p)
prefixarray := PathSplit(prefix)
return len(patharray) - len(prefixarray)
}
func PathIsDir(p string) bool {
return p[len(p)-1:] == "/" // a dir has a trailing slash in this context
}
// return the full list of "dependency" paths for a given path in reverse order
func PathSplitFullReversed(p string) []string {
var result []string
split := PathSplit(p)
count := len(split)
var x string
for i := 0; i < count; i++ {
x = "/" + path.Join(split[0:i+1]...)
if i != 0 && !(i+1 == count && !PathIsDir(p)) {
x += "/" // add trailing slash
}
result = append(result, x)
}
return ReverseStringList(result)
}
// add trailing slashes to any likely dirs in a package manager fileList
// if removeDirs is true, instead, don't keep the dirs in our output
func DirifyFileList(fileList []string, removeDirs bool) []string {
dirs := []string{}
for _, file := range fileList {
dir, _ := path.Split(file) // dir
dir = path.Clean(dir) // clean so cmp is easier
if !StrInList(dir, dirs) {
dirs = append(dirs, dir)
}
}
result := []string{}
for _, file := range fileList {
cleanFile := path.Clean(file)
if !StrInList(cleanFile, dirs) { // we're not a directory!
result = append(result, file) // pass through
} else if !removeDirs {
result = append(result, cleanFile+"/")
}
}
return result
}
// special version of time.After that blocks when given a negative integer
// when used in a case statement, the timer restarts on each select call to it
func TimeAfterOrBlock(t int) <-chan time.Time {
if t < 0 {
return make(chan time.Time) // blocks forever
}
return time.After(time.Duration(t) * time.Second)
}
// making using the private bus usable, should be upstream:
// TODO: https://github.com/godbus/dbus/issues/15
func SystemBusPrivateUsable() (conn *dbus.Conn, err error) {
conn, err = dbus.SystemBusPrivate()
if err != nil {
return nil, err
}
if err = conn.Auth(nil); err != nil {
conn.Close()
conn = nil
return
}
if err = conn.Hello(); err != nil {
conn.Close()
conn = nil
}
return conn, nil // success
}