From cffdb0618100ab6a281e850ac1565fc954f5e02f Mon Sep 17 00:00:00 2001 From: James Shubin Date: Wed, 21 Feb 2018 22:11:03 -0500 Subject: [PATCH] test, docs: Add a linter for testing markdown, and fix up our docs While writing docs, I couldn't remember what the correct style was supposed to be, and I remember someone complaining about this previously, so I decided to add a linter! I excluded a bunch of annoying style rules, but if we find more we can add those to the list too. Hopefully this gives us a more consistent feel throughout. --- .github/PULL_REQUEST_TEMPLATE.md | 8 ++++ README.md | 10 ++++- TODO.md | 18 ++++++++ docs/development.md | 23 +++++++--- docs/documentation.md | 35 +++++++++++++-- docs/faq.md | 3 +- docs/language-guide.md | 41 ++++++++++++++++- docs/on-the-web.md | 1 + docs/prometheus.md | 3 +- docs/puppet-guide.md | 4 +- docs/quick-start-guide.md | 68 ++++++++++++++++++++-------- docs/resource-guide.md | 41 ++++++++++++++++- docs/resources.md | 20 ++++++--- docs/style-guide.md | 1 + misc/make-deps.sh | 1 + test.sh | 1 + test/test-markdownlint.sh | 77 ++++++++++++++++++++++++++++++++ test/util.sh | 2 +- 18 files changed, 313 insertions(+), 44 deletions(-) create mode 100755 test/test-markdownlint.sh diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 39bf2484..fee150f3 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,20 +4,27 @@ [docs/style-guide.md](../docs/style-guide.md) * commit message titles must be in the form: + ```topic: Capitalized message with no trailing period``` + or: + ```topic, topic2: Capitalized message with no trailing period``` * golang code must be formatted according to the standard, please run: + ``` make gofmt # formats the entire project correctly ``` + or format a single golang file correctly: + ``` gofmt -w yourcode.go ``` * please rebase your patch against current git master: + ``` git checkout master git pull origin master @@ -28,6 +35,7 @@ hub pull-request # or submit with the github web ui ``` * after a patch review, please ping @purpleidea so we know to re-review: + ``` # make changes based on reviews... git add -p # add new changes diff --git a/README.md b/README.md index fcecbd5e..c424b3b2 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ [![Jenkins](https://img.shields.io/badge/jenkins-status-brightgreen.svg?style=flat-square)](https://ci.centos.org/job/purpleidea-mgmt/) ## Community: + Come join us in the `mgmt` community! | Medium | Link | @@ -18,6 +19,7 @@ Come join us in the `mgmt` community! | Mailing list | [mgmtconfig-list@redhat.com](https://www.redhat.com/mailman/listinfo/mgmtconfig-list) | ## Status: + Mgmt is a next generation automation tool. It has similarities to other tools in the configuration management space, but has a fast, modern, distributed systems approach. The project contains an engine and a language. @@ -28,6 +30,7 @@ With your help you'll be able to influence our design and get us to 1.0 sooner! Interested developers should read the [quick start guide](docs/quick-start-guide.md). ## Documentation: + Please read, enjoy and help improve our documentation! | Documentation | Additional Notes | @@ -44,25 +47,28 @@ Please read, enjoy and help improve our documentation! | [development](docs/development.md) | for mgmt developers | ## Questions: + Please ask in the [community](#community)! If you have a well phrased question that might benefit others, consider asking it by sending a patch to the [FAQ](docs/faq.md) section. I'll merge your question, and a patch with the answer! ## Roadmap: + Feel free to grab one of the straightforward [#mgmtlove](https://github.com/purpleidea/mgmt/labels/mgmtlove) issues if you're a first time contributor to the project or if you're unsure about what to hack on! Please see: [TODO.md](TODO.md) for a list of upcoming work and TODO items. Please get involved by working on one of these items or by suggesting something else! ## Bugs: + Please set the `DEBUG` constant in [main.go](https://github.com/purpleidea/mgmt/blob/master/main.go) to `true`, and post the logs when you report the [issue](https://github.com/purpleidea/mgmt/issues). Bonus points if you provide a [shell](https://github.com/purpleidea/mgmt/tree/master/test/shell) or [OMV](https://github.com/purpleidea/mgmt/tree/master/test/omv) reproducible test case. Feel free to read my article on [debugging golang programs](https://purpleidea.com/blog/2016/02/15/debugging-golang-programs/). ## Patches: + We'd love to have your patches! Please send them by email, or as a pull request. ## On the web: + [Read what people are saying and publishing about mgmt!](docs/on-the-web.md) -## - Happy hacking! diff --git a/TODO.md b/TODO.md index 0a7e66a4..fa4eeab3 100644 --- a/TODO.md +++ b/TODO.md @@ -1,4 +1,5 @@ # TODO + If you're looking for something to do, look here! Let us know if you're working on one of the items. If you'd like something to work on, ping @purpleidea and I'll create an issue @@ -6,54 +7,70 @@ tailored especially for you! Just let me know your approximate golang skill level and how many hours you'd like to spend on the patch. ## Package resource + - [ ] getfiles support on debian [bug](https://github.com/hughsie/PackageKit/issues/118) - [ ] directory info on fedora [bug](https://github.com/hughsie/PackageKit/issues/117) - [ ] dnf blocker [bug](https://github.com/hughsie/PackageKit/issues/110) ## File resource [bug](https://github.com/purpleidea/mgmt/issues/64) [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove) + - [ ] recurse limit support [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove) - [ ] fanotify support [bug](https://github.com/go-fsnotify/fsnotify/issues/114) ## Svc resource + - [ ] base resource improvements ## Exec resource + - [ ] base resource improvements ## Timer resource + - [ ] increment algorithm (linear, exponential, etc...) [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove) ## User/Group resource + - [ ] automatic edges to file resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove) ## Virt (libvirt) resource + - [ ] base resource improvements [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove) ## Net (systemd-networkd) resource + - [ ] base resource ## Nspawn (systemd-nspawn) resource + - [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove) ## Mount (systemd-mount) resource + - [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove) ## Cron (systemd-timer) resource + - [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove) ## Http resource + - [ ] base resource [:heart:](https://github.com/purpleidea/mgmt/labels/mgmtlove) ## Etcd improvements + - [ ] fix embedded etcd master race ## Torrent/dht file transfer + - [ ] base plumbing ## GPG/Auth improvements + - [ ] base plumbing ## Language improvements + - [ ] more core functions - [ ] automatic language formatter, ala `gofmt` - [ ] gedit/gnome-builder/gtksourceview syntax highlighting @@ -61,6 +78,7 @@ level and how many hours you'd like to spend on the patch. - [ ] emacs syntax highlighting ## Other + - [ ] better error/retry handling - [ ] deb package target in Makefile - [ ] reproducible builds diff --git a/docs/development.md b/docs/development.md index d8c94b28..c2b6013a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,12 +1,18 @@ # Development -This document contains some additional information and help regarding developing `mgmt`. Useful tools, conventions, etc. + +This document contains some additional information and help regarding +developing `mgmt`. Useful tools, conventions, etc. Be sure to read [quick start guide](docs/quick-start-guide.md) first. ## Testing -This project has both unit tests in the form of golang tests and integration tests using shell scripting. -Native golang tests are preferred over tests written in our shell testing framework. Please see https://golang.org/pkg/testing/ for more information. +This project has both unit tests in the form of golang tests and integration +tests using shell scripting. + +Native golang tests are preferred over tests written in our shell testing +framework. Please see [https://golang.org/pkg/testing/](https://golang.org/pkg/testing/) +for more information. To run all tests: @@ -14,9 +20,13 @@ To run all tests: make test ``` -There is a library of quick and small integration tests for the language and YAML related things, check out [`test/shell/`](/test/shell). Adding a test is as easy as copying one of the files in [`test/shell/`](/test/shell) and adapting it. +There is a library of quick and small integration tests for the language and +YAML related things, check out [`test/shell/`](/test/shell). Adding a test is as +easy as copying one of the files in [`test/shell/`](/test/shell) and adapting +it. -This test suite won't run by default (unless when on CI server) and needs to be called explictly using: +This test suite won't run by default (unless when on CI server) and needs to be +called explictly using: ``` make test-shell @@ -28,4 +38,5 @@ Or run an individual shell test using: make test-shell-load0 ``` -Tip: you can use TAB completion with `make` to quickly get a list of possible individual tests to run. +Tip: you can use TAB completion with `make` to quickly get a list of possible +individual tests to run. diff --git a/docs/documentation.md b/docs/documentation.md index 5a2696b7..cc4355f5 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -136,15 +136,15 @@ Invoke `mgmt` with the `--puppet` switch, which supports 3 variants: 1. Request the configuration from the Puppet Master (like `puppet agent` does) - mgmt run --puppet agent + `mgmt run --puppet agent` 2. Compile a local manifest file (like `puppet apply`) - mgmt run --puppet /path/to/my/manifest.pp + `mgmt run --puppet /path/to/my/manifest.pp` 3. Compile an ad hoc manifest from the commandline (like `puppet apply -e`) - mgmt run --puppet 'file { "/etc/ntp.conf": ensure => file }' + `mgmt run --puppet 'file { "/etc/ntp.conf": ensure => file }'` For more details and caveats see [Puppet.md](Puppet.md). @@ -154,33 +154,40 @@ An introductory post on the Puppet support is on [Felix's blog](http://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/). ## Reference + Please note that there are a number of undocumented options. For more information on these options, please view the source at: [https://github.com/purpleidea/mgmt/](https://github.com/purpleidea/mgmt/). If you feel that a well used option needs documenting here, please patch it! ### Overview of reference + * [Meta parameters](#meta-parameters): List of available resource meta parameters. * [Graph definition file](#graph-definition-file): Main graph definition file. * [Command line](#command-line): Command line parameters. * [Compilation options](#compilation-options): Compilation options. ### Meta parameters + These meta parameters are special parameters (or properties) which can apply to any resource. The usefulness of doing so will depend on the particular meta parameter and resource combination. #### AutoEdge + Boolean. Should we generate auto edges for this resource? #### AutoGroup + Boolean. Should we attempt to automatically group this resource with others? #### Noop + Boolean. Should the Apply portion of the CheckApply method of the resource make any changes? Noop is a concatenation of no-operation. #### Retry + Integer. The number of times to retry running the resource on error. Use -1 for infinite. This currently applies for both the Watch operation (which can fail) and for the CheckApply operation. While they could have separate values, I've @@ -188,6 +195,7 @@ decided to use the same ones for both until there's a proper reason to want to do something differently for the Watch errors. #### Delay + Integer. Number of milliseconds to wait between retries. The same value is shared between the Watch and CheckApply retries. This currently applies for both the Watch operation (which can fail) and for the CheckApply operation. While @@ -196,6 +204,7 @@ until there's a proper reason to want to do something differently for the Watch errors. #### Poll + Integer. Number of seconds to wait between `CheckApply` checks. If this is greater than zero, then the standard event based `Watch` mechanism for this resource is replaced with a simple polling mechanism. In general, this is not @@ -213,6 +222,7 @@ which is another way of saying that if the resource finally settles down to give the graph enough time, it can probably converge. #### Limit + Float. Maximum rate of `CheckApply` runs started per second. Useful to limit an especially _eventful_ process from causing excessive checks to run. This defaults to `+Infinity` which adds no limiting. If you change this value, you @@ -220,12 +230,14 @@ will also need to change the `Burst` value to a non-zero value. Please see the [rate](https://godoc.org/golang.org/x/time/rate) package for more information. #### Burst + Integer. Burst is the maximum number of runs which can happen without invoking the rate limiter as designated by the `Limit` value. If the `Limit` is not set to `+Infinity`, this must be a non-zero value. Please see the [rate](https://godoc.org/golang.org/x/time/rate) package for more information. #### Sema + List of string ids. Sema is a P/V style counting semaphore which can be used to limit parallelism during the CheckApply phase of resource execution. Each resource can have `N` different semaphores which share a graph global namespace. @@ -237,30 +249,37 @@ id's include: `some_id`, `hello:42`, `not:smart:4` and `:13`. It is expected that the last bare example be only used by the engine to add a global semaphore. ### Graph definition file + graph.yaml is the compiled graph definition file. The format is currently undocumented, but by looking through the [examples/](https://github.com/purpleidea/mgmt/tree/master/examples) you can probably figure out most of it, as it's fairly intuitive. ### Command line + The main interface to the `mgmt` tool is the command line. For the most recent documentation, please run `mgmt --help`. #### `--yaml ` + Point to a graph file to run. #### `--converged-timeout ` + Exit if the machine has converged for approximately this many seconds. #### `--max-runtime ` + Exit when the agent has run for approximately this many seconds. This is not generally recommended, but may be useful for users who know what they're doing. #### `--noop` + Globally force all resources into no-op mode. This also disables the export to etcd functionality, but does not disable resource collection, however all resources that are collected will have their individual noop settings set. #### `--sema ` + Globally add a counting semaphore of this size to each resource in the graph. The semaphore will get given an id of `:size`. In other words if you specify a size of 42, you can expect a semaphore if named: `:42`. It is expected that @@ -270,38 +289,46 @@ than zero at this time. The traditional non-parallel execution found in config management tools such as `Puppet` can be obtained with `--sema 1`. #### `--remote ` + Point to a graph file to run on the remote host specified within. This parameter can be used multiple times if you'd like to remotely run on multiple hosts in parallel. #### `--allow-interactive` + Allow interactive prompting for SSH passwords if there is no authentication method that works. #### `--ssh-priv-id-rsa` + Specify the path for finding SSH keys. This defaults to `~/.ssh/id_rsa`. To never use this method of authentication, set this to the empty string. #### `--cconns` + The maximum number of concurrent remote ssh connections to run. This defaults to `0`, which means unlimited. #### `--no-caching` + Don't allow remote caching of the remote execution binary. This will require the binary to be copied over for every remote execution, but it limits the likelihood that there is leftover information from the configuration process. #### `--prefix ` + Specify a path to a custom working directory prefix. This directory will get created if it does not exist. This usually defaults to `/var/lib/mgmt/`. This can't be combined with the `--tmp-prefix` option. It can be combined with the `--allow-tmp-prefix` option. #### `--tmp-prefix` + If this option is specified, a temporary prefix will be used instead of the default prefix. This can't be combined with the `--prefix` option. #### `--allow-tmp-prefix` + If this option is specified, we will attempt to fall back to a temporary prefix if the primary prefix couldn't be created. This is useful for avoiding failures in environments where the primary prefix may or may not be available, but you'd @@ -338,12 +365,14 @@ GOTAGS="noaugeas novirt" make build ``` ## Examples + For example configurations, please consult the [examples/](https://github.com/purpleidea/mgmt/tree/master/examples) directory in the git source repository. It is available from: [https://github.com/purpleidea/mgmt/tree/master/examples](https://github.com/purpleidea/mgmt/tree/master/examples) ### Systemd: + See [`misc/mgmt.service`](misc/mgmt.service) for a sample systemd unit file. This unit file is part of the RPM. diff --git a/docs/faq.md b/docs/faq.md index afd48c9e..63695c87 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,4 +1,4 @@ -# Frequently asked questions +## Frequently asked questions (Send your questions as a patch to this FAQ! I'll review it, merge it, and respond by commit with the answer.) @@ -98,6 +98,7 @@ shutdown after that time interval using the `--max-runtime` flag. This also requires a number of seconds as an argument. #### Example: + ``` ./mgmt run --lang examples/lang/hello0.mcl --converged-timeout=5 ``` diff --git a/docs/language-guide.md b/docs/language-guide.md index fdd7d309..f88e3706 100644 --- a/docs/language-guide.md +++ b/docs/language-guide.md @@ -1,17 +1,20 @@ # Language guide ## Overview + The `mgmt` tool has various frontends, each of which may produce a stream of between zero or more graphs that are passed to the engine for desired state application. In almost all scenarios, you're going to want to use the language frontend. This guide describes some of the internals of the language. ## Theory + The mgmt language is a declarative (immutable) functional, reactive programming language. It is implemented in `golang`. A longer introduction to the language is coming soon! ### Types + All expressions must have a type. A composite type such as a list of strings (`[]str`) is different from a list of integers (`[]int`). @@ -24,25 +27,31 @@ The implementation of the internal types can be found in [lang/types/](https://github.com/purpleidea/mgmt/tree/master/lang/types/). #### bool + A `true` or `false` value. #### str + Any `"string!"` enclosed in quotes. #### int + A number like `42` or `-13`. Integers are represented internally as golang's `int64`. #### float + A floating point number like: `3.1415926`. Float's are represented internally as golang's `float64`. #### list + An ordered collection of values of the same type, eg: `[6, 7, 8, 9,]`. It is worth mentioning that empty lists have a type, although without type hints it can be impossible to infer the item's type. #### map + An unordered set of unique keys of the same type and corresponding value pairs of another type, eg: `{"boiling" => 100, "freezing" => 0, "room" => "25", "house" => 22, "canada" => -30,}`. That is to say, all of the keys must have the same type, and all of the values @@ -50,6 +59,7 @@ must have the same type. You can use any type for either, although it is probably advisable to avoid using very complex types as map keys. #### struct + An ordered set of field names and corresponding values, each of their own type, eg: `struct{answer => "42", james => "awesome", is_mgmt_awesome => true,}`. These are useful for combining more than one type into the same value. Note the @@ -58,23 +68,27 @@ and as a result, string keys are enclosed in quotes, whereas struct _fields_ are not string values, and as such are bare and specified without quotes. #### func + An ordered set of optionally named, differently typed input arguments, and a return type, eg: `func(s str) int` or: `func(bool, []str, {str: float}) struct{foo str; bar int}`. ### Expressions + Expressions, and the `Expr` interface need to be better documented. For now please consume [lang/interfaces/ast.go](https://github.com/purpleidea/mgmt/tree/master/lang/interfaces/ast.go). These docs will be expanded on when things are more certain to be stable. ### Statements + Statements, and the `Stmt` interface need to be better documented. For now please consume [lang/interfaces/ast.go](https://github.com/purpleidea/mgmt/tree/master/lang/interfaces/ast.go). These docs will be expanded on when things are more certain to be stable. ### Stages + The mgmt compiler runs in a number of stages. In order of execution they are: * [Lexing](#lexing) * [Parsing](#parsing) @@ -93,6 +107,7 @@ to the engine as they are produced. What follows are some notes about each step. #### Lexing + Lexing is done using [nex](https://github.com/blynn/nex). It is a pure-golang implementation which is similar to _Lex_ or _Flex_, but which produces golang code instead of C. It integrates reasonably well with golang's _yacc_ which is @@ -101,6 +116,7 @@ used for parsing. The token definitions are in: Lexing and parsing run together by calling the `LexParse` method. #### Parsing + The parser used is golang's implementation of [yacc](https://godoc.org/golang.org/x/tools/cmd/goyacc). The documentation is quite abysmal, so it's helpful to rely on the documentation from standard yacc @@ -112,6 +128,7 @@ The yacc file exists at: Lexing and parsing run together by calling the `LexParse` method. #### Interpolation + Interpolation is used to transform the AST (which was produced from lexing and parsing) into one which is either identical or different. It expands strings which might contain expressions to be interpolated (eg: `"the answer is: ${foo}"`) @@ -120,6 +137,7 @@ be better represented by a larger AST. Most nodes in the AST simply return their own node address, and do not modify the AST. #### Scope propagation + Scope propagation passes the parent scope (starting with the top-level, built-in scope) down through the AST. This is necessary so that children nodes can access variables in the scope if needed. Most AST node's simply pass on the scope @@ -128,6 +146,7 @@ the `StmtProg` node cleverly passes the scope through in the order expected for the out-of-order bind logic to work. #### Type unification + Each expression must have a known type. The unpleasant option is to force the programmer to specify by annotation every type throughout their whole program so that each `Expr` node in the AST knows what to expect. Type annotation is @@ -174,6 +193,7 @@ alternate implementation if someone more skilled in the art of solver design would like to propose a more logical or performant variant. #### Function graph generation + At this point we have a fully type AST. The AST must now be transformed into a directed, acyclic graph (DAG) data structure that represents the flow of data as necessary for everything to be reactive. Note that this graph is *different* @@ -191,6 +211,7 @@ calling that expression's `Func` method. These are usually called by the function engine during function creation and validation. #### Function engine creation and validation + Finally we have a graph of the data flows. The function engine must first initialize which creates references to each of the necessary function implementations, and gets information about each one. It then needs to be type @@ -199,12 +220,14 @@ you were to pass an `int` to a function expecting a `bool`, this would be a problem. If all goes well, the program should get run shortly. #### Function engine running and interpret + At this point the function engine runs. It produces a stream of events which cause the `Output()` method of the top-level program to run, which produces the list of resources and edges. These are then transformed into the resource graph which is passed to the engine. ### Function API + If you'd like to create a built-in, core function, you'll need to implement the function API interface named `Func`. It can be found in [lang/interfaces/func.go](https://github.com/purpleidea/mgmt/tree/master/lang/interfaces/func.go). @@ -219,6 +242,7 @@ Failure to implement the API correctly can cause the function graph engine to block, or the program to panic. ### Info + ```golang Info() *Info ``` @@ -238,6 +262,7 @@ not depend on any other method being called first. Other methods must not depend on this method being called first. #### Example + ```golang func (obj *FooFunc) Info() *interfaces.Info { return &interfaces.Info{ @@ -247,6 +272,7 @@ func (obj *FooFunc) Info() *interfaces.Info { ``` ### Init + ```golang Init(*Init) error ``` @@ -279,12 +305,14 @@ or all of these info fields that you wish to use in the struct implementing this one value must be produced. #### Example + ```golang Please see the example functions in [lang/funcs/core/](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/core/). ``` ### Stream + ```golang Stream() error ``` @@ -298,12 +326,14 @@ channel when it has no more values to send. This may or may not influence whether or not you close the `Output` channel. #### Example + ```golang Please see the example functions in [lang/funcs/core/](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/core/). ``` ### Close + ```golang Close() error ``` @@ -312,12 +342,14 @@ Close asks the particular function to shutdown its `Stream()` function and return. #### Example + ```golang Please see the example functions in [lang/funcs/core/](https://github.com/purpleidea/mgmt/tree/master/lang/funcs/core/). ``` ### Polymorphic Function API + For some functions, it might be helpful to be able to implement a function once, but to have multiple polymorphic variants that can be chosen at compile time. For this more advanced topic, you will need to use the @@ -347,12 +379,15 @@ What follows are a few examples that might help you understand some of the language details. ##### Example Foo + TODO: please add an example here! ##### Example Bar + TODO: please add an example here! ## Frequently asked questions + (Send your questions as a patch to this FAQ! I'll review it, merge it, and respond by commit with the answer.) @@ -365,7 +400,8 @@ branches (a `then` and an `else` branch) which each contain one expression. The conditional. #### Example: -``` + +```mcl # this is an if expression, and both branches must exist $b = true $x = if $b { @@ -381,7 +417,8 @@ statement does not return any value, but it does produce output when it is evaluated. The output consists primarily of resources (vertices) and edges. #### Example: -``` + +```mcl # this is an if statement, and in this scenario the else branch was omitted $b = true if $b { diff --git a/docs/on-the-web.md b/docs/on-the-web.md index 115249f8..d238095b 100644 --- a/docs/on-the-web.md +++ b/docs/on-the-web.md @@ -4,6 +4,7 @@ Here is a list of places mgmt has appeared on the web. Feel free to send a patch if we missed something that you think is relevant! ## Links + | Author | Format | Subject | |---|---|---| | James Shubin | blog | [Next generation configuration mgmt](https://purpleidea.com/blog/2016/01/18/next-generation-configuration-mgmt/) | diff --git a/docs/prometheus.md b/docs/prometheus.md index d3f94517..e2c2fb06 100644 --- a/docs/prometheus.md +++ b/docs/prometheus.md @@ -57,8 +57,7 @@ We do not have grafana dashboards yet. Patches welcome! - [prometheus website](https://prometheus.io/) - [prometheus documentation](https://prometheus.io/docs/introduction/overview/) -- [prometheus best practices regarding metrics - naming](https://prometheus.io/docs/practices/naming/) +- [prometheus best practices regarding metrics naming](https://prometheus.io/docs/practices/naming/) - [grafana website](http://grafana.org/) [pgc]: https://github.com/prometheus/client_golang/blob/master/prometheus/go_collector.go diff --git a/docs/puppet-guide.md b/docs/puppet-guide.md index 9c4c4915..9b0bb546 100644 --- a/docs/puppet-guide.md +++ b/docs/puppet-guide.md @@ -109,8 +109,8 @@ file { "/tmp/mgmt-test": To avoid this, specify the parameter explicitly: -``` -$ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": backup => false }' +```bash +puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": backup => false }' ``` This is tedious in a more complex manifest. A good simplification is the diff --git a/docs/quick-start-guide.md b/docs/quick-start-guide.md index fdf422b2..9d1a7d73 100644 --- a/docs/quick-start-guide.md +++ b/docs/quick-start-guide.md @@ -1,6 +1,7 @@ # Quick start guide ## Introduction + This guide is intended for developers. Once `mgmt` is minimally viable, we'll publish a quick start guide for users too. If you're brand new to `mgmt`, it's probably a good idea to start by reading the @@ -11,24 +12,32 @@ Once you're familiar with the general idea, please start hacking... ## Quick start ### Installing golang + * You need golang version 1.9 or greater installed. -** To install on rpm style systems: `sudo dnf install golang` -** To install on apt style systems: `sudo apt install golang` -** To install on macOS systems install [Homebrew](https://brew.sh) and run: `brew install go` + * To install on rpm style systems: `sudo dnf install golang` + * To install on apt style systems: `sudo apt install golang` + * To install on macOS systems install [Homebrew](https://brew.sh) + and run: `brew install go` * You can run `go version` to check the golang version. -* If your distro is tool old, you may need to [download](https://golang.org/dl/) a newer golang version. +* If your distro is tool old, you may need to [download](https://golang.org/dl/) +a newer golang version. ### Setting up golang + * If you do not have a GOPATH yet, create one and export it: + ``` mkdir $HOME/gopath export GOPATH=$HOME/gopath ``` + * You might also want to add the GOPATH to your `~/.bashrc` or `~/.profile`. * For more information you can read the [GOPATH documentation](https://golang.org/cmd/go/#hdr-GOPATH_environment_variable). ### Getting the mgmt code and dependencies + * Download the `mgmt` code into the GOPATH, and switch to that directory: + ``` mkdir -p $GOPATH/src/github.com/purpleidea/ cd $GOPATH/src/github.com/purpleidea/ @@ -37,37 +46,49 @@ cd $GOPATH/src/github.com/purpleidea/mgmt ``` * Add $GOPATH/bin to $PATH + ``` export PATH=$PATH:$GOPATH/bin ``` -* Run `make deps` to install system and golang dependencies. Take a look at `misc/make-deps.sh` for details. +* Run `make deps` to install system and golang dependencies. Take a look at +`misc/make-deps.sh` for details. * Run `make build` to get a freshly built `mgmt` binary. ### Running mgmt -* Run `time ./mgmt run --lang examples/lang/hello0.mcl --tmp-prefix` to try out a very simple example! -* Look in that example file that you ran to see if you can figure out what it did! -* Have fun hacking on our future technology and get involved to shape the project! + +* Run `time ./mgmt run --lang examples/lang/hello0.mcl --tmp-prefix` to try out +a very simple example! +* Look in that example file that you ran to see if you can figure out what it +did! +* Have fun hacking on our future technology and get involved to shape the +project! ## Examples -Please look in the [examples/lang/](../examples/lang/) folder for some more examples! + +Please look in the [examples/lang/](../examples/lang/) folder for some more +examples! ## Vagrant + If you would like to avoid doing the above steps manually, we have prepared a [Vagrant](https://www.vagrantup.com/) environment for your convenience. From the project directory, run a `vagrant up`, and then a `vagrant status`. From there, you can `vagrant ssh` into the `mgmt` machine. The MOTD will explain the rest. ## Information about dependencies + Software projects have a few different kinds of dependencies. There are _build_ dependencies, _runtime_ dependencies, and additionally, a few extra dependencies required for running the _test_ suite. ### Build + * `golang` 1.9 or higher (required, available in some distros and distributed - as a binary officially by [golang.org](https://golang.org/dl/)) +as a binary officially by [golang.org](https://golang.org/dl/)) ### Runtime + A relatively modern GNU/Linux system should be able to run `mgmt` without any problems. Since `mgmt` runs as a single statically compiled binary, all of the library dependencies are included. It is expected, that certain advanced @@ -97,6 +118,7 @@ To build `mgmt` without augeas or libvirt support please run: `GOTAGS='noaugeas novirt' make build` ## Binary Package Installation + Installation of `mgmt` from distribution packages currently needs improvement. They are not always up-to-date with git master and as such are not recommended. At the moment we have: @@ -106,11 +128,19 @@ At the moment we have: Please contribute more! We'd especially like to see a Debian package! ## OSX/macOS/Darwin development -Developing and running `mgmt` on macOS is currently not supported (but not discouraged either). Meaning it might work but in the case it doesn't you would have to provide your own patches to fix problems (the project maintainer and community are glad to assist where needed). -There are currently some issues that make `mgmt` less suitable to run for provisioning macOS (eg: https://github.com/purpleidea/mgmt/issues/33). But as a client to provision remote servers it should run fine. +Developing and running `mgmt` on macOS is currently not supported (but not +discouraged either). Meaning it might work but in the case it doesn't you would +have to provide your own patches to fix problems (the project maintainer and +community are glad to assist where needed). -Since the primary supported systems are Linux and these are the environments tested for it is wise to run these suites during macOS development as well. To ease this Docker can be levaraged ((Docker for Mac)[https://docs.docker.com/docker-for-mac/]). +There are currently some issues that make `mgmt` less suitable to run for +provisioning macOS (eg: [https://github.com/purpleidea/mgmt/issues/33](https://github.com/purpleidea/mgmt/issues/33)). +But as a client to provision remote servers it should run fine. + +Since the primary supported systems are Linux and these are the environments +tested for it is wise to run these suites during macOS development as well. To +ease this Docker can be leveraged ([Docker for Mac](https://docs.docker.com/docker-for-mac/)). Before running any of the commands below create the development Docker image: @@ -124,15 +154,17 @@ Then to run the test suite: ``` docker run --rm -ti \ - -v $PWD:/go/src/github.com/purpleidea/mgmt/ \ - -w /go/src/github.com/purpleidea/mgmt/ \ - purpleidea/mgmt:development \ - make test + -v $PWD:/go/src/github.com/purpleidea/mgmt/ \ + -w /go/src/github.com/purpleidea/mgmt/ \ + purpleidea/mgmt:development \ + make test ``` For convenience this command is wrapped in `docker/scripts/exec-development`. -Basically any command can be executed this way. Because the repository source is mounted into the Docker container invocation will be quick and allow rapid testing, example: +Basically any command can be executed this way. Because the repository source is +mounted into the Docker container invocation will be quick and allow rapid +testing, example: ``` docker/scripts/exec-development test/test-shell.sh load0.sh diff --git a/docs/resource-guide.md b/docs/resource-guide.md index a34a16b0..2dc3cc09 100644 --- a/docs/resource-guide.md +++ b/docs/resource-guide.md @@ -27,6 +27,7 @@ interface. What follows are each of the method signatures and a description of each. ### Default + ```golang Default() Res ``` @@ -36,6 +37,7 @@ values which already have the correct default as the golang zero value. In general it is preferable if the zero values make for the correct defaults. #### Example + ```golang // Default returns some sensible defaults for this resource. func (obj *FooRes) Default() Res { @@ -46,6 +48,7 @@ func (obj *FooRes) Default() Res { ``` ### Validate + ```golang Validate() error ``` @@ -57,6 +60,7 @@ quite large, it might be an indication that you should reconsider the parameter list and interface to this resource. This method is called _before_ `Init`. #### Example + ```golang // Validate reports any problems with the struct definition. func (obj *FooRes) Validate() error { @@ -68,6 +72,7 @@ func (obj *FooRes) Validate() error { ``` ### Init + ```golang Init() error ``` @@ -77,6 +82,7 @@ return an error. It should do any resource specific work, and finish by calling the `Init` method of the base resource. #### Example + ```golang // Init initializes the Foo resource. func (obj *FooRes) Init() error { @@ -95,6 +101,7 @@ shouldn't allow `Init` to dangerously `rm -rf /$the_world` if your code only checks `$the_world` in `Validate`. Remember to always program safely! ### Close + ```golang Close() error ``` @@ -104,6 +111,7 @@ can be useful if you'd like to properly close a persistent connection that you opened in the `Init` method and were using throughout the resource. #### Example + ```golang // Close runs some cleanup code for this resource. func (obj *FooRes) Close() error { @@ -125,6 +133,7 @@ method! If you plan to return early if you hit an internal error, then at least call it with a defer! ### CheckApply + ```golang CheckApply(apply bool) (checkOK bool, err error) ``` @@ -150,6 +159,7 @@ facility will detect the change, ultimately resulting in a subsequent call to `CheckApply`. #### Example + ```golang // CheckApply does the idempotent work of checking and applying resource state. func (obj *FooRes) CheckApply(apply bool) (bool, error) { @@ -171,6 +181,7 @@ the documentation in case you are confused as to why a debug message you've added to the code isn't always printed. #### Refresh notifications + Some resources may choose to support receiving refresh notifications. In general these should be avoided if possible, but nevertheless, they do make sense in certain situations. Resources that support these need to verify if one was sent @@ -184,6 +195,7 @@ have enabled their propagation. Resources that currently perform some refresh action include `svc`, `timer`, and `password`. #### Paired execution + For many resources it is not uncommon to see `CheckApply` run twice in rapid succession. This is usually not a pathological occurrence, but rather a healthy pattern which is a consequence of the event system. When the state of the @@ -193,6 +205,7 @@ trigger the `Watch` code! In response, a second `CheckApply` is triggered, which will likely find the state to now be correct. #### Summary + * Anytime an error occurs during `CheckApply`, you should return `(false, err)`. * If the state is correct and no changes are needed, return `(true, nil)`. * You should only make changes to the system if `apply` is set to `true`. @@ -200,6 +213,7 @@ will likely find the state to now be correct. * Returning `(true, err)` is a programming error and will cause a `Fatal`. ### Watch + ```golang Watch() error ``` @@ -229,6 +243,7 @@ executed. As a result, the resource must still work even if the main loop is not running. #### Select + The lifetime of most resources `Watch` method should be spent in an infinite loop that is bounded by a `select` call. The `select` call is the point where our method hands back control to the engine (and the kernel) so that we can @@ -237,6 +252,7 @@ events from the engine via the `<-obj.Events()` call, and receive events for our resource itself! #### Events + If we receive an internal event from the `<-obj.Events()` method, we can read it with the ReadEvent helper function. This function tells us if we should shutdown our resource, and if we should generate an event. When we want to send an event, @@ -245,6 +261,7 @@ state as `dirty` if we believe it might have changed. We do this with the `StateOK(false)` function. #### Startup + Once the `Watch` function has finished starting up successfully, it is important to generate one event to notify the `mgmt` engine that we're now listening successfully, so that it can run an initial `CheckApply` to ensure we're safely @@ -252,6 +269,7 @@ tracking a healthy state and that we didn't miss anything when `Watch` was down or from before `mgmt` was running. It does this by calling the `Running` method. #### Converged + The engine might be asked to shutdown when the entire state of the system has not seen any changes for some duration of time. The engine can determine this automatically, but each resource can block this if it is absolutely necessary. @@ -270,6 +288,7 @@ prove to be useful if a resource wants to start off a long operation, but avoid sending out erroneous `Event` messages to keep things alive until it finishes. #### Example + ```golang // Watch is the listener and main loop for this resource. func (obj *FooRes) Watch() error { @@ -317,6 +336,7 @@ func (obj *FooRes) Watch() error { ``` #### Summary + * Remember to call the appropriate `converger` methods throughout the resource. * Remember to call `Startup` when the `Watch` is running successfully. * Remember to process internal events and shutdown promptly if asked to. @@ -324,6 +344,7 @@ func (obj *FooRes) Watch() error { * Have a look at the existing resources for a rough idea of how this all works. ### Compare + ```golang Compare(Res) bool ``` @@ -341,6 +362,7 @@ particular if they store some generated state, or if they aren't significant in some way. #### Example + ```golang // Compare two resources and return if they are equivalent. func (obj *FooRes) Compare(r Res) bool { @@ -368,6 +390,7 @@ func (obj *FooRes) Compare(r Res) bool { ``` ### UIDs + ```golang UIDs() []ResUID ``` @@ -377,6 +400,7 @@ particular resource uniquely. This is used with the AutoEdges API to determine if another resource can match a dependency to this one. ### AutoEdges + ```golang AutoEdges() (AutoEdge, error) ``` @@ -386,6 +410,7 @@ is used to match other resources that might be relevant dependencies for this resource. ### CollectPattern + ```golang CollectPattern() string ``` @@ -393,6 +418,7 @@ CollectPattern() string This is currently a stub and will be updated once the DSL is further along. ### UnmarshalYAML + ```golang UnmarshalYAML(unmarshal func(interface{}) error) error // optional ``` @@ -406,6 +432,7 @@ The signature intentionally matches what is required to satisfy the `go-yaml` [Unmarshaler](https://godoc.org/gopkg.in/yaml.v2#Unmarshaler) interface. #### Example + ```golang // UnmarshalYAML is the custom unmarshal handler for this struct. // It is primarily useful for setting the defaults. @@ -429,10 +456,12 @@ func (obj *FooRes) UnmarshalYAML(unmarshal func(interface{}) error) error { ``` ## Further considerations + There is some additional information that any resource writer will need to know. Each issue is listed separately below! ### Resource struct + Each resource will implement methods as pointer receivers on a resource struct. The resource struct must include an anonymous reference to the `BaseRes` struct. The naming convention for resources is that they end with a `Res` suffix. If @@ -440,6 +469,7 @@ you'd like your resource to be accessible by the `YAML` graph API (GAPI), then you'll need to include the appropriate YAML fields as shown below. #### Example + ```golang type FooRes struct { BaseRes `yaml:",inline"` // base properties @@ -453,6 +483,7 @@ type FooRes struct { ``` ### Resource registration + All resources must be registered with the engine so that they can be found. This also ensures they can be encoded and decoded. Make sure to include the following code snippet for this to work. @@ -465,6 +496,7 @@ func init() { // special golang method that runs once ``` ## Automatic edges + Automatic edges in `mgmt` are well described in [this article](https://purpleidea.com/blog/2016/03/14/automatic-edges-in-mgmt/). The best example of this technique can be seen in the `svc` resource. Unfortunately no further documentation about this subject has been written. To @@ -472,14 +504,15 @@ expand this section, please send a patch! Please contact us if you'd like to work on a resource that uses this feature, or to add it to an existing one! ## Automatic grouping + Automatic grouping in `mgmt` is well described in [this article](https://purpleidea.com/blog/2016/03/30/automatic-grouping-in-mgmt/). The best example of this technique can be seen in the `pkg` resource. Unfortunately no further documentation about this subject has been written. To expand this section, please send a patch! Please contact us if you'd like to work on a resource that uses this feature, or to add it to an existing one! - ## Send/Recv + In `mgmt` there is a novel concept called _Send/Recv_. For some background, please [read the introductory article](https://purpleidea.com/blog/2016/12/07/sendrecv-in-mgmt/). When using this feature, the engine will automatically send the user specified @@ -523,6 +556,7 @@ such as for cache invalidation. Remember, `Send/Recv` only changes your resource code if you cache state. ## Composite resources + Composite resources are resources which embed one or more existing resources. This is useful to prevent code duplication in higher level resource scenarios. The best example of this technique can be seen in the `nspawn` resource which @@ -532,10 +566,12 @@ expand this section, please send a patch! Please contact us if you'd like to work on a resource that uses this feature, or to add it to an existing one! ## Frequently asked questions + (Send your questions as a patch to this FAQ! I'll review it, merge it, and respond by commit with the answer.) ### Can I write resources in a different language? + Currently `golang` is the only supported language for built-in resources. We might consider allowing external resources to be imported in the future. This will likely require a language that can expose a C-like API, such as `python` or @@ -543,13 +579,16 @@ will likely require a language that can expose a C-like API, such as `python` or Higher level resource collections will be possible once the `mgmt` DSL is ready. ### What new resource primitives need writing? + There are still many ideas for new resources that haven't been written yet. If you'd like to contribute one, please contact us and tell us about your idea! ### Where can I find more information about mgmt? + Additional blog posts, videos and other material [is available!](https://github.com/purpleidea/mgmt/blob/master/docs/on-the-web.md). ## Suggestions + If you have any ideas for API changes or other improvements to resource writing, please let us know! We're still pre 1.0 and pre 0.1 and happy to break API in order to get it right! diff --git a/docs/resources.md b/docs/resources.md index 5fb5d30b..2523f5d4 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -46,12 +46,12 @@ identified by a trailing slash in their path name. File have no such slash. It has the following properties: -- `path`: file path (directories have a trailing slash here) -- `content`: raw file content -- `state`: either `exists` (the default value) or `absent` -- `mode`: octal unix file permissions -- `owner`: username or uid for the file owner -- `group`: group name or gid for the file group +* `path`: file path (directories have a trailing slash here) +* `content`: raw file content +* `state`: either `exists` (the default value) or `absent` +* `mode`: octal unix file permissions +* `owner`: username or uid for the file owner +* `group`: group name or gid for the file group ### Path @@ -88,21 +88,25 @@ The hostname resource manages static, transient/dynamic and pretty hostnames on the system and watches them for changes. ### static_hostname + The static hostname is the one configured in /etc/hostname or a similar file. It is chosen by the local user. It is not always in sync with the current host name as returned by the gethostname() system call. ### transient_hostname + The transient / dynamic hostname is the one configured via the kernel's sethostbyname(). It can be different from the static hostname in case DHCP or mDNS have been configured to change the name based on network information. ### pretty_hostname + The pretty hostname is a free-form UTF8 host name for presentation to the user. ### hostname + Hostname is the fallback value for all 3 fields above, if only `hostname` is specified, it will set all 3 fields to this value. @@ -116,17 +120,21 @@ refresh, then the stored value will be reset to the requested value even if the stored value is greater. ### Key + The string key used to store the key. ### Value + The string value to set. This can also be set via Send/Recv. ### SkipLessThan + If this parameter is set to `true`, then it will ignore updating the value as long as the database versions are greater than the requested value. The compare operation used is based on the `SkipCmpStyle` parameter. ### SkipCmpStyle + By default this converts the string values to integers and compares them as you would expect. diff --git a/docs/style-guide.md b/docs/style-guide.md index 2a25d6af..e1559017 100644 --- a/docs/style-guide.md +++ b/docs/style-guide.md @@ -91,5 +91,6 @@ When implementing code for the various types in the language, please follow this order: `bool`, `str`, `int`, `float`, `list`, `map`, `struct`, `func`. ## Suggestions + If you have any ideas for suggestions or other improvements to this guide, please let us know! diff --git a/misc/make-deps.sh b/misc/make-deps.sh index b6a76489..b582e577 100755 --- a/misc/make-deps.sh +++ b/misc/make-deps.sh @@ -94,4 +94,5 @@ go get golang.org/x/tools/cmd/stringer # for automatic stringer-ing go get github.com/tmthrgd/go-bindata/go-bindata # for compiling in non golang files go get github.com/golang/lint/golint # for `golint`-ing go get -u gopkg.in/alecthomas/gometalinter.v1 && mv "$(dirname $(command -v gometalinter.v1))/gometalinter.v1" "$(dirname $(command -v gometalinter.v1))/gometalinter" && gometalinter --install # bonus +command -v mdl &>/dev/null || gem install mdl || true # for linting markdown files cd "$XPWD" >/dev/null diff --git a/test.sh b/test.sh index c8455a66..6f9b34e8 100755 --- a/test.sh +++ b/test.sh @@ -48,6 +48,7 @@ run-testsuite ./test/test-gofmt.sh run-testsuite ./test/test-yamlfmt.sh run-testsuite ./test/test-bashfmt.sh run-testsuite ./test/test-headerfmt.sh +run-testsuite ./test/test-markdownlint.sh run-testsuite ./test/test-commit-message.sh run-testsuite ./test/test-govet.sh run-testsuite ./test/test-examples.sh diff --git a/test/test-markdownlint.sh b/test/test-markdownlint.sh new file mode 100755 index 00000000..2aa1e5b8 --- /dev/null +++ b/test/test-markdownlint.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# check for any markdown files that aren't in an ideal format + +echo running "$0 $@" +set -o errexit +set -o nounset +set -o pipefail + +#ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" # dir! +ROOT=$(dirname "${BASH_SOURCE}")/.. +cd "${ROOT}" +. test/util.sh + +MDL=`command -v mdl 2>/dev/null` +if [ -z $MDL ]; then + fail_test "The 'mdl' utility can't be found." +fi + +STYLE=$($mktemp) +# styles that we ignore... if they're too onerous, we can exclude them here... +cat << 'EOF' > $STYLE +all +exclude_rule 'MD010' # Hard tabs +exclude_rule 'MD032' # Lists should be surrounded by blank lines +exclude_rule 'MD013' # Line length +exclude_rule 'MD040' # Fenced code blocks should have a language specified +exclude_rule 'MD026' # Trailing punctuation in header +exclude_rule 'MD024' # Multiple headers with the same content +exclude_rule 'MD002' # First header should be a top level header +exclude_rule 'MD041' # First line in file should be a top level header +exclude_rule 'MD007' # Unordered list indentation + +# FIXME: no idea why this issue occurs +exclude_rule 'MD029' # Ordered list item prefix + +# FIXME: bug: https://github.com/markdownlint/markdownlint/issues/182 +exclude_rule 'MD039' # Spaces inside link text +EOF + +#STYLE="test/mdl.style" # style file + +find_files() { + git ls-files | grep '\.md$' +} + +F=${1:-} # only check this file from $1 is specified + +bad_files=$( + for i in $(find_files); do + if [ "$F" != "" ] && [ "$F" != "$i" ]; then + continue + fi + + # search for more than one leading space, to ensure we use tabs + if grep -q '^ ' "$i"; then + echo "$i: MDX042: Leading spaces found instead of tabs" 1>&2 + echo "$i" + fi + + # check the markdown format with the linter + if ! $MDL --style "$STYLE" "$i" 1>&2; then + echo "$i" + fi + done +) + +# cleanup +if [ -e "$STYLE" ]; then + rm "$STYLE" +fi + +if [[ -n "${bad_files}" ]]; then + # see a description of the rules at: + # https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md + fail_test "The following markdown files are not properly formatted:\n${bad_files}" +fi +echo 'PASS' diff --git a/test/util.sh b/test/util.sh index 3b8bf741..d7d639f9 100755 --- a/test/util.sh +++ b/test/util.sh @@ -18,7 +18,7 @@ fi fail_test() { - echo "FAIL: $@" + echo -e "FAIL: $@" exit 1 }