lourenco/mgmt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

gapi: Bring back puppet and langpuppet

Browse Source 
This reverts commit e767655ede.

In addition, it applies required changes to function with the new CLI backend.
This commit is contained in:
Felix Frank
2024-03-08 11:42:10 +00:00
committed by James Shubin
parent 22873b3c3f
commit 29ec867ac7
17 changed files with 1465 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -98,6 +98,7 @@ Please read, enjoy and help improve our documentation!
| [style guide](docs/style-guide.md) | for mgmt developers | | [style guide](docs/style-guide.md) | for mgmt developers |
| [godoc API reference](https://godoc.org/github.com/purpleidea/mgmt) | for mgmt developers | | [godoc API reference](https://godoc.org/github.com/purpleidea/mgmt) | for mgmt developers |
| [prometheus guide](docs/prometheus.md) | for everyone | | [prometheus guide](docs/prometheus.md) | for everyone |
| [puppet guide](docs/puppet-guide.md) | for puppet sysadmins |
| [development](docs/development.md) | for mgmt developers | | [development](docs/development.md) | for mgmt developers |
| [videos](docs/on-the-web.md) | for everyone | | [videos](docs/on-the-web.md) | for everyone |
| [blogs](docs/on-the-web.md) | for everyone | | [blogs](docs/on-the-web.md) | for everyone |

16
cli/deploy.go
View File

@@ -58,9 +58,11 @@ type DeployArgs struct {
NoGit bool `arg:"--no-git" help:"don't look at git commit id for safe deploys"` NoGit bool `arg:"--no-git" help:"don't look at git commit id for safe deploys"`
Force bool `arg:"--force" help:"force a new deploy, even if the safety chain would break"` Force bool `arg:"--force" help:"force a new deploy, even if the safety chain would break"`
DeployEmpty *cliUtil.EmptyArgs `arg:"subcommand:empty" help:"deploy empty payload"` DeployEmpty *cliUtil.EmptyArgs `arg:"subcommand:empty" help:"deploy empty payload"`
DeployLang *cliUtil.LangArgs `arg:"subcommand:lang" help:"deploy lang (mcl) payload"` DeployLang *cliUtil.LangArgs `arg:"subcommand:lang" help:"deploy lang (mcl) payload"`
DeployYaml *cliUtil.YamlArgs `arg:"subcommand:yaml" help:"deploy yaml graph payload"` DeployYaml *cliUtil.YamlArgs `arg:"subcommand:yaml" help:"deploy yaml graph payload"`
DeployPuppet *cliUtil.PuppetArgs `arg:"subcommand:puppet" help:"deploy puppet graph payload"`
DeployLangPuppet *cliUtil.LangPuppetArgs `arg:"subcommand:langpuppet" help:"deploy langpuppet graph payload"`
} }
// Run executes the correct subcommand. It errors if there's ever an error. It // Run executes the correct subcommand. It errors if there's ever an error. It
@@ -87,6 +89,14 @@ func (obj *DeployArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error
name = cliUtil.LookupSubcommand(obj, cmd) // "yaml" name = cliUtil.LookupSubcommand(obj, cmd) // "yaml"
args = cmd args = cmd
} }
if cmd := obj.DeployPuppet; cmd != nil {
name = cliUtil.LookupSubcommand(obj, cmd) // "puppet"
args = cmd
}
if cmd := obj.DeployLangPuppet; cmd != nil {
name = cliUtil.LookupSubcommand(obj, cmd) // "langpuppet"
args = cmd
}
// XXX: workaround https://github.com/alexflint/go-arg/issues/239 // XXX: workaround https://github.com/alexflint/go-arg/issues/239
gapiNames := gapi.Names() // list of registered names gapiNames := gapi.Names() // list of registered names

16
cli/run.go
View File

@@ -52,9 +52,11 @@ import (
type RunArgs struct { type RunArgs struct {
lib.Config // embedded config (can't be a pointer) https://github.com/alexflint/go-arg/issues/240 lib.Config // embedded config (can't be a pointer) https://github.com/alexflint/go-arg/issues/240
RunEmpty *cliUtil.EmptyArgs `arg:"subcommand:empty" help:"run empty payload"` RunEmpty *cliUtil.EmptyArgs `arg:"subcommand:empty" help:"run empty payload"`
RunLang *cliUtil.LangArgs `arg:"subcommand:lang" help:"run lang (mcl) payload"` RunLang *cliUtil.LangArgs `arg:"subcommand:lang" help:"run lang (mcl) payload"`
RunYaml *cliUtil.YamlArgs `arg:"subcommand:yaml" help:"run yaml graph payload"` RunYaml *cliUtil.YamlArgs `arg:"subcommand:yaml" help:"run yaml graph payload"`
RunPuppet *cliUtil.PuppetArgs `arg:"subcommand:puppet" help:"run puppet graph payload"`
RunLangPuppet *cliUtil.LangPuppetArgs `arg:"subcommand:langpuppet" help:"run a combined lang/puppet graph payload"`
} }
// Run executes the correct subcommand. It errors if there's ever an error. It // Run executes the correct subcommand. It errors if there's ever an error. It
@@ -81,6 +83,14 @@ func (obj *RunArgs) Run(ctx context.Context, data *cliUtil.Data) (bool, error) {
name = cliUtil.LookupSubcommand(obj, cmd) // "yaml" name = cliUtil.LookupSubcommand(obj, cmd) // "yaml"
args = cmd args = cmd
} }
if cmd := obj.RunPuppet; cmd != nil {
name = cliUtil.LookupSubcommand(obj, cmd) // "puppet"
args = cmd
}
if cmd := obj.RunLangPuppet; cmd != nil {
name = cliUtil.LookupSubcommand(obj, cmd) // "langpuppet"
args = cmd
}
// XXX: workaround https://github.com/alexflint/go-arg/issues/239 // XXX: workaround https://github.com/alexflint/go-arg/issues/239
lists := [][]string{ lists := [][]string{

47
cli/util/args.go
View File

@@ -102,3 +102,50 @@ type YamlArgs struct {
// Input is the input yaml code or file path or any input specification. // Input is the input yaml code or file path or any input specification.
Input string `arg:"positional,required"` Input string `arg:"positional,required"`
} }
// PuppetArgs is the puppet CLI parsing structure and type of the parsed result.
type PuppetArgs struct {
// Input is the input puppet code or file path or just "agent".
Input string `arg:"positional,required"`
// PuppetConf is the optional path to a puppet.conf config file.
PuppetConf string `arg:"--puppet-conf" help:"full path to the puppet.conf file to use"`
}
// LangPuppetArgs is the langpuppet CLI parsing structure and type of the parsed
// result.
type LangPuppetArgs struct {
// LangInput is the input mcl code or file path or any input specification.
LangInput string `arg:"--lang,required" help:"the input parameter for the lang module"`
// PuppetInput is the input puppet code or file path or just "agent".
PuppetInput string `arg:"--puppet,required" help:"the input parameter for the puppet module"`
// copy-pasted from PuppetArgs
// PuppetConf is the optional path to a puppet.conf config file.
PuppetConf string `arg:"--puppet-conf" help:"full path to the puppet.conf file to use"`
// end PuppetArgs
// copy-pasted from LangArgs
// TODO: removed (temporarily?)
//Stdin bool `arg:"--stdin" help:"use passthrough stdin"`
Download bool `arg:"--download" help:"download any missing imports"`
OnlyDownload bool `arg:"--only-download" help:"stop after downloading any missing imports"`
Update bool `arg:"--update" help:"update all dependencies to the latest versions"`
OnlyUnify bool `arg:"--only-unify" help:"stop after type unification"`
SkipUnify bool `arg:"--skip-unify" help:"skip type unification"`
Depth int `arg:"--depth" default:"-1" help:"max recursion depth limit (-1 is unlimited)"`
// The default of 0 means any error is a failure by default.
Retry int `arg:"--depth" help:"max number of retries (-1 is unlimited)"`
ModulePath string `arg:"--module-path,env:MGMT_MODULE_PATH" help:"choose the modules path (absolute)"`
// end LangArgs
}

29
docs/documentation.md
View File

@@ -131,6 +131,33 @@ execute via a `remote` resource.
You can read the introductory blog post about this topic here: You can read the introductory blog post about this topic here:
[https://purpleidea.com/blog/2016/10/07/remote-execution-in-mgmt/](https://purpleidea.com/blog/2016/10/07/remote-execution-in-mgmt/) [https://purpleidea.com/blog/2016/10/07/remote-execution-in-mgmt/](https://purpleidea.com/blog/2016/10/07/remote-execution-in-mgmt/)
### Puppet support
You can supply a puppet manifest instead of creating the (YAML) graph manually.
Puppet must be installed and in `mgmt`'s search path. You also need the
[ffrank-mgmtgraph puppet module](https://forge.puppet.com/ffrank/mgmtgraph).
Invoke `mgmt` with the `--puppet` switch, which supports 3 variants:
1. Request the configuration from the puppet server (like `puppet agent` does)
`mgmt run puppet --puppet agent`
2. Compile a local manifest file (like `puppet apply`)
`mgmt run puppet --puppet /path/to/my/manifest.pp`
3. Compile an ad hoc manifest from the commandline (like `puppet apply -e`)
`mgmt run puppet --puppet 'file { "/etc/ntp.conf": ensure => file }'`
For more details and caveats see [puppet-guide.md](puppet-guide.md).
#### Blog post
An introductory post on the puppet support is on
[Felix's blog](http://ffrank.github.io/features/2016/06/19/puppet-powered-mgmt/).
## Reference ## Reference
Please note that there are a number of undocumented options. For more Please note that there are a number of undocumented options. For more
@@ -335,7 +362,7 @@ size of 42, you can expect a semaphore if named: `:42`. It is expected that
consumers of the semaphore metaparameter always include a prefix to avoid a consumers of the semaphore metaparameter always include a prefix to avoid a
collision with this globally defined semaphore. The size value must be greater collision with this globally defined semaphore. The size value must be greater
than zero at this time. The traditional non-parallel execution found in config than zero at this time. The traditional non-parallel execution found in config
management tools such as `Puppet` can be obtained with `--sema 1`. management tools such as `puppet` can be obtained with `--sema 1`.
#### `--ssh-priv-id-rsa` #### `--ssh-priv-id-rsa`

1
docs/index.rst
View File

@@ -14,3 +14,4 @@ Welcome to mgmt's documentation!
quick-start-guide quick-start-guide
resource-guide resource-guide
prometheus prometheus
puppet-guide

316
docs/puppet-guide.md Normal file
View File

@@ -0,0 +1,316 @@
# Puppet guide
`mgmt` can use Puppet as its source for the configuration graph.
This document goes into detail on how this works, and lists
some pitfalls and limitations.
For basic instructions on how to use the Puppet support, see
the [main documentation](documentation.md#puppet-support).
## Prerequisites
You need Puppet installed in your system. It is not important how you
get it. On the most common Linux distributions, you can use packages
from the OS maintainer, or upstream Puppet repositories. An alternative
that will also work on OSX is the `puppet` Ruby gem. It also has the
advantage that you can install any desired version in your home directory
or any other location.
Any release of Puppet's 3.x and 4.x series should be suitable for use with
`mgmt`. Most importantly, make sure to install the `ffrank-mgmtgraph` Puppet
module (referred to below as "the translator module").
```
puppet module install ffrank-mgmtgraph
```
Please note that the module is not required on your Puppet master (if you
use a master/agent setup). It's needed on the machine that runs `mgmt`.
You can install the module on the master anyway, so that it gets distributed
to your agents through Puppet's `pluginsync` mechanism.
### Testing the Puppet side
The following command should run successfully and print a YAML hash on your
terminal:
```puppet
puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": ensure => present }'
```
You can use this CLI to test any manifests before handing them straight
to `mgmt`.
## Writing a suitable manifest
### Unsupported attributes
`mgmt` inherited its resource module from Puppet, so by and large, it's quite
possible to express `mgmt` graphs in terms of Puppet manifests. However,
there isn't (and likely never will be) full feature parity between the
respective resource types. In consequence, a manifest can have semantics that
cannot be transferred to `mgmt`.
For example, at the time of writing this, the `file` type in `mgmt` had no
notion of permissions (the file `mode`) yet. This lead to the following
warning (among others that will be discussed below):
```
$ puppet mgmtgraph print --code 'file { "/tmp/foo": mode => "0600" }'
Warning: cannot translate: File[/tmp/foo] { mode => "600" } (attribute is ignored)
```
This is a heads-up for the user, because the resulting `mgmt` graph will
in fact not pass this information to the `/tmp/foo` file resource, and
`mgmt` will ignore this file's permissions. Including such attributes in
manifests that are written expressly for `mgmt` is not sensible and should
be avoided.
### Unsupported resources
Puppet has a fairly large number of
[built-in types](https://docs.puppet.com/puppet/latest/reference/type.html),
and countless more are available through
[modules](https://forge.puppet.com/). It's unlikely that all of them will
eventually receive native counterparts in `mgmt`.
When encountering an unknown resource, the translator module will replace
it with an `exec` resource in its output. This resource will run the equivalent
of a `puppet resource` command to make Puppet apply the original resource
itself. This has quite abysmal performance, because processing such a
resource requires the forking of at least one Puppet process (two if it
is found to be out of sync). This comes with considerable overhead.
On most systems, starting up any Puppet command takes several seconds.
Compared to the split second that the actual work usually takes,
this overhead can amount to several orders of magnitude.
Avoid Puppet types that `mgmt` does not implement (yet).
### Avoiding common warnings
Many resource parameters in Puppet take default values. For the most part,
the translator module just ignores them. However, there are cases in which
Puppet will default to convenient behavior that `mgmt` cannot quite replicate.
For example, translating a plain `file` resource will lead to a warning message:
```
$ puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": }'
Warning: File[/tmp/mgmt-test] uses the 'puppet' file bucket, which mgmt cannot
do. There will be no backup copies!
```
The reason is that per default, Puppet assumes the following parameter value
(among others)
```puppet
file { "/tmp/mgmt-test":
backup => 'puppet',
}
```
To avoid this, specify the parameter explicitly:
```bash
puppet mgmtgraph print --code 'file { "/tmp/mgmt-test": backup => false }'
```
This is tedious in a more complex manifest. A good simplification is the
following [resource default](https://docs.puppet.com/puppet/latest/reference/lang_defaults.html)
anywhere on the top scope of your manifest:
```puppet
File { backup => false }
```
If you encounter similar warnings from other types and/or parameters,
use the same approach to silence them if possible.
## Configuring Puppet
Since `mgmt` uses an actual Puppet CLI behind the scenes, you might
need to tweak some of Puppet's runtime options in order to make it
do what you want. Reasons for this could be among the following:
* You use the `--puppet agent` variant and need to configure
`servername`, `certname` and other master/agent-related options.
* You don't want runtime information to end up in the `vardir`
that is used by your regular `puppet agent`.
* You install specific Puppet modules for `mgmt` in a non-standard
location.
`mgmt` exposes only one Puppet option in order to allow you to
control all of them, through its `--puppet-conf` option. It allows
you to specify which `puppet.conf` file should be used during
translation.
```
mgmt run puppet --puppet /opt/my-manifest.pp --puppet-conf /etc/mgmt/puppet.conf
```
Within this file, you can just specify any needed options in the
`[main]` section:
```
[main]
server=mgmt-master.example.net
vardir=/var/lib/mgmt/puppet
```
## Caveats
Please see the [README](https://github.com/ffrank/puppet-mgmtgraph/blob/master/README.md)
of the translator module for the current state of supported and unsupported
language features.
You should probably make sure to always use the latest release of
both `ffrank-mgmtgraph` and `ffrank-yamlresource` (the latter is
getting pulled in as a dependency of the former).
## Using Puppet in conjunction with the mcl lang
The graph that Puppet generates for `mgmt` can be united with a graph
that is created from native `mgmt` code in its mcl language. This is
useful when you are in the process of replacing Puppet with mgmt. You
can translate your custom modules into mgmt's language one by one,
and let mgmt run the current mix.
Instead of the usual `--puppet-conf` flag and argv for `puppet` and `mcl` input,
you need to use alternative flags to make this work:
* `--lp-lang` to specify the mcl input
* `--lp-puppet` to specify the puppet input
* `--lp-puppet-conf` to point to the optional puppet.conf file
`mgmt` will derive a graph that contains all edges and vertices from
both inputs. You essentially get two unrelated subgraphs that run in
parallel. To form edges between these subgraphs, you have to define
special vertices that will be merged. This works through a hard-coded
naming scheme.
### Mixed graph example 1 - No merges
```mcl
# lang
file "/tmp/mgmt_dir/" { state => "present" }
file "/tmp/mgmt_dir/a" { state => "present" }
```
```puppet
# puppet
file { "/tmp/puppet_dir": ensure => "directory" }
file { "/tmp/puppet_dir/a": ensure => "file" }
```
These very simple inputs (including implicit edges from directory to
respective file) result in two subgraphs that do not relate.
```
File[/tmp/mgmt_dir/] -> File[/tmp/mgmt_dir/a]
File[/tmp/puppet_dir] -> File[/tmp/puppet_dir/a]
```
### Mixed graph example 2 - Merged vertex
In order to have merged vertices in the resulting graph, you will
need to include special resources and classes in the respective
input code.
* On the lang side, add `noop` resources with names starting in `puppet_`.
* On the Puppet side, add **empty** classes with names starting in `mgmt_`.
```mcl
# lang
noop "puppet_handover_to_mgmt" {}
file "/tmp/mgmt_dir/" { state => "present" }
file "/tmp/mgmt_dir/a" { state => "present" }
Noop["puppet_handover_to_mgmt"] -> File["/tmp/mgmt_dir/"]
```
```puppet
# puppet
class mgmt_handover_to_mgmt {}
include mgmt_handover_to_mgmt
file { "/tmp/puppet_dir": ensure => "directory" }
file { "/tmp/puppet_dir/a": ensure => "file" }
File["/tmp/puppet_dir/a"] -> Class["mgmt_handover_to_mgmt"]
```
The new `noop` resource is merged with the new class, resulting in
the following graph:
```
File[/tmp/puppet_dir] -> File[/tmp/puppet_dir/a]
|
V
Noop[handover_to_mgmt]
|
V
File[/tmp/mgmt_dir/] -> File[/tmp/mgmt_dir/a]
```
You put all your ducks in a row, and the resources from the Puppet input
run before those from the mcl input.
**Note:** The names of the `noop` and the class must be identical after the
respective prefix. The common part (here, `handover_to_mgmt`) becomes the name
of the merged resource.
## Mixed graph example 3 - Multiple merges
In most scenarios, it will not be possible to define a single handover
point like in the previous example. For example, if some Puppet resources
need to run in between two stages of native resources, you need at least
two merged vertices:
```mcl
# lang
noop "puppet_handover" {}
noop "puppet_handback" {}
file "/tmp/mgmt_dir/" { state => "present" }
file "/tmp/mgmt_dir/a" { state => "present" }
file "/tmp/mgmt_dir/puppet_subtree/state-file" { state => "present" }
File["/tmp/mgmt_dir/"] -> Noop["puppet_handover"]
Noop["puppet_handback"] -> File["/tmp/mgmt_dir/puppet_subtree/state-file"]
```
```puppet
# puppet
class mgmt_handover {}
class mgmt_handback {}
include mgmt_handover, mgmt_handback
class important_stuff {
file { "/tmp/mgmt_dir/puppet_subtree":
ensure => "directory"
}
# ...
}
Class["mgmt_handover"] -> Class["important_stuff"] -> Class["mgmt_handback"]
```
The resulting graph looks roughly like this:
```
File[/tmp/mgmt_dir/] -> File[/tmp/mgmt_dir/a]
|
V
Noop[handover] -> ( class important_stuff resources )
|
V
Noop[handback]
|
V
File[/tmp/mgmt_dir/puppet_subtree/state-file]
```
You can add arbitrary numbers of merge pairs to your code bases,
with relationships as needed. From our limited experience, code
readability suffers quite a lot from these, however. We advise
to keep these structures simple.

7
docs/resource-guide.md
View File

@@ -60,7 +60,10 @@ it is not specified, but others cannot, and some might poorly infer if the
struct name is ambiguous. struct name is ambiguous.
If you'd like your resource to be accessible by the `YAML` graph API (GAPI), If you'd like your resource to be accessible by the `YAML` graph API (GAPI),
then you'll need to include the appropriate YAML fields as shown below. then you'll need to include the appropriate YAML fields as shown below. This is
used by the `puppet` compiler as well, so make sure you include these struct
tags if you want existing `puppet` code to be able to run using the `mgmt`
engine.
#### Example #### Example
@@ -620,7 +623,7 @@ func init() { // special golang method that runs once
To support YAML unmarshalling for your resource, you must implement an To support YAML unmarshalling for your resource, you must implement an
additional method. It is recommended if you want to use your resource with the additional method. It is recommended if you want to use your resource with the
`yaml` compiler. `puppet` compiler.
```golang ```golang
UnmarshalYAML(unmarshal func(interface{}) error) error // optional UnmarshalYAML(unmarshal func(interface{}) error) error // optional

9
examples/puppet/langpuppet/graph1.mcl Normal file
View File

@@ -0,0 +1,9 @@
noop "puppet_first_handover" {}
noop "puppet_second_handover" {}
print "first message" {}
print "third message" {}
Print["first message"] -> Noop["puppet_first_handover"]
Noop["puppet_second_handover"] -> Print["third message"]

10
examples/puppet/langpuppet/graph1.pp Normal file
View File

@@ -0,0 +1,10 @@
class mgmt_first_handover {}
class mgmt_second_handover {}
include mgmt_first_handover, mgmt_second_handover
Class["mgmt_first_handover"]
->
notify { "second message": }
->
Class["mgmt_second_handover"]

2
gapi/deploy.go
View File

@@ -47,7 +47,7 @@ func init() {
// TODO: add proper authentication with gpg key signing // TODO: add proper authentication with gpg key signing
type Deploy struct { type Deploy struct {
ID uint64 ID uint64
Name string // lang, yaml, etc... Name string // lang, puppet, yaml, etc...
//Sync bool // wait for everyone to close previous GAPI before switching //Sync bool // wait for everyone to close previous GAPI before switching
Noop bool Noop bool
Sema int // sema override Sema int // sema override

2
lib/main.go
View File

@@ -761,7 +761,7 @@ func (obj *Main) Run() error {
} }
var timing time.Time var timing time.Time
// make the graph from yaml, lib, or dsl! // make the graph from yaml, lib, puppet->yaml, or dsl!
timing = time.Now() timing = time.Now()
newGraph, err := gapiImpl.Graph() // generate graph! newGraph, err := gapiImpl.Graph() // generate graph!
if err != nil { if err != nil {

8
main.go
View File

@@ -39,9 +39,11 @@ import (
"github.com/purpleidea/mgmt/cli" "github.com/purpleidea/mgmt/cli"
cliUtil "github.com/purpleidea/mgmt/cli/util" cliUtil "github.com/purpleidea/mgmt/cli/util"
"github.com/purpleidea/mgmt/entry" "github.com/purpleidea/mgmt/entry"
_ "github.com/purpleidea/mgmt/gapi/empty" // import so the gapi registers _ "github.com/purpleidea/mgmt/gapi/empty" // import so the gapi registers
_ "github.com/purpleidea/mgmt/lang/gapi" // import so the gapi registers _ "github.com/purpleidea/mgmt/lang/gapi" // import so the gapi registers
_ "github.com/purpleidea/mgmt/yamlgraph" // import so the gapi registers _ "github.com/purpleidea/mgmt/puppet" // import so the gapi registers
_ "github.com/purpleidea/mgmt/puppet/langpuppet" // import so the gapi registers
_ "github.com/purpleidea/mgmt/yamlgraph" // import so the gapi registers
"go.etcd.io/etcd/server/v3/etcdmain" "go.etcd.io/etcd/server/v3/etcdmain"
) )

347
puppet/gapi.go Normal file
View File

@@ -0,0 +1,347 @@
// Mgmt
// Copyright (C) 2013-2024+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//
// Additional permission under GNU GPL version 3 section 7
//
// If you modify this program, or any covered work, by linking or combining it
// with embedded mcl code and modules (and that the embedded mcl code and
// modules which link with this program, contain a copy of their source code in
// the authoritative form) containing parts covered by the terms of any other
// license, the licensors of this program grant you additional permission to
// convey the resulting work. Furthermore, the licensors of this program grant
// the original author, James Shubin, additional permission to update this
// additional permission if he deems it necessary to achieve the goals of this
// additional permission.
package puppet
import (
"fmt"
"os"
"strings"
"sync"
"time"
cliUtil "github.com/purpleidea/mgmt/cli/util"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/gapi"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/util"
"github.com/purpleidea/mgmt/util/errwrap"
)
const (
// Name is the name of this frontend.
Name = "puppet"
// PuppetFile is the entry point filename that we use. It is arbitrary.
PuppetFile = "/file.pp"
// PuppetConf is the entry point config filename that we use.
PuppetConf = "/puppet.conf"
// PuppetSite is the entry point folder that we use. It is arbitrary.
PuppetSite = "/puppet/"
)
func init() {
gapi.Register(Name, func() gapi.GAPI { return &GAPI{} }) // register
}
// GAPI implements the main puppet GAPI interface.
type GAPI struct {
InputURI string
// Mode of operation can be: agent, file, string, dir.
Mode string
puppetFile string
puppetString string
puppetDir string
// The path to a dedicated puppet.conf file for mgmt.
puppetConf string
data *gapi.Data
initialized bool
closeChan chan struct{}
wg sync.WaitGroup
}
// Cli takes an *Info struct, and returns our deploy if activated, and if there
// are any validation problems, you should return an error. If there is no
// deploy, then you should return a nil deploy and a nil error.
func (obj *GAPI) Cli(info *gapi.Info) (*gapi.Deploy, error) {
args, ok := info.Args.(*cliUtil.PuppetArgs)
if !ok {
// programming error
return nil, fmt.Errorf("could not convert to our struct")
}
fs := info.Fs
writeableFS, ok := fs.(engine.WriteableFS)
if !ok {
return nil, fmt.Errorf("the FS was not writeable")
}
isDir := func(p string) (bool, error) {
if !strings.HasPrefix(p, "/") {
return false, nil
}
if !strings.HasSuffix(p, "/") {
return false, nil
}
fi, err := os.Stat(p)
if err != nil {
return false, err
}
return fi.IsDir(), nil
}
var mode string
if args.Input == "agent" {
mode = "agent"
} else if strings.HasSuffix(args.Input, ".pp") {
mode = "file"
if err := gapi.CopyFileToFs(writeableFS, args.Input, PuppetFile); err != nil {
return nil, errwrap.Wrapf(err, "can't copy code from `%s` to `%s`", args.Input, PuppetFile)
}
} else if exists, err := isDir(args.Input); err != nil {
return nil, errwrap.Wrapf(err, "can't read dir `%s`", args.Input)
} else if err == nil && exists { // from the isDir result...
// we have a whole directory of files to run
mode = "dir"
// TODO: this code path is untested! test and then rm this notice
if err := gapi.CopyDirToFs(fs, args.Input, PuppetSite); err != nil {
return nil, errwrap.Wrapf(err, "can't copy code to `%s`", PuppetSite)
}
} else {
mode = "string"
if err := gapi.CopyStringToFs(writeableFS, args.Input, PuppetFile); err != nil {
return nil, errwrap.Wrapf(err, "can't copy code to `%s`", PuppetFile)
}
}
// TODO: do we want to include this if we have mode == "dir" ?
if args.PuppetConf != "" {
if err := gapi.CopyFileToFs(writeableFS, args.PuppetConf, PuppetConf); err != nil {
return nil, errwrap.Wrapf(err, "can't copy puppet conf from `%s` to '%s'", args.PuppetConf, PuppetConf)
}
}
return &gapi.Deploy{
Name: Name,
Noop: info.Flags.Noop,
Sema: info.Flags.Sema,
GAPI: &GAPI{
InputURI: fs.URI(),
Mode: mode,
},
}, nil
}
// Init initializes the puppet GAPI struct.
func (obj *GAPI) Init(data *gapi.Data) error {
if obj.initialized {
return fmt.Errorf("already initialized")
}
if obj.InputURI == "" {
return fmt.Errorf("the InputURI param must be specified")
}
switch obj.Mode {
case "agent", "file", "string", "dir":
// pass
default:
return fmt.Errorf("the Mode param is invalid")
}
obj.data = data // store for later
fs, err := obj.data.World.Fs(obj.InputURI) // open the remote file system
if err != nil {
return errwrap.Wrapf(err, "can't load data from file system `%s`", obj.InputURI)
}
if obj.Mode == "file" {
b, err := fs.ReadFile(PuppetFile) // read the single file out of it
if err != nil {
return errwrap.Wrapf(err, "can't read code from file `%s`", PuppetFile)
}
// store the puppet file on disk for other binaries to see and use
prefix := fmt.Sprintf("%s-%s-%s", data.Program, data.Hostname, strings.Replace(PuppetFile, "/", "", -1))
tmpfile, err := os.CreateTemp("", prefix)
if err != nil {
return errwrap.Wrapf(err, "can't create temp file")
}
obj.puppetFile = tmpfile.Name() // path to temp file
defer tmpfile.Close()
if _, err := tmpfile.Write(b); err != nil {
return errwrap.Wrapf(err, "can't write file")
}
} else if obj.Mode == "string" {
b, err := fs.ReadFile(PuppetFile) // read the single code string out of it
if err != nil {
return errwrap.Wrapf(err, "can't read code from file `%s`", PuppetFile)
}
obj.puppetString = string(b)
} else if obj.Mode == "dir" {
// store the puppet files on disk for other binaries to see and use
prefix := fmt.Sprintf("%s-%s-%s", data.Program, data.Hostname, strings.Replace(PuppetSite, "/", "", -1))
tmpdirName, err := os.MkdirTemp("", prefix)
if err != nil {
return errwrap.Wrapf(err, "can't create temp dir")
}
if tmpdirName == "" || tmpdirName == "/" {
return fmt.Errorf("bad tmpdir created")
}
obj.puppetDir = tmpdirName // path to temp dir
// TODO: this code path is untested! test and then rm this notice
if err := util.CopyFsToDisk(fs, PuppetSite, tmpdirName, false); err != nil {
return errwrap.Wrapf(err, "can't copy dir")
}
}
if fi, err := fs.Stat(PuppetConf); err == nil && !fi.IsDir() { // if exists?
b, err := fs.ReadFile(PuppetConf) // read the single file out of it
if err != nil {
return errwrap.Wrapf(err, "can't read config from file `%s`", PuppetConf)
}
// store the puppet conf on disk for other binaries to see and use
prefix := fmt.Sprintf("%s-%s-%s", data.Program, data.Hostname, strings.Replace(PuppetConf, "/", "", -1))
tmpfile, err := os.CreateTemp("", prefix)
if err != nil {
return errwrap.Wrapf(err, "can't create temp file")
}
obj.puppetConf = tmpfile.Name() // path to temp file
defer tmpfile.Close()
if _, err := tmpfile.Write(b); err != nil {
return errwrap.Wrapf(err, "can't write file")
}
}
obj.closeChan = make(chan struct{})
obj.initialized = true
return nil
}
// Graph returns a current Graph.
func (obj *GAPI) Graph() (*pgraph.Graph, error) {
if !obj.initialized {
return nil, fmt.Errorf("%s: GAPI is not initialized", Name)
}
config, err := obj.ParseConfigFromPuppet()
if err != nil {
return nil, err
}
if config == nil {
return nil, fmt.Errorf("function ParseConfigFromPuppet returned nil")
}
g, err := config.NewGraphFromConfig(obj.data.Hostname, obj.data.World, obj.data.Noop)
return g, err
}
// Next returns nil errors every time there could be a new graph.
func (obj *GAPI) Next() chan gapi.Next {
puppetChan := func() <-chan time.Time { // helper function
return time.Tick(time.Duration(obj.refreshInterval()) * time.Second)
}
ch := make(chan gapi.Next)
obj.wg.Add(1)
go func() {
defer obj.wg.Done()
defer close(ch) // this will run before the obj.wg.Done()
if !obj.initialized {
next := gapi.Next{
Err: fmt.Errorf("%s: GAPI is not initialized", Name),
Exit: true, // exit, b/c programming error?
}
ch <- next
return
}
startChan := make(chan struct{}) // start signal
close(startChan) // kick it off!
var pChan <-chan time.Time
if obj.data.NoStreamWatch {
pChan = nil
} else {
pChan = puppetChan()
}
for {
select {
case <-startChan: // kick the loop once at start
startChan = nil // disable
// pass
case _, ok := <-pChan:
if !ok { // the channel closed!
return
}
case <-obj.closeChan:
return
}
obj.data.Logf("generating new graph...")
if obj.data.NoStreamWatch {
pChan = nil
} else {
pChan = puppetChan() // TODO: okay to update interval in case it changed?
}
next := gapi.Next{
//Exit: true, // TODO: for permanent shutdown!
Err: nil,
}
select {
case ch <- next: // trigger a run (send a msg)
// unblock if we exit while waiting to send!
case <-obj.closeChan:
return
}
}
}()
return ch
}
// Close shuts down the Puppet GAPI.
func (obj *GAPI) Close() error {
if !obj.initialized {
return fmt.Errorf("%s: GAPI is not initialized", Name)
}
if obj.puppetFile != "" {
os.Remove(obj.puppetFile) // clean up, don't bother with error
}
// make this as safe as possible, check we're removing a tempdir too!
if obj.puppetDir != "" && obj.puppetDir != "/" && strings.HasPrefix(obj.puppetDir, os.TempDir()) {
os.RemoveAll(obj.puppetDir)
}
obj.puppetString = "" // free!
if obj.puppetConf != "" {
os.Remove(obj.puppetConf)
}
close(obj.closeChan)
obj.wg.Wait()
obj.initialized = false // closed = true
return nil
}

328
puppet/langpuppet/gapi.go Normal file
View File

@@ -0,0 +1,328 @@
// Mgmt
// Copyright (C) 2013-2024+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//
// Additional permission under GNU GPL version 3 section 7
//
// If you modify this program, or any covered work, by linking or combining it
// with embedded mcl code and modules (and that the embedded mcl code and
// modules which link with this program, contain a copy of their source code in
// the authoritative form) containing parts covered by the terms of any other
// license, the licensors of this program grant you additional permission to
// convey the resulting work. Furthermore, the licensors of this program grant
// the original author, James Shubin, additional permission to update this
// additional permission if he deems it necessary to achieve the goals of this
// additional permission.
package langpuppet
import (
"fmt"
"sync"
cliUtil "github.com/purpleidea/mgmt/cli/util"
"github.com/purpleidea/mgmt/gapi"
lang "github.com/purpleidea/mgmt/lang/gapi"
"github.com/purpleidea/mgmt/pgraph"
"github.com/purpleidea/mgmt/puppet"
"github.com/purpleidea/mgmt/util/errwrap"
)
const (
// Name is the name of this frontend.
Name = "langpuppet"
)
func init() {
gapi.Register(Name, func() gapi.GAPI { return &GAPI{} }) // register
}
// GAPI implements the main langpuppet GAPI interface. It wraps the Puppet and
// Lang GAPIs and receives graphs from both. It then runs a merging algorithm
// that mainly just makes a union of both the sets of vertices and edges. Some
// vertices are merged using a naming convention. Details can be found in the
// langpuppet.mergeGraphs function.
type GAPI struct {
// the wrapped lang entrypoint
langGAPI gapi.GAPI
// the wrapped puppet entrypoint
puppetGAPI gapi.GAPI
// the most recent graph received from lang
currentLangGraph *pgraph.Graph
// the most recent graph received from puppet
currentPuppetGraph *pgraph.Graph
// flag to indicate that a new graph from lang is ready
langGraphReady bool
// flag to indicate that a new graph from puppet is ready
puppetGraphReady bool
graphFlagMutex *sync.Mutex
data *gapi.Data
initialized bool
closeChan chan struct{}
wg sync.WaitGroup // sync group for tunnel go routines
}
// Cli takes an *Info struct, and returns our deploy if activated, and if there
// are any validation problems, you should return an error. If there is no
// deploy, then you should return a nil deploy and a nil error.
func (obj *GAPI) Cli(info *gapi.Info) (*gapi.Deploy, error) {
args, ok := info.Args.(*cliUtil.LangPuppetArgs)
if !ok {
// programming error
return nil, fmt.Errorf("could not convert to our struct")
}
flags := info.Flags
fs := info.Fs
debug := info.Debug
logf := info.Logf
langInfo := &gapi.Info{
Args: &cliUtil.LangArgs{
Input: args.LangInput,
Download: args.Download,
OnlyDownload: args.OnlyDownload,
Update: args.Update,
OnlyUnify: args.OnlyUnify,
SkipUnify: args.SkipUnify,
Depth: args.Depth,
Retry: args.Retry,
ModulePath: args.ModulePath,
},
Flags: flags,
Fs: fs,
Debug: debug,
Logf: logf, // TODO: wrap logf?
}
puppetInfo := &gapi.Info{
Args: &cliUtil.PuppetArgs{
Input: args.PuppetInput,
PuppetConf: args.PuppetConf,
},
Flags: flags,
Fs: fs,
Debug: debug,
Logf: logf, // TODO: wrap logf?
}
var langDeploy *gapi.Deploy
var puppetDeploy *gapi.Deploy
var err error
if langDeploy, err = (&lang.GAPI{}).Cli(langInfo); err != nil {
return nil, err
}
if puppetDeploy, err = (&puppet.GAPI{}).Cli(puppetInfo); err != nil {
return nil, err
}
return &gapi.Deploy{
Name: Name,
Noop: info.Flags.Noop,
Sema: info.Flags.Sema,
GAPI: &GAPI{
langGAPI: langDeploy.GAPI,
puppetGAPI: puppetDeploy.GAPI,
},
}, nil
}
// Init initializes the langpuppet GAPI struct.
func (obj *GAPI) Init(data *gapi.Data) error {
if obj.initialized {
return fmt.Errorf("already initialized")
}
obj.data = data // store for later
obj.graphFlagMutex = &sync.Mutex{}
dataLang := &gapi.Data{
Program: obj.data.Program,
Version: obj.data.Version,
Hostname: obj.data.Hostname,
World: obj.data.World,
Noop: obj.data.Noop,
NoStreamWatch: obj.data.NoStreamWatch,
Debug: obj.data.Debug,
Logf: func(format string, v ...interface{}) {
obj.data.Logf(lang.Name+": "+format, v...)
},
}
dataPuppet := &gapi.Data{
Program: obj.data.Program,
Version: obj.data.Version,
Hostname: obj.data.Hostname,
World: obj.data.World,
Noop: obj.data.Noop,
NoStreamWatch: obj.data.NoStreamWatch,
Debug: obj.data.Debug,
Logf: func(format string, v ...interface{}) {
obj.data.Logf(puppet.Name+": "+format, v...)
},
}
if err := obj.langGAPI.Init(dataLang); err != nil {
return err
}
if err := obj.puppetGAPI.Init(dataPuppet); err != nil {
return err
}
obj.closeChan = make(chan struct{})
obj.initialized = true
return nil
}
// Graph returns a current Graph.
func (obj *GAPI) Graph() (*pgraph.Graph, error) {
if !obj.initialized {
return nil, fmt.Errorf("%s: GAPI is not initialized", Name)
}
var err error
obj.graphFlagMutex.Lock()
if obj.langGraphReady {
obj.langGraphReady = false
obj.graphFlagMutex.Unlock()
obj.currentLangGraph, err = obj.langGAPI.Graph()
if err != nil {
return nil, err
}
} else {
obj.graphFlagMutex.Unlock()
}
obj.graphFlagMutex.Lock()
if obj.puppetGraphReady {
obj.puppetGraphReady = false
obj.graphFlagMutex.Unlock()
obj.currentPuppetGraph, err = obj.puppetGAPI.Graph()
if err != nil {
return nil, err
}
} else {
obj.graphFlagMutex.Unlock()
}
g, err := mergeGraphs(obj.currentLangGraph, obj.currentPuppetGraph)
if obj.data.Debug {
obj.currentLangGraph.Logf(func(format string, v ...interface{}) {
obj.data.Logf("graph: "+lang.Name+": "+format, v...)
})
obj.currentPuppetGraph.Logf(func(format string, v ...interface{}) {
obj.data.Logf("graph: "+puppet.Name+": "+format, v...)
})
if err == nil {
g.Logf(func(format string, v ...interface{}) {
obj.data.Logf("graph: "+Name+": "+format, v...)
})
}
}
return g, err
}
// Next returns nil errors every time there could be a new graph.
func (obj *GAPI) Next() chan gapi.Next {
ch := make(chan gapi.Next)
obj.wg.Add(1)
go func() {
defer obj.wg.Done()
defer close(ch) // this will run before the obj.wg.Done()
if !obj.initialized {
next := gapi.Next{
Err: fmt.Errorf("%s: GAPI is not initialized", Name),
Exit: true, // exit, b/c programming error?
}
ch <- next
return
}
nextLang := obj.langGAPI.Next()
nextPuppet := obj.puppetGAPI.Next()
firstLang := false
firstPuppet := false
for {
var err error
exit := false
select {
case nextChild := <-nextLang:
if nextChild.Err != nil {
err = nextChild.Err
exit = nextChild.Exit
} else {
obj.graphFlagMutex.Lock()
obj.langGraphReady = true
obj.graphFlagMutex.Unlock()
firstLang = true
}
case nextChild := <-nextPuppet:
if nextChild.Err != nil {
err = nextChild.Err
exit = nextChild.Exit
} else {
obj.graphFlagMutex.Lock()
obj.puppetGraphReady = true
obj.graphFlagMutex.Unlock()
firstPuppet = true
}
case <-obj.closeChan:
return
}
if (!firstLang || !firstPuppet) && err == nil {
continue
}
if err == nil {
obj.data.Logf("generating new composite graph...")
}
next := gapi.Next{
Exit: exit,
Err: err,
}
select {
case ch <- next: // trigger a run (send a msg)
// unblock if we exit while waiting to send!
case <-obj.closeChan:
return
}
}
}()
return ch
}
// Close shuts down the Puppet GAPI.
func (obj *GAPI) Close() error {
if !obj.initialized {
return fmt.Errorf("%s: GAPI is not initialized", Name)
}
var err error
e1 := obj.langGAPI.Close()
err = errwrap.Append(err, errwrap.Wrapf(e1, "closing lang GAPI failed"))
e2 := obj.puppetGAPI.Close()
err = errwrap.Append(err, errwrap.Wrapf(e2, "closing Puppet GAPI failed"))
close(obj.closeChan)
obj.initialized = false // closed = true
return err
}

180
puppet/langpuppet/merge.go Normal file
View File

@@ -0,0 +1,180 @@
// Mgmt
// Copyright (C) 2013-2024+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//
// Additional permission under GNU GPL version 3 section 7
//
// If you modify this program, or any covered work, by linking or combining it
// with embedded mcl code and modules (and that the embedded mcl code and
// modules which link with this program, contain a copy of their source code in
// the authoritative form) containing parts covered by the terms of any other
// license, the licensors of this program grant you additional permission to
// convey the resulting work. Furthermore, the licensors of this program grant
// the original author, James Shubin, additional permission to update this
// additional permission if he deems it necessary to achieve the goals of this
// additional permission.
// Package langpuppet implements an integration entrypoint that combines lang
// and Puppet.
package langpuppet
import (
"fmt"
"strings"
"github.com/purpleidea/mgmt/engine"
"github.com/purpleidea/mgmt/pgraph"
)
const (
// MergePrefixLang is how a mergeable vertex name starts in mcl code.
MergePrefixLang = "puppet_"
// MergePrefixPuppet is how a mergeable Puppet class name starts.
MergePrefixPuppet = "mgmt_"
)
// mergeGraph returns the merged graph containing all vertices and edges found
// in the graphs produced by the lang and Puppet GAPIs associated with the
// wrapping GAPI. Vertices are merged if they adhere to the following rules (for
// any given value of POSTFIX): (1) The graph from lang contains a noop vertex
// named puppet_POSTFIX. (2) The graph from Puppet contains an empty class
// mgmt_POSTFIX. (3) The resulting graph will contain one noop vertex named
// POSTFIX that replaces all nodes mentioned in (1) and (2). All edges
// connecting to any of the vertices merged this way will be present in the
// merged graph.
func mergeGraphs(graphFromLang, graphFromPuppet *pgraph.Graph) (*pgraph.Graph, error) {
if graphFromLang == nil || graphFromPuppet == nil {
return nil, fmt.Errorf("cannot merge graphs until both child graphs are loaded")
}
result, err := pgraph.NewGraph(graphFromLang.Name + "+" + graphFromPuppet.Name)
if err != nil {
return nil, err
}
mergeTargets := make(map[string]pgraph.Vertex)
// first add all vertices from the lang graph
for _, vertex := range graphFromLang.Vertices() {
if strings.Index(vertex.String(), "noop["+MergePrefixLang) == 0 {
resource, ok := vertex.(engine.Res)
if !ok {
return nil, fmt.Errorf("vertex %s is not a named resource", vertex.String())
}
basename := strings.TrimPrefix(resource.Name(), MergePrefixLang)
resource.SetName(basename)
mergeTargets[basename] = vertex
}
result.AddVertex(vertex)
for _, neighbor := range graphFromLang.OutgoingGraphVertices(vertex) {
result.AddVertex(neighbor)
result.AddEdge(vertex, neighbor, graphFromLang.FindEdge(vertex, neighbor))
}
}
var anchor pgraph.Vertex
mergePairs := make(map[pgraph.Vertex]pgraph.Vertex)
// do a scan through the Puppet graph, and mark all vertices that will be
// subject to a merge, so it will be easier do generate the new edges
// in the final pass
for _, vertex := range graphFromPuppet.Vertices() {
if vertex.String() == "noop[admissible_Stage[main]]" {
// we can start a depth first search here
anchor = vertex
continue
}
// at this stage we don't distinguis between class start and end
if strings.Index(vertex.String(), "noop[admissible_Class["+strings.Title(MergePrefixPuppet)) != 0 &&
strings.Index(vertex.String(), "noop[completed_Class["+strings.Title(MergePrefixPuppet)) != 0 {
continue
}
resource, ok := vertex.(engine.Res)
if !ok {
return nil, fmt.Errorf("vertex %s is not a named resource", vertex.String())
}
// strip either prefix (plus the closing bracket)
basename := strings.TrimSuffix(
strings.TrimPrefix(
strings.TrimPrefix(resource.Name(),
"admissible_Class["+strings.Title(MergePrefixPuppet)),
"completed_Class["+strings.Title(MergePrefixPuppet)),
"]")
if _, found := mergeTargets[basename]; !found {
// FIXME: should be a warning not an error?
return nil, fmt.Errorf("puppet graph has unmatched class %s%s", MergePrefixPuppet, basename)
}
mergePairs[vertex] = mergeTargets[basename]
if strings.Index(resource.Name(), "admissible_Class["+strings.Title(MergePrefixPuppet)) != 0 {
continue
}
// is there more than one edge outgoing from the class start?
if graphFromPuppet.OutDegree()[vertex] > 1 {
return nil, fmt.Errorf("class %s is not empty", basename)
}
// does this edge not lead to the class end?
next := graphFromPuppet.OutgoingGraphVertices(vertex)[0]
if next.String() != "noop[completed_Class["+strings.Title(MergePrefixPuppet)+basename+"]]" {
return nil, fmt.Errorf("class %s%s is not empty, start is followed by %s", MergePrefixPuppet, basename, next.String())
}
}
merged := make(map[pgraph.Vertex]bool)
result.AddVertex(anchor)
// traverse the puppet graph, add all vertices and perform merges
// using DFS so we can be sure the "admissible" is visited before the "completed" vertex
for _, vertex := range graphFromPuppet.DFS(anchor) {
source := vertex
// when adding edges, the source might be a different vertex
// than the current one, if this is a merged vertex
if _, found := mergePairs[vertex]; found {
source = mergePairs[vertex]
}
// the current vertex has been added by previous iterations,
// we only add neighbors here
for _, neighbor := range graphFromPuppet.OutgoingGraphVertices(vertex) {
if strings.Index(neighbor.String(), "noop[admissible_Class["+strings.Title(MergePrefixPuppet)) == 0 {
result.AddEdge(source, mergePairs[neighbor], graphFromPuppet.FindEdge(vertex, neighbor))
continue
}
if strings.Index(neighbor.String(), "noop[completed_Class["+strings.Title(MergePrefixPuppet)) == 0 {
// mark target vertex as merged
merged[mergePairs[neighbor]] = true
continue
}
// if we reach here, this neighbor is a regular vertex
result.AddVertex(neighbor)
result.AddEdge(source, neighbor, graphFromPuppet.FindEdge(vertex, neighbor))
}
}
for _, vertex := range mergeTargets {
if !merged[vertex] {
// FIXME: should be a warning not an error?
return nil, fmt.Errorf("lang graph has unmatched %s", vertex.String())
}
}
return result, nil
}

160
puppet/puppet.go Normal file
View File

@@ -0,0 +1,160 @@
// Mgmt
// Copyright (C) 2013-2024+ James Shubin and the project contributors
// Written by James Shubin <james@shubin.ca> and the project contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
//
// Additional permission under GNU GPL version 3 section 7
//
// If you modify this program, or any covered work, by linking or combining it
// with embedded mcl code and modules (and that the embedded mcl code and
// modules which link with this program, contain a copy of their source code in
// the authoritative form) containing parts covered by the terms of any other
// license, the licensors of this program grant you additional permission to
// convey the resulting work. Furthermore, the licensors of this program grant
// the original author, James Shubin, additional permission to update this
// additional permission if he deems it necessary to achieve the goals of this
// additional permission.
// Package puppet provides the integration entrypoint for the puppet language.
package puppet
import (
"bufio"
"fmt"
"io"
"os/exec"
"strconv"
"strings"
"github.com/purpleidea/mgmt/util/errwrap"
"github.com/purpleidea/mgmt/yamlgraph"
)
const (
// PuppetYAMLBufferSize is the maximum buffer size for the yaml input data
PuppetYAMLBufferSize = 65535
)
func (obj *GAPI) runPuppetCommand(cmd *exec.Cmd) ([]byte, error) {
if obj.data.Debug {
obj.data.Logf("running command: %v", cmd)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return nil, errwrap.Wrapf(err, "error opening pipe to puppet command")
}
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, errwrap.Wrapf(err, "error opening error pipe to puppet command")
}
if err := cmd.Start(); err != nil {
return nil, errwrap.Wrapf(err, "error starting puppet command")
}
// XXX: the current implementation is likely prone to fail
// as soon as the YAML data overflows the buffer.
data := make([]byte, PuppetYAMLBufferSize)
var result []byte
for err == nil {
var count int
count, err = stdout.Read(data)
if err != nil && err != io.EOF {
obj.data.Logf("error reading YAML data from puppet: %v", err)
return nil, err
}
// Slicing down to the number of actual bytes is important, the YAML parser
// will choke on an oversized slice. http://stackoverflow.com/a/33726617/3356612
result = append(result, data[0:count]...)
}
if obj.data.Debug {
obj.data.Logf("read %d bytes of data from puppet", len(result))
}
for scanner := bufio.NewScanner(stderr); scanner.Scan(); {
obj.data.Logf("(output) %v", scanner.Text())
}
if err := cmd.Wait(); err != nil {
return nil, errwrap.Wrapf(err, "error waiting for puppet command to complete")
}
return result, nil
}
// ParseConfigFromPuppet returns the graph configuration structure from the mode
// and input values, including possibly some file and directory paths.
func (obj *GAPI) ParseConfigFromPuppet() (*yamlgraph.GraphConfig, error) {
var args []string
switch obj.Mode {
case "agent":
args = []string{"mgmtgraph", "print"}
case "file":
args = []string{"mgmtgraph", "print", "--manifest", obj.puppetFile}
case "string":
args = []string{"mgmtgraph", "print", "--code", obj.puppetString}
case "dir":
// TODO: run the code from the obj.puppetDir directory path
return nil, fmt.Errorf("not implemented") // XXX: not implemented
default:
panic(fmt.Sprintf("%s: unhandled case: %s", Name, obj.Mode))
}
if obj.puppetConf != "" {
args = append(args, "--config="+obj.puppetConf)
}
cmd := exec.Command("puppet", args...)
obj.data.Logf("launching translator")
var config yamlgraph.GraphConfig
if data, err := obj.runPuppetCommand(cmd); err != nil {
return nil, errwrap.Wrapf(err, "could not run puppet command")
} else if err := config.Parse(data); err != nil {
return nil, errwrap.Wrapf(err, "could not parse YAML output")
}
return &config, nil
}
// refreshInterval returns the graph refresh interval from the puppet
// configuration.
func (obj *GAPI) refreshInterval() int {
if obj.data.Debug {
obj.data.Logf("determining graph refresh interval")
}
var cmd *exec.Cmd
if obj.puppetConf != "" {
cmd = exec.Command("puppet", "config", "print", "runinterval", "--config", obj.puppetConf)
} else {
cmd = exec.Command("puppet", "config", "print", "runinterval")
}
obj.data.Logf("inspecting runinterval configuration")
interval := 1800
data, err := obj.runPuppetCommand(cmd)
if err != nil {
obj.data.Logf("could not determine configured run interval (%v), using default of %v", err, interval)
return interval
}
result, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 0)
if err != nil {
obj.data.Logf("error reading numeric runinterval value (%v), using default of %v", err, interval)
return interval
}
return int(result)
}