diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c323e329..6de62974 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -37,9 +37,8 @@ jobs: #fail-fast: false steps: - # Do not shallow fetch, will fail when building bindata/ - # The path can't be absolute, so we need to move it to the - # expected location later. + # Do not shallow fetch. The path can't be absolute, so we need to move it + # to the expected location later. - name: Clone mgmt uses: actions/checkout@v2 with: diff --git a/Makefile b/Makefile index f9e1c7ad..f029e68e 100644 --- a/Makefile +++ b/Makefile @@ -16,12 +16,12 @@ # along with this program. If not, see . SHELL = /usr/bin/env bash -.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs +.PHONY: all art cleanart version program lang path deps run race generate build build-debug crossbuild clean test gofmt yamlfmt format docs .PHONY: rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag .PHONY: mkosi mkosi_fedora-30 mkosi_fedora-29 mkosi_centos-7 mkosi_debian-10 mkosi_ubuntu-bionic mkosi_archlinux .PHONY: release releases_path release_fedora-30 release_fedora-29 release_centos-7 release_debian-10 release_ubuntu-bionic release_archlinux .PHONY: funcgen -.SILENT: clean bindata +.SILENT: clean # a large amount of output from this `find`, can cause `make` to be much slower! GO_FILES := $(shell find * -name '*.go' -not -path 'old/*' -not -path 'tmp/*') @@ -137,10 +137,6 @@ run: ## run mgmt race: find . -maxdepth 1 -type f -name '*.go' -not -name '*_test.go' | xargs go run -race -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)" -# generate go files from non-go source -bindata: ## generate go files from non-go sources - $(MAKE) --quiet -C lang/funcs - generate: go generate @@ -167,7 +163,7 @@ build-debug: $(PROGRAM) # extract os and arch from target pattern GOOS=$(firstword $(subst -, ,$*)) GOARCH=$(lastword $(subst -, ,$*)) -build/mgmt-%: $(GO_FILES) $(MCL_FILES) | bindata lang funcgen +build/mgmt-%: $(GO_FILES) $(MCL_FILES) | lang funcgen @echo "Building: $(PROGRAM), os/arch: $*, version: $(SVERSION)..." @time env GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS) diff --git a/lang/ast/structs.go b/lang/ast/structs.go index eadcff11..b0ca510e 100644 --- a/lang/ast/structs.go +++ b/lang/ast/structs.go @@ -30,7 +30,7 @@ import ( "github.com/purpleidea/mgmt/engine" engineUtil "github.com/purpleidea/mgmt/engine/util" "github.com/purpleidea/mgmt/lang/funcs" - "github.com/purpleidea/mgmt/lang/funcs/bindata" + "github.com/purpleidea/mgmt/lang/funcs/core" "github.com/purpleidea/mgmt/lang/funcs/structs" "github.com/purpleidea/mgmt/lang/inputs" "github.com/purpleidea/mgmt/lang/interfaces" @@ -3057,7 +3057,7 @@ func (obj *StmtProg) importScope(info *interfaces.ImportData, scope *interfaces. // importSystemScope takes the name of a built-in system scope (eg: "fmt") and // returns the scope struct for that built-in. This function is slightly less // trivial than expected, because the scope is built from both native mcl code -// and golang code as well. The native mcl code is compiled in as bindata. +// and golang code as well. The native mcl code is compiled in with "embed". // TODO: can we memoize? func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) { // this basically loop through the registeredFuncs and includes @@ -3096,7 +3096,10 @@ func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) { // to the remote machines as well, and we want to load off of it... // now add any compiled-in mcl code - paths := bindata.AssetNames() + paths, err := core.AssetNames() + if err != nil { + return nil, errwrap.Wrapf(err, "can't read asset names") + } // results are not sorted by default (ascertained by reading the code!) sort.Strings(paths) newScope := interfaces.EmptyScope() @@ -3113,7 +3116,7 @@ func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) { continue } - b, err := bindata.Asset(p) + b, err := core.Asset(p) if err != nil { return nil, errwrap.Wrapf(err, "can't read asset: `%s`", p) } diff --git a/lang/funcs/Makefile b/lang/funcs/Makefile deleted file mode 100644 index 8a0de2dc..00000000 --- a/lang/funcs/Makefile +++ /dev/null @@ -1,45 +0,0 @@ -# Mgmt -# Copyright (C) 2013-2022+ James Shubin and the project contributors -# Written by James Shubin and the project contributors -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# The bindata target generates go files from any source defined below. To use -# the files, import the generated "bindata" package and use: -# `bytes, err := bindata.Asset("FILEPATH")` -# where FILEPATH is the path of the original input file relative to `bindata/`. -# To get a list of files stored in this "bindata" package, you can use: -# `paths := bindata.AssetNames()` and `paths, err := bindata.AssetDir(name)` -# to get a list of files with a directory prefix. - -.PHONY: build clean -default: build - -MCL_FILES := $(shell find * -name '*.mcl' -not -path 'old/*' -not -path 'tmp/*') -GENERATED = bindata/bindata.go - -build: $(GENERATED) - -# add more input files as dependencies at the end here... -$(GENERATED): $(MCL_FILES) - @echo "Generating: native mcl..." - @# go-bindata --pkg bindata -o - go-bindata --pkg bindata -o ./$@ $^ - @# gofmt the output file - gofmt -s -w $@ - @ROOT=$$(dirname "$${BASH_SOURCE}")/../.. && $$ROOT/misc/header.sh '$@' - -clean: - @# remove generated bindata/bindata.go - @ROOT=$$(dirname "$${BASH_SOURCE}")/../.. && rm -f $(GENERATED) diff --git a/lang/funcs/bindata.mcl b/lang/funcs/bindata.mcl deleted file mode 100644 index 5c597b88..00000000 --- a/lang/funcs/bindata.mcl +++ /dev/null @@ -1,2 +0,0 @@ -# You can add *.mcl files alongside the *.go files into the core/ directory. -# They will get compiled into the binary when it is built. diff --git a/lang/funcs/bindata/.gitignore b/lang/funcs/bindata/.gitignore deleted file mode 100644 index b8836fcf..00000000 --- a/lang/funcs/bindata/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# this file gets generated here -bindata.go diff --git a/lang/funcs/bindata/doc.go b/lang/funcs/bindata/doc.go deleted file mode 100644 index 9de1bf23..00000000 --- a/lang/funcs/bindata/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -// Mgmt -// Copyright (C) 2013-2022+ James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -// Package bindata stores core mcl code that is built-in at compile time. -package bindata diff --git a/lang/funcs/core/core.go b/lang/funcs/core/core.go index 0d059403..ecfa85b0 100644 --- a/lang/funcs/core/core.go +++ b/lang/funcs/core/core.go @@ -18,6 +18,9 @@ package core import ( + "embed" + "io/fs" + // import so the funcs register _ "github.com/purpleidea/mgmt/lang/funcs/core/convert" _ "github.com/purpleidea/mgmt/lang/funcs/core/datetime" @@ -34,3 +37,34 @@ import ( _ "github.com/purpleidea/mgmt/lang/funcs/core/sys" _ "github.com/purpleidea/mgmt/lang/funcs/core/world" ) + +// TODO: Instead of doing this one-level embed, we could give each package an +// API that it calls to "register" the private embed.FS that it wants to share. + +//go:embed */*.mcl +var mcl embed.FS + +// AssetNames returns a flattened list of embedded .mcl file paths. +func AssetNames() ([]string, error) { + fileSystem := mcl + paths := []string{} + err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { // skip the dirs + return nil + } + paths = append(paths, path) + return nil + }) + if err != nil { + return nil, err + } + return paths, nil +} + +// Asset returns the contents of an embedded .mcl file. +func Asset(name string) ([]byte, error) { + return mcl.ReadFile(name) +} diff --git a/lang/funcs/funcs.go b/lang/funcs/funcs.go index 60cef143..b4ed143d 100644 --- a/lang/funcs/funcs.go +++ b/lang/funcs/funcs.go @@ -41,7 +41,7 @@ const ( // NOTE: the template library will panic if it is one of: .-# ReplaceChar = "_" - // CoreDir is the directory prefix where core bindata mcl code is added. + // CoreDir is the directory prefix where core mcl code is embedded. CoreDir = "core/" ) diff --git a/misc/make-deps.sh b/misc/make-deps.sh index 1583e2e8..48542a44 100755 --- a/misc/make-deps.sh +++ b/misc/make-deps.sh @@ -150,7 +150,6 @@ go get golang.org/x/tools/cmd/goyacc # formerly `go tool yacc` go get golang.org/x/tools/cmd/stringer # for automatic stringer-ing go get golang.org/x/lint/golint # for `golint`-ing go get golang.org/x/tools/cmd/goimports # for fmt -go get github.com/kevinburke/go-bindata/go-bindata # for compiling in non golang files go get github.com/dvyukov/go-fuzz/go-fuzz # for fuzzing the mcl lang bits if in_ci; then go get -u gopkg.in/alecthomas/gometalinter.v1 && \ diff --git a/test/test-golint.sh b/test/test-golint.sh index a7d4e22d..21b479ff 100755 --- a/test/test-golint.sh +++ b/test/test-golint.sh @@ -27,7 +27,7 @@ if [ "$COMMITS" != "" ] && [ "$COMMITS" -gt "1" ]; then fi # find all go files, exluding temporary directories and generated files -LINT=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'bindata/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'lang/interpolate/parse.generated.go' -not -path 'lang/types/kind_stringer.go' -not -path 'vendor/*' -exec golint {} \;) # current golint output +LINT=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'lang/interpolate/parse.generated.go' -not -path 'lang/types/kind_stringer.go' -not -path 'vendor/*' -exec golint {} \;) # current golint output COUNT=`echo -e "$LINT" | wc -l` # number of golint problems in current branch [ "$LINT" = "" ] && echo PASS && exit # everything is "perfect" @@ -55,7 +55,7 @@ while read -r line; do done <<< "$NUMSTAT1" # three < is the secret to putting a variable into read git checkout "$PREVIOUS" &>/dev/null # previous commit -LINT1=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'bindata/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'vendor/*' -exec golint {} \;) +LINT1=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'vendor/*' -exec golint {} \;) COUNT1=`echo -e "$LINT1" | wc -l` # number of golint problems in older branch # clean up diff --git a/test/test-gometalinter.sh b/test/test-gometalinter.sh index 4f8dfb67..f15dd3c9 100755 --- a/test/test-gometalinter.sh +++ b/test/test-gometalinter.sh @@ -47,7 +47,6 @@ gml="$gml --enable=misspell" # TODO: at least until https://github.com/alecthomas/gometalinter/issues/270 gml="$gml --exclude=lang/parser/lexer.nn.go" gml="$gml --exclude=lang/parser/y.go" -gml="$gml --exclude=bindata/bindata.go" gml="$gml --exclude=lang/types/kind_stringer.go" gml="$gml --exclude=lang/interpolate/parse.generated.go" diff --git a/test/test-govet.sh b/test/test-govet.sh index ce92235c..bc7da063 100755 --- a/test/test-govet.sh +++ b/test/test-govet.sh @@ -60,10 +60,6 @@ function naked-error() { # catch errors that start with a capital function lowercase-errors() { - if [[ $1 == *"/bindata.go" ]]; then # ends with bindata.go ? - return 0 # skip those generated files - fi - if grep -E 'errors\.New\(\"[A-Z]' "$1"; then return 1 fi @@ -101,10 +97,6 @@ function consistent-imports() { } function reflowed-comments() { - if [[ $1 == *"/bindata.go" ]]; then # ends with bindata.go ? - return 0 # skip those generated files - fi - if [ "$1" = './lang/funcs/core/generated_funcs.go' ]; then return 0 fi