From 6bfd78194742ec893816dd9e010f664d1e7ac6cd Mon Sep 17 00:00:00 2001 From: James Shubin Date: Sun, 11 Sep 2022 20:17:46 -0400 Subject: [PATCH] lang: Replace the go-bindata usage with embed This doesn't let us have nested mcl at the moment, but we could improve on this with an embed API for each package. For now this makes building the project easier. --- .github/workflows/test.yaml | 5 ++-- Makefile | 10 +++----- lang/ast/structs.go | 11 +++++---- lang/funcs/Makefile | 45 ----------------------------------- lang/funcs/bindata.mcl | 2 -- lang/funcs/bindata/.gitignore | 2 -- lang/funcs/bindata/doc.go | 19 --------------- lang/funcs/core/core.go | 34 ++++++++++++++++++++++++++ lang/funcs/funcs.go | 2 +- misc/make-deps.sh | 1 - test/test-golint.sh | 4 ++-- test/test-gometalinter.sh | 1 - test/test-govet.sh | 8 ------- 13 files changed, 49 insertions(+), 95 deletions(-) delete mode 100644 lang/funcs/Makefile delete mode 100644 lang/funcs/bindata.mcl delete mode 100644 lang/funcs/bindata/.gitignore delete mode 100644 lang/funcs/bindata/doc.go diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c323e329..6de62974 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -37,9 +37,8 @@ jobs: #fail-fast: false steps: - # Do not shallow fetch, will fail when building bindata/ - # The path can't be absolute, so we need to move it to the - # expected location later. + # Do not shallow fetch. The path can't be absolute, so we need to move it + # to the expected location later. - name: Clone mgmt uses: actions/checkout@v2 with: diff --git a/Makefile b/Makefile index f9e1c7ad..f029e68e 100644 --- a/Makefile +++ b/Makefile @@ -16,12 +16,12 @@ # along with this program. If not, see . SHELL = /usr/bin/env bash -.PHONY: all art cleanart version program lang path deps run race bindata generate build build-debug crossbuild clean test gofmt yamlfmt format docs +.PHONY: all art cleanart version program lang path deps run race generate build build-debug crossbuild clean test gofmt yamlfmt format docs .PHONY: rpmbuild mkdirs rpm srpm spec tar upload upload-sources upload-srpms upload-rpms upload-releases copr tag .PHONY: mkosi mkosi_fedora-30 mkosi_fedora-29 mkosi_centos-7 mkosi_debian-10 mkosi_ubuntu-bionic mkosi_archlinux .PHONY: release releases_path release_fedora-30 release_fedora-29 release_centos-7 release_debian-10 release_ubuntu-bionic release_archlinux .PHONY: funcgen -.SILENT: clean bindata +.SILENT: clean # a large amount of output from this `find`, can cause `make` to be much slower! GO_FILES := $(shell find * -name '*.go' -not -path 'old/*' -not -path 'tmp/*') @@ -137,10 +137,6 @@ run: ## run mgmt race: find . -maxdepth 1 -type f -name '*.go' -not -name '*_test.go' | xargs go run -race -ldflags "-X main.program=$(PROGRAM) -X main.version=$(SVERSION)" -# generate go files from non-go source -bindata: ## generate go files from non-go sources - $(MAKE) --quiet -C lang/funcs - generate: go generate @@ -167,7 +163,7 @@ build-debug: $(PROGRAM) # extract os and arch from target pattern GOOS=$(firstword $(subst -, ,$*)) GOARCH=$(lastword $(subst -, ,$*)) -build/mgmt-%: $(GO_FILES) $(MCL_FILES) | bindata lang funcgen +build/mgmt-%: $(GO_FILES) $(MCL_FILES) | lang funcgen @echo "Building: $(PROGRAM), os/arch: $*, version: $(SVERSION)..." @time env GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags=$(PKGNAME)="-X main.program=$(PROGRAM) -X main.version=$(SVERSION) ${LDFLAGS}" -o $@ $(BUILD_FLAGS) diff --git a/lang/ast/structs.go b/lang/ast/structs.go index eadcff11..b0ca510e 100644 --- a/lang/ast/structs.go +++ b/lang/ast/structs.go @@ -30,7 +30,7 @@ import ( "github.com/purpleidea/mgmt/engine" engineUtil "github.com/purpleidea/mgmt/engine/util" "github.com/purpleidea/mgmt/lang/funcs" - "github.com/purpleidea/mgmt/lang/funcs/bindata" + "github.com/purpleidea/mgmt/lang/funcs/core" "github.com/purpleidea/mgmt/lang/funcs/structs" "github.com/purpleidea/mgmt/lang/inputs" "github.com/purpleidea/mgmt/lang/interfaces" @@ -3057,7 +3057,7 @@ func (obj *StmtProg) importScope(info *interfaces.ImportData, scope *interfaces. // importSystemScope takes the name of a built-in system scope (eg: "fmt") and // returns the scope struct for that built-in. This function is slightly less // trivial than expected, because the scope is built from both native mcl code -// and golang code as well. The native mcl code is compiled in as bindata. +// and golang code as well. The native mcl code is compiled in with "embed". // TODO: can we memoize? func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) { // this basically loop through the registeredFuncs and includes @@ -3096,7 +3096,10 @@ func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) { // to the remote machines as well, and we want to load off of it... // now add any compiled-in mcl code - paths := bindata.AssetNames() + paths, err := core.AssetNames() + if err != nil { + return nil, errwrap.Wrapf(err, "can't read asset names") + } // results are not sorted by default (ascertained by reading the code!) sort.Strings(paths) newScope := interfaces.EmptyScope() @@ -3113,7 +3116,7 @@ func (obj *StmtProg) importSystemScope(name string) (*interfaces.Scope, error) { continue } - b, err := bindata.Asset(p) + b, err := core.Asset(p) if err != nil { return nil, errwrap.Wrapf(err, "can't read asset: `%s`", p) } diff --git a/lang/funcs/Makefile b/lang/funcs/Makefile deleted file mode 100644 index 8a0de2dc..00000000 --- a/lang/funcs/Makefile +++ /dev/null @@ -1,45 +0,0 @@ -# Mgmt -# Copyright (C) 2013-2022+ James Shubin and the project contributors -# Written by James Shubin and the project contributors -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -# The bindata target generates go files from any source defined below. To use -# the files, import the generated "bindata" package and use: -# `bytes, err := bindata.Asset("FILEPATH")` -# where FILEPATH is the path of the original input file relative to `bindata/`. -# To get a list of files stored in this "bindata" package, you can use: -# `paths := bindata.AssetNames()` and `paths, err := bindata.AssetDir(name)` -# to get a list of files with a directory prefix. - -.PHONY: build clean -default: build - -MCL_FILES := $(shell find * -name '*.mcl' -not -path 'old/*' -not -path 'tmp/*') -GENERATED = bindata/bindata.go - -build: $(GENERATED) - -# add more input files as dependencies at the end here... -$(GENERATED): $(MCL_FILES) - @echo "Generating: native mcl..." - @# go-bindata --pkg bindata -o - go-bindata --pkg bindata -o ./$@ $^ - @# gofmt the output file - gofmt -s -w $@ - @ROOT=$$(dirname "$${BASH_SOURCE}")/../.. && $$ROOT/misc/header.sh '$@' - -clean: - @# remove generated bindata/bindata.go - @ROOT=$$(dirname "$${BASH_SOURCE}")/../.. && rm -f $(GENERATED) diff --git a/lang/funcs/bindata.mcl b/lang/funcs/bindata.mcl deleted file mode 100644 index 5c597b88..00000000 --- a/lang/funcs/bindata.mcl +++ /dev/null @@ -1,2 +0,0 @@ -# You can add *.mcl files alongside the *.go files into the core/ directory. -# They will get compiled into the binary when it is built. diff --git a/lang/funcs/bindata/.gitignore b/lang/funcs/bindata/.gitignore deleted file mode 100644 index b8836fcf..00000000 --- a/lang/funcs/bindata/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# this file gets generated here -bindata.go diff --git a/lang/funcs/bindata/doc.go b/lang/funcs/bindata/doc.go deleted file mode 100644 index 9de1bf23..00000000 --- a/lang/funcs/bindata/doc.go +++ /dev/null @@ -1,19 +0,0 @@ -// Mgmt -// Copyright (C) 2013-2022+ James Shubin and the project contributors -// Written by James Shubin and the project contributors -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -// Package bindata stores core mcl code that is built-in at compile time. -package bindata diff --git a/lang/funcs/core/core.go b/lang/funcs/core/core.go index 0d059403..ecfa85b0 100644 --- a/lang/funcs/core/core.go +++ b/lang/funcs/core/core.go @@ -18,6 +18,9 @@ package core import ( + "embed" + "io/fs" + // import so the funcs register _ "github.com/purpleidea/mgmt/lang/funcs/core/convert" _ "github.com/purpleidea/mgmt/lang/funcs/core/datetime" @@ -34,3 +37,34 @@ import ( _ "github.com/purpleidea/mgmt/lang/funcs/core/sys" _ "github.com/purpleidea/mgmt/lang/funcs/core/world" ) + +// TODO: Instead of doing this one-level embed, we could give each package an +// API that it calls to "register" the private embed.FS that it wants to share. + +//go:embed */*.mcl +var mcl embed.FS + +// AssetNames returns a flattened list of embedded .mcl file paths. +func AssetNames() ([]string, error) { + fileSystem := mcl + paths := []string{} + err := fs.WalkDir(fileSystem, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { // skip the dirs + return nil + } + paths = append(paths, path) + return nil + }) + if err != nil { + return nil, err + } + return paths, nil +} + +// Asset returns the contents of an embedded .mcl file. +func Asset(name string) ([]byte, error) { + return mcl.ReadFile(name) +} diff --git a/lang/funcs/funcs.go b/lang/funcs/funcs.go index 60cef143..b4ed143d 100644 --- a/lang/funcs/funcs.go +++ b/lang/funcs/funcs.go @@ -41,7 +41,7 @@ const ( // NOTE: the template library will panic if it is one of: .-# ReplaceChar = "_" - // CoreDir is the directory prefix where core bindata mcl code is added. + // CoreDir is the directory prefix where core mcl code is embedded. CoreDir = "core/" ) diff --git a/misc/make-deps.sh b/misc/make-deps.sh index 1583e2e8..48542a44 100755 --- a/misc/make-deps.sh +++ b/misc/make-deps.sh @@ -150,7 +150,6 @@ go get golang.org/x/tools/cmd/goyacc # formerly `go tool yacc` go get golang.org/x/tools/cmd/stringer # for automatic stringer-ing go get golang.org/x/lint/golint # for `golint`-ing go get golang.org/x/tools/cmd/goimports # for fmt -go get github.com/kevinburke/go-bindata/go-bindata # for compiling in non golang files go get github.com/dvyukov/go-fuzz/go-fuzz # for fuzzing the mcl lang bits if in_ci; then go get -u gopkg.in/alecthomas/gometalinter.v1 && \ diff --git a/test/test-golint.sh b/test/test-golint.sh index a7d4e22d..21b479ff 100755 --- a/test/test-golint.sh +++ b/test/test-golint.sh @@ -27,7 +27,7 @@ if [ "$COMMITS" != "" ] && [ "$COMMITS" -gt "1" ]; then fi # find all go files, exluding temporary directories and generated files -LINT=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'bindata/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'lang/interpolate/parse.generated.go' -not -path 'lang/types/kind_stringer.go' -not -path 'vendor/*' -exec golint {} \;) # current golint output +LINT=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'lang/interpolate/parse.generated.go' -not -path 'lang/types/kind_stringer.go' -not -path 'vendor/*' -exec golint {} \;) # current golint output COUNT=`echo -e "$LINT" | wc -l` # number of golint problems in current branch [ "$LINT" = "" ] && echo PASS && exit # everything is "perfect" @@ -55,7 +55,7 @@ while read -r line; do done <<< "$NUMSTAT1" # three < is the secret to putting a variable into read git checkout "$PREVIOUS" &>/dev/null # previous commit -LINT1=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'bindata/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'vendor/*' -exec golint {} \;) +LINT1=$(find * -maxdepth 9 -iname '*.go' -not -path 'old/*' -not -path 'tmp/*' -not -path 'lang/parser/y.go' -not -path 'lang/parser/lexer.nn.go' -not -path 'vendor/*' -exec golint {} \;) COUNT1=`echo -e "$LINT1" | wc -l` # number of golint problems in older branch # clean up diff --git a/test/test-gometalinter.sh b/test/test-gometalinter.sh index 4f8dfb67..f15dd3c9 100755 --- a/test/test-gometalinter.sh +++ b/test/test-gometalinter.sh @@ -47,7 +47,6 @@ gml="$gml --enable=misspell" # TODO: at least until https://github.com/alecthomas/gometalinter/issues/270 gml="$gml --exclude=lang/parser/lexer.nn.go" gml="$gml --exclude=lang/parser/y.go" -gml="$gml --exclude=bindata/bindata.go" gml="$gml --exclude=lang/types/kind_stringer.go" gml="$gml --exclude=lang/interpolate/parse.generated.go" diff --git a/test/test-govet.sh b/test/test-govet.sh index ce92235c..bc7da063 100755 --- a/test/test-govet.sh +++ b/test/test-govet.sh @@ -60,10 +60,6 @@ function naked-error() { # catch errors that start with a capital function lowercase-errors() { - if [[ $1 == *"/bindata.go" ]]; then # ends with bindata.go ? - return 0 # skip those generated files - fi - if grep -E 'errors\.New\(\"[A-Z]' "$1"; then return 1 fi @@ -101,10 +97,6 @@ function consistent-imports() { } function reflowed-comments() { - if [[ $1 == *"/bindata.go" ]]; then # ends with bindata.go ? - return 0 # skip those generated files - fi - if [ "$1" = './lang/funcs/core/generated_funcs.go' ]; then return 0 fi