diff --git a/lang/lexparse.go b/lang/lexparse.go index 486e0edf..248e2176 100644 --- a/lang/lexparse.go +++ b/lang/lexparse.go @@ -21,6 +21,7 @@ import ( "bufio" "fmt" "io" + "net/url" "path" "sort" "strings" @@ -32,6 +33,14 @@ import ( errwrap "github.com/pkg/errors" ) +const ( + // ModuleMagicPrefix is the prefix which, if found as a prefix to the + // last token in an import path, will be removed silently if there are + // remaining characters following the name. If this is the empty string + // then it will be ignored. + ModuleMagicPrefix = "mgmt-" +) + // These constants represent the different possible lexer/parser errors. const ( ErrLexerUnrecognized = interfaces.Error("unrecognized") @@ -221,3 +230,109 @@ func DirectoryReader(fs engine.Fs, dir string) (io.Reader, map[uint64]string, er return io.MultiReader(readers...), offsets, nil } + +// ImportData is the result of parsing a string import when it has not errored. +type ImportData struct { + // Name is the original input that produced this struct. It is stored + // here so that you can parse it once and pass this struct around + // without having to include a copy of the original data if needed. + Name string + + // Alias is the name identifier that should be used for this import. + Alias string + + // System specifies that this is a system import. + System bool + + // Local represents if a module is either local or a remote import. + Local bool + + // Path represents the relative path to the directory that this import + // points to. Since it specifies a directory, it will end with a + // trailing slash which makes detection more obvious for other helpers. + // If this points to a local import, that directory is probably not + // expected to contain a metadata file, and it will be a simple path + // addition relative to the current file this import was parsed from. If + // this is a remote import, then it's likely that the file will be found + // in a more distinct path, such as a search path that contains the full + // fqdn of the import. + // TODO: should system imports put something here? + Path string +} + +// ParseImportName parses an import name and returns the default namespace name +// that should be used with it. For example, if the import name was: +// "git://example.com/purpleidea/Module-Name", this might return an alias of +// "module_name". It also returns a bunch of other data about the parsed import. +// TODO: check for invalid or unwanted special characters +func ParseImportName(name string) (*ImportData, error) { + magicPrefix := ModuleMagicPrefix + if name == "" { + return nil, fmt.Errorf("empty name") + } + if strings.HasPrefix(name, "/") { + return nil, fmt.Errorf("absolute paths are not allowed") + } + + u, err := url.Parse(name) + if err != nil { + return nil, errwrap.Wrapf(err, "name is not a valid url") + } + if u.Path == "" { + return nil, fmt.Errorf("empty path") + } + p := u.Path + + for strings.HasSuffix(p, "/") { // remove trailing slashes + p = p[:len(p)-len("/")] + } + + split := strings.Split(p, "/") // take last chunk if slash separated + s := split[0] + if len(split) > 1 { + s = split[len(split)-1] // pick last chunk + } + + // TODO: should we treat a special name: "purpleidea/mgmt-foo" as "foo"? + if magicPrefix != "" && strings.HasPrefix(s, magicPrefix) && len(s) > len(magicPrefix) { + s = s[len(magicPrefix):] + } + + s = strings.Replace(s, "-", "_", -1) + if strings.HasPrefix(s, "_") || strings.HasSuffix(s, "_") { + return nil, fmt.Errorf("name can't begin or end with dash or underscore") + } + alias := strings.ToLower(s) + + // if this is a local import, it's a straight directory path + // if it's an fqdn import, it should contain a metadata file + + // if there's no protocol prefix, then this must be a local path + local := u.Scheme == "" + system := local && !strings.HasSuffix(u.Path, "/") + xpath := u.Path // magic path + if system { + xpath = "" + } + if !local { + host := u.Host // host or host:port + split := strings.Split(host, ":") + if l := len(split); l == 1 || l == 2 { + host = split[0] + } else { + return nil, fmt.Errorf("incorrect number of colons (%d) in hostname", l) + } + xpath = path.Join(host, xpath) + } + if !local && !strings.HasSuffix(xpath, "/") { + xpath = xpath + "/" + } + + return &ImportData{ + Name: name, // save the original value here + Alias: alias, + System: system, + Local: local, + Path: xpath, + }, nil +} diff --git a/lang/lexparse_test.go b/lang/lexparse_test.go index a8ca3ff9..27474a0e 100644 --- a/lang/lexparse_test.go +++ b/lang/lexparse_test.go @@ -20,6 +20,7 @@ package lang import ( + "fmt" "io" "reflect" "strings" @@ -1578,3 +1579,213 @@ func TestLexParseWithOffsets1(t *testing.T) { t.Logf("output: %+v", err) // this will be 1-indexed, instead of zero-indexed } } + +func TestImportParsing0(t *testing.T) { + type test struct { // an individual test + name string + fail bool + alias string + system bool + local bool + path string + } + testCases := []test{} + testCases = append(testCases, test{ // index: 0 + name: "", + fail: true, // can't be empty + }) + testCases = append(testCases, test{ + name: "/", + fail: true, // can't be root + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/mgmt", + alias: "mgmt", + local: false, + path: "example.com/purpleidea/mgmt/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/mgmt/", + alias: "mgmt", + local: false, + path: "example.com/purpleidea/mgmt/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/mgmt/foo/bar/", + alias: "bar", + local: false, + path: "example.com/purpleidea/mgmt/foo/bar/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/mgmt-foo", + alias: "foo", // prefix is magic + local: false, + path: "example.com/purpleidea/mgmt-foo/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/foo-bar", + alias: "foo_bar", + local: false, + path: "example.com/purpleidea/foo-bar/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/FOO-bar", + alias: "foo_bar", + local: false, + path: "example.com/purpleidea/FOO-bar/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/foo-BAR", + alias: "foo_bar", + local: false, + path: "example.com/purpleidea/foo-BAR/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/foo-BAR-baz", + alias: "foo_bar_baz", + local: false, + path: "example.com/purpleidea/foo-BAR-baz/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/Module-Name", + alias: "module_name", + local: false, + path: "example.com/purpleidea/Module-Name/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/foo-", + fail: true, // trailing dash or underscore + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/foo_", + fail: true, // trailing dash or underscore + }) + testCases = append(testCases, test{ + name: "/var/lib/mgmt", + alias: "mgmt", + fail: true, // don't allow absolute paths + //local: true, + //path: "/var/lib/mgmt", + }) + testCases = append(testCases, test{ + name: "/var/lib/mgmt/", + alias: "mgmt", + fail: true, // don't allow absolute paths + //local: true, + //path: "/var/lib/mgmt/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/Module-Name?foo=bar&baz=42", + alias: "module_name", + local: false, + path: "example.com/purpleidea/Module-Name/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/Module-Name/?foo=bar&baz=42", + alias: "module_name", + local: false, + path: "example.com/purpleidea/Module-Name/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/Module-Name/?sha1=25ad05cce36d55ce1c55fd7e70a3ab74e321b66e", + alias: "module_name", + local: false, + path: "example.com/purpleidea/Module-Name/", + }) + testCases = append(testCases, test{ + name: "git://example.com/purpleidea/Module-Name/subpath/foo", + alias: "foo", + local: false, + path: "example.com/purpleidea/Module-Name/subpath/foo/", + }) + testCases = append(testCases, test{ + name: "foo/", + alias: "foo", + local: true, + path: "foo/", + }) + testCases = append(testCases, test{ + name: "foo/bar", + alias: "bar", + system: true, // system because not a dir (no trailing slash) + local: true, // not really used, but this is what we return + }) + testCases = append(testCases, test{ + name: "foo/bar/baz", + alias: "baz", + system: true, // system because not a dir (no trailing slash) + local: true, // not really used, but this is what we return + }) + testCases = append(testCases, test{ + name: "fmt", + alias: "fmt", + system: true, + local: true, // not really used, but this is what we return + }) + testCases = append(testCases, test{ + name: "blah", + alias: "blah", + system: true, // even modules that don't exist return true here + local: true, + }) + + t.Logf("ModuleMagicPrefix: %s", ModuleMagicPrefix) + names := []string{} + for index, tc := range testCases { // run all the tests + if util.StrInList(tc.name, names) { + t.Errorf("test #%d: duplicate sub test name of: %s", index, tc.name) + continue + } + names = append(names, tc.name) + t.Run(fmt.Sprintf("test #%d (%s)", index, tc.name), func(t *testing.T) { + name, fail, alias, system, local, path := tc.name, tc.fail, tc.alias, tc.system, tc.local, tc.path + + output, err := ParseImportName(name) + if !fail && err != nil { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: ParseImportName failed with: %+v", index, err) + return + } + if fail && err == nil { + t.Errorf("test #%d: FAIL", index) + t.Errorf("test #%d: ParseImportName expected error, not nil", index) + return + } + if fail { // we failed as expected, don't continue... + return + } + + if alias != output.Alias { + t.Errorf("test #%d: unexpected value for: `Alias`", index) + //t.Logf("test #%d: input: %s", index, name) + t.Logf("test #%d: output: %+v", index, output) + t.Logf("test #%d: alias: %s", index, alias) + return + } + if system != output.System { + t.Errorf("test #%d: unexpected value for: `System`", index) + //t.Logf("test #%d: input: %s", index, name) + t.Logf("test #%d: output: %+v", index, output) + t.Logf("test #%d: system: %t", index, system) + return + + } + if local != output.Local { + t.Errorf("test #%d: unexpected value for: `Local`", index) + //t.Logf("test #%d: input: %s", index, name) + t.Logf("test #%d: output: %+v", index, output) + t.Logf("test #%d: local: %t", index, local) + return + + } + if path != output.Path { + t.Errorf("test #%d: unexpected value for: `Path`", index) + //t.Logf("test #%d: input: %s", index, name) + t.Logf("test #%d: output: %+v", index, output) + t.Logf("test #%d: path: %s", index, path) + return + + } + }) + } +}