diff --git a/tools/tgdoc/extract.go b/tools/tgdoc/extract.go new file mode 100644 index 000000000..12d2fa311 --- /dev/null +++ b/tools/tgdoc/extract.go @@ -0,0 +1,124 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "go/ast" + "go/build/constraint" + "go/doc" + "go/parser" + "go/token" + "os" + "path/filepath" + "strings" +) + +// buildTagSet creates the full set of tags that should evaluate to true +// for a given target, matching TinyGo's Config.BuildTags() behavior. +func buildTagSet(t Target) map[string]bool { + tags := make(map[string]bool) + for _, tag := range t.BuildTags { + tags[tag] = true + } + if t.GOOS != "" { + tags[t.GOOS] = true + } + if t.GOARCH != "" { + tags[t.GOARCH] = true + } + tags["tinygo"] = true + tags["purego"] = true + tags["osusergo"] = true + tags["math_big_pure_go"] = true + if t.GC != "" { + tags["gc."+t.GC] = true + } + if t.Scheduler != "" { + tags["scheduler."+t.Scheduler] = true + } + if t.Serial != "" { + tags["serial."+t.Serial] = true + } + // Go version tags — TinyGo currently tracks Go 1.22. + for i := 1; i <= 22; i++ { + tags[fmt.Sprintf("go1.%d", i)] = true + } + return tags +} + +// extractBuildConstraint reads the file header (before the package clause) +// looking for a //go:build constraint line. +func extractBuildConstraint(data []byte) (constraint.Expr, error) { + scanner := bufio.NewScanner(bytes.NewReader(data)) + for scanner.Scan() { + line := scanner.Text() + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "package ") { + break + } + if constraint.IsGoBuild(trimmed) { + return constraint.Parse(trimmed) + } + } + return nil, nil +} + +// ExtractDocs filters .go files in pkgDir by the target's build tags, +// parses them, and returns a *doc.Package via go/doc.NewFromFiles. +func ExtractDocs(t Target, pkgDir string, allDecls bool) (*doc.Package, *token.FileSet, error) { + tags := buildTagSet(t) + + entries, err := os.ReadDir(pkgDir) + if err != nil { + return nil, nil, err + } + + fset := token.NewFileSet() + var files []*ast.File + + for _, e := range entries { + name := e.Name() + if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + + path := filepath.Join(pkgDir, name) + data, err := os.ReadFile(path) + if err != nil { + return nil, nil, err + } + + expr, err := extractBuildConstraint(data) + if err != nil { + continue // skip files with unparseable constraints + } + + if expr != nil && !expr.Eval(func(tag string) bool { return tags[tag] }) { + continue + } + + f, err := parser.ParseFile(fset, path, data, parser.ParseComments) + if err != nil { + continue // skip files with syntax errors + } + + files = append(files, f) + } + + if len(files) == 0 { + return nil, nil, fmt.Errorf("no matching files for target %s", t.Name) + } + + var opts []any + if allDecls { + opts = append(opts, doc.AllDecls) + } + + docPkg, err := doc.NewFromFiles(fset, files, "machine", opts...) + if err != nil { + return nil, nil, err + } + + return docPkg, fset, nil +} diff --git a/tools/tgdoc/main.go b/tools/tgdoc/main.go new file mode 100644 index 000000000..2fb5a66db --- /dev/null +++ b/tools/tgdoc/main.go @@ -0,0 +1,172 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "go/ast" + "go/doc" + "go/format" + "go/token" + "log" + "os" + "sort" + "strings" +) + +// Result holds the extracted documentation for a single target. +type Result struct { + Target Target + Pkg *doc.Package + Fset *token.FileSet +} + +// JSON-serializable API types. + +type PackageAPI struct { + Target string `json:"target"` + Package string `json:"package"` + Doc string `json:"doc,omitempty"` + Constants []APIValue `json:"constants,omitempty"` + Variables []APIValue `json:"variables,omitempty"` + Types []APIType `json:"types,omitempty"` + Functions []APIFunc `json:"functions,omitempty"` +} + +type APIValue struct { + Doc string `json:"doc,omitempty"` + Names []string `json:"names"` + Decl string `json:"decl"` +} + +type APIType struct { + Doc string `json:"doc,omitempty"` + Name string `json:"name"` + Decl string `json:"decl"` + Constants []APIValue `json:"constants,omitempty"` + Variables []APIValue `json:"variables,omitempty"` + Functions []APIFunc `json:"functions,omitempty"` + Methods []APIFunc `json:"methods,omitempty"` +} + +type APIFunc struct { + Doc string `json:"doc,omitempty"` + Name string `json:"name"` + Decl string `json:"decl"` + Recv string `json:"recv,omitempty"` +} + +func formatNode(fset *token.FileSet, node ast.Node) string { + var buf bytes.Buffer + format.Node(&buf, fset, node) + return buf.String() +} + +func convertValues(fset *token.FileSet, vals []*doc.Value) []APIValue { + out := make([]APIValue, 0, len(vals)) + for _, v := range vals { + out = append(out, APIValue{ + Doc: v.Doc, + Names: v.Names, + Decl: formatNode(fset, v.Decl), + }) + } + return out +} + +func convertFuncs(fset *token.FileSet, funcs []*doc.Func) []APIFunc { + out := make([]APIFunc, 0, len(funcs)) + for _, f := range funcs { + out = append(out, APIFunc{ + Doc: f.Doc, + Name: f.Name, + Decl: formatNode(fset, f.Decl), + Recv: f.Recv, + }) + } + return out +} + +func convertPackage(targetName string, pkg *doc.Package, fset *token.FileSet) *PackageAPI { + api := &PackageAPI{ + Target: targetName, + Package: pkg.Name, + Doc: pkg.Doc, + Constants: convertValues(fset, pkg.Consts), + Variables: convertValues(fset, pkg.Vars), + Functions: convertFuncs(fset, pkg.Funcs), + } + for _, t := range pkg.Types { + api.Types = append(api.Types, APIType{ + Doc: t.Doc, + Name: t.Name, + Decl: formatNode(fset, t.Decl), + Constants: convertValues(fset, t.Consts), + Variables: convertValues(fset, t.Vars), + Functions: convertFuncs(fset, t.Funcs), + Methods: convertFuncs(fset, t.Methods), + }) + } + return api +} + +func main() { + jsonOutput := flag.Bool("json", false, "output JSON to stdout") + httpAddr := flag.String("http", "", "serve HTTP documentation (e.g. :8080)") + allDecls := flag.Bool("all", false, "include unexported identifiers") + includeBase := flag.Bool("base", false, "include base/parent targets") + flag.Parse() + + args := flag.Args() + if len(args) < 2 { + fmt.Fprintf(os.Stderr, "usage: tgdoc [flags] \n") + fmt.Fprintf(os.Stderr, "\nexample: tgdoc -http :8080 ./targets ./src/machine\n") + fmt.Fprintf(os.Stderr, "\nFlags:\n") + flag.PrintDefaults() + os.Exit(1) + } + + targetsDir := args[0] + pkgDir := args[1] + + if *httpAddr != "" && !strings.Contains(*httpAddr, ":") { + fmt.Fprintf(os.Stderr, "error: -http value %q doesn't look like an address (expected e.g. :8080)\n", *httpAddr) + os.Exit(1) + } + + targets, err := LoadTargets(targetsDir, *includeBase) + if err != nil { + log.Fatal(err) + } + + var results []Result + for _, t := range targets { + pkg, fset, err := ExtractDocs(t, pkgDir, *allDecls) + if err != nil { + fmt.Fprintf(os.Stderr, "warning: %s: %v\n", t.Name, err) + continue + } + results = append(results, Result{t, pkg, fset}) + } + + sort.Slice(results, func(i, j int) bool { return results[i].Target.Name < results[j].Target.Name }) + + if *httpAddr != "" { + serve(*httpAddr, results) + return + } + + // Default: JSON output. + _ = jsonOutput + apis := make([]*PackageAPI, 0, len(results)) + for _, r := range results { + apis = append(apis, convertPackage(r.Target.Name, r.Pkg, r.Fset)) + } + + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(apis); err != nil { + log.Fatal(err) + } +} diff --git a/tools/tgdoc/server.go b/tools/tgdoc/server.go new file mode 100644 index 000000000..c22894425 --- /dev/null +++ b/tools/tgdoc/server.go @@ -0,0 +1,445 @@ +package main + +import ( + "bytes" + "go/ast" + "go/doc" + "go/format" + "go/token" + "html/template" + "log" + "net/http" + "sort" + "strings" +) + +// IdentIndex is the precomputed identifier index across all targets. +type IdentIndex struct { + Entries []*IdentEntry + ByKey map[string]*IdentEntry + Total int // total number of targets +} + +// IdentEntry is one identifier (type, func, method, const, or var) across all targets. +type IdentEntry struct { + Name string + Kind string // "type", "func", "method", "const", "var" + Count int + Key string // URL key: "kind/name" + Groups []IdentGroup +} + +// IdentGroup is a set of targets with identical declaration for an identifier. +type IdentGroup struct { + Decl string // representative formatted declaration + Targets []string // sorted target names +} + +func buildIdentIndex(results []Result) *IdentIndex { + type occ struct { + target string + decl string // grouping key + display + } + type ikey struct { + name, kind string + } + + idx := make(map[ikey][]occ) + + addFunc := func(target string, fset *token.FileSet, f *doc.Func, prefix string) { + name := f.Name + if prefix != "" { + name = prefix + "." + f.Name + } + kind := "func" + if f.Recv != "" { + kind = "method" + } + decl := fmtNode(fset, f.Decl) + idx[ikey{name, kind}] = append(idx[ikey{name, kind}], occ{target, decl}) + } + + addValues := func(target string, fset *token.FileSet, vals []*doc.Value, kind string) { + for _, v := range vals { + for _, name := range v.Names { + typeKey := resolveValueType(fset, v.Decl, name) + idx[ikey{name, kind}] = append(idx[ikey{name, kind}], occ{target, typeKey}) + } + } + } + + for _, r := range results { + t := r.Target.Name + fset := r.Fset + pkg := r.Pkg + + for _, ty := range pkg.Types { + decl := fmtNode(fset, ty.Decl) + idx[ikey{ty.Name, "type"}] = append(idx[ikey{ty.Name, "type"}], occ{t, decl}) + + for _, m := range ty.Methods { + addFunc(t, fset, m, ty.Name) + } + for _, f := range ty.Funcs { + addFunc(t, fset, f, "") + } + addValues(t, fset, ty.Consts, "const") + addValues(t, fset, ty.Vars, "var") + } + + for _, f := range pkg.Funcs { + addFunc(t, fset, f, "") + } + addValues(t, fset, pkg.Consts, "const") + addValues(t, fset, pkg.Vars, "var") + } + + index := &IdentIndex{ + ByKey: make(map[string]*IdentEntry), + Total: len(results), + } + + for key, occs := range idx { + groups := make(map[string][]string) // decl -> targets + for _, o := range occs { + groups[o.decl] = append(groups[o.decl], o.target) + } + + var gs []IdentGroup + for decl, targets := range groups { + sort.Strings(targets) + gs = append(gs, IdentGroup{Decl: decl, Targets: targets}) + } + sort.Slice(gs, func(i, j int) bool { return len(gs[i].Targets) > len(gs[j].Targets) }) + + urlKey := key.kind + "/" + key.name + entry := &IdentEntry{ + Name: key.name, + Kind: key.kind, + Count: len(occs), + Key: urlKey, + Groups: gs, + } + index.Entries = append(index.Entries, entry) + index.ByKey[urlKey] = entry + } + + sort.Slice(index.Entries, func(i, j int) bool { + if index.Entries[i].Count != index.Entries[j].Count { + return index.Entries[i].Count > index.Entries[j].Count + } + return index.Entries[i].Name < index.Entries[j].Name + }) + + return index +} + +// resolveValueType returns the effective type string for a const/var name +// within a GenDecl, handling iota type inheritance for constants. +func resolveValueType(fset *token.FileSet, gd *ast.GenDecl, name string) string { + isConst := gd.Tok == token.CONST + var lastType ast.Expr + for _, spec := range gd.Specs { + vs := spec.(*ast.ValueSpec) + if vs.Type != nil { + lastType = vs.Type + } + for _, n := range vs.Names { + if n.Name != name { + continue + } + effType := vs.Type + if effType == nil && isConst { + effType = lastType + } + if effType != nil { + return fmtNode(fset, effType) + } + return "(untyped)" + } + } + return "(unknown)" +} + +// fmtNode formats an AST node to Go source. Separate from formatNode in main.go +// to make it clear this is the server's version (same logic). +func fmtNode(fset *token.FileSet, node ast.Node) string { + var buf bytes.Buffer + format.Node(&buf, fset, node) + return buf.String() +} + +// --- HTTP Server --- + +func serve(addr string, results []Result) { + index := buildIdentIndex(results) + + byTarget := make(map[string]Result, len(results)) + for _, r := range results { + byTarget[r.Target.Name] = r + } + + mux := http.NewServeMux() + + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + data := pageData{Index: index} + pageTmpl.Execute(w, data) + }) + + mux.HandleFunc("/id/", func(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/id/") + entry, ok := index.ByKey[key] + if !ok { + http.NotFound(w, r) + return + } + data := pageData{Index: index, Selected: entry, SelectedKey: key} + pageTmpl.Execute(w, data) + }) + + mux.HandleFunc("/target/", func(w http.ResponseWriter, r *http.Request) { + name := strings.TrimPrefix(r.URL.Path, "/target/") + res, ok := byTarget[name] + if !ok { + http.NotFound(w, r) + return + } + api := convertPackage(res.Target.Name, res.Pkg, res.Fset) + targetTmpl.Execute(w, api) + }) + + log.Printf("serving documentation on http://localhost%s", addr) + log.Fatal(http.ListenAndServe(addr, mux)) +} + +type pageData struct { + Index *IdentIndex + Selected *IdentEntry + SelectedKey string +} + +// --- Templates --- + +var pageTmpl = template.Must(template.New("page").Funcs(template.FuncMap{ + "kindBadge": func(kind string) string { + switch kind { + case "type": + return "T" + case "func": + return "F" + case "method": + return "M" + case "const": + return "C" + case "var": + return "V" + } + return "?" + }, + "pre": func(s string) template.HTML { + return template.HTML("
" + template.HTMLEscapeString(s) + "
") + }, + "kindClass": func(kind string) string { return "kind-" + kind }, +}).Parse(` + +tgdoc{{if .Selected}} — {{.Selected.Name}}{{end}} + + +
+ +
+{{if .Selected}} +

{{.Selected.Kind}} {{.Selected.Name}}

+

Present in {{.Selected.Count}} / {{.Index.Total}} targets +{{if eq (len .Selected.Groups) 1}}— identical across all targets{{else}}— {{len .Selected.Groups}} distinct signatures{{end}}

+ +{{range .Selected.Groups}} +
+
{{len .Targets}} target{{if ne (len .Targets) 1}}s{{end}}
+{{pre .Decl}} +
{{range .Targets}}{{.}} {{end}}
+
+{{end}} + +{{else}} +
+

tgdoc

+

{{len .Index.Entries}} identifiers across {{.Index.Total}} targets

+

Select an identifier from the sidebar to see its appearances across targets.

+

Browse by target:

+ +
+{{end}} +
+
+ +`)) + +var targetTmpl = template.Must(template.New("target").Funcs(template.FuncMap{ + "pre": func(s string) template.HTML { + return template.HTML("
" + template.HTMLEscapeString(s) + "
") + }, +}).Parse(` + +{{.Target}} — tgdoc + + +

← identifier index

+

{{.Target}}

+

package {{.Package}}

+{{if .Doc}}
{{.Doc}}
{{end}} + +{{if .Constants}} +

Constants

+{{range .Constants}} +{{if .Doc}}
{{.Doc}}
{{end}} +{{pre .Decl}} +{{end}} +{{end}} + +{{if .Variables}} +

Variables

+{{range .Variables}} +{{if .Doc}}
{{.Doc}}
{{end}} +{{pre .Decl}} +{{end}} +{{end}} + +{{if .Functions}} +

Functions

+{{range .Functions}} +

func {{.Name}}

+{{if .Doc}}
{{.Doc}}
{{end}} +{{pre .Decl}} +{{end}} +{{end}} + +{{if .Types}} +

Types

+{{range .Types}} +

type {{.Name}}

+{{if .Doc}}
{{.Doc}}
{{end}} +{{pre .Decl}} +{{if .Constants}}

Associated Constants

+{{range .Constants}}{{pre .Decl}}{{end}}
{{end}} +{{if .Variables}}

Associated Variables

+{{range .Variables}}{{pre .Decl}}{{end}}
{{end}} +{{if .Functions}}

Constructors

+{{range .Functions}} +{{if .Doc}}
{{.Doc}}
{{end}} +{{pre .Decl}} +{{end}}
{{end}} +{{if .Methods}}

Methods

+{{range .Methods}} +{{if .Doc}}
{{.Doc}}
{{end}} +{{pre .Decl}} +{{end}}
{{end}} +{{end}} +{{end}} + +`)) diff --git a/tools/tgdoc/target.go b/tools/tgdoc/target.go new file mode 100644 index 000000000..6d102a56d --- /dev/null +++ b/tools/tgdoc/target.go @@ -0,0 +1,168 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// targetSpec is the minimal subset of TinyGo's target JSON we need. +type targetSpec struct { + Inherits []string `json:"inherits,omitempty"` + BuildTags []string `json:"build-tags,omitempty"` + GOOS string `json:"goos,omitempty"` + GOARCH string `json:"goarch,omitempty"` + GC string `json:"gc,omitempty"` + Scheduler string `json:"scheduler,omitempty"` + Serial string `json:"serial,omitempty"` + FlashMethod string `json:"flash-method,omitempty"` + FlashCommand string `json:"flash-command,omitempty"` + Emulator string `json:"emulator,omitempty"` +} + +// Target is a resolved target with inheritance applied. +type Target struct { + Name string + BuildTags []string + GOOS string + GOARCH string + GC string + Scheduler string + Serial string +} + +// override merges src properties into dst following TinyGo's semantics: +// strings override if non-empty, slices append with dedup. +func (dst *targetSpec) override(src *targetSpec) { + if src.GOOS != "" { + dst.GOOS = src.GOOS + } + if src.GOARCH != "" { + dst.GOARCH = src.GOARCH + } + if src.GC != "" { + dst.GC = src.GC + } + if src.Scheduler != "" { + dst.Scheduler = src.Scheduler + } + if src.Serial != "" { + dst.Serial = src.Serial + } + if src.FlashMethod != "" { + dst.FlashMethod = src.FlashMethod + } + if src.FlashCommand != "" { + dst.FlashCommand = src.FlashCommand + } + if src.Emulator != "" { + dst.Emulator = src.Emulator + } + dst.BuildTags = appendUnique(dst.BuildTags, src.BuildTags...) +} + +func appendUnique(dst []string, src ...string) []string { + seen := make(map[string]bool, len(dst)) + for _, s := range dst { + seen[s] = true + } + for _, s := range src { + if !seen[s] { + dst = append(dst, s) + seen[s] = true + } + } + return dst +} + +func loadRawTargets(dir string) (map[string]*targetSpec, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + specs := make(map[string]*targetSpec) + for _, e := range entries { + if !e.Type().IsRegular() || !strings.HasSuffix(e.Name(), ".json") { + continue + } + name := strings.TrimSuffix(e.Name(), ".json") + data, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + return nil, err + } + var spec targetSpec + if err := json.Unmarshal(data, &spec); err != nil { + return nil, fmt.Errorf("parsing %s: %w", e.Name(), err) + } + specs[name] = &spec + } + return specs, nil +} + +func resolveSpec(name string, raw map[string]*targetSpec, cache map[string]*targetSpec, resolving map[string]bool) (*targetSpec, error) { + if cached, ok := cache[name]; ok { + return cached, nil + } + if resolving[name] { + return nil, fmt.Errorf("circular inheritance: %s", name) + } + spec, ok := raw[name] + if !ok { + return nil, fmt.Errorf("unknown target: %s", name) + } + + resolving[name] = true + defer delete(resolving, name) + + result := &targetSpec{} + for _, parent := range spec.Inherits { + resolved, err := resolveSpec(parent, raw, cache, resolving) + if err != nil { + return nil, fmt.Errorf("resolving %s parent %s: %w", name, parent, err) + } + result.override(resolved) + } + result.override(spec) + + cache[name] = result + return result, nil +} + +// LoadTargets reads all target JSONs from dir, resolves inheritance, +// and returns the resolved targets. If includeBase is false, targets +// without flash/emulator configuration are excluded (base/parent targets). +func LoadTargets(dir string, includeBase bool) ([]Target, error) { + raw, err := loadRawTargets(dir) + if err != nil { + return nil, err + } + + cache := make(map[string]*targetSpec) + resolving := make(map[string]bool) + + var targets []Target + for name := range raw { + resolved, err := resolveSpec(name, raw, cache, resolving) + if err != nil { + return nil, err + } + if !includeBase && resolved.FlashMethod == "" && resolved.FlashCommand == "" && resolved.Emulator == "" { + continue + } + targets = append(targets, Target{ + Name: name, + BuildTags: resolved.BuildTags, + GOOS: resolved.GOOS, + GOARCH: resolved.GOARCH, + GC: resolved.GC, + Scheduler: resolved.Scheduler, + Serial: resolved.Serial, + }) + } + + sort.Slice(targets, func(i, j int) bool { return targets[i].Name < targets[j].Name }) + return targets, nil +}