From 43a123c1ae6613b3efeed291fa552ecd909d3acf Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Tue, 16 Apr 2024 21:23:18 +0200 Subject: Adding upstream version 1.20.14. Signed-off-by: Daniel Baumann --- src/go/ast/ast.go | 1074 ++++++++ src/go/ast/ast_test.go | 82 + src/go/ast/commentmap.go | 330 +++ src/go/ast/commentmap_test.go | 169 ++ src/go/ast/example_test.go | 208 ++ src/go/ast/filter.go | 495 ++++ src/go/ast/filter_test.go | 85 + src/go/ast/import.go | 230 ++ src/go/ast/issues_test.go | 42 + src/go/ast/print.go | 254 ++ src/go/ast/print_test.go | 96 + src/go/ast/resolve.go | 173 ++ src/go/ast/scope.go | 156 ++ src/go/ast/walk.go | 398 +++ src/go/build/build.go | 2012 ++++++++++++++ src/go/build/build_test.go | 803 ++++++ src/go/build/constraint/expr.go | 574 ++++ src/go/build/constraint/expr_test.go | 321 +++ src/go/build/deps_test.go | 774 ++++++ src/go/build/doc.go | 98 + src/go/build/gc.go | 17 + src/go/build/gccgo.go | 14 + src/go/build/read.go | 600 ++++ src/go/build/read_test.go | 352 +++ src/go/build/syslist.go | 80 + src/go/build/syslist_test.go | 62 + src/go/build/testdata/alltags/alltags.go | 5 + src/go/build/testdata/alltags/x_netbsd_arm.go | 5 + src/go/build/testdata/bads/bad.s | 1 + src/go/build/testdata/cgo_disabled/cgo_disabled.go | 5 + src/go/build/testdata/cgo_disabled/empty.go | 1 + src/go/build/testdata/doc/a_test.go | 2 + src/go/build/testdata/doc/b_test.go | 1 + src/go/build/testdata/doc/c_test.go | 1 + src/go/build/testdata/doc/d_test.go | 2 + src/go/build/testdata/doc/e.go | 1 + src/go/build/testdata/doc/f.go | 2 + src/go/build/testdata/empty/dummy | 0 src/go/build/testdata/multi/file.go | 5 + src/go/build/testdata/multi/file_appengine.go | 5 + .../testdata/non_source_tags/non_source_tags.go | 5 + .../build/testdata/non_source_tags/x_arm.go.ignore | 5 + src/go/build/testdata/other/file/file.go | 5 + src/go/build/testdata/other/main.go | 11 + src/go/build/testdata/withvendor/src/a/b/b.go | 3 + .../testdata/withvendor/src/a/vendor/c/d/d.go | 1 + src/go/constant/example_test.go | 180 ++ src/go/constant/kind_string.go | 28 + src/go/constant/value.go | 1410 ++++++++++ src/go/constant/value_test.go | 729 +++++ src/go/doc/Makefile | 7 + src/go/doc/comment.go | 71 + src/go/doc/comment/doc.go | 36 + src/go/doc/comment/html.go | 169 ++ src/go/doc/comment/markdown.go | 188 ++ src/go/doc/comment/mkstd.sh | 24 + src/go/doc/comment/old_test.go | 80 + src/go/doc/comment/parse.go | 1262 +++++++++ src/go/doc/comment/parse_test.go | 12 + src/go/doc/comment/print.go | 290 ++ src/go/doc/comment/std.go | 44 + src/go/doc/comment/std_test.go | 34 + src/go/doc/comment/testdata/README.md | 42 + src/go/doc/comment/testdata/blank.txt | 12 + src/go/doc/comment/testdata/code.txt | 94 + src/go/doc/comment/testdata/code2.txt | 31 + src/go/doc/comment/testdata/code3.txt | 33 + src/go/doc/comment/testdata/code4.txt | 38 + src/go/doc/comment/testdata/code5.txt | 21 + src/go/doc/comment/testdata/code6.txt | 24 + src/go/doc/comment/testdata/crash1.txt | 16 + src/go/doc/comment/testdata/doclink.txt | 21 + src/go/doc/comment/testdata/doclink2.txt | 8 + src/go/doc/comment/testdata/doclink3.txt | 8 + src/go/doc/comment/testdata/doclink4.txt | 7 + src/go/doc/comment/testdata/doclink5.txt | 5 + src/go/doc/comment/testdata/doclink6.txt | 5 + src/go/doc/comment/testdata/doclink7.txt | 4 + src/go/doc/comment/testdata/escape.txt | 55 + src/go/doc/comment/testdata/head.txt | 92 + src/go/doc/comment/testdata/head2.txt | 36 + src/go/doc/comment/testdata/head3.txt | 7 + src/go/doc/comment/testdata/hello.txt | 35 + src/go/doc/comment/testdata/link.txt | 17 + src/go/doc/comment/testdata/link2.txt | 31 + src/go/doc/comment/testdata/link3.txt | 14 + src/go/doc/comment/testdata/link4.txt | 77 + src/go/doc/comment/testdata/link5.txt | 36 + src/go/doc/comment/testdata/link6.txt | 50 + src/go/doc/comment/testdata/link7.txt | 25 + src/go/doc/comment/testdata/linklist.txt | 18 + src/go/doc/comment/testdata/linklist2.txt | 39 + src/go/doc/comment/testdata/linklist3.txt | 31 + src/go/doc/comment/testdata/linklist4.txt | 36 + src/go/doc/comment/testdata/list.txt | 48 + src/go/doc/comment/testdata/list10.txt | 13 + src/go/doc/comment/testdata/list2.txt | 57 + src/go/doc/comment/testdata/list3.txt | 32 + src/go/doc/comment/testdata/list4.txt | 38 + src/go/doc/comment/testdata/list5.txt | 40 + src/go/doc/comment/testdata/list6.txt | 129 + src/go/doc/comment/testdata/list7.txt | 98 + src/go/doc/comment/testdata/list8.txt | 56 + src/go/doc/comment/testdata/list9.txt | 30 + src/go/doc/comment/testdata/para.txt | 17 + src/go/doc/comment/testdata/quote.txt | 15 + src/go/doc/comment/testdata/text.txt | 62 + src/go/doc/comment/testdata/text2.txt | 14 + src/go/doc/comment/testdata/text3.txt | 28 + src/go/doc/comment/testdata/text4.txt | 29 + src/go/doc/comment/testdata/text5.txt | 38 + src/go/doc/comment/testdata/text6.txt | 18 + src/go/doc/comment/testdata/text7.txt | 21 + src/go/doc/comment/testdata/text8.txt | 94 + src/go/doc/comment/testdata/text9.txt | 12 + src/go/doc/comment/testdata/words.txt | 10 + src/go/doc/comment/testdata_test.go | 202 ++ src/go/doc/comment/text.go | 337 +++ src/go/doc/comment/wrap_test.go | 141 + src/go/doc/comment_test.go | 67 + src/go/doc/doc.go | 354 +++ src/go/doc/doc_test.go | 292 ++ src/go/doc/example.go | 722 +++++ src/go/doc/example_internal_test.go | 121 + src/go/doc/example_test.go | 336 +++ src/go/doc/exports.go | 324 +++ src/go/doc/filter.go | 106 + src/go/doc/headscan.go | 109 + src/go/doc/reader.go | 1029 +++++++ src/go/doc/synopsis.go | 78 + src/go/doc/synopsis_test.go | 52 + src/go/doc/testdata/a.0.golden | 52 + src/go/doc/testdata/a.1.golden | 52 + src/go/doc/testdata/a.2.golden | 52 + src/go/doc/testdata/a0.go | 40 + src/go/doc/testdata/a1.go | 12 + src/go/doc/testdata/b.0.golden | 74 + src/go/doc/testdata/b.1.golden | 89 + src/go/doc/testdata/b.2.golden | 74 + src/go/doc/testdata/b.go | 64 + src/go/doc/testdata/benchmark.go | 293 ++ src/go/doc/testdata/blank.0.golden | 62 + src/go/doc/testdata/blank.1.golden | 83 + src/go/doc/testdata/blank.2.golden | 62 + src/go/doc/testdata/blank.go | 75 + src/go/doc/testdata/bugpara.0.golden | 20 + src/go/doc/testdata/bugpara.1.golden | 20 + src/go/doc/testdata/bugpara.2.golden | 20 + src/go/doc/testdata/bugpara.go | 9 + src/go/doc/testdata/c.0.golden | 48 + src/go/doc/testdata/c.1.golden | 48 + src/go/doc/testdata/c.2.golden | 48 + src/go/doc/testdata/c.go | 62 + src/go/doc/testdata/d.0.golden | 104 + src/go/doc/testdata/d.1.golden | 104 + src/go/doc/testdata/d.2.golden | 104 + src/go/doc/testdata/d1.go | 57 + src/go/doc/testdata/d2.go | 45 + src/go/doc/testdata/e.0.golden | 109 + src/go/doc/testdata/e.1.golden | 144 + src/go/doc/testdata/e.2.golden | 130 + src/go/doc/testdata/e.go | 147 + src/go/doc/testdata/error1.0.golden | 30 + src/go/doc/testdata/error1.1.golden | 32 + src/go/doc/testdata/error1.2.golden | 30 + src/go/doc/testdata/error1.go | 24 + src/go/doc/testdata/error2.0.golden | 27 + src/go/doc/testdata/error2.1.golden | 37 + src/go/doc/testdata/error2.2.golden | 27 + src/go/doc/testdata/error2.go | 29 + src/go/doc/testdata/example.go | 81 + src/go/doc/testdata/examples/README.md | 12 + src/go/doc/testdata/examples/empty.go | 8 + src/go/doc/testdata/examples/empty.golden | 6 + .../doc/testdata/examples/generic_constraints.go | 38 + .../testdata/examples/generic_constraints.golden | 39 + src/go/doc/testdata/examples/import_groups.go | 23 + src/go/doc/testdata/examples/import_groups.golden | 27 + .../doc/testdata/examples/import_groups_named.go | 23 + .../testdata/examples/import_groups_named.golden | 27 + src/go/doc/testdata/examples/inspect_signature.go | 23 + .../doc/testdata/examples/inspect_signature.golden | 24 + src/go/doc/testdata/examples/iota.go | 34 + src/go/doc/testdata/examples/iota.golden | 23 + src/go/doc/testdata/examples/issue43658.go | 223 ++ src/go/doc/testdata/examples/issue43658.golden | 156 ++ src/go/doc/testdata/examples/multiple.go | 98 + src/go/doc/testdata/examples/multiple.golden | 129 + src/go/doc/testdata/examples/values.go | 22 + src/go/doc/testdata/examples/values.golden | 21 + src/go/doc/testdata/examples/whole_file.go | 23 + src/go/doc/testdata/examples/whole_file.golden | 21 + src/go/doc/testdata/examples/whole_function.go | 13 + src/go/doc/testdata/examples/whole_function.golden | 11 + .../testdata/examples/whole_function_external.go | 12 + .../examples/whole_function_external.golden | 8 + src/go/doc/testdata/f.0.golden | 13 + src/go/doc/testdata/f.1.golden | 16 + src/go/doc/testdata/f.2.golden | 13 + src/go/doc/testdata/f.go | 14 + src/go/doc/testdata/g.0.golden | 32 + src/go/doc/testdata/g.1.golden | 34 + src/go/doc/testdata/g.2.golden | 32 + src/go/doc/testdata/g.go | 25 + src/go/doc/testdata/generics.0.golden | 76 + src/go/doc/testdata/generics.1.golden | 66 + src/go/doc/testdata/generics.2.golden | 76 + src/go/doc/testdata/generics.go | 74 + src/go/doc/testdata/issue12839.0.golden | 51 + src/go/doc/testdata/issue12839.1.golden | 54 + src/go/doc/testdata/issue12839.2.golden | 51 + src/go/doc/testdata/issue12839.go | 69 + src/go/doc/testdata/issue13742.0.golden | 25 + src/go/doc/testdata/issue13742.1.golden | 25 + src/go/doc/testdata/issue13742.2.golden | 25 + src/go/doc/testdata/issue13742.go | 18 + src/go/doc/testdata/issue16153.0.golden | 32 + src/go/doc/testdata/issue16153.1.golden | 34 + src/go/doc/testdata/issue16153.2.golden | 32 + src/go/doc/testdata/issue16153.go | 27 + src/go/doc/testdata/issue17788.0.golden | 8 + src/go/doc/testdata/issue17788.1.golden | 8 + src/go/doc/testdata/issue17788.2.golden | 8 + src/go/doc/testdata/issue17788.go | 8 + src/go/doc/testdata/issue22856.0.golden | 45 + src/go/doc/testdata/issue22856.1.golden | 45 + src/go/doc/testdata/issue22856.2.golden | 45 + src/go/doc/testdata/issue22856.go | 27 + src/go/doc/testdata/pkgdoc/doc.go | 24 + src/go/doc/testdata/predeclared.0.golden | 8 + src/go/doc/testdata/predeclared.1.golden | 22 + src/go/doc/testdata/predeclared.2.golden | 8 + src/go/doc/testdata/predeclared.go | 22 + src/go/doc/testdata/template.txt | 68 + src/go/doc/testdata/testing.0.golden | 156 ++ src/go/doc/testdata/testing.1.golden | 298 ++ src/go/doc/testdata/testing.2.golden | 156 ++ src/go/doc/testdata/testing.go | 404 +++ src/go/format/benchmark_test.go | 91 + src/go/format/example_test.go | 39 + src/go/format/format.go | 133 + src/go/format/format_test.go | 187 ++ src/go/format/internal.go | 176 ++ src/go/importer/importer.go | 122 + src/go/importer/importer_test.go | 96 + src/go/internal/gccgoimporter/ar.go | 171 ++ src/go/internal/gccgoimporter/gccgoinstallation.go | 97 + .../gccgoimporter/gccgoinstallation_test.go | 192 ++ src/go/internal/gccgoimporter/importer.go | 261 ++ src/go/internal/gccgoimporter/importer_test.go | 199 ++ src/go/internal/gccgoimporter/parser.go | 1283 +++++++++ src/go/internal/gccgoimporter/parser_test.go | 78 + src/go/internal/gccgoimporter/testdata/aliases.go | 65 + src/go/internal/gccgoimporter/testdata/aliases.gox | 33 + .../internal/gccgoimporter/testdata/complexnums.go | 6 + .../gccgoimporter/testdata/complexnums.gox | 8 + .../internal/gccgoimporter/testdata/conversions.go | 5 + .../gccgoimporter/testdata/conversions.gox | 6 + .../internal/gccgoimporter/testdata/escapeinfo.go | 13 + .../internal/gccgoimporter/testdata/escapeinfo.gox | 9 + src/go/internal/gccgoimporter/testdata/imports.go | 5 + src/go/internal/gccgoimporter/testdata/imports.gox | 7 + .../internal/gccgoimporter/testdata/issue27856.go | 9 + .../internal/gccgoimporter/testdata/issue27856.gox | 9 + .../internal/gccgoimporter/testdata/issue29198.go | 37 + .../internal/gccgoimporter/testdata/issue29198.gox | 86 + .../internal/gccgoimporter/testdata/issue30628.go | 18 + .../internal/gccgoimporter/testdata/issue30628.gox | 28 + .../internal/gccgoimporter/testdata/issue31540.go | 26 + .../internal/gccgoimporter/testdata/issue31540.gox | 16 + .../internal/gccgoimporter/testdata/issue34182.go | 17 + .../internal/gccgoimporter/testdata/issue34182.gox | 13 + .../internal/gccgoimporter/testdata/libimportsar.a | Bin 0 -> 9302 bytes .../internal/gccgoimporter/testdata/nointerface.go | 12 + .../gccgoimporter/testdata/nointerface.gox | 8 + .../internal/gccgoimporter/testdata/notinheap.go | 4 + .../internal/gccgoimporter/testdata/notinheap.gox | 7 + src/go/internal/gccgoimporter/testdata/pointer.go | 3 + src/go/internal/gccgoimporter/testdata/pointer.gox | 4 + src/go/internal/gccgoimporter/testdata/time.gox | 142 + src/go/internal/gccgoimporter/testdata/unicode.gox | 273 ++ .../internal/gccgoimporter/testdata/v1reflect.gox | 184 ++ src/go/internal/gcimporter/exportdata.go | 92 + src/go/internal/gcimporter/gcimporter.go | 245 ++ src/go/internal/gcimporter/gcimporter_test.go | 832 ++++++ src/go/internal/gcimporter/iimport.go | 817 ++++++ src/go/internal/gcimporter/support.go | 183 ++ src/go/internal/gcimporter/testdata/a.go | 14 + src/go/internal/gcimporter/testdata/b.go | 11 + src/go/internal/gcimporter/testdata/exports.go | 91 + src/go/internal/gcimporter/testdata/g.go | 23 + src/go/internal/gcimporter/testdata/generics.go | 29 + src/go/internal/gcimporter/testdata/issue15920.go | 11 + src/go/internal/gcimporter/testdata/issue20046.go | 9 + src/go/internal/gcimporter/testdata/issue25301.go | 17 + src/go/internal/gcimporter/testdata/issue25596.go | 13 + src/go/internal/gcimporter/testdata/issue57015.go | 16 + src/go/internal/gcimporter/testdata/p.go | 13 + .../internal/gcimporter/testdata/versions/test.go | 28 + .../gcimporter/testdata/versions/test_go1.11_0i.a | Bin 0 -> 2420 bytes .../gcimporter/testdata/versions/test_go1.11_6b.a | Bin 0 -> 2426 bytes .../testdata/versions/test_go1.11_999b.a | Bin 0 -> 2600 bytes .../testdata/versions/test_go1.11_999i.a | Bin 0 -> 2420 bytes .../gcimporter/testdata/versions/test_go1.7_0.a | Bin 0 -> 1862 bytes .../gcimporter/testdata/versions/test_go1.7_1.a | Bin 0 -> 2316 bytes .../gcimporter/testdata/versions/test_go1.8_4.a | Bin 0 -> 1658 bytes .../gcimporter/testdata/versions/test_go1.8_5.a | Bin 0 -> 1658 bytes src/go/internal/gcimporter/ureader.go | 682 +++++ src/go/internal/srcimporter/srcimporter.go | 269 ++ src/go/internal/srcimporter/srcimporter_test.go | 253 ++ .../srcimporter/testdata/issue20855/issue20855.go | 7 + .../srcimporter/testdata/issue23092/issue23092.go | 5 + .../srcimporter/testdata/issue24392/issue24392.go | 5 + src/go/internal/typeparams/typeparams.go | 54 + src/go/parser/error_test.go | 202 ++ src/go/parser/example_test.go | 43 + src/go/parser/interface.go | 238 ++ src/go/parser/parser.go | 2864 ++++++++++++++++++++ src/go/parser/parser_test.go | 782 ++++++ src/go/parser/performance_test.go | 54 + src/go/parser/resolver.go | 612 +++++ src/go/parser/resolver_test.go | 172 ++ src/go/parser/short_test.go | 214 ++ src/go/parser/testdata/chans.go2 | 62 + src/go/parser/testdata/commas.src | 19 + src/go/parser/testdata/interface.go2 | 76 + src/go/parser/testdata/issue11377.src | 27 + src/go/parser/testdata/issue23434.src | 25 + src/go/parser/testdata/issue3106.src | 46 + src/go/parser/testdata/issue34946.src | 22 + .../testdata/issue42951/not_a_file.go/invalid.go | 1 + src/go/parser/testdata/issue44504.src | 13 + src/go/parser/testdata/issue49174.go2 | 8 + src/go/parser/testdata/issue49175.go2 | 13 + src/go/parser/testdata/issue49482.go2 | 34 + src/go/parser/testdata/issue50427.go2 | 19 + src/go/parser/testdata/linalg.go2 | 83 + src/go/parser/testdata/map.go2 | 109 + src/go/parser/testdata/metrics.go2 | 58 + src/go/parser/testdata/resolution/issue45136.src | 27 + src/go/parser/testdata/resolution/issue45160.src | 25 + src/go/parser/testdata/resolution/resolution.src | 63 + src/go/parser/testdata/resolution/typeparams.go2 | 51 + src/go/parser/testdata/set.go2 | 31 + src/go/parser/testdata/slices.go2 | 31 + src/go/parser/testdata/sort.go2 | 27 + src/go/parser/testdata/tparams.go2 | 54 + src/go/parser/testdata/typeset.go2 | 72 + src/go/printer/comment.go | 155 ++ src/go/printer/example_test.go | 67 + src/go/printer/gobuild.go | 170 ++ src/go/printer/nodes.go | 2001 ++++++++++++++ src/go/printer/performance_test.go | 91 + src/go/printer/printer.go | 1436 ++++++++++ src/go/printer/printer_test.go | 827 ++++++ src/go/printer/testdata/alignment.golden | 172 ++ src/go/printer/testdata/alignment.input | 179 ++ src/go/printer/testdata/comments.golden | 774 ++++++ src/go/printer/testdata/comments.input | 773 ++++++ src/go/printer/testdata/comments.x | 55 + src/go/printer/testdata/comments2.golden | 163 ++ src/go/printer/testdata/comments2.input | 168 ++ src/go/printer/testdata/complit.input | 65 + src/go/printer/testdata/complit.x | 62 + src/go/printer/testdata/declarations.golden | 1008 +++++++ src/go/printer/testdata/declarations.input | 1021 +++++++ src/go/printer/testdata/doc.golden | 21 + src/go/printer/testdata/doc.input | 20 + src/go/printer/testdata/empty.golden | 5 + src/go/printer/testdata/empty.input | 5 + src/go/printer/testdata/expressions.golden | 743 +++++ src/go/printer/testdata/expressions.input | 771 ++++++ src/go/printer/testdata/expressions.raw | 743 +++++ src/go/printer/testdata/generics.golden | 109 + src/go/printer/testdata/generics.input | 105 + src/go/printer/testdata/go2numbers.golden | 186 ++ src/go/printer/testdata/go2numbers.input | 186 ++ src/go/printer/testdata/go2numbers.norm | 186 ++ src/go/printer/testdata/gobuild1.golden | 6 + src/go/printer/testdata/gobuild1.input | 7 + src/go/printer/testdata/gobuild2.golden | 8 + src/go/printer/testdata/gobuild2.input | 9 + src/go/printer/testdata/gobuild3.golden | 10 + src/go/printer/testdata/gobuild3.input | 11 + src/go/printer/testdata/gobuild4.golden | 6 + src/go/printer/testdata/gobuild4.input | 5 + src/go/printer/testdata/gobuild5.golden | 4 + src/go/printer/testdata/gobuild5.input | 4 + src/go/printer/testdata/gobuild6.golden | 5 + src/go/printer/testdata/gobuild6.input | 4 + src/go/printer/testdata/gobuild7.golden | 11 + src/go/printer/testdata/gobuild7.input | 11 + src/go/printer/testdata/linebreaks.golden | 295 ++ src/go/printer/testdata/linebreaks.input | 291 ++ src/go/printer/testdata/parser.go | 2148 +++++++++++++++ src/go/printer/testdata/slow.golden | 85 + src/go/printer/testdata/slow.input | 85 + src/go/printer/testdata/statements.golden | 644 +++++ src/go/printer/testdata/statements.input | 555 ++++ src/go/scanner/errors.go | 120 + src/go/scanner/example_test.go | 46 + src/go/scanner/scanner.go | 957 +++++++ src/go/scanner/scanner_test.go | 1125 ++++++++ src/go/token/example_test.go | 77 + src/go/token/position.go | 556 ++++ src/go/token/position_bench_test.go | 24 + src/go/token/position_test.go | 477 ++++ src/go/token/serialize.go | 70 + src/go/token/serialize_test.go | 105 + src/go/token/token.go | 341 +++ src/go/token/token_test.go | 33 + src/go/types/api.go | 492 ++++ src/go/types/api_test.go | 2596 ++++++++++++++++++ src/go/types/array.go | 25 + src/go/types/assignments.go | 475 ++++ src/go/types/basic.go | 82 + src/go/types/builtins.go | 1017 +++++++ src/go/types/builtins_test.go | 254 ++ src/go/types/call.go | 817 ++++++ src/go/types/chan.go | 35 + src/go/types/check.go | 578 ++++ src/go/types/check_test.go | 425 +++ src/go/types/context.go | 144 + src/go/types/context_test.go | 70 + src/go/types/conversions.go | 296 ++ src/go/types/decl.go | 937 +++++++ src/go/types/errorcalls_test.go | 53 + src/go/types/errors.go | 390 +++ src/go/types/errors_test.go | 47 + src/go/types/eval.go | 99 + src/go/types/eval_test.go | 297 ++ src/go/types/example_test.go | 326 +++ src/go/types/expr.go | 1826 +++++++++++++ src/go/types/exprstring.go | 238 ++ src/go/types/exprstring_test.go | 121 + src/go/types/gccgosizes.go | 41 + src/go/types/gotype.go | 356 +++ src/go/types/hilbert_test.go | 222 ++ src/go/types/index.go | 456 ++++ src/go/types/infer.go | 773 ++++++ src/go/types/initorder.go | 325 +++ src/go/types/instantiate.go | 366 +++ src/go/types/instantiate_test.go | 238 ++ src/go/types/interface.go | 233 ++ src/go/types/issues_test.go | 854 ++++++ src/go/types/labels.go | 274 ++ src/go/types/lookup.go | 543 ++++ src/go/types/lookup_test.go | 58 + src/go/types/main_test.go | 17 + src/go/types/map.go | 24 + src/go/types/methodset.go | 238 ++ src/go/types/methodset_test.go | 156 ++ src/go/types/mono.go | 337 +++ src/go/types/mono_test.go | 91 + src/go/types/named.go | 656 +++++ src/go/types/named_test.go | 129 + src/go/types/object.go | 568 ++++ src/go/types/object_test.go | 166 ++ src/go/types/objset.go | 31 + src/go/types/operand.go | 368 +++ src/go/types/package.go | 74 + src/go/types/pointer.go | 19 + src/go/types/predicates.go | 497 ++++ src/go/types/resolver.go | 726 +++++ src/go/types/resolver_test.go | 211 ++ src/go/types/return.go | 184 ++ src/go/types/scope.go | 292 ++ src/go/types/selection.go | 142 + src/go/types/self_test.go | 122 + src/go/types/signature.go | 321 +++ src/go/types/sizeof_test.go | 62 + src/go/types/sizes.go | 296 ++ src/go/types/sizes_test.go | 142 + src/go/types/slice.go | 19 + src/go/types/stdlib_test.go | 355 +++ src/go/types/stmt.go | 962 +++++++ src/go/types/struct.go | 218 ++ src/go/types/subst.go | 421 +++ src/go/types/termlist.go | 161 ++ src/go/types/termlist_test.go | 284 ++ src/go/types/testdata/local/issue47996.go | 9 + src/go/types/testdata/local/shifts.go | 27 + src/go/types/testdata/manual.go | 9 + src/go/types/token_test.go | 47 + src/go/types/tuple.go | 34 + src/go/types/type.go | 124 + src/go/types/typelists.go | 69 + src/go/types/typeparam.go | 158 ++ src/go/types/typeset.go | 417 +++ src/go/types/typeset_test.go | 81 + src/go/types/typestring.go | 491 ++++ src/go/types/typestring_test.go | 167 ++ src/go/types/typeterm.go | 165 ++ src/go/types/typeterm_test.go | 240 ++ src/go/types/typexpr.go | 521 ++++ src/go/types/unify.go | 583 ++++ src/go/types/union.go | 200 ++ src/go/types/universe.go | 284 ++ src/go/types/validtype.go | 256 ++ src/go/types/version.go | 83 + 500 files changed, 89313 insertions(+) create mode 100644 src/go/ast/ast.go create mode 100644 src/go/ast/ast_test.go create mode 100644 src/go/ast/commentmap.go create mode 100644 src/go/ast/commentmap_test.go create mode 100644 src/go/ast/example_test.go create mode 100644 src/go/ast/filter.go create mode 100644 src/go/ast/filter_test.go create mode 100644 src/go/ast/import.go create mode 100644 src/go/ast/issues_test.go create mode 100644 src/go/ast/print.go create mode 100644 src/go/ast/print_test.go create mode 100644 src/go/ast/resolve.go create mode 100644 src/go/ast/scope.go create mode 100644 src/go/ast/walk.go create mode 100644 src/go/build/build.go create mode 100644 src/go/build/build_test.go create mode 100644 src/go/build/constraint/expr.go create mode 100644 src/go/build/constraint/expr_test.go create mode 100644 src/go/build/deps_test.go create mode 100644 src/go/build/doc.go create mode 100644 src/go/build/gc.go create mode 100644 src/go/build/gccgo.go create mode 100644 src/go/build/read.go create mode 100644 src/go/build/read_test.go create mode 100644 src/go/build/syslist.go create mode 100644 src/go/build/syslist_test.go create mode 100644 src/go/build/testdata/alltags/alltags.go create mode 100644 src/go/build/testdata/alltags/x_netbsd_arm.go create mode 100644 src/go/build/testdata/bads/bad.s create mode 100644 src/go/build/testdata/cgo_disabled/cgo_disabled.go create mode 100644 src/go/build/testdata/cgo_disabled/empty.go create mode 100644 src/go/build/testdata/doc/a_test.go create mode 100644 src/go/build/testdata/doc/b_test.go create mode 100644 src/go/build/testdata/doc/c_test.go create mode 100644 src/go/build/testdata/doc/d_test.go create mode 100644 src/go/build/testdata/doc/e.go create mode 100644 src/go/build/testdata/doc/f.go create mode 100644 src/go/build/testdata/empty/dummy create mode 100644 src/go/build/testdata/multi/file.go create mode 100644 src/go/build/testdata/multi/file_appengine.go create mode 100644 src/go/build/testdata/non_source_tags/non_source_tags.go create mode 100644 src/go/build/testdata/non_source_tags/x_arm.go.ignore create mode 100644 src/go/build/testdata/other/file/file.go create mode 100644 src/go/build/testdata/other/main.go create mode 100644 src/go/build/testdata/withvendor/src/a/b/b.go create mode 100644 src/go/build/testdata/withvendor/src/a/vendor/c/d/d.go create mode 100644 src/go/constant/example_test.go create mode 100644 src/go/constant/kind_string.go create mode 100644 src/go/constant/value.go create mode 100644 src/go/constant/value_test.go create mode 100644 src/go/doc/Makefile create mode 100644 src/go/doc/comment.go create mode 100644 src/go/doc/comment/doc.go create mode 100644 src/go/doc/comment/html.go create mode 100644 src/go/doc/comment/markdown.go create mode 100755 src/go/doc/comment/mkstd.sh create mode 100644 src/go/doc/comment/old_test.go create mode 100644 src/go/doc/comment/parse.go create mode 100644 src/go/doc/comment/parse_test.go create mode 100644 src/go/doc/comment/print.go create mode 100644 src/go/doc/comment/std.go create mode 100644 src/go/doc/comment/std_test.go create mode 100644 src/go/doc/comment/testdata/README.md create mode 100644 src/go/doc/comment/testdata/blank.txt create mode 100644 src/go/doc/comment/testdata/code.txt create mode 100644 src/go/doc/comment/testdata/code2.txt create mode 100644 src/go/doc/comment/testdata/code3.txt create mode 100644 src/go/doc/comment/testdata/code4.txt create mode 100644 src/go/doc/comment/testdata/code5.txt create mode 100644 src/go/doc/comment/testdata/code6.txt create mode 100644 src/go/doc/comment/testdata/crash1.txt create mode 100644 src/go/doc/comment/testdata/doclink.txt create mode 100644 src/go/doc/comment/testdata/doclink2.txt create mode 100644 src/go/doc/comment/testdata/doclink3.txt create mode 100644 src/go/doc/comment/testdata/doclink4.txt create mode 100644 src/go/doc/comment/testdata/doclink5.txt create mode 100644 src/go/doc/comment/testdata/doclink6.txt create mode 100644 src/go/doc/comment/testdata/doclink7.txt create mode 100644 src/go/doc/comment/testdata/escape.txt create mode 100644 src/go/doc/comment/testdata/head.txt create mode 100644 src/go/doc/comment/testdata/head2.txt create mode 100644 src/go/doc/comment/testdata/head3.txt create mode 100644 src/go/doc/comment/testdata/hello.txt create mode 100644 src/go/doc/comment/testdata/link.txt create mode 100644 src/go/doc/comment/testdata/link2.txt create mode 100644 src/go/doc/comment/testdata/link3.txt create mode 100644 src/go/doc/comment/testdata/link4.txt create mode 100644 src/go/doc/comment/testdata/link5.txt create mode 100644 src/go/doc/comment/testdata/link6.txt create mode 100644 src/go/doc/comment/testdata/link7.txt create mode 100644 src/go/doc/comment/testdata/linklist.txt create mode 100644 src/go/doc/comment/testdata/linklist2.txt create mode 100644 src/go/doc/comment/testdata/linklist3.txt create mode 100644 src/go/doc/comment/testdata/linklist4.txt create mode 100644 src/go/doc/comment/testdata/list.txt create mode 100644 src/go/doc/comment/testdata/list10.txt create mode 100644 src/go/doc/comment/testdata/list2.txt create mode 100644 src/go/doc/comment/testdata/list3.txt create mode 100644 src/go/doc/comment/testdata/list4.txt create mode 100644 src/go/doc/comment/testdata/list5.txt create mode 100644 src/go/doc/comment/testdata/list6.txt create mode 100644 src/go/doc/comment/testdata/list7.txt create mode 100644 src/go/doc/comment/testdata/list8.txt create mode 100644 src/go/doc/comment/testdata/list9.txt create mode 100644 src/go/doc/comment/testdata/para.txt create mode 100644 src/go/doc/comment/testdata/quote.txt create mode 100644 src/go/doc/comment/testdata/text.txt create mode 100644 src/go/doc/comment/testdata/text2.txt create mode 100644 src/go/doc/comment/testdata/text3.txt create mode 100644 src/go/doc/comment/testdata/text4.txt create mode 100644 src/go/doc/comment/testdata/text5.txt create mode 100644 src/go/doc/comment/testdata/text6.txt create mode 100644 src/go/doc/comment/testdata/text7.txt create mode 100644 src/go/doc/comment/testdata/text8.txt create mode 100644 src/go/doc/comment/testdata/text9.txt create mode 100644 src/go/doc/comment/testdata/words.txt create mode 100644 src/go/doc/comment/testdata_test.go create mode 100644 src/go/doc/comment/text.go create mode 100644 src/go/doc/comment/wrap_test.go create mode 100644 src/go/doc/comment_test.go create mode 100644 src/go/doc/doc.go create mode 100644 src/go/doc/doc_test.go create mode 100644 src/go/doc/example.go create mode 100644 src/go/doc/example_internal_test.go create mode 100644 src/go/doc/example_test.go create mode 100644 src/go/doc/exports.go create mode 100644 src/go/doc/filter.go create mode 100644 src/go/doc/headscan.go create mode 100644 src/go/doc/reader.go create mode 100644 src/go/doc/synopsis.go create mode 100644 src/go/doc/synopsis_test.go create mode 100644 src/go/doc/testdata/a.0.golden create mode 100644 src/go/doc/testdata/a.1.golden create mode 100644 src/go/doc/testdata/a.2.golden create mode 100644 src/go/doc/testdata/a0.go create mode 100644 src/go/doc/testdata/a1.go create mode 100644 src/go/doc/testdata/b.0.golden create mode 100644 src/go/doc/testdata/b.1.golden create mode 100644 src/go/doc/testdata/b.2.golden create mode 100644 src/go/doc/testdata/b.go create mode 100644 src/go/doc/testdata/benchmark.go create mode 100644 src/go/doc/testdata/blank.0.golden create mode 100644 src/go/doc/testdata/blank.1.golden create mode 100644 src/go/doc/testdata/blank.2.golden create mode 100644 src/go/doc/testdata/blank.go create mode 100644 src/go/doc/testdata/bugpara.0.golden create mode 100644 src/go/doc/testdata/bugpara.1.golden create mode 100644 src/go/doc/testdata/bugpara.2.golden create mode 100644 src/go/doc/testdata/bugpara.go create mode 100644 src/go/doc/testdata/c.0.golden create mode 100644 src/go/doc/testdata/c.1.golden create mode 100644 src/go/doc/testdata/c.2.golden create mode 100644 src/go/doc/testdata/c.go create mode 100644 src/go/doc/testdata/d.0.golden create mode 100644 src/go/doc/testdata/d.1.golden create mode 100644 src/go/doc/testdata/d.2.golden create mode 100644 src/go/doc/testdata/d1.go create mode 100644 src/go/doc/testdata/d2.go create mode 100644 src/go/doc/testdata/e.0.golden create mode 100644 src/go/doc/testdata/e.1.golden create mode 100644 src/go/doc/testdata/e.2.golden create mode 100644 src/go/doc/testdata/e.go create mode 100644 src/go/doc/testdata/error1.0.golden create mode 100644 src/go/doc/testdata/error1.1.golden create mode 100644 src/go/doc/testdata/error1.2.golden create mode 100644 src/go/doc/testdata/error1.go create mode 100644 src/go/doc/testdata/error2.0.golden create mode 100644 src/go/doc/testdata/error2.1.golden create mode 100644 src/go/doc/testdata/error2.2.golden create mode 100644 src/go/doc/testdata/error2.go create mode 100644 src/go/doc/testdata/example.go create mode 100644 src/go/doc/testdata/examples/README.md create mode 100644 src/go/doc/testdata/examples/empty.go create mode 100644 src/go/doc/testdata/examples/empty.golden create mode 100644 src/go/doc/testdata/examples/generic_constraints.go create mode 100644 src/go/doc/testdata/examples/generic_constraints.golden create mode 100644 src/go/doc/testdata/examples/import_groups.go create mode 100644 src/go/doc/testdata/examples/import_groups.golden create mode 100644 src/go/doc/testdata/examples/import_groups_named.go create mode 100644 src/go/doc/testdata/examples/import_groups_named.golden create mode 100644 src/go/doc/testdata/examples/inspect_signature.go create mode 100644 src/go/doc/testdata/examples/inspect_signature.golden create mode 100644 src/go/doc/testdata/examples/iota.go create mode 100644 src/go/doc/testdata/examples/iota.golden create mode 100644 src/go/doc/testdata/examples/issue43658.go create mode 100644 src/go/doc/testdata/examples/issue43658.golden create mode 100644 src/go/doc/testdata/examples/multiple.go create mode 100644 src/go/doc/testdata/examples/multiple.golden create mode 100644 src/go/doc/testdata/examples/values.go create mode 100644 src/go/doc/testdata/examples/values.golden create mode 100644 src/go/doc/testdata/examples/whole_file.go create mode 100644 src/go/doc/testdata/examples/whole_file.golden create mode 100644 src/go/doc/testdata/examples/whole_function.go create mode 100644 src/go/doc/testdata/examples/whole_function.golden create mode 100644 src/go/doc/testdata/examples/whole_function_external.go create mode 100644 src/go/doc/testdata/examples/whole_function_external.golden create mode 100644 src/go/doc/testdata/f.0.golden create mode 100644 src/go/doc/testdata/f.1.golden create mode 100644 src/go/doc/testdata/f.2.golden create mode 100644 src/go/doc/testdata/f.go create mode 100644 src/go/doc/testdata/g.0.golden create mode 100644 src/go/doc/testdata/g.1.golden create mode 100644 src/go/doc/testdata/g.2.golden create mode 100644 src/go/doc/testdata/g.go create mode 100644 src/go/doc/testdata/generics.0.golden create mode 100644 src/go/doc/testdata/generics.1.golden create mode 100644 src/go/doc/testdata/generics.2.golden create mode 100644 src/go/doc/testdata/generics.go create mode 100644 src/go/doc/testdata/issue12839.0.golden create mode 100644 src/go/doc/testdata/issue12839.1.golden create mode 100644 src/go/doc/testdata/issue12839.2.golden create mode 100644 src/go/doc/testdata/issue12839.go create mode 100644 src/go/doc/testdata/issue13742.0.golden create mode 100644 src/go/doc/testdata/issue13742.1.golden create mode 100644 src/go/doc/testdata/issue13742.2.golden create mode 100644 src/go/doc/testdata/issue13742.go create mode 100644 src/go/doc/testdata/issue16153.0.golden create mode 100644 src/go/doc/testdata/issue16153.1.golden create mode 100644 src/go/doc/testdata/issue16153.2.golden create mode 100644 src/go/doc/testdata/issue16153.go create mode 100644 src/go/doc/testdata/issue17788.0.golden create mode 100644 src/go/doc/testdata/issue17788.1.golden create mode 100644 src/go/doc/testdata/issue17788.2.golden create mode 100644 src/go/doc/testdata/issue17788.go create mode 100644 src/go/doc/testdata/issue22856.0.golden create mode 100644 src/go/doc/testdata/issue22856.1.golden create mode 100644 src/go/doc/testdata/issue22856.2.golden create mode 100644 src/go/doc/testdata/issue22856.go create mode 100644 src/go/doc/testdata/pkgdoc/doc.go create mode 100644 src/go/doc/testdata/predeclared.0.golden create mode 100644 src/go/doc/testdata/predeclared.1.golden create mode 100644 src/go/doc/testdata/predeclared.2.golden create mode 100644 src/go/doc/testdata/predeclared.go create mode 100644 src/go/doc/testdata/template.txt create mode 100644 src/go/doc/testdata/testing.0.golden create mode 100644 src/go/doc/testdata/testing.1.golden create mode 100644 src/go/doc/testdata/testing.2.golden create mode 100644 src/go/doc/testdata/testing.go create mode 100644 src/go/format/benchmark_test.go create mode 100644 src/go/format/example_test.go create mode 100644 src/go/format/format.go create mode 100644 src/go/format/format_test.go create mode 100644 src/go/format/internal.go create mode 100644 src/go/importer/importer.go create mode 100644 src/go/importer/importer_test.go create mode 100644 src/go/internal/gccgoimporter/ar.go create mode 100644 src/go/internal/gccgoimporter/gccgoinstallation.go create mode 100644 src/go/internal/gccgoimporter/gccgoinstallation_test.go create mode 100644 src/go/internal/gccgoimporter/importer.go create mode 100644 src/go/internal/gccgoimporter/importer_test.go create mode 100644 src/go/internal/gccgoimporter/parser.go create mode 100644 src/go/internal/gccgoimporter/parser_test.go create mode 100644 src/go/internal/gccgoimporter/testdata/aliases.go create mode 100644 src/go/internal/gccgoimporter/testdata/aliases.gox create mode 100644 src/go/internal/gccgoimporter/testdata/complexnums.go create mode 100644 src/go/internal/gccgoimporter/testdata/complexnums.gox create mode 100644 src/go/internal/gccgoimporter/testdata/conversions.go create mode 100644 src/go/internal/gccgoimporter/testdata/conversions.gox create mode 100644 src/go/internal/gccgoimporter/testdata/escapeinfo.go create mode 100644 src/go/internal/gccgoimporter/testdata/escapeinfo.gox create mode 100644 src/go/internal/gccgoimporter/testdata/imports.go create mode 100644 src/go/internal/gccgoimporter/testdata/imports.gox create mode 100644 src/go/internal/gccgoimporter/testdata/issue27856.go create mode 100644 src/go/internal/gccgoimporter/testdata/issue27856.gox create mode 100644 src/go/internal/gccgoimporter/testdata/issue29198.go create mode 100644 src/go/internal/gccgoimporter/testdata/issue29198.gox create mode 100644 src/go/internal/gccgoimporter/testdata/issue30628.go create mode 100644 src/go/internal/gccgoimporter/testdata/issue30628.gox create mode 100644 src/go/internal/gccgoimporter/testdata/issue31540.go create mode 100644 src/go/internal/gccgoimporter/testdata/issue31540.gox create mode 100644 src/go/internal/gccgoimporter/testdata/issue34182.go create mode 100644 src/go/internal/gccgoimporter/testdata/issue34182.gox create mode 100644 src/go/internal/gccgoimporter/testdata/libimportsar.a create mode 100644 src/go/internal/gccgoimporter/testdata/nointerface.go create mode 100644 src/go/internal/gccgoimporter/testdata/nointerface.gox create mode 100644 src/go/internal/gccgoimporter/testdata/notinheap.go create mode 100644 src/go/internal/gccgoimporter/testdata/notinheap.gox create mode 100644 src/go/internal/gccgoimporter/testdata/pointer.go create mode 100644 src/go/internal/gccgoimporter/testdata/pointer.gox create mode 100644 src/go/internal/gccgoimporter/testdata/time.gox create mode 100644 src/go/internal/gccgoimporter/testdata/unicode.gox create mode 100644 src/go/internal/gccgoimporter/testdata/v1reflect.gox create mode 100644 src/go/internal/gcimporter/exportdata.go create mode 100644 src/go/internal/gcimporter/gcimporter.go create mode 100644 src/go/internal/gcimporter/gcimporter_test.go create mode 100644 src/go/internal/gcimporter/iimport.go create mode 100644 src/go/internal/gcimporter/support.go create mode 100644 src/go/internal/gcimporter/testdata/a.go create mode 100644 src/go/internal/gcimporter/testdata/b.go create mode 100644 src/go/internal/gcimporter/testdata/exports.go create mode 100644 src/go/internal/gcimporter/testdata/g.go create mode 100644 src/go/internal/gcimporter/testdata/generics.go create mode 100644 src/go/internal/gcimporter/testdata/issue15920.go create mode 100644 src/go/internal/gcimporter/testdata/issue20046.go create mode 100644 src/go/internal/gcimporter/testdata/issue25301.go create mode 100644 src/go/internal/gcimporter/testdata/issue25596.go create mode 100644 src/go/internal/gcimporter/testdata/issue57015.go create mode 100644 src/go/internal/gcimporter/testdata/p.go create mode 100644 src/go/internal/gcimporter/testdata/versions/test.go create mode 100644 src/go/internal/gcimporter/testdata/versions/test_go1.11_0i.a create mode 100644 src/go/internal/gcimporter/testdata/versions/test_go1.11_6b.a create mode 100644 src/go/internal/gcimporter/testdata/versions/test_go1.11_999b.a create mode 100644 src/go/internal/gcimporter/testdata/versions/test_go1.11_999i.a create mode 100644 src/go/internal/gcimporter/testdata/versions/test_go1.7_0.a create mode 100644 src/go/internal/gcimporter/testdata/versions/test_go1.7_1.a create mode 100644 src/go/internal/gcimporter/testdata/versions/test_go1.8_4.a create mode 100644 src/go/internal/gcimporter/testdata/versions/test_go1.8_5.a create mode 100644 src/go/internal/gcimporter/ureader.go create mode 100644 src/go/internal/srcimporter/srcimporter.go create mode 100644 src/go/internal/srcimporter/srcimporter_test.go create mode 100644 src/go/internal/srcimporter/testdata/issue20855/issue20855.go create mode 100644 src/go/internal/srcimporter/testdata/issue23092/issue23092.go create mode 100644 src/go/internal/srcimporter/testdata/issue24392/issue24392.go create mode 100644 src/go/internal/typeparams/typeparams.go create mode 100644 src/go/parser/error_test.go create mode 100644 src/go/parser/example_test.go create mode 100644 src/go/parser/interface.go create mode 100644 src/go/parser/parser.go create mode 100644 src/go/parser/parser_test.go create mode 100644 src/go/parser/performance_test.go create mode 100644 src/go/parser/resolver.go create mode 100644 src/go/parser/resolver_test.go create mode 100644 src/go/parser/short_test.go create mode 100644 src/go/parser/testdata/chans.go2 create mode 100644 src/go/parser/testdata/commas.src create mode 100644 src/go/parser/testdata/interface.go2 create mode 100644 src/go/parser/testdata/issue11377.src create mode 100644 src/go/parser/testdata/issue23434.src create mode 100644 src/go/parser/testdata/issue3106.src create mode 100644 src/go/parser/testdata/issue34946.src create mode 100644 src/go/parser/testdata/issue42951/not_a_file.go/invalid.go create mode 100644 src/go/parser/testdata/issue44504.src create mode 100644 src/go/parser/testdata/issue49174.go2 create mode 100644 src/go/parser/testdata/issue49175.go2 create mode 100644 src/go/parser/testdata/issue49482.go2 create mode 100644 src/go/parser/testdata/issue50427.go2 create mode 100644 src/go/parser/testdata/linalg.go2 create mode 100644 src/go/parser/testdata/map.go2 create mode 100644 src/go/parser/testdata/metrics.go2 create mode 100644 src/go/parser/testdata/resolution/issue45136.src create mode 100644 src/go/parser/testdata/resolution/issue45160.src create mode 100644 src/go/parser/testdata/resolution/resolution.src create mode 100644 src/go/parser/testdata/resolution/typeparams.go2 create mode 100644 src/go/parser/testdata/set.go2 create mode 100644 src/go/parser/testdata/slices.go2 create mode 100644 src/go/parser/testdata/sort.go2 create mode 100644 src/go/parser/testdata/tparams.go2 create mode 100644 src/go/parser/testdata/typeset.go2 create mode 100644 src/go/printer/comment.go create mode 100644 src/go/printer/example_test.go create mode 100644 src/go/printer/gobuild.go create mode 100644 src/go/printer/nodes.go create mode 100644 src/go/printer/performance_test.go create mode 100644 src/go/printer/printer.go create mode 100644 src/go/printer/printer_test.go create mode 100644 src/go/printer/testdata/alignment.golden create mode 100644 src/go/printer/testdata/alignment.input create mode 100644 src/go/printer/testdata/comments.golden create mode 100644 src/go/printer/testdata/comments.input create mode 100644 src/go/printer/testdata/comments.x create mode 100644 src/go/printer/testdata/comments2.golden create mode 100644 src/go/printer/testdata/comments2.input create mode 100644 src/go/printer/testdata/complit.input create mode 100644 src/go/printer/testdata/complit.x create mode 100644 src/go/printer/testdata/declarations.golden create mode 100644 src/go/printer/testdata/declarations.input create mode 100644 src/go/printer/testdata/doc.golden create mode 100644 src/go/printer/testdata/doc.input create mode 100644 src/go/printer/testdata/empty.golden create mode 100644 src/go/printer/testdata/empty.input create mode 100644 src/go/printer/testdata/expressions.golden create mode 100644 src/go/printer/testdata/expressions.input create mode 100644 src/go/printer/testdata/expressions.raw create mode 100644 src/go/printer/testdata/generics.golden create mode 100644 src/go/printer/testdata/generics.input create mode 100644 src/go/printer/testdata/go2numbers.golden create mode 100644 src/go/printer/testdata/go2numbers.input create mode 100644 src/go/printer/testdata/go2numbers.norm create mode 100644 src/go/printer/testdata/gobuild1.golden create mode 100644 src/go/printer/testdata/gobuild1.input create mode 100644 src/go/printer/testdata/gobuild2.golden create mode 100644 src/go/printer/testdata/gobuild2.input create mode 100644 src/go/printer/testdata/gobuild3.golden create mode 100644 src/go/printer/testdata/gobuild3.input create mode 100644 src/go/printer/testdata/gobuild4.golden create mode 100644 src/go/printer/testdata/gobuild4.input create mode 100644 src/go/printer/testdata/gobuild5.golden create mode 100644 src/go/printer/testdata/gobuild5.input create mode 100644 src/go/printer/testdata/gobuild6.golden create mode 100644 src/go/printer/testdata/gobuild6.input create mode 100644 src/go/printer/testdata/gobuild7.golden create mode 100644 src/go/printer/testdata/gobuild7.input create mode 100644 src/go/printer/testdata/linebreaks.golden create mode 100644 src/go/printer/testdata/linebreaks.input create mode 100644 src/go/printer/testdata/parser.go create mode 100644 src/go/printer/testdata/slow.golden create mode 100644 src/go/printer/testdata/slow.input create mode 100644 src/go/printer/testdata/statements.golden create mode 100644 src/go/printer/testdata/statements.input create mode 100644 src/go/scanner/errors.go create mode 100644 src/go/scanner/example_test.go create mode 100644 src/go/scanner/scanner.go create mode 100644 src/go/scanner/scanner_test.go create mode 100644 src/go/token/example_test.go create mode 100644 src/go/token/position.go create mode 100644 src/go/token/position_bench_test.go create mode 100644 src/go/token/position_test.go create mode 100644 src/go/token/serialize.go create mode 100644 src/go/token/serialize_test.go create mode 100644 src/go/token/token.go create mode 100644 src/go/token/token_test.go create mode 100644 src/go/types/api.go create mode 100644 src/go/types/api_test.go create mode 100644 src/go/types/array.go create mode 100644 src/go/types/assignments.go create mode 100644 src/go/types/basic.go create mode 100644 src/go/types/builtins.go create mode 100644 src/go/types/builtins_test.go create mode 100644 src/go/types/call.go create mode 100644 src/go/types/chan.go create mode 100644 src/go/types/check.go create mode 100644 src/go/types/check_test.go create mode 100644 src/go/types/context.go create mode 100644 src/go/types/context_test.go create mode 100644 src/go/types/conversions.go create mode 100644 src/go/types/decl.go create mode 100644 src/go/types/errorcalls_test.go create mode 100644 src/go/types/errors.go create mode 100644 src/go/types/errors_test.go create mode 100644 src/go/types/eval.go create mode 100644 src/go/types/eval_test.go create mode 100644 src/go/types/example_test.go create mode 100644 src/go/types/expr.go create mode 100644 src/go/types/exprstring.go create mode 100644 src/go/types/exprstring_test.go create mode 100644 src/go/types/gccgosizes.go create mode 100644 src/go/types/gotype.go create mode 100644 src/go/types/hilbert_test.go create mode 100644 src/go/types/index.go create mode 100644 src/go/types/infer.go create mode 100644 src/go/types/initorder.go create mode 100644 src/go/types/instantiate.go create mode 100644 src/go/types/instantiate_test.go create mode 100644 src/go/types/interface.go create mode 100644 src/go/types/issues_test.go create mode 100644 src/go/types/labels.go create mode 100644 src/go/types/lookup.go create mode 100644 src/go/types/lookup_test.go create mode 100644 src/go/types/main_test.go create mode 100644 src/go/types/map.go create mode 100644 src/go/types/methodset.go create mode 100644 src/go/types/methodset_test.go create mode 100644 src/go/types/mono.go create mode 100644 src/go/types/mono_test.go create mode 100644 src/go/types/named.go create mode 100644 src/go/types/named_test.go create mode 100644 src/go/types/object.go create mode 100644 src/go/types/object_test.go create mode 100644 src/go/types/objset.go create mode 100644 src/go/types/operand.go create mode 100644 src/go/types/package.go create mode 100644 src/go/types/pointer.go create mode 100644 src/go/types/predicates.go create mode 100644 src/go/types/resolver.go create mode 100644 src/go/types/resolver_test.go create mode 100644 src/go/types/return.go create mode 100644 src/go/types/scope.go create mode 100644 src/go/types/selection.go create mode 100644 src/go/types/self_test.go create mode 100644 src/go/types/signature.go create mode 100644 src/go/types/sizeof_test.go create mode 100644 src/go/types/sizes.go create mode 100644 src/go/types/sizes_test.go create mode 100644 src/go/types/slice.go create mode 100644 src/go/types/stdlib_test.go create mode 100644 src/go/types/stmt.go create mode 100644 src/go/types/struct.go create mode 100644 src/go/types/subst.go create mode 100644 src/go/types/termlist.go create mode 100644 src/go/types/termlist_test.go create mode 100644 src/go/types/testdata/local/issue47996.go create mode 100644 src/go/types/testdata/local/shifts.go create mode 100644 src/go/types/testdata/manual.go create mode 100644 src/go/types/token_test.go create mode 100644 src/go/types/tuple.go create mode 100644 src/go/types/type.go create mode 100644 src/go/types/typelists.go create mode 100644 src/go/types/typeparam.go create mode 100644 src/go/types/typeset.go create mode 100644 src/go/types/typeset_test.go create mode 100644 src/go/types/typestring.go create mode 100644 src/go/types/typestring_test.go create mode 100644 src/go/types/typeterm.go create mode 100644 src/go/types/typeterm_test.go create mode 100644 src/go/types/typexpr.go create mode 100644 src/go/types/unify.go create mode 100644 src/go/types/union.go create mode 100644 src/go/types/universe.go create mode 100644 src/go/types/validtype.go create mode 100644 src/go/types/version.go (limited to 'src/go') diff --git a/src/go/ast/ast.go b/src/go/ast/ast.go new file mode 100644 index 0000000..9baf72f --- /dev/null +++ b/src/go/ast/ast.go @@ -0,0 +1,1074 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package ast declares the types used to represent syntax trees for Go +// packages. +package ast + +import ( + "go/token" + "strings" +) + +// ---------------------------------------------------------------------------- +// Interfaces +// +// There are 3 main classes of nodes: Expressions and type nodes, +// statement nodes, and declaration nodes. The node names usually +// match the corresponding Go spec production names to which they +// correspond. The node fields correspond to the individual parts +// of the respective productions. +// +// All nodes contain position information marking the beginning of +// the corresponding source text segment; it is accessible via the +// Pos accessor method. Nodes may contain additional position info +// for language constructs where comments may be found between parts +// of the construct (typically any larger, parenthesized subpart). +// That position information is needed to properly position comments +// when printing the construct. + +// All node types implement the Node interface. +type Node interface { + Pos() token.Pos // position of first character belonging to the node + End() token.Pos // position of first character immediately after the node +} + +// All expression nodes implement the Expr interface. +type Expr interface { + Node + exprNode() +} + +// All statement nodes implement the Stmt interface. +type Stmt interface { + Node + stmtNode() +} + +// All declaration nodes implement the Decl interface. +type Decl interface { + Node + declNode() +} + +// ---------------------------------------------------------------------------- +// Comments + +// A Comment node represents a single //-style or /*-style comment. +// +// The Text field contains the comment text without carriage returns (\r) that +// may have been present in the source. Because a comment's end position is +// computed using len(Text), the position reported by End() does not match the +// true source end position for comments containing carriage returns. +type Comment struct { + Slash token.Pos // position of "/" starting the comment + Text string // comment text (excluding '\n' for //-style comments) +} + +func (c *Comment) Pos() token.Pos { return c.Slash } +func (c *Comment) End() token.Pos { return token.Pos(int(c.Slash) + len(c.Text)) } + +// A CommentGroup represents a sequence of comments +// with no other tokens and no empty lines between. +type CommentGroup struct { + List []*Comment // len(List) > 0 +} + +func (g *CommentGroup) Pos() token.Pos { return g.List[0].Pos() } +func (g *CommentGroup) End() token.Pos { return g.List[len(g.List)-1].End() } + +func isWhitespace(ch byte) bool { return ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' } + +func stripTrailingWhitespace(s string) string { + i := len(s) + for i > 0 && isWhitespace(s[i-1]) { + i-- + } + return s[0:i] +} + +// Text returns the text of the comment. +// Comment markers (//, /*, and */), the first space of a line comment, and +// leading and trailing empty lines are removed. +// Comment directives like "//line" and "//go:noinline" are also removed. +// Multiple empty lines are reduced to one, and trailing space on lines is trimmed. +// Unless the result is empty, it is newline-terminated. +func (g *CommentGroup) Text() string { + if g == nil { + return "" + } + comments := make([]string, len(g.List)) + for i, c := range g.List { + comments[i] = c.Text + } + + lines := make([]string, 0, 10) // most comments are less than 10 lines + for _, c := range comments { + // Remove comment markers. + // The parser has given us exactly the comment text. + switch c[1] { + case '/': + //-style comment (no newline at the end) + c = c[2:] + if len(c) == 0 { + // empty line + break + } + if c[0] == ' ' { + // strip first space - required for Example tests + c = c[1:] + break + } + if isDirective(c) { + // Ignore //go:noinline, //line, and so on. + continue + } + case '*': + /*-style comment */ + c = c[2 : len(c)-2] + } + + // Split on newlines. + cl := strings.Split(c, "\n") + + // Walk lines, stripping trailing white space and adding to list. + for _, l := range cl { + lines = append(lines, stripTrailingWhitespace(l)) + } + } + + // Remove leading blank lines; convert runs of + // interior blank lines to a single blank line. + n := 0 + for _, line := range lines { + if line != "" || n > 0 && lines[n-1] != "" { + lines[n] = line + n++ + } + } + lines = lines[0:n] + + // Add final "" entry to get trailing newline from Join. + if n > 0 && lines[n-1] != "" { + lines = append(lines, "") + } + + return strings.Join(lines, "\n") +} + +// isDirective reports whether c is a comment directive. +// This code is also in go/printer. +func isDirective(c string) bool { + // "//line " is a line directive. + // "//extern " is for gccgo. + // "//export " is for cgo. + // (The // has been removed.) + if strings.HasPrefix(c, "line ") || strings.HasPrefix(c, "extern ") || strings.HasPrefix(c, "export ") { + return true + } + + // "//[a-z0-9]+:[a-z0-9]" + // (The // has been removed.) + colon := strings.Index(c, ":") + if colon <= 0 || colon+1 >= len(c) { + return false + } + for i := 0; i <= colon+1; i++ { + if i == colon { + continue + } + b := c[i] + if !('a' <= b && b <= 'z' || '0' <= b && b <= '9') { + return false + } + } + return true +} + +// ---------------------------------------------------------------------------- +// Expressions and types + +// A Field represents a Field declaration list in a struct type, +// a method list in an interface type, or a parameter/result declaration +// in a signature. +// Field.Names is nil for unnamed parameters (parameter lists which only contain types) +// and embedded struct fields. In the latter case, the field name is the type name. +type Field struct { + Doc *CommentGroup // associated documentation; or nil + Names []*Ident // field/method/(type) parameter names; or nil + Type Expr // field/method/parameter type; or nil + Tag *BasicLit // field tag; or nil + Comment *CommentGroup // line comments; or nil +} + +func (f *Field) Pos() token.Pos { + if len(f.Names) > 0 { + return f.Names[0].Pos() + } + if f.Type != nil { + return f.Type.Pos() + } + return token.NoPos +} + +func (f *Field) End() token.Pos { + if f.Tag != nil { + return f.Tag.End() + } + if f.Type != nil { + return f.Type.End() + } + if len(f.Names) > 0 { + return f.Names[len(f.Names)-1].End() + } + return token.NoPos +} + +// A FieldList represents a list of Fields, enclosed by parentheses, +// curly braces, or square brackets. +type FieldList struct { + Opening token.Pos // position of opening parenthesis/brace/bracket, if any + List []*Field // field list; or nil + Closing token.Pos // position of closing parenthesis/brace/bracket, if any +} + +func (f *FieldList) Pos() token.Pos { + if f.Opening.IsValid() { + return f.Opening + } + // the list should not be empty in this case; + // be conservative and guard against bad ASTs + if len(f.List) > 0 { + return f.List[0].Pos() + } + return token.NoPos +} + +func (f *FieldList) End() token.Pos { + if f.Closing.IsValid() { + return f.Closing + 1 + } + // the list should not be empty in this case; + // be conservative and guard against bad ASTs + if n := len(f.List); n > 0 { + return f.List[n-1].End() + } + return token.NoPos +} + +// NumFields returns the number of parameters or struct fields represented by a FieldList. +func (f *FieldList) NumFields() int { + n := 0 + if f != nil { + for _, g := range f.List { + m := len(g.Names) + if m == 0 { + m = 1 + } + n += m + } + } + return n +} + +// An expression is represented by a tree consisting of one +// or more of the following concrete expression nodes. +type ( + // A BadExpr node is a placeholder for an expression containing + // syntax errors for which a correct expression node cannot be + // created. + // + BadExpr struct { + From, To token.Pos // position range of bad expression + } + + // An Ident node represents an identifier. + Ident struct { + NamePos token.Pos // identifier position + Name string // identifier name + Obj *Object // denoted object; or nil + } + + // An Ellipsis node stands for the "..." type in a + // parameter list or the "..." length in an array type. + // + Ellipsis struct { + Ellipsis token.Pos // position of "..." + Elt Expr // ellipsis element type (parameter lists only); or nil + } + + // A BasicLit node represents a literal of basic type. + BasicLit struct { + ValuePos token.Pos // literal position + Kind token.Token // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING + Value string // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o` + } + + // A FuncLit node represents a function literal. + FuncLit struct { + Type *FuncType // function type + Body *BlockStmt // function body + } + + // A CompositeLit node represents a composite literal. + CompositeLit struct { + Type Expr // literal type; or nil + Lbrace token.Pos // position of "{" + Elts []Expr // list of composite elements; or nil + Rbrace token.Pos // position of "}" + Incomplete bool // true if (source) expressions are missing in the Elts list + } + + // A ParenExpr node represents a parenthesized expression. + ParenExpr struct { + Lparen token.Pos // position of "(" + X Expr // parenthesized expression + Rparen token.Pos // position of ")" + } + + // A SelectorExpr node represents an expression followed by a selector. + SelectorExpr struct { + X Expr // expression + Sel *Ident // field selector + } + + // An IndexExpr node represents an expression followed by an index. + IndexExpr struct { + X Expr // expression + Lbrack token.Pos // position of "[" + Index Expr // index expression + Rbrack token.Pos // position of "]" + } + + // An IndexListExpr node represents an expression followed by multiple + // indices. + IndexListExpr struct { + X Expr // expression + Lbrack token.Pos // position of "[" + Indices []Expr // index expressions + Rbrack token.Pos // position of "]" + } + + // A SliceExpr node represents an expression followed by slice indices. + SliceExpr struct { + X Expr // expression + Lbrack token.Pos // position of "[" + Low Expr // begin of slice range; or nil + High Expr // end of slice range; or nil + Max Expr // maximum capacity of slice; or nil + Slice3 bool // true if 3-index slice (2 colons present) + Rbrack token.Pos // position of "]" + } + + // A TypeAssertExpr node represents an expression followed by a + // type assertion. + // + TypeAssertExpr struct { + X Expr // expression + Lparen token.Pos // position of "(" + Type Expr // asserted type; nil means type switch X.(type) + Rparen token.Pos // position of ")" + } + + // A CallExpr node represents an expression followed by an argument list. + CallExpr struct { + Fun Expr // function expression + Lparen token.Pos // position of "(" + Args []Expr // function arguments; or nil + Ellipsis token.Pos // position of "..." (token.NoPos if there is no "...") + Rparen token.Pos // position of ")" + } + + // A StarExpr node represents an expression of the form "*" Expression. + // Semantically it could be a unary "*" expression, or a pointer type. + // + StarExpr struct { + Star token.Pos // position of "*" + X Expr // operand + } + + // A UnaryExpr node represents a unary expression. + // Unary "*" expressions are represented via StarExpr nodes. + // + UnaryExpr struct { + OpPos token.Pos // position of Op + Op token.Token // operator + X Expr // operand + } + + // A BinaryExpr node represents a binary expression. + BinaryExpr struct { + X Expr // left operand + OpPos token.Pos // position of Op + Op token.Token // operator + Y Expr // right operand + } + + // A KeyValueExpr node represents (key : value) pairs + // in composite literals. + // + KeyValueExpr struct { + Key Expr + Colon token.Pos // position of ":" + Value Expr + } +) + +// The direction of a channel type is indicated by a bit +// mask including one or both of the following constants. +type ChanDir int + +const ( + SEND ChanDir = 1 << iota + RECV +) + +// A type is represented by a tree consisting of one +// or more of the following type-specific expression +// nodes. +type ( + // An ArrayType node represents an array or slice type. + ArrayType struct { + Lbrack token.Pos // position of "[" + Len Expr // Ellipsis node for [...]T array types, nil for slice types + Elt Expr // element type + } + + // A StructType node represents a struct type. + StructType struct { + Struct token.Pos // position of "struct" keyword + Fields *FieldList // list of field declarations + Incomplete bool // true if (source) fields are missing in the Fields list + } + + // Pointer types are represented via StarExpr nodes. + + // A FuncType node represents a function type. + FuncType struct { + Func token.Pos // position of "func" keyword (token.NoPos if there is no "func") + TypeParams *FieldList // type parameters; or nil + Params *FieldList // (incoming) parameters; non-nil + Results *FieldList // (outgoing) results; or nil + } + + // An InterfaceType node represents an interface type. + InterfaceType struct { + Interface token.Pos // position of "interface" keyword + Methods *FieldList // list of embedded interfaces, methods, or types + Incomplete bool // true if (source) methods or types are missing in the Methods list + } + + // A MapType node represents a map type. + MapType struct { + Map token.Pos // position of "map" keyword + Key Expr + Value Expr + } + + // A ChanType node represents a channel type. + ChanType struct { + Begin token.Pos // position of "chan" keyword or "<-" (whichever comes first) + Arrow token.Pos // position of "<-" (token.NoPos if there is no "<-") + Dir ChanDir // channel direction + Value Expr // value type + } +) + +// Pos and End implementations for expression/type nodes. + +func (x *BadExpr) Pos() token.Pos { return x.From } +func (x *Ident) Pos() token.Pos { return x.NamePos } +func (x *Ellipsis) Pos() token.Pos { return x.Ellipsis } +func (x *BasicLit) Pos() token.Pos { return x.ValuePos } +func (x *FuncLit) Pos() token.Pos { return x.Type.Pos() } +func (x *CompositeLit) Pos() token.Pos { + if x.Type != nil { + return x.Type.Pos() + } + return x.Lbrace +} +func (x *ParenExpr) Pos() token.Pos { return x.Lparen } +func (x *SelectorExpr) Pos() token.Pos { return x.X.Pos() } +func (x *IndexExpr) Pos() token.Pos { return x.X.Pos() } +func (x *IndexListExpr) Pos() token.Pos { return x.X.Pos() } +func (x *SliceExpr) Pos() token.Pos { return x.X.Pos() } +func (x *TypeAssertExpr) Pos() token.Pos { return x.X.Pos() } +func (x *CallExpr) Pos() token.Pos { return x.Fun.Pos() } +func (x *StarExpr) Pos() token.Pos { return x.Star } +func (x *UnaryExpr) Pos() token.Pos { return x.OpPos } +func (x *BinaryExpr) Pos() token.Pos { return x.X.Pos() } +func (x *KeyValueExpr) Pos() token.Pos { return x.Key.Pos() } +func (x *ArrayType) Pos() token.Pos { return x.Lbrack } +func (x *StructType) Pos() token.Pos { return x.Struct } +func (x *FuncType) Pos() token.Pos { + if x.Func.IsValid() || x.Params == nil { // see issue 3870 + return x.Func + } + return x.Params.Pos() // interface method declarations have no "func" keyword +} +func (x *InterfaceType) Pos() token.Pos { return x.Interface } +func (x *MapType) Pos() token.Pos { return x.Map } +func (x *ChanType) Pos() token.Pos { return x.Begin } + +func (x *BadExpr) End() token.Pos { return x.To } +func (x *Ident) End() token.Pos { return token.Pos(int(x.NamePos) + len(x.Name)) } +func (x *Ellipsis) End() token.Pos { + if x.Elt != nil { + return x.Elt.End() + } + return x.Ellipsis + 3 // len("...") +} +func (x *BasicLit) End() token.Pos { return token.Pos(int(x.ValuePos) + len(x.Value)) } +func (x *FuncLit) End() token.Pos { return x.Body.End() } +func (x *CompositeLit) End() token.Pos { return x.Rbrace + 1 } +func (x *ParenExpr) End() token.Pos { return x.Rparen + 1 } +func (x *SelectorExpr) End() token.Pos { return x.Sel.End() } +func (x *IndexExpr) End() token.Pos { return x.Rbrack + 1 } +func (x *IndexListExpr) End() token.Pos { return x.Rbrack + 1 } +func (x *SliceExpr) End() token.Pos { return x.Rbrack + 1 } +func (x *TypeAssertExpr) End() token.Pos { return x.Rparen + 1 } +func (x *CallExpr) End() token.Pos { return x.Rparen + 1 } +func (x *StarExpr) End() token.Pos { return x.X.End() } +func (x *UnaryExpr) End() token.Pos { return x.X.End() } +func (x *BinaryExpr) End() token.Pos { return x.Y.End() } +func (x *KeyValueExpr) End() token.Pos { return x.Value.End() } +func (x *ArrayType) End() token.Pos { return x.Elt.End() } +func (x *StructType) End() token.Pos { return x.Fields.End() } +func (x *FuncType) End() token.Pos { + if x.Results != nil { + return x.Results.End() + } + return x.Params.End() +} +func (x *InterfaceType) End() token.Pos { return x.Methods.End() } +func (x *MapType) End() token.Pos { return x.Value.End() } +func (x *ChanType) End() token.Pos { return x.Value.End() } + +// exprNode() ensures that only expression/type nodes can be +// assigned to an Expr. +func (*BadExpr) exprNode() {} +func (*Ident) exprNode() {} +func (*Ellipsis) exprNode() {} +func (*BasicLit) exprNode() {} +func (*FuncLit) exprNode() {} +func (*CompositeLit) exprNode() {} +func (*ParenExpr) exprNode() {} +func (*SelectorExpr) exprNode() {} +func (*IndexExpr) exprNode() {} +func (*IndexListExpr) exprNode() {} +func (*SliceExpr) exprNode() {} +func (*TypeAssertExpr) exprNode() {} +func (*CallExpr) exprNode() {} +func (*StarExpr) exprNode() {} +func (*UnaryExpr) exprNode() {} +func (*BinaryExpr) exprNode() {} +func (*KeyValueExpr) exprNode() {} + +func (*ArrayType) exprNode() {} +func (*StructType) exprNode() {} +func (*FuncType) exprNode() {} +func (*InterfaceType) exprNode() {} +func (*MapType) exprNode() {} +func (*ChanType) exprNode() {} + +// ---------------------------------------------------------------------------- +// Convenience functions for Idents + +// NewIdent creates a new Ident without position. +// Useful for ASTs generated by code other than the Go parser. +func NewIdent(name string) *Ident { return &Ident{token.NoPos, name, nil} } + +// IsExported reports whether name starts with an upper-case letter. +func IsExported(name string) bool { return token.IsExported(name) } + +// IsExported reports whether id starts with an upper-case letter. +func (id *Ident) IsExported() bool { return token.IsExported(id.Name) } + +func (id *Ident) String() string { + if id != nil { + return id.Name + } + return "" +} + +// ---------------------------------------------------------------------------- +// Statements + +// A statement is represented by a tree consisting of one +// or more of the following concrete statement nodes. +type ( + // A BadStmt node is a placeholder for statements containing + // syntax errors for which no correct statement nodes can be + // created. + // + BadStmt struct { + From, To token.Pos // position range of bad statement + } + + // A DeclStmt node represents a declaration in a statement list. + DeclStmt struct { + Decl Decl // *GenDecl with CONST, TYPE, or VAR token + } + + // An EmptyStmt node represents an empty statement. + // The "position" of the empty statement is the position + // of the immediately following (explicit or implicit) semicolon. + // + EmptyStmt struct { + Semicolon token.Pos // position of following ";" + Implicit bool // if set, ";" was omitted in the source + } + + // A LabeledStmt node represents a labeled statement. + LabeledStmt struct { + Label *Ident + Colon token.Pos // position of ":" + Stmt Stmt + } + + // An ExprStmt node represents a (stand-alone) expression + // in a statement list. + // + ExprStmt struct { + X Expr // expression + } + + // A SendStmt node represents a send statement. + SendStmt struct { + Chan Expr + Arrow token.Pos // position of "<-" + Value Expr + } + + // An IncDecStmt node represents an increment or decrement statement. + IncDecStmt struct { + X Expr + TokPos token.Pos // position of Tok + Tok token.Token // INC or DEC + } + + // An AssignStmt node represents an assignment or + // a short variable declaration. + // + AssignStmt struct { + Lhs []Expr + TokPos token.Pos // position of Tok + Tok token.Token // assignment token, DEFINE + Rhs []Expr + } + + // A GoStmt node represents a go statement. + GoStmt struct { + Go token.Pos // position of "go" keyword + Call *CallExpr + } + + // A DeferStmt node represents a defer statement. + DeferStmt struct { + Defer token.Pos // position of "defer" keyword + Call *CallExpr + } + + // A ReturnStmt node represents a return statement. + ReturnStmt struct { + Return token.Pos // position of "return" keyword + Results []Expr // result expressions; or nil + } + + // A BranchStmt node represents a break, continue, goto, + // or fallthrough statement. + // + BranchStmt struct { + TokPos token.Pos // position of Tok + Tok token.Token // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH) + Label *Ident // label name; or nil + } + + // A BlockStmt node represents a braced statement list. + BlockStmt struct { + Lbrace token.Pos // position of "{" + List []Stmt + Rbrace token.Pos // position of "}", if any (may be absent due to syntax error) + } + + // An IfStmt node represents an if statement. + IfStmt struct { + If token.Pos // position of "if" keyword + Init Stmt // initialization statement; or nil + Cond Expr // condition + Body *BlockStmt + Else Stmt // else branch; or nil + } + + // A CaseClause represents a case of an expression or type switch statement. + CaseClause struct { + Case token.Pos // position of "case" or "default" keyword + List []Expr // list of expressions or types; nil means default case + Colon token.Pos // position of ":" + Body []Stmt // statement list; or nil + } + + // A SwitchStmt node represents an expression switch statement. + SwitchStmt struct { + Switch token.Pos // position of "switch" keyword + Init Stmt // initialization statement; or nil + Tag Expr // tag expression; or nil + Body *BlockStmt // CaseClauses only + } + + // A TypeSwitchStmt node represents a type switch statement. + TypeSwitchStmt struct { + Switch token.Pos // position of "switch" keyword + Init Stmt // initialization statement; or nil + Assign Stmt // x := y.(type) or y.(type) + Body *BlockStmt // CaseClauses only + } + + // A CommClause node represents a case of a select statement. + CommClause struct { + Case token.Pos // position of "case" or "default" keyword + Comm Stmt // send or receive statement; nil means default case + Colon token.Pos // position of ":" + Body []Stmt // statement list; or nil + } + + // A SelectStmt node represents a select statement. + SelectStmt struct { + Select token.Pos // position of "select" keyword + Body *BlockStmt // CommClauses only + } + + // A ForStmt represents a for statement. + ForStmt struct { + For token.Pos // position of "for" keyword + Init Stmt // initialization statement; or nil + Cond Expr // condition; or nil + Post Stmt // post iteration statement; or nil + Body *BlockStmt + } + + // A RangeStmt represents a for statement with a range clause. + RangeStmt struct { + For token.Pos // position of "for" keyword + Key, Value Expr // Key, Value may be nil + TokPos token.Pos // position of Tok; invalid if Key == nil + Tok token.Token // ILLEGAL if Key == nil, ASSIGN, DEFINE + Range token.Pos // position of "range" keyword + X Expr // value to range over + Body *BlockStmt + } +) + +// Pos and End implementations for statement nodes. + +func (s *BadStmt) Pos() token.Pos { return s.From } +func (s *DeclStmt) Pos() token.Pos { return s.Decl.Pos() } +func (s *EmptyStmt) Pos() token.Pos { return s.Semicolon } +func (s *LabeledStmt) Pos() token.Pos { return s.Label.Pos() } +func (s *ExprStmt) Pos() token.Pos { return s.X.Pos() } +func (s *SendStmt) Pos() token.Pos { return s.Chan.Pos() } +func (s *IncDecStmt) Pos() token.Pos { return s.X.Pos() } +func (s *AssignStmt) Pos() token.Pos { return s.Lhs[0].Pos() } +func (s *GoStmt) Pos() token.Pos { return s.Go } +func (s *DeferStmt) Pos() token.Pos { return s.Defer } +func (s *ReturnStmt) Pos() token.Pos { return s.Return } +func (s *BranchStmt) Pos() token.Pos { return s.TokPos } +func (s *BlockStmt) Pos() token.Pos { return s.Lbrace } +func (s *IfStmt) Pos() token.Pos { return s.If } +func (s *CaseClause) Pos() token.Pos { return s.Case } +func (s *SwitchStmt) Pos() token.Pos { return s.Switch } +func (s *TypeSwitchStmt) Pos() token.Pos { return s.Switch } +func (s *CommClause) Pos() token.Pos { return s.Case } +func (s *SelectStmt) Pos() token.Pos { return s.Select } +func (s *ForStmt) Pos() token.Pos { return s.For } +func (s *RangeStmt) Pos() token.Pos { return s.For } + +func (s *BadStmt) End() token.Pos { return s.To } +func (s *DeclStmt) End() token.Pos { return s.Decl.End() } +func (s *EmptyStmt) End() token.Pos { + if s.Implicit { + return s.Semicolon + } + return s.Semicolon + 1 /* len(";") */ +} +func (s *LabeledStmt) End() token.Pos { return s.Stmt.End() } +func (s *ExprStmt) End() token.Pos { return s.X.End() } +func (s *SendStmt) End() token.Pos { return s.Value.End() } +func (s *IncDecStmt) End() token.Pos { + return s.TokPos + 2 /* len("++") */ +} +func (s *AssignStmt) End() token.Pos { return s.Rhs[len(s.Rhs)-1].End() } +func (s *GoStmt) End() token.Pos { return s.Call.End() } +func (s *DeferStmt) End() token.Pos { return s.Call.End() } +func (s *ReturnStmt) End() token.Pos { + if n := len(s.Results); n > 0 { + return s.Results[n-1].End() + } + return s.Return + 6 // len("return") +} +func (s *BranchStmt) End() token.Pos { + if s.Label != nil { + return s.Label.End() + } + return token.Pos(int(s.TokPos) + len(s.Tok.String())) +} +func (s *BlockStmt) End() token.Pos { + if s.Rbrace.IsValid() { + return s.Rbrace + 1 + } + if n := len(s.List); n > 0 { + return s.List[n-1].End() + } + return s.Lbrace + 1 +} +func (s *IfStmt) End() token.Pos { + if s.Else != nil { + return s.Else.End() + } + return s.Body.End() +} +func (s *CaseClause) End() token.Pos { + if n := len(s.Body); n > 0 { + return s.Body[n-1].End() + } + return s.Colon + 1 +} +func (s *SwitchStmt) End() token.Pos { return s.Body.End() } +func (s *TypeSwitchStmt) End() token.Pos { return s.Body.End() } +func (s *CommClause) End() token.Pos { + if n := len(s.Body); n > 0 { + return s.Body[n-1].End() + } + return s.Colon + 1 +} +func (s *SelectStmt) End() token.Pos { return s.Body.End() } +func (s *ForStmt) End() token.Pos { return s.Body.End() } +func (s *RangeStmt) End() token.Pos { return s.Body.End() } + +// stmtNode() ensures that only statement nodes can be +// assigned to a Stmt. +func (*BadStmt) stmtNode() {} +func (*DeclStmt) stmtNode() {} +func (*EmptyStmt) stmtNode() {} +func (*LabeledStmt) stmtNode() {} +func (*ExprStmt) stmtNode() {} +func (*SendStmt) stmtNode() {} +func (*IncDecStmt) stmtNode() {} +func (*AssignStmt) stmtNode() {} +func (*GoStmt) stmtNode() {} +func (*DeferStmt) stmtNode() {} +func (*ReturnStmt) stmtNode() {} +func (*BranchStmt) stmtNode() {} +func (*BlockStmt) stmtNode() {} +func (*IfStmt) stmtNode() {} +func (*CaseClause) stmtNode() {} +func (*SwitchStmt) stmtNode() {} +func (*TypeSwitchStmt) stmtNode() {} +func (*CommClause) stmtNode() {} +func (*SelectStmt) stmtNode() {} +func (*ForStmt) stmtNode() {} +func (*RangeStmt) stmtNode() {} + +// ---------------------------------------------------------------------------- +// Declarations + +// A Spec node represents a single (non-parenthesized) import, +// constant, type, or variable declaration. +type ( + // The Spec type stands for any of *ImportSpec, *ValueSpec, and *TypeSpec. + Spec interface { + Node + specNode() + } + + // An ImportSpec node represents a single package import. + ImportSpec struct { + Doc *CommentGroup // associated documentation; or nil + Name *Ident // local package name (including "."); or nil + Path *BasicLit // import path + Comment *CommentGroup // line comments; or nil + EndPos token.Pos // end of spec (overrides Path.Pos if nonzero) + } + + // A ValueSpec node represents a constant or variable declaration + // (ConstSpec or VarSpec production). + // + ValueSpec struct { + Doc *CommentGroup // associated documentation; or nil + Names []*Ident // value names (len(Names) > 0) + Type Expr // value type; or nil + Values []Expr // initial values; or nil + Comment *CommentGroup // line comments; or nil + } + + // A TypeSpec node represents a type declaration (TypeSpec production). + TypeSpec struct { + Doc *CommentGroup // associated documentation; or nil + Name *Ident // type name + TypeParams *FieldList // type parameters; or nil + Assign token.Pos // position of '=', if any + Type Expr // *Ident, *ParenExpr, *SelectorExpr, *StarExpr, or any of the *XxxTypes + Comment *CommentGroup // line comments; or nil + } +) + +// Pos and End implementations for spec nodes. + +func (s *ImportSpec) Pos() token.Pos { + if s.Name != nil { + return s.Name.Pos() + } + return s.Path.Pos() +} +func (s *ValueSpec) Pos() token.Pos { return s.Names[0].Pos() } +func (s *TypeSpec) Pos() token.Pos { return s.Name.Pos() } + +func (s *ImportSpec) End() token.Pos { + if s.EndPos != 0 { + return s.EndPos + } + return s.Path.End() +} + +func (s *ValueSpec) End() token.Pos { + if n := len(s.Values); n > 0 { + return s.Values[n-1].End() + } + if s.Type != nil { + return s.Type.End() + } + return s.Names[len(s.Names)-1].End() +} +func (s *TypeSpec) End() token.Pos { return s.Type.End() } + +// specNode() ensures that only spec nodes can be +// assigned to a Spec. +func (*ImportSpec) specNode() {} +func (*ValueSpec) specNode() {} +func (*TypeSpec) specNode() {} + +// A declaration is represented by one of the following declaration nodes. +type ( + // A BadDecl node is a placeholder for a declaration containing + // syntax errors for which a correct declaration node cannot be + // created. + // + BadDecl struct { + From, To token.Pos // position range of bad declaration + } + + // A GenDecl node (generic declaration node) represents an import, + // constant, type or variable declaration. A valid Lparen position + // (Lparen.IsValid()) indicates a parenthesized declaration. + // + // Relationship between Tok value and Specs element type: + // + // token.IMPORT *ImportSpec + // token.CONST *ValueSpec + // token.TYPE *TypeSpec + // token.VAR *ValueSpec + // + GenDecl struct { + Doc *CommentGroup // associated documentation; or nil + TokPos token.Pos // position of Tok + Tok token.Token // IMPORT, CONST, TYPE, or VAR + Lparen token.Pos // position of '(', if any + Specs []Spec + Rparen token.Pos // position of ')', if any + } + + // A FuncDecl node represents a function declaration. + FuncDecl struct { + Doc *CommentGroup // associated documentation; or nil + Recv *FieldList // receiver (methods); or nil (functions) + Name *Ident // function/method name + Type *FuncType // function signature: type and value parameters, results, and position of "func" keyword + Body *BlockStmt // function body; or nil for external (non-Go) function + } +) + +// Pos and End implementations for declaration nodes. + +func (d *BadDecl) Pos() token.Pos { return d.From } +func (d *GenDecl) Pos() token.Pos { return d.TokPos } +func (d *FuncDecl) Pos() token.Pos { return d.Type.Pos() } + +func (d *BadDecl) End() token.Pos { return d.To } +func (d *GenDecl) End() token.Pos { + if d.Rparen.IsValid() { + return d.Rparen + 1 + } + return d.Specs[0].End() +} +func (d *FuncDecl) End() token.Pos { + if d.Body != nil { + return d.Body.End() + } + return d.Type.End() +} + +// declNode() ensures that only declaration nodes can be +// assigned to a Decl. +func (*BadDecl) declNode() {} +func (*GenDecl) declNode() {} +func (*FuncDecl) declNode() {} + +// ---------------------------------------------------------------------------- +// Files and packages + +// A File node represents a Go source file. +// +// The Comments list contains all comments in the source file in order of +// appearance, including the comments that are pointed to from other nodes +// via Doc and Comment fields. +// +// For correct printing of source code containing comments (using packages +// go/format and go/printer), special care must be taken to update comments +// when a File's syntax tree is modified: For printing, comments are interspersed +// between tokens based on their position. If syntax tree nodes are +// removed or moved, relevant comments in their vicinity must also be removed +// (from the File.Comments list) or moved accordingly (by updating their +// positions). A CommentMap may be used to facilitate some of these operations. +// +// Whether and how a comment is associated with a node depends on the +// interpretation of the syntax tree by the manipulating program: Except for Doc +// and Comment comments directly associated with nodes, the remaining comments +// are "free-floating" (see also issues #18593, #20744). +type File struct { + Doc *CommentGroup // associated documentation; or nil + Package token.Pos // position of "package" keyword + Name *Ident // package name + Decls []Decl // top-level declarations; or nil + + FileStart, FileEnd token.Pos // start and end of entire file + Scope *Scope // package scope (this file only) + Imports []*ImportSpec // imports in this file + Unresolved []*Ident // unresolved identifiers in this file + Comments []*CommentGroup // list of all comments in the source file +} + +// Pos returns the position of the package declaration. +// (Use FileStart for the start of the entire file.) +func (f *File) Pos() token.Pos { return f.Package } + +// End returns the end of the last declaration in the file. +// (Use FileEnd for the end of the entire file.) +func (f *File) End() token.Pos { + if n := len(f.Decls); n > 0 { + return f.Decls[n-1].End() + } + return f.Name.End() +} + +// A Package node represents a set of source files +// collectively building a Go package. +type Package struct { + Name string // package name + Scope *Scope // package scope across all files + Imports map[string]*Object // map of package id -> package object + Files map[string]*File // Go source files by filename +} + +func (p *Package) Pos() token.Pos { return token.NoPos } +func (p *Package) End() token.Pos { return token.NoPos } diff --git a/src/go/ast/ast_test.go b/src/go/ast/ast_test.go new file mode 100644 index 0000000..66ae884 --- /dev/null +++ b/src/go/ast/ast_test.go @@ -0,0 +1,82 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import ( + "testing" +) + +var comments = []struct { + list []string + text string +}{ + {[]string{"//"}, ""}, + {[]string{"// "}, ""}, + {[]string{"//", "//", "// "}, ""}, + {[]string{"// foo "}, "foo\n"}, + {[]string{"//", "//", "// foo"}, "foo\n"}, + {[]string{"// foo bar "}, "foo bar\n"}, + {[]string{"// foo", "// bar"}, "foo\nbar\n"}, + {[]string{"// foo", "//", "//", "//", "// bar"}, "foo\n\nbar\n"}, + {[]string{"// foo", "/* bar */"}, "foo\n bar\n"}, + {[]string{"//", "//", "//", "// foo", "//", "//", "//"}, "foo\n"}, + + {[]string{"/**/"}, ""}, + {[]string{"/* */"}, ""}, + {[]string{"/**/", "/**/", "/* */"}, ""}, + {[]string{"/* Foo */"}, " Foo\n"}, + {[]string{"/* Foo Bar */"}, " Foo Bar\n"}, + {[]string{"/* Foo*/", "/* Bar*/"}, " Foo\n Bar\n"}, + {[]string{"/* Foo*/", "/**/", "/**/", "/**/", "// Bar"}, " Foo\n\nBar\n"}, + {[]string{"/* Foo*/", "/*\n*/", "//", "/*\n*/", "// Bar"}, " Foo\n\nBar\n"}, + {[]string{"/* Foo*/", "// Bar"}, " Foo\nBar\n"}, + {[]string{"/* Foo\n Bar*/"}, " Foo\n Bar\n"}, + + {[]string{"// foo", "//go:noinline", "// bar", "//:baz"}, "foo\nbar\n:baz\n"}, + {[]string{"// foo", "//lint123:ignore", "// bar"}, "foo\nbar\n"}, +} + +func TestCommentText(t *testing.T) { + for i, c := range comments { + list := make([]*Comment, len(c.list)) + for i, s := range c.list { + list[i] = &Comment{Text: s} + } + + text := (&CommentGroup{list}).Text() + if text != c.text { + t.Errorf("case %d: got %q; expected %q", i, text, c.text) + } + } +} + +var isDirectiveTests = []struct { + in string + ok bool +}{ + {"abc", false}, + {"go:inline", true}, + {"Go:inline", false}, + {"go:Inline", false}, + {":inline", false}, + {"lint:ignore", true}, + {"lint:1234", true}, + {"1234:lint", true}, + {"go: inline", false}, + {"go:", false}, + {"go:*", false}, + {"go:x*", true}, + {"export foo", true}, + {"extern foo", true}, + {"expert foo", false}, +} + +func TestIsDirective(t *testing.T) { + for _, tt := range isDirectiveTests { + if ok := isDirective(tt.in); ok != tt.ok { + t.Errorf("isDirective(%q) = %v, want %v", tt.in, ok, tt.ok) + } + } +} diff --git a/src/go/ast/commentmap.go b/src/go/ast/commentmap.go new file mode 100644 index 0000000..4196e47 --- /dev/null +++ b/src/go/ast/commentmap.go @@ -0,0 +1,330 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import ( + "bytes" + "fmt" + "go/token" + "sort" + "strings" +) + +type byPos []*CommentGroup + +func (a byPos) Len() int { return len(a) } +func (a byPos) Less(i, j int) bool { return a[i].Pos() < a[j].Pos() } +func (a byPos) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// sortComments sorts the list of comment groups in source order. +func sortComments(list []*CommentGroup) { + // TODO(gri): Does it make sense to check for sorted-ness + // first (because we know that sorted-ness is + // very likely)? + if orderedList := byPos(list); !sort.IsSorted(orderedList) { + sort.Sort(orderedList) + } +} + +// A CommentMap maps an AST node to a list of comment groups +// associated with it. See NewCommentMap for a description of +// the association. +type CommentMap map[Node][]*CommentGroup + +func (cmap CommentMap) addComment(n Node, c *CommentGroup) { + list := cmap[n] + if len(list) == 0 { + list = []*CommentGroup{c} + } else { + list = append(list, c) + } + cmap[n] = list +} + +type byInterval []Node + +func (a byInterval) Len() int { return len(a) } +func (a byInterval) Less(i, j int) bool { + pi, pj := a[i].Pos(), a[j].Pos() + return pi < pj || pi == pj && a[i].End() > a[j].End() +} +func (a byInterval) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// nodeList returns the list of nodes of the AST n in source order. +func nodeList(n Node) []Node { + var list []Node + Inspect(n, func(n Node) bool { + // don't collect comments + switch n.(type) { + case nil, *CommentGroup, *Comment: + return false + } + list = append(list, n) + return true + }) + // Note: The current implementation assumes that Inspect traverses the + // AST in depth-first and thus _source_ order. If AST traversal + // does not follow source order, the sorting call below will be + // required. + // sort.Sort(byInterval(list)) + return list +} + +// A commentListReader helps iterating through a list of comment groups. +type commentListReader struct { + fset *token.FileSet + list []*CommentGroup + index int + comment *CommentGroup // comment group at current index + pos, end token.Position // source interval of comment group at current index +} + +func (r *commentListReader) eol() bool { + return r.index >= len(r.list) +} + +func (r *commentListReader) next() { + if !r.eol() { + r.comment = r.list[r.index] + r.pos = r.fset.Position(r.comment.Pos()) + r.end = r.fset.Position(r.comment.End()) + r.index++ + } +} + +// A nodeStack keeps track of nested nodes. +// A node lower on the stack lexically contains the nodes higher on the stack. +type nodeStack []Node + +// push pops all nodes that appear lexically before n +// and then pushes n on the stack. +func (s *nodeStack) push(n Node) { + s.pop(n.Pos()) + *s = append((*s), n) +} + +// pop pops all nodes that appear lexically before pos +// (i.e., whose lexical extent has ended before or at pos). +// It returns the last node popped. +func (s *nodeStack) pop(pos token.Pos) (top Node) { + i := len(*s) + for i > 0 && (*s)[i-1].End() <= pos { + top = (*s)[i-1] + i-- + } + *s = (*s)[0:i] + return top +} + +// NewCommentMap creates a new comment map by associating comment groups +// of the comments list with the nodes of the AST specified by node. +// +// A comment group g is associated with a node n if: +// +// - g starts on the same line as n ends +// - g starts on the line immediately following n, and there is +// at least one empty line after g and before the next node +// - g starts before n and is not associated to the node before n +// via the previous rules +// +// NewCommentMap tries to associate a comment group to the "largest" +// node possible: For instance, if the comment is a line comment +// trailing an assignment, the comment is associated with the entire +// assignment rather than just the last operand in the assignment. +func NewCommentMap(fset *token.FileSet, node Node, comments []*CommentGroup) CommentMap { + if len(comments) == 0 { + return nil // no comments to map + } + + cmap := make(CommentMap) + + // set up comment reader r + tmp := make([]*CommentGroup, len(comments)) + copy(tmp, comments) // don't change incoming comments + sortComments(tmp) + r := commentListReader{fset: fset, list: tmp} // !r.eol() because len(comments) > 0 + r.next() + + // create node list in lexical order + nodes := nodeList(node) + nodes = append(nodes, nil) // append sentinel + + // set up iteration variables + var ( + p Node // previous node + pend token.Position // end of p + pg Node // previous node group (enclosing nodes of "importance") + pgend token.Position // end of pg + stack nodeStack // stack of node groups + ) + + for _, q := range nodes { + var qpos token.Position + if q != nil { + qpos = fset.Position(q.Pos()) // current node position + } else { + // set fake sentinel position to infinity so that + // all comments get processed before the sentinel + const infinity = 1 << 30 + qpos.Offset = infinity + qpos.Line = infinity + } + + // process comments before current node + for r.end.Offset <= qpos.Offset { + // determine recent node group + if top := stack.pop(r.comment.Pos()); top != nil { + pg = top + pgend = fset.Position(pg.End()) + } + // Try to associate a comment first with a node group + // (i.e., a node of "importance" such as a declaration); + // if that fails, try to associate it with the most recent + // node. + // TODO(gri) try to simplify the logic below + var assoc Node + switch { + case pg != nil && + (pgend.Line == r.pos.Line || + pgend.Line+1 == r.pos.Line && r.end.Line+1 < qpos.Line): + // 1) comment starts on same line as previous node group ends, or + // 2) comment starts on the line immediately after the + // previous node group and there is an empty line before + // the current node + // => associate comment with previous node group + assoc = pg + case p != nil && + (pend.Line == r.pos.Line || + pend.Line+1 == r.pos.Line && r.end.Line+1 < qpos.Line || + q == nil): + // same rules apply as above for p rather than pg, + // but also associate with p if we are at the end (q == nil) + assoc = p + default: + // otherwise, associate comment with current node + if q == nil { + // we can only reach here if there was no p + // which would imply that there were no nodes + panic("internal error: no comments should be associated with sentinel") + } + assoc = q + } + cmap.addComment(assoc, r.comment) + if r.eol() { + return cmap + } + r.next() + } + + // update previous node + p = q + pend = fset.Position(p.End()) + + // update previous node group if we see an "important" node + switch q.(type) { + case *File, *Field, Decl, Spec, Stmt: + stack.push(q) + } + } + + return cmap +} + +// Update replaces an old node in the comment map with the new node +// and returns the new node. Comments that were associated with the +// old node are associated with the new node. +func (cmap CommentMap) Update(old, new Node) Node { + if list := cmap[old]; len(list) > 0 { + delete(cmap, old) + cmap[new] = append(cmap[new], list...) + } + return new +} + +// Filter returns a new comment map consisting of only those +// entries of cmap for which a corresponding node exists in +// the AST specified by node. +func (cmap CommentMap) Filter(node Node) CommentMap { + umap := make(CommentMap) + Inspect(node, func(n Node) bool { + if g := cmap[n]; len(g) > 0 { + umap[n] = g + } + return true + }) + return umap +} + +// Comments returns the list of comment groups in the comment map. +// The result is sorted in source order. +func (cmap CommentMap) Comments() []*CommentGroup { + list := make([]*CommentGroup, 0, len(cmap)) + for _, e := range cmap { + list = append(list, e...) + } + sortComments(list) + return list +} + +func summary(list []*CommentGroup) string { + const maxLen = 40 + var buf bytes.Buffer + + // collect comments text +loop: + for _, group := range list { + // Note: CommentGroup.Text() does too much work for what we + // need and would only replace this innermost loop. + // Just do it explicitly. + for _, comment := range group.List { + if buf.Len() >= maxLen { + break loop + } + buf.WriteString(comment.Text) + } + } + + // truncate if too long + if buf.Len() > maxLen { + buf.Truncate(maxLen - 3) + buf.WriteString("...") + } + + // replace any invisibles with blanks + bytes := buf.Bytes() + for i, b := range bytes { + switch b { + case '\t', '\n', '\r': + bytes[i] = ' ' + } + } + + return string(bytes) +} + +func (cmap CommentMap) String() string { + // print map entries in sorted order + var nodes []Node + for node := range cmap { + nodes = append(nodes, node) + } + sort.Sort(byInterval(nodes)) + + var buf strings.Builder + fmt.Fprintln(&buf, "CommentMap {") + for _, node := range nodes { + comment := cmap[node] + // print name of identifiers; print node type for other nodes + var s string + if ident, ok := node.(*Ident); ok { + s = ident.Name + } else { + s = fmt.Sprintf("%T", node) + } + fmt.Fprintf(&buf, "\t%p %20s: %s\n", node, s, summary(comment)) + } + fmt.Fprintln(&buf, "}") + return buf.String() +} diff --git a/src/go/ast/commentmap_test.go b/src/go/ast/commentmap_test.go new file mode 100644 index 0000000..f0faeed --- /dev/null +++ b/src/go/ast/commentmap_test.go @@ -0,0 +1,169 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// To avoid a cyclic dependency with go/parser, this file is in a separate package. + +package ast_test + +import ( + "fmt" + . "go/ast" + "go/parser" + "go/token" + "sort" + "strings" + "testing" +) + +const src = ` +// the very first comment + +// package p +package p /* the name is p */ + +// imports +import ( + "bytes" // bytes + "fmt" // fmt + "go/ast" + "go/parser" +) + +// T +type T struct { + a, b, c int // associated with a, b, c + // associated with x, y + x, y float64 // float values + z complex128 // complex value +} +// also associated with T + +// x +var x = 0 // x = 0 +// also associated with x + +// f1 +func f1() { + /* associated with s1 */ + s1() + // also associated with s1 + + // associated with s2 + + // also associated with s2 + s2() // line comment for s2 +} +// associated with f1 +// also associated with f1 + +// associated with f2 + +// f2 +func f2() { +} + +func f3() { + i := 1 /* 1 */ + 2 // addition + _ = i +} + +// the very last comment +` + +// res maps a key of the form "line number: node type" +// to the associated comments' text. +var res = map[string]string{ + " 5: *ast.File": "the very first comment\npackage p\n", + " 5: *ast.Ident": " the name is p\n", + " 8: *ast.GenDecl": "imports\n", + " 9: *ast.ImportSpec": "bytes\n", + "10: *ast.ImportSpec": "fmt\n", + "16: *ast.GenDecl": "T\nalso associated with T\n", + "17: *ast.Field": "associated with a, b, c\n", + "19: *ast.Field": "associated with x, y\nfloat values\n", + "20: *ast.Field": "complex value\n", + "25: *ast.GenDecl": "x\nx = 0\nalso associated with x\n", + "29: *ast.FuncDecl": "f1\nassociated with f1\nalso associated with f1\n", + "31: *ast.ExprStmt": " associated with s1\nalso associated with s1\n", + "37: *ast.ExprStmt": "associated with s2\nalso associated with s2\nline comment for s2\n", + "45: *ast.FuncDecl": "associated with f2\nf2\n", + "49: *ast.AssignStmt": "addition\n", + "49: *ast.BasicLit": " 1\n", + "50: *ast.Ident": "the very last comment\n", +} + +func ctext(list []*CommentGroup) string { + var buf strings.Builder + for _, g := range list { + buf.WriteString(g.Text()) + } + return buf.String() +} + +func TestCommentMap(t *testing.T) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + cmap := NewCommentMap(fset, f, f.Comments) + + // very correct association of comments + for n, list := range cmap { + key := fmt.Sprintf("%2d: %T", fset.Position(n.Pos()).Line, n) + got := ctext(list) + want := res[key] + if got != want { + t.Errorf("%s: got %q; want %q", key, got, want) + } + } + + // verify that no comments got lost + if n := len(cmap.Comments()); n != len(f.Comments) { + t.Errorf("got %d comment groups in map; want %d", n, len(f.Comments)) + } + + // support code to update test: + // set genMap to true to generate res map + const genMap = false + if genMap { + out := make([]string, 0, len(cmap)) + for n, list := range cmap { + out = append(out, fmt.Sprintf("\t\"%2d: %T\":\t%q,", fset.Position(n.Pos()).Line, n, ctext(list))) + } + sort.Strings(out) + for _, s := range out { + fmt.Println(s) + } + } +} + +func TestFilter(t *testing.T) { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + cmap := NewCommentMap(fset, f, f.Comments) + + // delete variable declaration + for i, decl := range f.Decls { + if gen, ok := decl.(*GenDecl); ok && gen.Tok == token.VAR { + copy(f.Decls[i:], f.Decls[i+1:]) + f.Decls = f.Decls[:len(f.Decls)-1] + break + } + } + + // check if comments are filtered correctly + cc := cmap.Filter(f) + for n, list := range cc { + key := fmt.Sprintf("%2d: %T", fset.Position(n.Pos()).Line, n) + got := ctext(list) + want := res[key] + if key == "25: *ast.GenDecl" || got != want { + t.Errorf("%s: got %q; want %q", key, got, want) + } + } +} diff --git a/src/go/ast/example_test.go b/src/go/ast/example_test.go new file mode 100644 index 0000000..c6904be --- /dev/null +++ b/src/go/ast/example_test.go @@ -0,0 +1,208 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast_test + +import ( + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "strings" +) + +// This example demonstrates how to inspect the AST of a Go program. +func ExampleInspect() { + // src is the input for which we want to inspect the AST. + src := ` +package p +const c = 1.0 +var X = f(3.14)*2 + c +` + + // Create the AST by parsing src. + fset := token.NewFileSet() // positions are relative to fset + f, err := parser.ParseFile(fset, "src.go", src, 0) + if err != nil { + panic(err) + } + + // Inspect the AST and print all identifiers and literals. + ast.Inspect(f, func(n ast.Node) bool { + var s string + switch x := n.(type) { + case *ast.BasicLit: + s = x.Value + case *ast.Ident: + s = x.Name + } + if s != "" { + fmt.Printf("%s:\t%s\n", fset.Position(n.Pos()), s) + } + return true + }) + + // Output: + // src.go:2:9: p + // src.go:3:7: c + // src.go:3:11: 1.0 + // src.go:4:5: X + // src.go:4:9: f + // src.go:4:11: 3.14 + // src.go:4:17: 2 + // src.go:4:21: c +} + +// This example shows what an AST looks like when printed for debugging. +func ExamplePrint() { + // src is the input for which we want to print the AST. + src := ` +package main +func main() { + println("Hello, World!") +} +` + + // Create the AST by parsing src. + fset := token.NewFileSet() // positions are relative to fset + f, err := parser.ParseFile(fset, "", src, 0) + if err != nil { + panic(err) + } + + // Print the AST. + ast.Print(fset, f) + + // Output: + // 0 *ast.File { + // 1 . Package: 2:1 + // 2 . Name: *ast.Ident { + // 3 . . NamePos: 2:9 + // 4 . . Name: "main" + // 5 . } + // 6 . Decls: []ast.Decl (len = 1) { + // 7 . . 0: *ast.FuncDecl { + // 8 . . . Name: *ast.Ident { + // 9 . . . . NamePos: 3:6 + // 10 . . . . Name: "main" + // 11 . . . . Obj: *ast.Object { + // 12 . . . . . Kind: func + // 13 . . . . . Name: "main" + // 14 . . . . . Decl: *(obj @ 7) + // 15 . . . . } + // 16 . . . } + // 17 . . . Type: *ast.FuncType { + // 18 . . . . Func: 3:1 + // 19 . . . . Params: *ast.FieldList { + // 20 . . . . . Opening: 3:10 + // 21 . . . . . Closing: 3:11 + // 22 . . . . } + // 23 . . . } + // 24 . . . Body: *ast.BlockStmt { + // 25 . . . . Lbrace: 3:13 + // 26 . . . . List: []ast.Stmt (len = 1) { + // 27 . . . . . 0: *ast.ExprStmt { + // 28 . . . . . . X: *ast.CallExpr { + // 29 . . . . . . . Fun: *ast.Ident { + // 30 . . . . . . . . NamePos: 4:2 + // 31 . . . . . . . . Name: "println" + // 32 . . . . . . . } + // 33 . . . . . . . Lparen: 4:9 + // 34 . . . . . . . Args: []ast.Expr (len = 1) { + // 35 . . . . . . . . 0: *ast.BasicLit { + // 36 . . . . . . . . . ValuePos: 4:10 + // 37 . . . . . . . . . Kind: STRING + // 38 . . . . . . . . . Value: "\"Hello, World!\"" + // 39 . . . . . . . . } + // 40 . . . . . . . } + // 41 . . . . . . . Ellipsis: - + // 42 . . . . . . . Rparen: 4:25 + // 43 . . . . . . } + // 44 . . . . . } + // 45 . . . . } + // 46 . . . . Rbrace: 5:1 + // 47 . . . } + // 48 . . } + // 49 . } + // 50 . FileStart: 1:1 + // 51 . FileEnd: 5:3 + // 52 . Scope: *ast.Scope { + // 53 . . Objects: map[string]*ast.Object (len = 1) { + // 54 . . . "main": *(obj @ 11) + // 55 . . } + // 56 . } + // 57 . Unresolved: []*ast.Ident (len = 1) { + // 58 . . 0: *(obj @ 29) + // 59 . } + // 60 } +} + +// This example illustrates how to remove a variable declaration +// in a Go program while maintaining correct comment association +// using an ast.CommentMap. +func ExampleCommentMap() { + // src is the input for which we create the AST that we + // are going to manipulate. + src := ` +// This is the package comment. +package main + +// This comment is associated with the hello constant. +const hello = "Hello, World!" // line comment 1 + +// This comment is associated with the foo variable. +var foo = hello // line comment 2 + +// This comment is associated with the main function. +func main() { + fmt.Println(hello) // line comment 3 +} +` + + // Create the AST by parsing src. + fset := token.NewFileSet() // positions are relative to fset + f, err := parser.ParseFile(fset, "src.go", src, parser.ParseComments) + if err != nil { + panic(err) + } + + // Create an ast.CommentMap from the ast.File's comments. + // This helps keeping the association between comments + // and AST nodes. + cmap := ast.NewCommentMap(fset, f, f.Comments) + + // Remove the first variable declaration from the list of declarations. + for i, decl := range f.Decls { + if gen, ok := decl.(*ast.GenDecl); ok && gen.Tok == token.VAR { + copy(f.Decls[i:], f.Decls[i+1:]) + f.Decls = f.Decls[:len(f.Decls)-1] + break + } + } + + // Use the comment map to filter comments that don't belong anymore + // (the comments associated with the variable declaration), and create + // the new comments list. + f.Comments = cmap.Filter(f).Comments() + + // Print the modified AST. + var buf strings.Builder + if err := format.Node(&buf, fset, f); err != nil { + panic(err) + } + fmt.Printf("%s", buf.String()) + + // Output: + // // This is the package comment. + // package main + // + // // This comment is associated with the hello constant. + // const hello = "Hello, World!" // line comment 1 + // + // // This comment is associated with the main function. + // func main() { + // fmt.Println(hello) // line comment 3 + // } +} diff --git a/src/go/ast/filter.go b/src/go/ast/filter.go new file mode 100644 index 0000000..7d2a11e --- /dev/null +++ b/src/go/ast/filter.go @@ -0,0 +1,495 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import ( + "go/token" + "sort" +) + +// ---------------------------------------------------------------------------- +// Export filtering + +// exportFilter is a special filter function to extract exported nodes. +func exportFilter(name string) bool { + return IsExported(name) +} + +// FileExports trims the AST for a Go source file in place such that +// only exported nodes remain: all top-level identifiers which are not exported +// and their associated information (such as type, initial value, or function +// body) are removed. Non-exported fields and methods of exported types are +// stripped. The File.Comments list is not changed. +// +// FileExports reports whether there are exported declarations. +func FileExports(src *File) bool { + return filterFile(src, exportFilter, true) +} + +// PackageExports trims the AST for a Go package in place such that +// only exported nodes remain. The pkg.Files list is not changed, so that +// file names and top-level package comments don't get lost. +// +// PackageExports reports whether there are exported declarations; +// it returns false otherwise. +func PackageExports(pkg *Package) bool { + return filterPackage(pkg, exportFilter, true) +} + +// ---------------------------------------------------------------------------- +// General filtering + +type Filter func(string) bool + +func filterIdentList(list []*Ident, f Filter) []*Ident { + j := 0 + for _, x := range list { + if f(x.Name) { + list[j] = x + j++ + } + } + return list[0:j] +} + +// fieldName assumes that x is the type of an anonymous field and +// returns the corresponding field name. If x is not an acceptable +// anonymous field, the result is nil. +func fieldName(x Expr) *Ident { + switch t := x.(type) { + case *Ident: + return t + case *SelectorExpr: + if _, ok := t.X.(*Ident); ok { + return t.Sel + } + case *StarExpr: + return fieldName(t.X) + } + return nil +} + +func filterFieldList(fields *FieldList, filter Filter, export bool) (removedFields bool) { + if fields == nil { + return false + } + list := fields.List + j := 0 + for _, f := range list { + keepField := false + if len(f.Names) == 0 { + // anonymous field + name := fieldName(f.Type) + keepField = name != nil && filter(name.Name) + } else { + n := len(f.Names) + f.Names = filterIdentList(f.Names, filter) + if len(f.Names) < n { + removedFields = true + } + keepField = len(f.Names) > 0 + } + if keepField { + if export { + filterType(f.Type, filter, export) + } + list[j] = f + j++ + } + } + if j < len(list) { + removedFields = true + } + fields.List = list[0:j] + return +} + +func filterCompositeLit(lit *CompositeLit, filter Filter, export bool) { + n := len(lit.Elts) + lit.Elts = filterExprList(lit.Elts, filter, export) + if len(lit.Elts) < n { + lit.Incomplete = true + } +} + +func filterExprList(list []Expr, filter Filter, export bool) []Expr { + j := 0 + for _, exp := range list { + switch x := exp.(type) { + case *CompositeLit: + filterCompositeLit(x, filter, export) + case *KeyValueExpr: + if x, ok := x.Key.(*Ident); ok && !filter(x.Name) { + continue + } + if x, ok := x.Value.(*CompositeLit); ok { + filterCompositeLit(x, filter, export) + } + } + list[j] = exp + j++ + } + return list[0:j] +} + +func filterParamList(fields *FieldList, filter Filter, export bool) bool { + if fields == nil { + return false + } + var b bool + for _, f := range fields.List { + if filterType(f.Type, filter, export) { + b = true + } + } + return b +} + +func filterType(typ Expr, f Filter, export bool) bool { + switch t := typ.(type) { + case *Ident: + return f(t.Name) + case *ParenExpr: + return filterType(t.X, f, export) + case *ArrayType: + return filterType(t.Elt, f, export) + case *StructType: + if filterFieldList(t.Fields, f, export) { + t.Incomplete = true + } + return len(t.Fields.List) > 0 + case *FuncType: + b1 := filterParamList(t.Params, f, export) + b2 := filterParamList(t.Results, f, export) + return b1 || b2 + case *InterfaceType: + if filterFieldList(t.Methods, f, export) { + t.Incomplete = true + } + return len(t.Methods.List) > 0 + case *MapType: + b1 := filterType(t.Key, f, export) + b2 := filterType(t.Value, f, export) + return b1 || b2 + case *ChanType: + return filterType(t.Value, f, export) + } + return false +} + +func filterSpec(spec Spec, f Filter, export bool) bool { + switch s := spec.(type) { + case *ValueSpec: + s.Names = filterIdentList(s.Names, f) + s.Values = filterExprList(s.Values, f, export) + if len(s.Names) > 0 { + if export { + filterType(s.Type, f, export) + } + return true + } + case *TypeSpec: + if f(s.Name.Name) { + if export { + filterType(s.Type, f, export) + } + return true + } + if !export { + // For general filtering (not just exports), + // filter type even if name is not filtered + // out. + // If the type contains filtered elements, + // keep the declaration. + return filterType(s.Type, f, export) + } + } + return false +} + +func filterSpecList(list []Spec, f Filter, export bool) []Spec { + j := 0 + for _, s := range list { + if filterSpec(s, f, export) { + list[j] = s + j++ + } + } + return list[0:j] +} + +// FilterDecl trims the AST for a Go declaration in place by removing +// all names (including struct field and interface method names, but +// not from parameter lists) that don't pass through the filter f. +// +// FilterDecl reports whether there are any declared names left after +// filtering. +func FilterDecl(decl Decl, f Filter) bool { + return filterDecl(decl, f, false) +} + +func filterDecl(decl Decl, f Filter, export bool) bool { + switch d := decl.(type) { + case *GenDecl: + d.Specs = filterSpecList(d.Specs, f, export) + return len(d.Specs) > 0 + case *FuncDecl: + return f(d.Name.Name) + } + return false +} + +// FilterFile trims the AST for a Go file in place by removing all +// names from top-level declarations (including struct field and +// interface method names, but not from parameter lists) that don't +// pass through the filter f. If the declaration is empty afterwards, +// the declaration is removed from the AST. Import declarations are +// always removed. The File.Comments list is not changed. +// +// FilterFile reports whether there are any top-level declarations +// left after filtering. +func FilterFile(src *File, f Filter) bool { + return filterFile(src, f, false) +} + +func filterFile(src *File, f Filter, export bool) bool { + j := 0 + for _, d := range src.Decls { + if filterDecl(d, f, export) { + src.Decls[j] = d + j++ + } + } + src.Decls = src.Decls[0:j] + return j > 0 +} + +// FilterPackage trims the AST for a Go package in place by removing +// all names from top-level declarations (including struct field and +// interface method names, but not from parameter lists) that don't +// pass through the filter f. If the declaration is empty afterwards, +// the declaration is removed from the AST. The pkg.Files list is not +// changed, so that file names and top-level package comments don't get +// lost. +// +// FilterPackage reports whether there are any top-level declarations +// left after filtering. +func FilterPackage(pkg *Package, f Filter) bool { + return filterPackage(pkg, f, false) +} + +func filterPackage(pkg *Package, f Filter, export bool) bool { + hasDecls := false + for _, src := range pkg.Files { + if filterFile(src, f, export) { + hasDecls = true + } + } + return hasDecls +} + +// ---------------------------------------------------------------------------- +// Merging of package files + +// The MergeMode flags control the behavior of MergePackageFiles. +type MergeMode uint + +const ( + // If set, duplicate function declarations are excluded. + FilterFuncDuplicates MergeMode = 1 << iota + // If set, comments that are not associated with a specific + // AST node (as Doc or Comment) are excluded. + FilterUnassociatedComments + // If set, duplicate import declarations are excluded. + FilterImportDuplicates +) + +// nameOf returns the function (foo) or method name (foo.bar) for +// the given function declaration. If the AST is incorrect for the +// receiver, it assumes a function instead. +func nameOf(f *FuncDecl) string { + if r := f.Recv; r != nil && len(r.List) == 1 { + // looks like a correct receiver declaration + t := r.List[0].Type + // dereference pointer receiver types + if p, _ := t.(*StarExpr); p != nil { + t = p.X + } + // the receiver type must be a type name + if p, _ := t.(*Ident); p != nil { + return p.Name + "." + f.Name.Name + } + // otherwise assume a function instead + } + return f.Name.Name +} + +// separator is an empty //-style comment that is interspersed between +// different comment groups when they are concatenated into a single group +var separator = &Comment{token.NoPos, "//"} + +// MergePackageFiles creates a file AST by merging the ASTs of the +// files belonging to a package. The mode flags control merging behavior. +func MergePackageFiles(pkg *Package, mode MergeMode) *File { + // Count the number of package docs, comments and declarations across + // all package files. Also, compute sorted list of filenames, so that + // subsequent iterations can always iterate in the same order. + ndocs := 0 + ncomments := 0 + ndecls := 0 + filenames := make([]string, len(pkg.Files)) + var minPos, maxPos token.Pos + i := 0 + for filename, f := range pkg.Files { + filenames[i] = filename + i++ + if f.Doc != nil { + ndocs += len(f.Doc.List) + 1 // +1 for separator + } + ncomments += len(f.Comments) + ndecls += len(f.Decls) + if i == 0 || f.FileStart < minPos { + minPos = f.FileStart + } + if i == 0 || f.FileEnd > maxPos { + maxPos = f.FileEnd + } + } + sort.Strings(filenames) + + // Collect package comments from all package files into a single + // CommentGroup - the collected package documentation. In general + // there should be only one file with a package comment; but it's + // better to collect extra comments than drop them on the floor. + var doc *CommentGroup + var pos token.Pos + if ndocs > 0 { + list := make([]*Comment, ndocs-1) // -1: no separator before first group + i := 0 + for _, filename := range filenames { + f := pkg.Files[filename] + if f.Doc != nil { + if i > 0 { + // not the first group - add separator + list[i] = separator + i++ + } + for _, c := range f.Doc.List { + list[i] = c + i++ + } + if f.Package > pos { + // Keep the maximum package clause position as + // position for the package clause of the merged + // files. + pos = f.Package + } + } + } + doc = &CommentGroup{list} + } + + // Collect declarations from all package files. + var decls []Decl + if ndecls > 0 { + decls = make([]Decl, ndecls) + funcs := make(map[string]int) // map of func name -> decls index + i := 0 // current index + n := 0 // number of filtered entries + for _, filename := range filenames { + f := pkg.Files[filename] + for _, d := range f.Decls { + if mode&FilterFuncDuplicates != 0 { + // A language entity may be declared multiple + // times in different package files; only at + // build time declarations must be unique. + // For now, exclude multiple declarations of + // functions - keep the one with documentation. + // + // TODO(gri): Expand this filtering to other + // entities (const, type, vars) if + // multiple declarations are common. + if f, isFun := d.(*FuncDecl); isFun { + name := nameOf(f) + if j, exists := funcs[name]; exists { + // function declared already + if decls[j] != nil && decls[j].(*FuncDecl).Doc == nil { + // existing declaration has no documentation; + // ignore the existing declaration + decls[j] = nil + } else { + // ignore the new declaration + d = nil + } + n++ // filtered an entry + } else { + funcs[name] = i + } + } + } + decls[i] = d + i++ + } + } + + // Eliminate nil entries from the decls list if entries were + // filtered. We do this using a 2nd pass in order to not disturb + // the original declaration order in the source (otherwise, this + // would also invalidate the monotonically increasing position + // info within a single file). + if n > 0 { + i = 0 + for _, d := range decls { + if d != nil { + decls[i] = d + i++ + } + } + decls = decls[0:i] + } + } + + // Collect import specs from all package files. + var imports []*ImportSpec + if mode&FilterImportDuplicates != 0 { + seen := make(map[string]bool) + for _, filename := range filenames { + f := pkg.Files[filename] + for _, imp := range f.Imports { + if path := imp.Path.Value; !seen[path] { + // TODO: consider handling cases where: + // - 2 imports exist with the same import path but + // have different local names (one should probably + // keep both of them) + // - 2 imports exist but only one has a comment + // - 2 imports exist and they both have (possibly + // different) comments + imports = append(imports, imp) + seen[path] = true + } + } + } + } else { + // Iterate over filenames for deterministic order. + for _, filename := range filenames { + f := pkg.Files[filename] + imports = append(imports, f.Imports...) + } + } + + // Collect comments from all package files. + var comments []*CommentGroup + if mode&FilterUnassociatedComments == 0 { + comments = make([]*CommentGroup, ncomments) + i := 0 + for _, filename := range filenames { + f := pkg.Files[filename] + i += copy(comments[i:], f.Comments) + } + } + + // TODO(gri) need to compute unresolved identifiers! + return &File{doc, pos, NewIdent(pkg.Name), decls, minPos, maxPos, pkg.Scope, imports, nil, comments} +} diff --git a/src/go/ast/filter_test.go b/src/go/ast/filter_test.go new file mode 100644 index 0000000..d5cb0c2 --- /dev/null +++ b/src/go/ast/filter_test.go @@ -0,0 +1,85 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// To avoid a cyclic dependency with go/parser, this file is in a separate package. + +package ast_test + +import ( + "go/ast" + "go/format" + "go/parser" + "go/token" + "strings" + "testing" +) + +const input = `package p + +type t1 struct{} +type t2 struct{} + +func f1() {} +func f1() {} +func f2() {} + +func (*t1) f1() {} +func (t1) f1() {} +func (t1) f2() {} + +func (t2) f1() {} +func (t2) f2() {} +func (x *t2) f2() {} +` + +// Calling ast.MergePackageFiles with ast.FilterFuncDuplicates +// keeps a duplicate entry with attached documentation in favor +// of one without, and it favors duplicate entries appearing +// later in the source over ones appearing earlier. This is why +// (*t2).f2 is kept and t2.f2 is eliminated in this test case. +const golden = `package p + +type t1 struct{} +type t2 struct{} + +func f1() {} +func f2() {} + +func (t1) f1() {} +func (t1) f2() {} + +func (t2) f1() {} + +func (x *t2) f2() {} +` + +func TestFilterDuplicates(t *testing.T) { + // parse input + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "", input, 0) + if err != nil { + t.Fatal(err) + } + + // create package + files := map[string]*ast.File{"": file} + pkg, err := ast.NewPackage(fset, files, nil, nil) + if err != nil { + t.Fatal(err) + } + + // filter + merged := ast.MergePackageFiles(pkg, ast.FilterFuncDuplicates) + + // pretty-print + var buf strings.Builder + if err := format.Node(&buf, fset, merged); err != nil { + t.Fatal(err) + } + output := buf.String() + + if output != golden { + t.Errorf("incorrect output:\n%s", output) + } +} diff --git a/src/go/ast/import.go b/src/go/ast/import.go new file mode 100644 index 0000000..7fdf137 --- /dev/null +++ b/src/go/ast/import.go @@ -0,0 +1,230 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import ( + "go/token" + "sort" + "strconv" +) + +// SortImports sorts runs of consecutive import lines in import blocks in f. +// It also removes duplicate imports when it is possible to do so without data loss. +func SortImports(fset *token.FileSet, f *File) { + for _, d := range f.Decls { + d, ok := d.(*GenDecl) + if !ok || d.Tok != token.IMPORT { + // Not an import declaration, so we're done. + // Imports are always first. + break + } + + if !d.Lparen.IsValid() { + // Not a block: sorted by default. + continue + } + + // Identify and sort runs of specs on successive lines. + i := 0 + specs := d.Specs[:0] + for j, s := range d.Specs { + if j > i && lineAt(fset, s.Pos()) > 1+lineAt(fset, d.Specs[j-1].End()) { + // j begins a new run. End this one. + specs = append(specs, sortSpecs(fset, f, d.Specs[i:j])...) + i = j + } + } + specs = append(specs, sortSpecs(fset, f, d.Specs[i:])...) + d.Specs = specs + + // Deduping can leave a blank line before the rparen; clean that up. + if len(d.Specs) > 0 { + lastSpec := d.Specs[len(d.Specs)-1] + lastLine := lineAt(fset, lastSpec.Pos()) + rParenLine := lineAt(fset, d.Rparen) + for rParenLine > lastLine+1 { + rParenLine-- + fset.File(d.Rparen).MergeLine(rParenLine) + } + } + } +} + +func lineAt(fset *token.FileSet, pos token.Pos) int { + return fset.PositionFor(pos, false).Line +} + +func importPath(s Spec) string { + t, err := strconv.Unquote(s.(*ImportSpec).Path.Value) + if err == nil { + return t + } + return "" +} + +func importName(s Spec) string { + n := s.(*ImportSpec).Name + if n == nil { + return "" + } + return n.Name +} + +func importComment(s Spec) string { + c := s.(*ImportSpec).Comment + if c == nil { + return "" + } + return c.Text() +} + +// collapse indicates whether prev may be removed, leaving only next. +func collapse(prev, next Spec) bool { + if importPath(next) != importPath(prev) || importName(next) != importName(prev) { + return false + } + return prev.(*ImportSpec).Comment == nil +} + +type posSpan struct { + Start token.Pos + End token.Pos +} + +type cgPos struct { + left bool // true if comment is to the left of the spec, false otherwise. + cg *CommentGroup +} + +func sortSpecs(fset *token.FileSet, f *File, specs []Spec) []Spec { + // Can't short-circuit here even if specs are already sorted, + // since they might yet need deduplication. + // A lone import, however, may be safely ignored. + if len(specs) <= 1 { + return specs + } + + // Record positions for specs. + pos := make([]posSpan, len(specs)) + for i, s := range specs { + pos[i] = posSpan{s.Pos(), s.End()} + } + + // Identify comments in this range. + begSpecs := pos[0].Start + endSpecs := pos[len(pos)-1].End + beg := fset.File(begSpecs).LineStart(lineAt(fset, begSpecs)) + endLine := lineAt(fset, endSpecs) + endFile := fset.File(endSpecs) + var end token.Pos + if endLine == endFile.LineCount() { + end = endSpecs + } else { + end = endFile.LineStart(endLine + 1) // beginning of next line + } + first := len(f.Comments) + last := -1 + for i, g := range f.Comments { + if g.End() >= end { + break + } + // g.End() < end + if beg <= g.Pos() { + // comment is within the range [beg, end[ of import declarations + if i < first { + first = i + } + if i > last { + last = i + } + } + } + + var comments []*CommentGroup + if last >= 0 { + comments = f.Comments[first : last+1] + } + + // Assign each comment to the import spec on the same line. + importComments := map[*ImportSpec][]cgPos{} + specIndex := 0 + for _, g := range comments { + for specIndex+1 < len(specs) && pos[specIndex+1].Start <= g.Pos() { + specIndex++ + } + var left bool + // A block comment can appear before the first import spec. + if specIndex == 0 && pos[specIndex].Start > g.Pos() { + left = true + } else if specIndex+1 < len(specs) && // Or it can appear on the left of an import spec. + lineAt(fset, pos[specIndex].Start)+1 == lineAt(fset, g.Pos()) { + specIndex++ + left = true + } + s := specs[specIndex].(*ImportSpec) + importComments[s] = append(importComments[s], cgPos{left: left, cg: g}) + } + + // Sort the import specs by import path. + // Remove duplicates, when possible without data loss. + // Reassign the import paths to have the same position sequence. + // Reassign each comment to the spec on the same line. + // Sort the comments by new position. + sort.Slice(specs, func(i, j int) bool { + ipath := importPath(specs[i]) + jpath := importPath(specs[j]) + if ipath != jpath { + return ipath < jpath + } + iname := importName(specs[i]) + jname := importName(specs[j]) + if iname != jname { + return iname < jname + } + return importComment(specs[i]) < importComment(specs[j]) + }) + + // Dedup. Thanks to our sorting, we can just consider + // adjacent pairs of imports. + deduped := specs[:0] + for i, s := range specs { + if i == len(specs)-1 || !collapse(s, specs[i+1]) { + deduped = append(deduped, s) + } else { + p := s.Pos() + fset.File(p).MergeLine(lineAt(fset, p)) + } + } + specs = deduped + + // Fix up comment positions + for i, s := range specs { + s := s.(*ImportSpec) + if s.Name != nil { + s.Name.NamePos = pos[i].Start + } + s.Path.ValuePos = pos[i].Start + s.EndPos = pos[i].End + for _, g := range importComments[s] { + for _, c := range g.cg.List { + if g.left { + c.Slash = pos[i].Start - 1 + } else { + // An import spec can have both block comment and a line comment + // to its right. In that case, both of them will have the same pos. + // But while formatting the AST, the line comment gets moved to + // after the block comment. + c.Slash = pos[i].End + } + } + } + } + + sort.Slice(comments, func(i, j int) bool { + return comments[i].Pos() < comments[j].Pos() + }) + + return specs +} diff --git a/src/go/ast/issues_test.go b/src/go/ast/issues_test.go new file mode 100644 index 0000000..788c557 --- /dev/null +++ b/src/go/ast/issues_test.go @@ -0,0 +1,42 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast_test + +import ( + "go/ast" + "go/parser" + "go/token" + "testing" +) + +func TestIssue33649(t *testing.T) { + for _, src := range []string{ + `package p; func _()`, + `package p; func _() {`, + `package p; func _() { _ = 0`, + `package p; func _() { _ = 0 }`, + } { + fset := token.NewFileSet() + f, _ := parser.ParseFile(fset, "", src, parser.AllErrors) + if f == nil { + panic("invalid test setup: parser didn't return an AST") + } + + // find corresponding token.File + var tf *token.File + fset.Iterate(func(f *token.File) bool { + tf = f + return true + }) + tfEnd := tf.Base() + tf.Size() + + fd := f.Decls[len(f.Decls)-1].(*ast.FuncDecl) + fdEnd := int(fd.End()) + + if fdEnd != tfEnd { + t.Errorf("%q: got fdEnd = %d; want %d (base = %d, size = %d)", src, fdEnd, tfEnd, tf.Base(), tf.Size()) + } + } +} diff --git a/src/go/ast/print.go b/src/go/ast/print.go new file mode 100644 index 0000000..85e6943 --- /dev/null +++ b/src/go/ast/print.go @@ -0,0 +1,254 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains printing support for ASTs. + +package ast + +import ( + "fmt" + "go/token" + "io" + "os" + "reflect" +) + +// A FieldFilter may be provided to Fprint to control the output. +type FieldFilter func(name string, value reflect.Value) bool + +// NotNilFilter returns true for field values that are not nil; +// it returns false otherwise. +func NotNilFilter(_ string, v reflect.Value) bool { + switch v.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + return !v.IsNil() + } + return true +} + +// Fprint prints the (sub-)tree starting at AST node x to w. +// If fset != nil, position information is interpreted relative +// to that file set. Otherwise positions are printed as integer +// values (file set specific offsets). +// +// A non-nil FieldFilter f may be provided to control the output: +// struct fields for which f(fieldname, fieldvalue) is true are +// printed; all others are filtered from the output. Unexported +// struct fields are never printed. +func Fprint(w io.Writer, fset *token.FileSet, x any, f FieldFilter) error { + return fprint(w, fset, x, f) +} + +func fprint(w io.Writer, fset *token.FileSet, x any, f FieldFilter) (err error) { + // setup printer + p := printer{ + output: w, + fset: fset, + filter: f, + ptrmap: make(map[any]int), + last: '\n', // force printing of line number on first line + } + + // install error handler + defer func() { + if e := recover(); e != nil { + err = e.(localError).err // re-panics if it's not a localError + } + }() + + // print x + if x == nil { + p.printf("nil\n") + return + } + p.print(reflect.ValueOf(x)) + p.printf("\n") + + return +} + +// Print prints x to standard output, skipping nil fields. +// Print(fset, x) is the same as Fprint(os.Stdout, fset, x, NotNilFilter). +func Print(fset *token.FileSet, x any) error { + return Fprint(os.Stdout, fset, x, NotNilFilter) +} + +type printer struct { + output io.Writer + fset *token.FileSet + filter FieldFilter + ptrmap map[any]int // *T -> line number + indent int // current indentation level + last byte // the last byte processed by Write + line int // current line number +} + +var indent = []byte(". ") + +func (p *printer) Write(data []byte) (n int, err error) { + var m int + for i, b := range data { + // invariant: data[0:n] has been written + if b == '\n' { + m, err = p.output.Write(data[n : i+1]) + n += m + if err != nil { + return + } + p.line++ + } else if p.last == '\n' { + _, err = fmt.Fprintf(p.output, "%6d ", p.line) + if err != nil { + return + } + for j := p.indent; j > 0; j-- { + _, err = p.output.Write(indent) + if err != nil { + return + } + } + } + p.last = b + } + if len(data) > n { + m, err = p.output.Write(data[n:]) + n += m + } + return +} + +// localError wraps locally caught errors so we can distinguish +// them from genuine panics which we don't want to return as errors. +type localError struct { + err error +} + +// printf is a convenience wrapper that takes care of print errors. +func (p *printer) printf(format string, args ...any) { + if _, err := fmt.Fprintf(p, format, args...); err != nil { + panic(localError{err}) + } +} + +// Implementation note: Print is written for AST nodes but could be +// used to print arbitrary data structures; such a version should +// probably be in a different package. +// +// Note: This code detects (some) cycles created via pointers but +// not cycles that are created via slices or maps containing the +// same slice or map. Code for general data structures probably +// should catch those as well. + +func (p *printer) print(x reflect.Value) { + if !NotNilFilter("", x) { + p.printf("nil") + return + } + + switch x.Kind() { + case reflect.Interface: + p.print(x.Elem()) + + case reflect.Map: + p.printf("%s (len = %d) {", x.Type(), x.Len()) + if x.Len() > 0 { + p.indent++ + p.printf("\n") + for _, key := range x.MapKeys() { + p.print(key) + p.printf(": ") + p.print(x.MapIndex(key)) + p.printf("\n") + } + p.indent-- + } + p.printf("}") + + case reflect.Pointer: + p.printf("*") + // type-checked ASTs may contain cycles - use ptrmap + // to keep track of objects that have been printed + // already and print the respective line number instead + ptr := x.Interface() + if line, exists := p.ptrmap[ptr]; exists { + p.printf("(obj @ %d)", line) + } else { + p.ptrmap[ptr] = p.line + p.print(x.Elem()) + } + + case reflect.Array: + p.printf("%s {", x.Type()) + if x.Len() > 0 { + p.indent++ + p.printf("\n") + for i, n := 0, x.Len(); i < n; i++ { + p.printf("%d: ", i) + p.print(x.Index(i)) + p.printf("\n") + } + p.indent-- + } + p.printf("}") + + case reflect.Slice: + if s, ok := x.Interface().([]byte); ok { + p.printf("%#q", s) + return + } + p.printf("%s (len = %d) {", x.Type(), x.Len()) + if x.Len() > 0 { + p.indent++ + p.printf("\n") + for i, n := 0, x.Len(); i < n; i++ { + p.printf("%d: ", i) + p.print(x.Index(i)) + p.printf("\n") + } + p.indent-- + } + p.printf("}") + + case reflect.Struct: + t := x.Type() + p.printf("%s {", t) + p.indent++ + first := true + for i, n := 0, t.NumField(); i < n; i++ { + // exclude non-exported fields because their + // values cannot be accessed via reflection + if name := t.Field(i).Name; IsExported(name) { + value := x.Field(i) + if p.filter == nil || p.filter(name, value) { + if first { + p.printf("\n") + first = false + } + p.printf("%s: ", name) + p.print(value) + p.printf("\n") + } + } + } + p.indent-- + p.printf("}") + + default: + v := x.Interface() + switch v := v.(type) { + case string: + // print strings in quotes + p.printf("%q", v) + return + case token.Pos: + // position values can be printed nicely if we have a file set + if p.fset != nil { + p.printf("%s", p.fset.Position(v)) + return + } + } + // default + p.printf("%v", v) + } +} diff --git a/src/go/ast/print_test.go b/src/go/ast/print_test.go new file mode 100644 index 0000000..94b515b --- /dev/null +++ b/src/go/ast/print_test.go @@ -0,0 +1,96 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import ( + "strings" + "testing" +) + +var tests = []struct { + x any // x is printed as s + s string +}{ + // basic types + {nil, "0 nil"}, + {true, "0 true"}, + {42, "0 42"}, + {3.14, "0 3.14"}, + {1 + 2.718i, "0 (1+2.718i)"}, + {"foobar", "0 \"foobar\""}, + + // maps + {map[Expr]string{}, `0 map[ast.Expr]string (len = 0) {}`}, + {map[string]int{"a": 1}, + `0 map[string]int (len = 1) { + 1 . "a": 1 + 2 }`}, + + // pointers + {new(int), "0 *0"}, + + // arrays + {[0]int{}, `0 [0]int {}`}, + {[3]int{1, 2, 3}, + `0 [3]int { + 1 . 0: 1 + 2 . 1: 2 + 3 . 2: 3 + 4 }`}, + {[...]int{42}, + `0 [1]int { + 1 . 0: 42 + 2 }`}, + + // slices + {[]int{}, `0 []int (len = 0) {}`}, + {[]int{1, 2, 3}, + `0 []int (len = 3) { + 1 . 0: 1 + 2 . 1: 2 + 3 . 2: 3 + 4 }`}, + + // structs + {struct{}{}, `0 struct {} {}`}, + {struct{ x int }{007}, `0 struct { x int } {}`}, + {struct{ X, y int }{42, 991}, + `0 struct { X int; y int } { + 1 . X: 42 + 2 }`}, + {struct{ X, Y int }{42, 991}, + `0 struct { X int; Y int } { + 1 . X: 42 + 2 . Y: 991 + 3 }`}, +} + +// Split s into lines, trim whitespace from all lines, and return +// the concatenated non-empty lines. +func trim(s string) string { + lines := strings.Split(s, "\n") + i := 0 + for _, line := range lines { + line = strings.TrimSpace(line) + if line != "" { + lines[i] = line + i++ + } + } + return strings.Join(lines[0:i], "\n") +} + +func TestPrint(t *testing.T) { + var buf strings.Builder + for _, test := range tests { + buf.Reset() + if err := Fprint(&buf, nil, test.x, nil); err != nil { + t.Errorf("Fprint failed: %s", err) + } + if s, ts := trim(buf.String()), trim(test.s); s != ts { + t.Errorf("got:\n%s\nexpected:\n%s\n", s, ts) + } + } +} diff --git a/src/go/ast/resolve.go b/src/go/ast/resolve.go new file mode 100644 index 0000000..970aa88 --- /dev/null +++ b/src/go/ast/resolve.go @@ -0,0 +1,173 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements NewPackage. + +package ast + +import ( + "fmt" + "go/scanner" + "go/token" + "strconv" +) + +type pkgBuilder struct { + fset *token.FileSet + errors scanner.ErrorList +} + +func (p *pkgBuilder) error(pos token.Pos, msg string) { + p.errors.Add(p.fset.Position(pos), msg) +} + +func (p *pkgBuilder) errorf(pos token.Pos, format string, args ...any) { + p.error(pos, fmt.Sprintf(format, args...)) +} + +func (p *pkgBuilder) declare(scope, altScope *Scope, obj *Object) { + alt := scope.Insert(obj) + if alt == nil && altScope != nil { + // see if there is a conflicting declaration in altScope + alt = altScope.Lookup(obj.Name) + } + if alt != nil { + prevDecl := "" + if pos := alt.Pos(); pos.IsValid() { + prevDecl = fmt.Sprintf("\n\tprevious declaration at %s", p.fset.Position(pos)) + } + p.error(obj.Pos(), fmt.Sprintf("%s redeclared in this block%s", obj.Name, prevDecl)) + } +} + +func resolve(scope *Scope, ident *Ident) bool { + for ; scope != nil; scope = scope.Outer { + if obj := scope.Lookup(ident.Name); obj != nil { + ident.Obj = obj + return true + } + } + return false +} + +// An Importer resolves import paths to package Objects. +// The imports map records the packages already imported, +// indexed by package id (canonical import path). +// An Importer must determine the canonical import path and +// check the map to see if it is already present in the imports map. +// If so, the Importer can return the map entry. Otherwise, the +// Importer should load the package data for the given path into +// a new *Object (pkg), record pkg in the imports map, and then +// return pkg. +type Importer func(imports map[string]*Object, path string) (pkg *Object, err error) + +// NewPackage creates a new Package node from a set of File nodes. It resolves +// unresolved identifiers across files and updates each file's Unresolved list +// accordingly. If a non-nil importer and universe scope are provided, they are +// used to resolve identifiers not declared in any of the package files. Any +// remaining unresolved identifiers are reported as undeclared. If the files +// belong to different packages, one package name is selected and files with +// different package names are reported and then ignored. +// The result is a package node and a scanner.ErrorList if there were errors. +func NewPackage(fset *token.FileSet, files map[string]*File, importer Importer, universe *Scope) (*Package, error) { + var p pkgBuilder + p.fset = fset + + // complete package scope + pkgName := "" + pkgScope := NewScope(universe) + for _, file := range files { + // package names must match + switch name := file.Name.Name; { + case pkgName == "": + pkgName = name + case name != pkgName: + p.errorf(file.Package, "package %s; expected %s", name, pkgName) + continue // ignore this file + } + + // collect top-level file objects in package scope + for _, obj := range file.Scope.Objects { + p.declare(pkgScope, nil, obj) + } + } + + // package global mapping of imported package ids to package objects + imports := make(map[string]*Object) + + // complete file scopes with imports and resolve identifiers + for _, file := range files { + // ignore file if it belongs to a different package + // (error has already been reported) + if file.Name.Name != pkgName { + continue + } + + // build file scope by processing all imports + importErrors := false + fileScope := NewScope(pkgScope) + for _, spec := range file.Imports { + if importer == nil { + importErrors = true + continue + } + path, _ := strconv.Unquote(spec.Path.Value) + pkg, err := importer(imports, path) + if err != nil { + p.errorf(spec.Path.Pos(), "could not import %s (%s)", path, err) + importErrors = true + continue + } + // TODO(gri) If a local package name != "." is provided, + // global identifier resolution could proceed even if the + // import failed. Consider adjusting the logic here a bit. + + // local name overrides imported package name + name := pkg.Name + if spec.Name != nil { + name = spec.Name.Name + } + + // add import to file scope + if name == "." { + // merge imported scope with file scope + for _, obj := range pkg.Data.(*Scope).Objects { + p.declare(fileScope, pkgScope, obj) + } + } else if name != "_" { + // declare imported package object in file scope + // (do not re-use pkg in the file scope but create + // a new object instead; the Decl field is different + // for different files) + obj := NewObj(Pkg, name) + obj.Decl = spec + obj.Data = pkg.Data + p.declare(fileScope, pkgScope, obj) + } + } + + // resolve identifiers + if importErrors { + // don't use the universe scope without correct imports + // (objects in the universe may be shadowed by imports; + // with missing imports, identifiers might get resolved + // incorrectly to universe objects) + pkgScope.Outer = nil + } + i := 0 + for _, ident := range file.Unresolved { + if !resolve(fileScope, ident) { + p.errorf(ident.Pos(), "undeclared name: %s", ident.Name) + file.Unresolved[i] = ident + i++ + } + + } + file.Unresolved = file.Unresolved[0:i] + pkgScope.Outer = universe // reset universe scope + } + + p.errors.Sort() + return &Package{pkgName, pkgScope, imports, files}, p.errors.Err() +} diff --git a/src/go/ast/scope.go b/src/go/ast/scope.go new file mode 100644 index 0000000..8882212 --- /dev/null +++ b/src/go/ast/scope.go @@ -0,0 +1,156 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements scopes and the objects they contain. + +package ast + +import ( + "fmt" + "go/token" + "strings" +) + +// A Scope maintains the set of named language entities declared +// in the scope and a link to the immediately surrounding (outer) +// scope. +type Scope struct { + Outer *Scope + Objects map[string]*Object +} + +// NewScope creates a new scope nested in the outer scope. +func NewScope(outer *Scope) *Scope { + const n = 4 // initial scope capacity + return &Scope{outer, make(map[string]*Object, n)} +} + +// Lookup returns the object with the given name if it is +// found in scope s, otherwise it returns nil. Outer scopes +// are ignored. +func (s *Scope) Lookup(name string) *Object { + return s.Objects[name] +} + +// Insert attempts to insert a named object obj into the scope s. +// If the scope already contains an object alt with the same name, +// Insert leaves the scope unchanged and returns alt. Otherwise +// it inserts obj and returns nil. +func (s *Scope) Insert(obj *Object) (alt *Object) { + if alt = s.Objects[obj.Name]; alt == nil { + s.Objects[obj.Name] = obj + } + return +} + +// Debugging support +func (s *Scope) String() string { + var buf strings.Builder + fmt.Fprintf(&buf, "scope %p {", s) + if s != nil && len(s.Objects) > 0 { + fmt.Fprintln(&buf) + for _, obj := range s.Objects { + fmt.Fprintf(&buf, "\t%s %s\n", obj.Kind, obj.Name) + } + } + fmt.Fprintf(&buf, "}\n") + return buf.String() +} + +// ---------------------------------------------------------------------------- +// Objects + +// An Object describes a named language entity such as a package, +// constant, type, variable, function (incl. methods), or label. +// +// The Data fields contains object-specific data: +// +// Kind Data type Data value +// Pkg *Scope package scope +// Con int iota for the respective declaration +type Object struct { + Kind ObjKind + Name string // declared name + Decl any // corresponding Field, XxxSpec, FuncDecl, LabeledStmt, AssignStmt, Scope; or nil + Data any // object-specific data; or nil + Type any // placeholder for type information; may be nil +} + +// NewObj creates a new object of a given kind and name. +func NewObj(kind ObjKind, name string) *Object { + return &Object{Kind: kind, Name: name} +} + +// Pos computes the source position of the declaration of an object name. +// The result may be an invalid position if it cannot be computed +// (obj.Decl may be nil or not correct). +func (obj *Object) Pos() token.Pos { + name := obj.Name + switch d := obj.Decl.(type) { + case *Field: + for _, n := range d.Names { + if n.Name == name { + return n.Pos() + } + } + case *ImportSpec: + if d.Name != nil && d.Name.Name == name { + return d.Name.Pos() + } + return d.Path.Pos() + case *ValueSpec: + for _, n := range d.Names { + if n.Name == name { + return n.Pos() + } + } + case *TypeSpec: + if d.Name.Name == name { + return d.Name.Pos() + } + case *FuncDecl: + if d.Name.Name == name { + return d.Name.Pos() + } + case *LabeledStmt: + if d.Label.Name == name { + return d.Label.Pos() + } + case *AssignStmt: + for _, x := range d.Lhs { + if ident, isIdent := x.(*Ident); isIdent && ident.Name == name { + return ident.Pos() + } + } + case *Scope: + // predeclared object - nothing to do for now + } + return token.NoPos +} + +// ObjKind describes what an object represents. +type ObjKind int + +// The list of possible Object kinds. +const ( + Bad ObjKind = iota // for error handling + Pkg // package + Con // constant + Typ // type + Var // variable + Fun // function or method + Lbl // label +) + +var objKindStrings = [...]string{ + Bad: "bad", + Pkg: "package", + Con: "const", + Typ: "type", + Var: "var", + Fun: "func", + Lbl: "label", +} + +func (kind ObjKind) String() string { return objKindStrings[kind] } diff --git a/src/go/ast/walk.go b/src/go/ast/walk.go new file mode 100644 index 0000000..a293c99 --- /dev/null +++ b/src/go/ast/walk.go @@ -0,0 +1,398 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package ast + +import "fmt" + +// A Visitor's Visit method is invoked for each node encountered by Walk. +// If the result visitor w is not nil, Walk visits each of the children +// of node with the visitor w, followed by a call of w.Visit(nil). +type Visitor interface { + Visit(node Node) (w Visitor) +} + +// Helper functions for common node lists. They may be empty. + +func walkIdentList(v Visitor, list []*Ident) { + for _, x := range list { + Walk(v, x) + } +} + +func walkExprList(v Visitor, list []Expr) { + for _, x := range list { + Walk(v, x) + } +} + +func walkStmtList(v Visitor, list []Stmt) { + for _, x := range list { + Walk(v, x) + } +} + +func walkDeclList(v Visitor, list []Decl) { + for _, x := range list { + Walk(v, x) + } +} + +// TODO(gri): Investigate if providing a closure to Walk leads to +// simpler use (and may help eliminate Inspect in turn). + +// Walk traverses an AST in depth-first order: It starts by calling +// v.Visit(node); node must not be nil. If the visitor w returned by +// v.Visit(node) is not nil, Walk is invoked recursively with visitor +// w for each of the non-nil children of node, followed by a call of +// w.Visit(nil). +func Walk(v Visitor, node Node) { + if v = v.Visit(node); v == nil { + return + } + + // walk children + // (the order of the cases matches the order + // of the corresponding node types in ast.go) + switch n := node.(type) { + // Comments and fields + case *Comment: + // nothing to do + + case *CommentGroup: + for _, c := range n.List { + Walk(v, c) + } + + case *Field: + if n.Doc != nil { + Walk(v, n.Doc) + } + walkIdentList(v, n.Names) + if n.Type != nil { + Walk(v, n.Type) + } + if n.Tag != nil { + Walk(v, n.Tag) + } + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *FieldList: + for _, f := range n.List { + Walk(v, f) + } + + // Expressions + case *BadExpr, *Ident, *BasicLit: + // nothing to do + + case *Ellipsis: + if n.Elt != nil { + Walk(v, n.Elt) + } + + case *FuncLit: + Walk(v, n.Type) + Walk(v, n.Body) + + case *CompositeLit: + if n.Type != nil { + Walk(v, n.Type) + } + walkExprList(v, n.Elts) + + case *ParenExpr: + Walk(v, n.X) + + case *SelectorExpr: + Walk(v, n.X) + Walk(v, n.Sel) + + case *IndexExpr: + Walk(v, n.X) + Walk(v, n.Index) + + case *IndexListExpr: + Walk(v, n.X) + for _, index := range n.Indices { + Walk(v, index) + } + + case *SliceExpr: + Walk(v, n.X) + if n.Low != nil { + Walk(v, n.Low) + } + if n.High != nil { + Walk(v, n.High) + } + if n.Max != nil { + Walk(v, n.Max) + } + + case *TypeAssertExpr: + Walk(v, n.X) + if n.Type != nil { + Walk(v, n.Type) + } + + case *CallExpr: + Walk(v, n.Fun) + walkExprList(v, n.Args) + + case *StarExpr: + Walk(v, n.X) + + case *UnaryExpr: + Walk(v, n.X) + + case *BinaryExpr: + Walk(v, n.X) + Walk(v, n.Y) + + case *KeyValueExpr: + Walk(v, n.Key) + Walk(v, n.Value) + + // Types + case *ArrayType: + if n.Len != nil { + Walk(v, n.Len) + } + Walk(v, n.Elt) + + case *StructType: + Walk(v, n.Fields) + + case *FuncType: + if n.TypeParams != nil { + Walk(v, n.TypeParams) + } + if n.Params != nil { + Walk(v, n.Params) + } + if n.Results != nil { + Walk(v, n.Results) + } + + case *InterfaceType: + Walk(v, n.Methods) + + case *MapType: + Walk(v, n.Key) + Walk(v, n.Value) + + case *ChanType: + Walk(v, n.Value) + + // Statements + case *BadStmt: + // nothing to do + + case *DeclStmt: + Walk(v, n.Decl) + + case *EmptyStmt: + // nothing to do + + case *LabeledStmt: + Walk(v, n.Label) + Walk(v, n.Stmt) + + case *ExprStmt: + Walk(v, n.X) + + case *SendStmt: + Walk(v, n.Chan) + Walk(v, n.Value) + + case *IncDecStmt: + Walk(v, n.X) + + case *AssignStmt: + walkExprList(v, n.Lhs) + walkExprList(v, n.Rhs) + + case *GoStmt: + Walk(v, n.Call) + + case *DeferStmt: + Walk(v, n.Call) + + case *ReturnStmt: + walkExprList(v, n.Results) + + case *BranchStmt: + if n.Label != nil { + Walk(v, n.Label) + } + + case *BlockStmt: + walkStmtList(v, n.List) + + case *IfStmt: + if n.Init != nil { + Walk(v, n.Init) + } + Walk(v, n.Cond) + Walk(v, n.Body) + if n.Else != nil { + Walk(v, n.Else) + } + + case *CaseClause: + walkExprList(v, n.List) + walkStmtList(v, n.Body) + + case *SwitchStmt: + if n.Init != nil { + Walk(v, n.Init) + } + if n.Tag != nil { + Walk(v, n.Tag) + } + Walk(v, n.Body) + + case *TypeSwitchStmt: + if n.Init != nil { + Walk(v, n.Init) + } + Walk(v, n.Assign) + Walk(v, n.Body) + + case *CommClause: + if n.Comm != nil { + Walk(v, n.Comm) + } + walkStmtList(v, n.Body) + + case *SelectStmt: + Walk(v, n.Body) + + case *ForStmt: + if n.Init != nil { + Walk(v, n.Init) + } + if n.Cond != nil { + Walk(v, n.Cond) + } + if n.Post != nil { + Walk(v, n.Post) + } + Walk(v, n.Body) + + case *RangeStmt: + if n.Key != nil { + Walk(v, n.Key) + } + if n.Value != nil { + Walk(v, n.Value) + } + Walk(v, n.X) + Walk(v, n.Body) + + // Declarations + case *ImportSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + if n.Name != nil { + Walk(v, n.Name) + } + Walk(v, n.Path) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *ValueSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + walkIdentList(v, n.Names) + if n.Type != nil { + Walk(v, n.Type) + } + walkExprList(v, n.Values) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *TypeSpec: + if n.Doc != nil { + Walk(v, n.Doc) + } + Walk(v, n.Name) + if n.TypeParams != nil { + Walk(v, n.TypeParams) + } + Walk(v, n.Type) + if n.Comment != nil { + Walk(v, n.Comment) + } + + case *BadDecl: + // nothing to do + + case *GenDecl: + if n.Doc != nil { + Walk(v, n.Doc) + } + for _, s := range n.Specs { + Walk(v, s) + } + + case *FuncDecl: + if n.Doc != nil { + Walk(v, n.Doc) + } + if n.Recv != nil { + Walk(v, n.Recv) + } + Walk(v, n.Name) + Walk(v, n.Type) + if n.Body != nil { + Walk(v, n.Body) + } + + // Files and packages + case *File: + if n.Doc != nil { + Walk(v, n.Doc) + } + Walk(v, n.Name) + walkDeclList(v, n.Decls) + // don't walk n.Comments - they have been + // visited already through the individual + // nodes + + case *Package: + for _, f := range n.Files { + Walk(v, f) + } + + default: + panic(fmt.Sprintf("ast.Walk: unexpected node type %T", n)) + } + + v.Visit(nil) +} + +type inspector func(Node) bool + +func (f inspector) Visit(node Node) Visitor { + if f(node) { + return f + } + return nil +} + +// Inspect traverses an AST in depth-first order: It starts by calling +// f(node); node must not be nil. If f returns true, Inspect invokes f +// recursively for each of the non-nil children of node, followed by a +// call of f(nil). +func Inspect(node Node, f func(Node) bool) { + Walk(inspector(f), node) +} diff --git a/src/go/build/build.go b/src/go/build/build.go new file mode 100644 index 0000000..420873c --- /dev/null +++ b/src/go/build/build.go @@ -0,0 +1,2012 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/build/constraint" + "go/doc" + "go/token" + "internal/buildcfg" + "internal/godebug" + "internal/goroot" + "internal/goversion" + "io" + "io/fs" + "os" + "os/exec" + pathpkg "path" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// A Context specifies the supporting context for a build. +type Context struct { + GOARCH string // target architecture + GOOS string // target operating system + GOROOT string // Go root + GOPATH string // Go paths + + // Dir is the caller's working directory, or the empty string to use + // the current directory of the running process. In module mode, this is used + // to locate the main module. + // + // If Dir is non-empty, directories passed to Import and ImportDir must + // be absolute. + Dir string + + CgoEnabled bool // whether cgo files are included + UseAllFiles bool // use files regardless of go:build lines, file names + Compiler string // compiler to assume when computing target paths + + // The build, tool, and release tags specify build constraints + // that should be considered satisfied when processing go:build lines. + // Clients creating a new context may customize BuildTags, which + // defaults to empty, but it is usually an error to customize ToolTags or ReleaseTags. + // ToolTags defaults to build tags appropriate to the current Go toolchain configuration. + // ReleaseTags defaults to the list of Go releases the current release is compatible with. + // BuildTags is not set for the Default build Context. + // In addition to the BuildTags, ToolTags, and ReleaseTags, build constraints + // consider the values of GOARCH and GOOS as satisfied tags. + // The last element in ReleaseTags is assumed to be the current release. + BuildTags []string + ToolTags []string + ReleaseTags []string + + // The install suffix specifies a suffix to use in the name of the installation + // directory. By default it is empty, but custom builds that need to keep + // their outputs separate can set InstallSuffix to do so. For example, when + // using the race detector, the go command uses InstallSuffix = "race", so + // that on a Linux/386 system, packages are written to a directory named + // "linux_386_race" instead of the usual "linux_386". + InstallSuffix string + + // By default, Import uses the operating system's file system calls + // to read directories and files. To read from other sources, + // callers can set the following functions. They all have default + // behaviors that use the local file system, so clients need only set + // the functions whose behaviors they wish to change. + + // JoinPath joins the sequence of path fragments into a single path. + // If JoinPath is nil, Import uses filepath.Join. + JoinPath func(elem ...string) string + + // SplitPathList splits the path list into a slice of individual paths. + // If SplitPathList is nil, Import uses filepath.SplitList. + SplitPathList func(list string) []string + + // IsAbsPath reports whether path is an absolute path. + // If IsAbsPath is nil, Import uses filepath.IsAbs. + IsAbsPath func(path string) bool + + // IsDir reports whether the path names a directory. + // If IsDir is nil, Import calls os.Stat and uses the result's IsDir method. + IsDir func(path string) bool + + // HasSubdir reports whether dir is lexically a subdirectory of + // root, perhaps multiple levels below. It does not try to check + // whether dir exists. + // If so, HasSubdir sets rel to a slash-separated path that + // can be joined to root to produce a path equivalent to dir. + // If HasSubdir is nil, Import uses an implementation built on + // filepath.EvalSymlinks. + HasSubdir func(root, dir string) (rel string, ok bool) + + // ReadDir returns a slice of fs.FileInfo, sorted by Name, + // describing the content of the named directory. + // If ReadDir is nil, Import uses os.ReadDir. + ReadDir func(dir string) ([]fs.FileInfo, error) + + // OpenFile opens a file (not a directory) for reading. + // If OpenFile is nil, Import uses os.Open. + OpenFile func(path string) (io.ReadCloser, error) +} + +// joinPath calls ctxt.JoinPath (if not nil) or else filepath.Join. +func (ctxt *Context) joinPath(elem ...string) string { + if f := ctxt.JoinPath; f != nil { + return f(elem...) + } + return filepath.Join(elem...) +} + +// splitPathList calls ctxt.SplitPathList (if not nil) or else filepath.SplitList. +func (ctxt *Context) splitPathList(s string) []string { + if f := ctxt.SplitPathList; f != nil { + return f(s) + } + return filepath.SplitList(s) +} + +// isAbsPath calls ctxt.IsAbsPath (if not nil) or else filepath.IsAbs. +func (ctxt *Context) isAbsPath(path string) bool { + if f := ctxt.IsAbsPath; f != nil { + return f(path) + } + return filepath.IsAbs(path) +} + +// isDir calls ctxt.IsDir (if not nil) or else uses os.Stat. +func (ctxt *Context) isDir(path string) bool { + if f := ctxt.IsDir; f != nil { + return f(path) + } + fi, err := os.Stat(path) + return err == nil && fi.IsDir() +} + +// hasSubdir calls ctxt.HasSubdir (if not nil) or else uses +// the local file system to answer the question. +func (ctxt *Context) hasSubdir(root, dir string) (rel string, ok bool) { + if f := ctxt.HasSubdir; f != nil { + return f(root, dir) + } + + // Try using paths we received. + if rel, ok = hasSubdir(root, dir); ok { + return + } + + // Try expanding symlinks and comparing + // expanded against unexpanded and + // expanded against expanded. + rootSym, _ := filepath.EvalSymlinks(root) + dirSym, _ := filepath.EvalSymlinks(dir) + + if rel, ok = hasSubdir(rootSym, dir); ok { + return + } + if rel, ok = hasSubdir(root, dirSym); ok { + return + } + return hasSubdir(rootSym, dirSym) +} + +// hasSubdir reports if dir is within root by performing lexical analysis only. +func hasSubdir(root, dir string) (rel string, ok bool) { + const sep = string(filepath.Separator) + root = filepath.Clean(root) + if !strings.HasSuffix(root, sep) { + root += sep + } + dir = filepath.Clean(dir) + after, found := strings.CutPrefix(dir, root) + if !found { + return "", false + } + return filepath.ToSlash(after), true +} + +// readDir calls ctxt.ReadDir (if not nil) or else os.ReadDir. +func (ctxt *Context) readDir(path string) ([]fs.DirEntry, error) { + // TODO: add a fs.DirEntry version of Context.ReadDir + if f := ctxt.ReadDir; f != nil { + fis, err := f(path) + if err != nil { + return nil, err + } + des := make([]fs.DirEntry, len(fis)) + for i, fi := range fis { + des[i] = fs.FileInfoToDirEntry(fi) + } + return des, nil + } + return os.ReadDir(path) +} + +// openFile calls ctxt.OpenFile (if not nil) or else os.Open. +func (ctxt *Context) openFile(path string) (io.ReadCloser, error) { + if fn := ctxt.OpenFile; fn != nil { + return fn(path) + } + + f, err := os.Open(path) + if err != nil { + return nil, err // nil interface + } + return f, nil +} + +// isFile determines whether path is a file by trying to open it. +// It reuses openFile instead of adding another function to the +// list in Context. +func (ctxt *Context) isFile(path string) bool { + f, err := ctxt.openFile(path) + if err != nil { + return false + } + f.Close() + return true +} + +// gopath returns the list of Go path directories. +func (ctxt *Context) gopath() []string { + var all []string + for _, p := range ctxt.splitPathList(ctxt.GOPATH) { + if p == "" || p == ctxt.GOROOT { + // Empty paths are uninteresting. + // If the path is the GOROOT, ignore it. + // People sometimes set GOPATH=$GOROOT. + // Do not get confused by this common mistake. + continue + } + if strings.HasPrefix(p, "~") { + // Path segments starting with ~ on Unix are almost always + // users who have incorrectly quoted ~ while setting GOPATH, + // preventing it from expanding to $HOME. + // The situation is made more confusing by the fact that + // bash allows quoted ~ in $PATH (most shells do not). + // Do not get confused by this, and do not try to use the path. + // It does not exist, and printing errors about it confuses + // those users even more, because they think "sure ~ exists!". + // The go command diagnoses this situation and prints a + // useful error. + // On Windows, ~ is used in short names, such as c:\progra~1 + // for c:\program files. + continue + } + all = append(all, p) + } + return all +} + +// SrcDirs returns a list of package source root directories. +// It draws from the current Go root and Go path but omits directories +// that do not exist. +func (ctxt *Context) SrcDirs() []string { + var all []string + if ctxt.GOROOT != "" && ctxt.Compiler != "gccgo" { + dir := ctxt.joinPath(ctxt.GOROOT, "src") + if ctxt.isDir(dir) { + all = append(all, dir) + } + } + for _, p := range ctxt.gopath() { + dir := ctxt.joinPath(p, "src") + if ctxt.isDir(dir) { + all = append(all, dir) + } + } + return all +} + +// Default is the default Context for builds. +// It uses the GOARCH, GOOS, GOROOT, and GOPATH environment variables +// if set, or else the compiled code's GOARCH, GOOS, and GOROOT. +var Default Context = defaultContext() + +func defaultGOPATH() string { + env := "HOME" + if runtime.GOOS == "windows" { + env = "USERPROFILE" + } else if runtime.GOOS == "plan9" { + env = "home" + } + if home := os.Getenv(env); home != "" { + def := filepath.Join(home, "go") + if filepath.Clean(def) == filepath.Clean(runtime.GOROOT()) { + // Don't set the default GOPATH to GOROOT, + // as that will trigger warnings from the go tool. + return "" + } + return def + } + return "" +} + +var defaultToolTags, defaultReleaseTags []string + +func defaultContext() Context { + var c Context + + c.GOARCH = buildcfg.GOARCH + c.GOOS = buildcfg.GOOS + if goroot := runtime.GOROOT(); goroot != "" { + c.GOROOT = filepath.Clean(goroot) + } + c.GOPATH = envOr("GOPATH", defaultGOPATH()) + c.Compiler = runtime.Compiler + c.ToolTags = append(c.ToolTags, buildcfg.ToolTags...) + + defaultToolTags = append([]string{}, c.ToolTags...) // our own private copy + + // Each major Go release in the Go 1.x series adds a new + // "go1.x" release tag. That is, the go1.x tag is present in + // all releases >= Go 1.x. Code that requires Go 1.x or later + // should say "go:build go1.x", and code that should only be + // built before Go 1.x (perhaps it is the stub to use in that + // case) should say "go:build !go1.x". + // The last element in ReleaseTags is the current release. + for i := 1; i <= goversion.Version; i++ { + c.ReleaseTags = append(c.ReleaseTags, "go1."+strconv.Itoa(i)) + } + + defaultReleaseTags = append([]string{}, c.ReleaseTags...) // our own private copy + + env := os.Getenv("CGO_ENABLED") + if env == "" { + env = defaultCGO_ENABLED + } + switch env { + case "1": + c.CgoEnabled = true + case "0": + c.CgoEnabled = false + default: + // cgo must be explicitly enabled for cross compilation builds + if runtime.GOARCH == c.GOARCH && runtime.GOOS == c.GOOS { + c.CgoEnabled = cgoEnabled[c.GOOS+"/"+c.GOARCH] + break + } + c.CgoEnabled = false + } + + return c +} + +func envOr(name, def string) string { + s := os.Getenv(name) + if s == "" { + return def + } + return s +} + +// An ImportMode controls the behavior of the Import method. +type ImportMode uint + +const ( + // If FindOnly is set, Import stops after locating the directory + // that should contain the sources for a package. It does not + // read any files in the directory. + FindOnly ImportMode = 1 << iota + + // If AllowBinary is set, Import can be satisfied by a compiled + // package object without corresponding sources. + // + // Deprecated: + // The supported way to create a compiled-only package is to + // write source code containing a //go:binary-only-package comment at + // the top of the file. Such a package will be recognized + // regardless of this flag setting (because it has source code) + // and will have BinaryOnly set to true in the returned Package. + AllowBinary + + // If ImportComment is set, parse import comments on package statements. + // Import returns an error if it finds a comment it cannot understand + // or finds conflicting comments in multiple source files. + // See golang.org/s/go14customimport for more information. + ImportComment + + // By default, Import searches vendor directories + // that apply in the given source directory before searching + // the GOROOT and GOPATH roots. + // If an Import finds and returns a package using a vendor + // directory, the resulting ImportPath is the complete path + // to the package, including the path elements leading up + // to and including "vendor". + // For example, if Import("y", "x/subdir", 0) finds + // "x/vendor/y", the returned package's ImportPath is "x/vendor/y", + // not plain "y". + // See golang.org/s/go15vendor for more information. + // + // Setting IgnoreVendor ignores vendor directories. + // + // In contrast to the package's ImportPath, + // the returned package's Imports, TestImports, and XTestImports + // are always the exact import paths from the source files: + // Import makes no attempt to resolve or check those paths. + IgnoreVendor +) + +// A Package describes the Go package found in a directory. +type Package struct { + Dir string // directory containing package sources + Name string // package name + ImportComment string // path in import comment on package statement + Doc string // documentation synopsis + ImportPath string // import path of package ("" if unknown) + Root string // root of Go tree where this package lives + SrcRoot string // package source root directory ("" if unknown) + PkgRoot string // package install root directory ("" if unknown) + PkgTargetRoot string // architecture dependent install root directory ("" if unknown) + BinDir string // command install directory ("" if unknown) + Goroot bool // package found in Go root + PkgObj string // installed .a file + AllTags []string // tags that can influence file selection in this directory + ConflictDir string // this directory shadows Dir in $GOPATH + BinaryOnly bool // cannot be rebuilt from source (has //go:binary-only-package comment) + + // Source files + GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string // .go source files that import "C" + IgnoredGoFiles []string // .go source files ignored for this build (including ignored _test.go files) + InvalidGoFiles []string // .go source files with detected problems (parse error, wrong package name, and so on) + IgnoredOtherFiles []string // non-.go source files ignored for this build + CFiles []string // .c source files + CXXFiles []string // .cc, .cpp and .cxx source files + MFiles []string // .m (Objective-C) source files + HFiles []string // .h, .hh, .hpp and .hxx source files + FFiles []string // .f, .F, .for and .f90 Fortran source files + SFiles []string // .s source files + SwigFiles []string // .swig files + SwigCXXFiles []string // .swigcxx files + SysoFiles []string // .syso system object files to add to archive + + // Cgo directives + CgoCFLAGS []string // Cgo CFLAGS directives + CgoCPPFLAGS []string // Cgo CPPFLAGS directives + CgoCXXFLAGS []string // Cgo CXXFLAGS directives + CgoFFLAGS []string // Cgo FFLAGS directives + CgoLDFLAGS []string // Cgo LDFLAGS directives + CgoPkgConfig []string // Cgo pkg-config directives + + // Test information + TestGoFiles []string // _test.go files in package + XTestGoFiles []string // _test.go files outside package + + // Dependency information + Imports []string // import paths from GoFiles, CgoFiles + ImportPos map[string][]token.Position // line information for Imports + TestImports []string // import paths from TestGoFiles + TestImportPos map[string][]token.Position // line information for TestImports + XTestImports []string // import paths from XTestGoFiles + XTestImportPos map[string][]token.Position // line information for XTestImports + + // //go:embed patterns found in Go source files + // For example, if a source file says + // //go:embed a* b.c + // then the list will contain those two strings as separate entries. + // (See package embed for more details about //go:embed.) + EmbedPatterns []string // patterns from GoFiles, CgoFiles + EmbedPatternPos map[string][]token.Position // line information for EmbedPatterns + TestEmbedPatterns []string // patterns from TestGoFiles + TestEmbedPatternPos map[string][]token.Position // line information for TestEmbedPatterns + XTestEmbedPatterns []string // patterns from XTestGoFiles + XTestEmbedPatternPos map[string][]token.Position // line information for XTestEmbedPatternPos +} + +// IsCommand reports whether the package is considered a +// command to be installed (not just a library). +// Packages named "main" are treated as commands. +func (p *Package) IsCommand() bool { + return p.Name == "main" +} + +// ImportDir is like Import but processes the Go package found in +// the named directory. +func (ctxt *Context) ImportDir(dir string, mode ImportMode) (*Package, error) { + return ctxt.Import(".", dir, mode) +} + +// NoGoError is the error used by Import to describe a directory +// containing no buildable Go source files. (It may still contain +// test files, files hidden by build tags, and so on.) +type NoGoError struct { + Dir string +} + +func (e *NoGoError) Error() string { + return "no buildable Go source files in " + e.Dir +} + +// MultiplePackageError describes a directory containing +// multiple buildable Go source files for multiple packages. +type MultiplePackageError struct { + Dir string // directory containing files + Packages []string // package names found + Files []string // corresponding files: Files[i] declares package Packages[i] +} + +func (e *MultiplePackageError) Error() string { + // Error string limited to two entries for compatibility. + return fmt.Sprintf("found packages %s (%s) and %s (%s) in %s", e.Packages[0], e.Files[0], e.Packages[1], e.Files[1], e.Dir) +} + +func nameExt(name string) string { + i := strings.LastIndex(name, ".") + if i < 0 { + return "" + } + return name[i:] +} + +var installgoroot = godebug.New("installgoroot") + +// Import returns details about the Go package named by the import path, +// interpreting local import paths relative to the srcDir directory. +// If the path is a local import path naming a package that can be imported +// using a standard import path, the returned package will set p.ImportPath +// to that path. +// +// In the directory containing the package, .go, .c, .h, and .s files are +// considered part of the package except for: +// +// - .go files in package documentation +// - files starting with _ or . (likely editor temporary files) +// - files with build constraints not satisfied by the context +// +// If an error occurs, Import returns a non-nil error and a non-nil +// *Package containing partial information. +func (ctxt *Context) Import(path string, srcDir string, mode ImportMode) (*Package, error) { + p := &Package{ + ImportPath: path, + } + if path == "" { + return p, fmt.Errorf("import %q: invalid import path", path) + } + + var pkgtargetroot string + var pkga string + var pkgerr error + suffix := "" + if ctxt.InstallSuffix != "" { + suffix = "_" + ctxt.InstallSuffix + } + switch ctxt.Compiler { + case "gccgo": + pkgtargetroot = "pkg/gccgo_" + ctxt.GOOS + "_" + ctxt.GOARCH + suffix + case "gc": + pkgtargetroot = "pkg/" + ctxt.GOOS + "_" + ctxt.GOARCH + suffix + default: + // Save error for end of function. + pkgerr = fmt.Errorf("import %q: unknown compiler %q", path, ctxt.Compiler) + } + setPkga := func() { + switch ctxt.Compiler { + case "gccgo": + dir, elem := pathpkg.Split(p.ImportPath) + pkga = pkgtargetroot + "/" + dir + "lib" + elem + ".a" + case "gc": + pkga = pkgtargetroot + "/" + p.ImportPath + ".a" + } + } + setPkga() + + binaryOnly := false + if IsLocalImport(path) { + pkga = "" // local imports have no installed path + if srcDir == "" { + return p, fmt.Errorf("import %q: import relative to unknown directory", path) + } + if !ctxt.isAbsPath(path) { + p.Dir = ctxt.joinPath(srcDir, path) + } + // p.Dir directory may or may not exist. Gather partial information first, check if it exists later. + // Determine canonical import path, if any. + // Exclude results where the import path would include /testdata/. + inTestdata := func(sub string) bool { + return strings.Contains(sub, "/testdata/") || strings.HasSuffix(sub, "/testdata") || strings.HasPrefix(sub, "testdata/") || sub == "testdata" + } + if ctxt.GOROOT != "" { + root := ctxt.joinPath(ctxt.GOROOT, "src") + if sub, ok := ctxt.hasSubdir(root, p.Dir); ok && !inTestdata(sub) { + p.Goroot = true + p.ImportPath = sub + p.Root = ctxt.GOROOT + setPkga() // p.ImportPath changed + goto Found + } + } + all := ctxt.gopath() + for i, root := range all { + rootsrc := ctxt.joinPath(root, "src") + if sub, ok := ctxt.hasSubdir(rootsrc, p.Dir); ok && !inTestdata(sub) { + // We found a potential import path for dir, + // but check that using it wouldn't find something + // else first. + if ctxt.GOROOT != "" && ctxt.Compiler != "gccgo" { + if dir := ctxt.joinPath(ctxt.GOROOT, "src", sub); ctxt.isDir(dir) { + p.ConflictDir = dir + goto Found + } + } + for _, earlyRoot := range all[:i] { + if dir := ctxt.joinPath(earlyRoot, "src", sub); ctxt.isDir(dir) { + p.ConflictDir = dir + goto Found + } + } + + // sub would not name some other directory instead of this one. + // Record it. + p.ImportPath = sub + p.Root = root + setPkga() // p.ImportPath changed + goto Found + } + } + // It's okay that we didn't find a root containing dir. + // Keep going with the information we have. + } else { + if strings.HasPrefix(path, "/") { + return p, fmt.Errorf("import %q: cannot import absolute path", path) + } + + if err := ctxt.importGo(p, path, srcDir, mode); err == nil { + goto Found + } else if err != errNoModules { + return p, err + } + + gopath := ctxt.gopath() // needed twice below; avoid computing many times + + // tried records the location of unsuccessful package lookups + var tried struct { + vendor []string + goroot string + gopath []string + } + + // Vendor directories get first chance to satisfy import. + if mode&IgnoreVendor == 0 && srcDir != "" { + searchVendor := func(root string, isGoroot bool) bool { + sub, ok := ctxt.hasSubdir(root, srcDir) + if !ok || !strings.HasPrefix(sub, "src/") || strings.Contains(sub, "/testdata/") { + return false + } + for { + vendor := ctxt.joinPath(root, sub, "vendor") + if ctxt.isDir(vendor) { + dir := ctxt.joinPath(vendor, path) + if ctxt.isDir(dir) && hasGoFiles(ctxt, dir) { + p.Dir = dir + p.ImportPath = strings.TrimPrefix(pathpkg.Join(sub, "vendor", path), "src/") + p.Goroot = isGoroot + p.Root = root + setPkga() // p.ImportPath changed + return true + } + tried.vendor = append(tried.vendor, dir) + } + i := strings.LastIndex(sub, "/") + if i < 0 { + break + } + sub = sub[:i] + } + return false + } + if ctxt.Compiler != "gccgo" && ctxt.GOROOT != "" && searchVendor(ctxt.GOROOT, true) { + goto Found + } + for _, root := range gopath { + if searchVendor(root, false) { + goto Found + } + } + } + + // Determine directory from import path. + if ctxt.GOROOT != "" { + // If the package path starts with "vendor/", only search GOROOT before + // GOPATH if the importer is also within GOROOT. That way, if the user has + // vendored in a package that is subsequently included in the standard + // distribution, they'll continue to pick up their own vendored copy. + gorootFirst := srcDir == "" || !strings.HasPrefix(path, "vendor/") + if !gorootFirst { + _, gorootFirst = ctxt.hasSubdir(ctxt.GOROOT, srcDir) + } + if gorootFirst { + dir := ctxt.joinPath(ctxt.GOROOT, "src", path) + if ctxt.Compiler != "gccgo" { + isDir := ctxt.isDir(dir) + binaryOnly = !isDir && mode&AllowBinary != 0 && pkga != "" && ctxt.isFile(ctxt.joinPath(ctxt.GOROOT, pkga)) + if isDir || binaryOnly { + p.Dir = dir + p.Goroot = true + p.Root = ctxt.GOROOT + goto Found + } + } + tried.goroot = dir + } + if ctxt.Compiler == "gccgo" && goroot.IsStandardPackage(ctxt.GOROOT, ctxt.Compiler, path) { + // TODO(bcmills): Setting p.Dir here is misleading, because gccgo + // doesn't actually load its standard-library packages from this + // directory. See if we can leave it unset. + p.Dir = ctxt.joinPath(ctxt.GOROOT, "src", path) + p.Goroot = true + p.Root = ctxt.GOROOT + goto Found + } + } + for _, root := range gopath { + dir := ctxt.joinPath(root, "src", path) + isDir := ctxt.isDir(dir) + binaryOnly = !isDir && mode&AllowBinary != 0 && pkga != "" && ctxt.isFile(ctxt.joinPath(root, pkga)) + if isDir || binaryOnly { + p.Dir = dir + p.Root = root + goto Found + } + tried.gopath = append(tried.gopath, dir) + } + + // If we tried GOPATH first due to a "vendor/" prefix, fall back to GOPATH. + // That way, the user can still get useful results from 'go list' for + // standard-vendored paths passed on the command line. + if ctxt.GOROOT != "" && tried.goroot == "" { + dir := ctxt.joinPath(ctxt.GOROOT, "src", path) + if ctxt.Compiler != "gccgo" { + isDir := ctxt.isDir(dir) + binaryOnly = !isDir && mode&AllowBinary != 0 && pkga != "" && ctxt.isFile(ctxt.joinPath(ctxt.GOROOT, pkga)) + if isDir || binaryOnly { + p.Dir = dir + p.Goroot = true + p.Root = ctxt.GOROOT + goto Found + } + } + tried.goroot = dir + } + + // package was not found + var paths []string + format := "\t%s (vendor tree)" + for _, dir := range tried.vendor { + paths = append(paths, fmt.Sprintf(format, dir)) + format = "\t%s" + } + if tried.goroot != "" { + paths = append(paths, fmt.Sprintf("\t%s (from $GOROOT)", tried.goroot)) + } else { + paths = append(paths, "\t($GOROOT not set)") + } + format = "\t%s (from $GOPATH)" + for _, dir := range tried.gopath { + paths = append(paths, fmt.Sprintf(format, dir)) + format = "\t%s" + } + if len(tried.gopath) == 0 { + paths = append(paths, "\t($GOPATH not set. For more details see: 'go help gopath')") + } + return p, fmt.Errorf("cannot find package %q in any of:\n%s", path, strings.Join(paths, "\n")) + } + +Found: + if p.Root != "" { + p.SrcRoot = ctxt.joinPath(p.Root, "src") + p.PkgRoot = ctxt.joinPath(p.Root, "pkg") + p.BinDir = ctxt.joinPath(p.Root, "bin") + if pkga != "" { + // Always set PkgTargetRoot. It might be used when building in shared + // mode. + p.PkgTargetRoot = ctxt.joinPath(p.Root, pkgtargetroot) + + // Set the install target if applicable. + if !p.Goroot || (installgoroot.Value() == "all" && p.ImportPath != "unsafe" && p.ImportPath != "builtin") { + p.PkgObj = ctxt.joinPath(p.Root, pkga) + } + } + } + + // If it's a local import path, by the time we get here, we still haven't checked + // that p.Dir directory exists. This is the right time to do that check. + // We can't do it earlier, because we want to gather partial information for the + // non-nil *Package returned when an error occurs. + // We need to do this before we return early on FindOnly flag. + if IsLocalImport(path) && !ctxt.isDir(p.Dir) { + if ctxt.Compiler == "gccgo" && p.Goroot { + // gccgo has no sources for GOROOT packages. + return p, nil + } + + // package was not found + return p, fmt.Errorf("cannot find package %q in:\n\t%s", p.ImportPath, p.Dir) + } + + if mode&FindOnly != 0 { + return p, pkgerr + } + if binaryOnly && (mode&AllowBinary) != 0 { + return p, pkgerr + } + + if ctxt.Compiler == "gccgo" && p.Goroot { + // gccgo has no sources for GOROOT packages. + return p, nil + } + + dirs, err := ctxt.readDir(p.Dir) + if err != nil { + return p, err + } + + var badGoError error + badGoFiles := make(map[string]bool) + badGoFile := func(name string, err error) { + if badGoError == nil { + badGoError = err + } + if !badGoFiles[name] { + p.InvalidGoFiles = append(p.InvalidGoFiles, name) + badGoFiles[name] = true + } + } + + var Sfiles []string // files with ".S"(capital S)/.sx(capital s equivalent for case insensitive filesystems) + var firstFile, firstCommentFile string + embedPos := make(map[string][]token.Position) + testEmbedPos := make(map[string][]token.Position) + xTestEmbedPos := make(map[string][]token.Position) + importPos := make(map[string][]token.Position) + testImportPos := make(map[string][]token.Position) + xTestImportPos := make(map[string][]token.Position) + allTags := make(map[string]bool) + fset := token.NewFileSet() + for _, d := range dirs { + if d.IsDir() { + continue + } + if d.Type() == fs.ModeSymlink { + if ctxt.isDir(ctxt.joinPath(p.Dir, d.Name())) { + // Symlinks to directories are not source files. + continue + } + } + + name := d.Name() + ext := nameExt(name) + + info, err := ctxt.matchFile(p.Dir, name, allTags, &p.BinaryOnly, fset) + if err != nil && strings.HasSuffix(name, ".go") { + badGoFile(name, err) + continue + } + if info == nil { + if strings.HasPrefix(name, "_") || strings.HasPrefix(name, ".") { + // not due to build constraints - don't report + } else if ext == ".go" { + p.IgnoredGoFiles = append(p.IgnoredGoFiles, name) + } else if fileListForExt(p, ext) != nil { + p.IgnoredOtherFiles = append(p.IgnoredOtherFiles, name) + } + continue + } + + // Going to save the file. For non-Go files, can stop here. + switch ext { + case ".go": + // keep going + case ".S", ".sx": + // special case for cgo, handled at end + Sfiles = append(Sfiles, name) + continue + default: + if list := fileListForExt(p, ext); list != nil { + *list = append(*list, name) + } + continue + } + + data, filename := info.header, info.name + + if info.parseErr != nil { + badGoFile(name, info.parseErr) + // Fall through: we might still have a partial AST in info.parsed, + // and we want to list files with parse errors anyway. + } + + var pkg string + if info.parsed != nil { + pkg = info.parsed.Name.Name + if pkg == "documentation" { + p.IgnoredGoFiles = append(p.IgnoredGoFiles, name) + continue + } + } + + isTest := strings.HasSuffix(name, "_test.go") + isXTest := false + if isTest && strings.HasSuffix(pkg, "_test") && p.Name != pkg { + isXTest = true + pkg = pkg[:len(pkg)-len("_test")] + } + + if p.Name == "" { + p.Name = pkg + firstFile = name + } else if pkg != p.Name { + // TODO(#45999): The choice of p.Name is arbitrary based on file iteration + // order. Instead of resolving p.Name arbitrarily, we should clear out the + // existing name and mark the existing files as also invalid. + badGoFile(name, &MultiplePackageError{ + Dir: p.Dir, + Packages: []string{p.Name, pkg}, + Files: []string{firstFile, name}, + }) + } + // Grab the first package comment as docs, provided it is not from a test file. + if info.parsed != nil && info.parsed.Doc != nil && p.Doc == "" && !isTest && !isXTest { + p.Doc = doc.Synopsis(info.parsed.Doc.Text()) + } + + if mode&ImportComment != 0 { + qcom, line := findImportComment(data) + if line != 0 { + com, err := strconv.Unquote(qcom) + if err != nil { + badGoFile(name, fmt.Errorf("%s:%d: cannot parse import comment", filename, line)) + } else if p.ImportComment == "" { + p.ImportComment = com + firstCommentFile = name + } else if p.ImportComment != com { + badGoFile(name, fmt.Errorf("found import comments %q (%s) and %q (%s) in %s", p.ImportComment, firstCommentFile, com, name, p.Dir)) + } + } + } + + // Record imports and information about cgo. + isCgo := false + for _, imp := range info.imports { + if imp.path == "C" { + if isTest { + badGoFile(name, fmt.Errorf("use of cgo in test %s not supported", filename)) + continue + } + isCgo = true + if imp.doc != nil { + if err := ctxt.saveCgo(filename, p, imp.doc); err != nil { + badGoFile(name, err) + } + } + } + } + + var fileList *[]string + var importMap, embedMap map[string][]token.Position + switch { + case isCgo: + allTags["cgo"] = true + if ctxt.CgoEnabled { + fileList = &p.CgoFiles + importMap = importPos + embedMap = embedPos + } else { + // Ignore imports and embeds from cgo files if cgo is disabled. + fileList = &p.IgnoredGoFiles + } + case isXTest: + fileList = &p.XTestGoFiles + importMap = xTestImportPos + embedMap = xTestEmbedPos + case isTest: + fileList = &p.TestGoFiles + importMap = testImportPos + embedMap = testEmbedPos + default: + fileList = &p.GoFiles + importMap = importPos + embedMap = embedPos + } + *fileList = append(*fileList, name) + if importMap != nil { + for _, imp := range info.imports { + importMap[imp.path] = append(importMap[imp.path], fset.Position(imp.pos)) + } + } + if embedMap != nil { + for _, emb := range info.embeds { + embedMap[emb.pattern] = append(embedMap[emb.pattern], emb.pos) + } + } + } + + for tag := range allTags { + p.AllTags = append(p.AllTags, tag) + } + sort.Strings(p.AllTags) + + p.EmbedPatterns, p.EmbedPatternPos = cleanDecls(embedPos) + p.TestEmbedPatterns, p.TestEmbedPatternPos = cleanDecls(testEmbedPos) + p.XTestEmbedPatterns, p.XTestEmbedPatternPos = cleanDecls(xTestEmbedPos) + + p.Imports, p.ImportPos = cleanDecls(importPos) + p.TestImports, p.TestImportPos = cleanDecls(testImportPos) + p.XTestImports, p.XTestImportPos = cleanDecls(xTestImportPos) + + // add the .S/.sx files only if we are using cgo + // (which means gcc will compile them). + // The standard assemblers expect .s files. + if len(p.CgoFiles) > 0 { + p.SFiles = append(p.SFiles, Sfiles...) + sort.Strings(p.SFiles) + } else { + p.IgnoredOtherFiles = append(p.IgnoredOtherFiles, Sfiles...) + sort.Strings(p.IgnoredOtherFiles) + } + + if badGoError != nil { + return p, badGoError + } + if len(p.GoFiles)+len(p.CgoFiles)+len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 { + return p, &NoGoError{p.Dir} + } + return p, pkgerr +} + +func fileListForExt(p *Package, ext string) *[]string { + switch ext { + case ".c": + return &p.CFiles + case ".cc", ".cpp", ".cxx": + return &p.CXXFiles + case ".m": + return &p.MFiles + case ".h", ".hh", ".hpp", ".hxx": + return &p.HFiles + case ".f", ".F", ".for", ".f90": + return &p.FFiles + case ".s", ".S", ".sx": + return &p.SFiles + case ".swig": + return &p.SwigFiles + case ".swigcxx": + return &p.SwigCXXFiles + case ".syso": + return &p.SysoFiles + } + return nil +} + +func uniq(list []string) []string { + if list == nil { + return nil + } + out := make([]string, len(list)) + copy(out, list) + sort.Strings(out) + uniq := out[:0] + for _, x := range out { + if len(uniq) == 0 || uniq[len(uniq)-1] != x { + uniq = append(uniq, x) + } + } + return uniq +} + +var errNoModules = errors.New("not using modules") + +// importGo checks whether it can use the go command to find the directory for path. +// If using the go command is not appropriate, importGo returns errNoModules. +// Otherwise, importGo tries using the go command and reports whether that succeeded. +// Using the go command lets build.Import and build.Context.Import find code +// in Go modules. In the long term we want tools to use go/packages (currently golang.org/x/tools/go/packages), +// which will also use the go command. +// Invoking the go command here is not very efficient in that it computes information +// about the requested package and all dependencies and then only reports about the requested package. +// Then we reinvoke it for every dependency. But this is still better than not working at all. +// See golang.org/issue/26504. +func (ctxt *Context) importGo(p *Package, path, srcDir string, mode ImportMode) error { + // To invoke the go command, + // we must not being doing special things like AllowBinary or IgnoreVendor, + // and all the file system callbacks must be nil (we're meant to use the local file system). + if mode&AllowBinary != 0 || mode&IgnoreVendor != 0 || + ctxt.JoinPath != nil || ctxt.SplitPathList != nil || ctxt.IsAbsPath != nil || ctxt.IsDir != nil || ctxt.HasSubdir != nil || ctxt.ReadDir != nil || ctxt.OpenFile != nil || !equal(ctxt.ToolTags, defaultToolTags) || !equal(ctxt.ReleaseTags, defaultReleaseTags) { + return errNoModules + } + + // If ctxt.GOROOT is not set, we don't know which go command to invoke, + // and even if we did we might return packages in GOROOT that we wouldn't otherwise find + // (because we don't know to search in 'go env GOROOT' otherwise). + if ctxt.GOROOT == "" { + return errNoModules + } + + // Predict whether module aware mode is enabled by checking the value of + // GO111MODULE and looking for a go.mod file in the source directory or + // one of its parents. Running 'go env GOMOD' in the source directory would + // give a canonical answer, but we'd prefer not to execute another command. + go111Module := os.Getenv("GO111MODULE") + switch go111Module { + case "off": + return errNoModules + default: // "", "on", "auto", anything else + // Maybe use modules. + } + + if srcDir != "" { + var absSrcDir string + if filepath.IsAbs(srcDir) { + absSrcDir = srcDir + } else if ctxt.Dir != "" { + return fmt.Errorf("go/build: Dir is non-empty, so relative srcDir is not allowed: %v", srcDir) + } else { + // Find the absolute source directory. hasSubdir does not handle + // relative paths (and can't because the callbacks don't support this). + var err error + absSrcDir, err = filepath.Abs(srcDir) + if err != nil { + return errNoModules + } + } + + // If the source directory is in GOROOT, then the in-process code works fine + // and we should keep using it. Moreover, the 'go list' approach below doesn't + // take standard-library vendoring into account and will fail. + if _, ok := ctxt.hasSubdir(filepath.Join(ctxt.GOROOT, "src"), absSrcDir); ok { + return errNoModules + } + } + + // For efficiency, if path is a standard library package, let the usual lookup code handle it. + if dir := ctxt.joinPath(ctxt.GOROOT, "src", path); ctxt.isDir(dir) { + return errNoModules + } + + // If GO111MODULE=auto, look to see if there is a go.mod. + // Since go1.13, it doesn't matter if we're inside GOPATH. + if go111Module == "auto" { + var ( + parent string + err error + ) + if ctxt.Dir == "" { + parent, err = os.Getwd() + if err != nil { + // A nonexistent working directory can't be in a module. + return errNoModules + } + } else { + parent, err = filepath.Abs(ctxt.Dir) + if err != nil { + // If the caller passed a bogus Dir explicitly, that's materially + // different from not having modules enabled. + return err + } + } + for { + if f, err := ctxt.openFile(ctxt.joinPath(parent, "go.mod")); err == nil { + buf := make([]byte, 100) + _, err := f.Read(buf) + f.Close() + if err == nil || err == io.EOF { + // go.mod exists and is readable (is a file, not a directory). + break + } + } + d := filepath.Dir(parent) + if len(d) >= len(parent) { + return errNoModules // reached top of file system, no go.mod + } + parent = d + } + } + + goCmd := filepath.Join(ctxt.GOROOT, "bin", "go") + cmd := exec.Command(goCmd, "list", "-e", "-compiler="+ctxt.Compiler, "-tags="+strings.Join(ctxt.BuildTags, ","), "-installsuffix="+ctxt.InstallSuffix, "-f={{.Dir}}\n{{.ImportPath}}\n{{.Root}}\n{{.Goroot}}\n{{if .Error}}{{.Error}}{{end}}\n", "--", path) + + if ctxt.Dir != "" { + cmd.Dir = ctxt.Dir + } + + var stdout, stderr strings.Builder + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + cgo := "0" + if ctxt.CgoEnabled { + cgo = "1" + } + cmd.Env = append(cmd.Environ(), + "GOOS="+ctxt.GOOS, + "GOARCH="+ctxt.GOARCH, + "GOROOT="+ctxt.GOROOT, + "GOPATH="+ctxt.GOPATH, + "CGO_ENABLED="+cgo, + ) + + if err := cmd.Run(); err != nil { + return fmt.Errorf("go/build: go list %s: %v\n%s\n", path, err, stderr.String()) + } + + f := strings.SplitN(stdout.String(), "\n", 5) + if len(f) != 5 { + return fmt.Errorf("go/build: importGo %s: unexpected output:\n%s\n", path, stdout.String()) + } + dir := f[0] + errStr := strings.TrimSpace(f[4]) + if errStr != "" && dir == "" { + // If 'go list' could not locate the package (dir is empty), + // return the same error that 'go list' reported. + return errors.New(errStr) + } + + // If 'go list' did locate the package, ignore the error. + // It was probably related to loading source files, and we'll + // encounter it ourselves shortly if the FindOnly flag isn't set. + p.Dir = dir + p.ImportPath = f[1] + p.Root = f[2] + p.Goroot = f[3] == "true" + return nil +} + +func equal(x, y []string) bool { + if len(x) != len(y) { + return false + } + for i, xi := range x { + if xi != y[i] { + return false + } + } + return true +} + +// hasGoFiles reports whether dir contains any files with names ending in .go. +// For a vendor check we must exclude directories that contain no .go files. +// Otherwise it is not possible to vendor just a/b/c and still import the +// non-vendored a/b. See golang.org/issue/13832. +func hasGoFiles(ctxt *Context, dir string) bool { + ents, _ := ctxt.readDir(dir) + for _, ent := range ents { + if !ent.IsDir() && strings.HasSuffix(ent.Name(), ".go") { + return true + } + } + return false +} + +func findImportComment(data []byte) (s string, line int) { + // expect keyword package + word, data := parseWord(data) + if string(word) != "package" { + return "", 0 + } + + // expect package name + _, data = parseWord(data) + + // now ready for import comment, a // or /* */ comment + // beginning and ending on the current line. + for len(data) > 0 && (data[0] == ' ' || data[0] == '\t' || data[0] == '\r') { + data = data[1:] + } + + var comment []byte + switch { + case bytes.HasPrefix(data, slashSlash): + comment, _, _ = bytes.Cut(data[2:], newline) + case bytes.HasPrefix(data, slashStar): + var ok bool + comment, _, ok = bytes.Cut(data[2:], starSlash) + if !ok { + // malformed comment + return "", 0 + } + if bytes.Contains(comment, newline) { + return "", 0 + } + } + comment = bytes.TrimSpace(comment) + + // split comment into `import`, `"pkg"` + word, arg := parseWord(comment) + if string(word) != "import" { + return "", 0 + } + + line = 1 + bytes.Count(data[:cap(data)-cap(arg)], newline) + return strings.TrimSpace(string(arg)), line +} + +var ( + slashSlash = []byte("//") + slashStar = []byte("/*") + starSlash = []byte("*/") + newline = []byte("\n") +) + +// skipSpaceOrComment returns data with any leading spaces or comments removed. +func skipSpaceOrComment(data []byte) []byte { + for len(data) > 0 { + switch data[0] { + case ' ', '\t', '\r', '\n': + data = data[1:] + continue + case '/': + if bytes.HasPrefix(data, slashSlash) { + i := bytes.Index(data, newline) + if i < 0 { + return nil + } + data = data[i+1:] + continue + } + if bytes.HasPrefix(data, slashStar) { + data = data[2:] + i := bytes.Index(data, starSlash) + if i < 0 { + return nil + } + data = data[i+2:] + continue + } + } + break + } + return data +} + +// parseWord skips any leading spaces or comments in data +// and then parses the beginning of data as an identifier or keyword, +// returning that word and what remains after the word. +func parseWord(data []byte) (word, rest []byte) { + data = skipSpaceOrComment(data) + + // Parse past leading word characters. + rest = data + for { + r, size := utf8.DecodeRune(rest) + if unicode.IsLetter(r) || '0' <= r && r <= '9' || r == '_' { + rest = rest[size:] + continue + } + break + } + + word = data[:len(data)-len(rest)] + if len(word) == 0 { + return nil, nil + } + + return word, rest +} + +// MatchFile reports whether the file with the given name in the given directory +// matches the context and would be included in a Package created by ImportDir +// of that directory. +// +// MatchFile considers the name of the file and may use ctxt.OpenFile to +// read some or all of the file's content. +func (ctxt *Context) MatchFile(dir, name string) (match bool, err error) { + info, err := ctxt.matchFile(dir, name, nil, nil, nil) + return info != nil, err +} + +var dummyPkg Package + +// fileInfo records information learned about a file included in a build. +type fileInfo struct { + name string // full name including dir + header []byte + fset *token.FileSet + parsed *ast.File + parseErr error + imports []fileImport + embeds []fileEmbed +} + +type fileImport struct { + path string + pos token.Pos + doc *ast.CommentGroup +} + +type fileEmbed struct { + pattern string + pos token.Position +} + +// matchFile determines whether the file with the given name in the given directory +// should be included in the package being constructed. +// If the file should be included, matchFile returns a non-nil *fileInfo (and a nil error). +// Non-nil errors are reserved for unexpected problems. +// +// If name denotes a Go program, matchFile reads until the end of the +// imports and returns that section of the file in the fileInfo's header field, +// even though it only considers text until the first non-comment +// for go:build lines. +// +// If allTags is non-nil, matchFile records any encountered build tag +// by setting allTags[tag] = true. +func (ctxt *Context) matchFile(dir, name string, allTags map[string]bool, binaryOnly *bool, fset *token.FileSet) (*fileInfo, error) { + if strings.HasPrefix(name, "_") || + strings.HasPrefix(name, ".") { + return nil, nil + } + + i := strings.LastIndex(name, ".") + if i < 0 { + i = len(name) + } + ext := name[i:] + + if ext != ".go" && fileListForExt(&dummyPkg, ext) == nil { + // skip + return nil, nil + } + + if !ctxt.goodOSArchFile(name, allTags) && !ctxt.UseAllFiles { + return nil, nil + } + + info := &fileInfo{name: ctxt.joinPath(dir, name), fset: fset} + if ext == ".syso" { + // binary, no reading + return info, nil + } + + f, err := ctxt.openFile(info.name) + if err != nil { + return nil, err + } + + if strings.HasSuffix(name, ".go") { + err = readGoInfo(f, info) + if strings.HasSuffix(name, "_test.go") { + binaryOnly = nil // ignore //go:binary-only-package comments in _test.go files + } + } else { + binaryOnly = nil // ignore //go:binary-only-package comments in non-Go sources + info.header, err = readComments(f) + } + f.Close() + if err != nil { + return info, fmt.Errorf("read %s: %v", info.name, err) + } + + // Look for go:build comments to accept or reject the file. + ok, sawBinaryOnly, err := ctxt.shouldBuild(info.header, allTags) + if err != nil { + return nil, fmt.Errorf("%s: %v", name, err) + } + if !ok && !ctxt.UseAllFiles { + return nil, nil + } + + if binaryOnly != nil && sawBinaryOnly { + *binaryOnly = true + } + + return info, nil +} + +func cleanDecls(m map[string][]token.Position) ([]string, map[string][]token.Position) { + all := make([]string, 0, len(m)) + for path := range m { + all = append(all, path) + } + sort.Strings(all) + return all, m +} + +// Import is shorthand for Default.Import. +func Import(path, srcDir string, mode ImportMode) (*Package, error) { + return Default.Import(path, srcDir, mode) +} + +// ImportDir is shorthand for Default.ImportDir. +func ImportDir(dir string, mode ImportMode) (*Package, error) { + return Default.ImportDir(dir, mode) +} + +var ( + plusBuild = []byte("+build") + + goBuildComment = []byte("//go:build") + + errMultipleGoBuild = errors.New("multiple //go:build comments") +) + +func isGoBuildComment(line []byte) bool { + if !bytes.HasPrefix(line, goBuildComment) { + return false + } + line = bytes.TrimSpace(line) + rest := line[len(goBuildComment):] + return len(rest) == 0 || len(bytes.TrimSpace(rest)) < len(rest) +} + +// Special comment denoting a binary-only package. +// See https://golang.org/design/2775-binary-only-packages +// for more about the design of binary-only packages. +var binaryOnlyComment = []byte("//go:binary-only-package") + +// shouldBuild reports whether it is okay to use this file, +// The rule is that in the file's leading run of // comments +// and blank lines, which must be followed by a blank line +// (to avoid including a Go package clause doc comment), +// lines beginning with '//go:build' are taken as build directives. +// +// The file is accepted only if each such line lists something +// matching the file. For example: +// +// //go:build windows linux +// +// marks the file as applicable only on Windows and Linux. +// +// For each build tag it consults, shouldBuild sets allTags[tag] = true. +// +// shouldBuild reports whether the file should be built +// and whether a //go:binary-only-package comment was found. +func (ctxt *Context) shouldBuild(content []byte, allTags map[string]bool) (shouldBuild, binaryOnly bool, err error) { + // Identify leading run of // comments and blank lines, + // which must be followed by a blank line. + // Also identify any //go:build comments. + content, goBuild, sawBinaryOnly, err := parseFileHeader(content) + if err != nil { + return false, false, err + } + + // If //go:build line is present, it controls. + // Otherwise fall back to +build processing. + switch { + case goBuild != nil: + x, err := constraint.Parse(string(goBuild)) + if err != nil { + return false, false, fmt.Errorf("parsing //go:build line: %v", err) + } + shouldBuild = ctxt.eval(x, allTags) + + default: + shouldBuild = true + p := content + for len(p) > 0 { + line := p + if i := bytes.IndexByte(line, '\n'); i >= 0 { + line, p = line[:i], p[i+1:] + } else { + p = p[len(p):] + } + line = bytes.TrimSpace(line) + if !bytes.HasPrefix(line, slashSlash) || !bytes.Contains(line, plusBuild) { + continue + } + text := string(line) + if !constraint.IsPlusBuild(text) { + continue + } + if x, err := constraint.Parse(text); err == nil { + if !ctxt.eval(x, allTags) { + shouldBuild = false + } + } + } + } + + return shouldBuild, sawBinaryOnly, nil +} + +func parseFileHeader(content []byte) (trimmed, goBuild []byte, sawBinaryOnly bool, err error) { + end := 0 + p := content + ended := false // found non-blank, non-// line, so stopped accepting //go:build lines + inSlashStar := false // in /* */ comment + +Lines: + for len(p) > 0 { + line := p + if i := bytes.IndexByte(line, '\n'); i >= 0 { + line, p = line[:i], p[i+1:] + } else { + p = p[len(p):] + } + line = bytes.TrimSpace(line) + if len(line) == 0 && !ended { // Blank line + // Remember position of most recent blank line. + // When we find the first non-blank, non-// line, + // this "end" position marks the latest file position + // where a //go:build line can appear. + // (It must appear _before_ a blank line before the non-blank, non-// line. + // Yes, that's confusing, which is part of why we moved to //go:build lines.) + // Note that ended==false here means that inSlashStar==false, + // since seeing a /* would have set ended==true. + end = len(content) - len(p) + continue Lines + } + if !bytes.HasPrefix(line, slashSlash) { // Not comment line + ended = true + } + + if !inSlashStar && isGoBuildComment(line) { + if goBuild != nil { + return nil, nil, false, errMultipleGoBuild + } + goBuild = line + } + if !inSlashStar && bytes.Equal(line, binaryOnlyComment) { + sawBinaryOnly = true + } + + Comments: + for len(line) > 0 { + if inSlashStar { + if i := bytes.Index(line, starSlash); i >= 0 { + inSlashStar = false + line = bytes.TrimSpace(line[i+len(starSlash):]) + continue Comments + } + continue Lines + } + if bytes.HasPrefix(line, slashSlash) { + continue Lines + } + if bytes.HasPrefix(line, slashStar) { + inSlashStar = true + line = bytes.TrimSpace(line[len(slashStar):]) + continue Comments + } + // Found non-comment text. + break Lines + } + } + + return content[:end], goBuild, sawBinaryOnly, nil +} + +// saveCgo saves the information from the #cgo lines in the import "C" comment. +// These lines set CFLAGS, CPPFLAGS, CXXFLAGS and LDFLAGS and pkg-config directives +// that affect the way cgo's C code is built. +func (ctxt *Context) saveCgo(filename string, di *Package, cg *ast.CommentGroup) error { + text := cg.Text() + for _, line := range strings.Split(text, "\n") { + orig := line + + // Line is + // #cgo [GOOS/GOARCH...] LDFLAGS: stuff + // + line = strings.TrimSpace(line) + if len(line) < 5 || line[:4] != "#cgo" || (line[4] != ' ' && line[4] != '\t') { + continue + } + + // Split at colon. + line, argstr, ok := strings.Cut(strings.TrimSpace(line[4:]), ":") + if !ok { + return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig) + } + + // Parse GOOS/GOARCH stuff. + f := strings.Fields(line) + if len(f) < 1 { + return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig) + } + + cond, verb := f[:len(f)-1], f[len(f)-1] + if len(cond) > 0 { + ok := false + for _, c := range cond { + if ctxt.matchAuto(c, nil) { + ok = true + break + } + } + if !ok { + continue + } + } + + args, err := splitQuoted(argstr) + if err != nil { + return fmt.Errorf("%s: invalid #cgo line: %s", filename, orig) + } + for i, arg := range args { + if arg, ok = expandSrcDir(arg, di.Dir); !ok { + return fmt.Errorf("%s: malformed #cgo argument: %s", filename, arg) + } + args[i] = arg + } + + switch verb { + case "CFLAGS", "CPPFLAGS", "CXXFLAGS", "FFLAGS", "LDFLAGS": + // Change relative paths to absolute. + ctxt.makePathsAbsolute(args, di.Dir) + } + + switch verb { + case "CFLAGS": + di.CgoCFLAGS = append(di.CgoCFLAGS, args...) + case "CPPFLAGS": + di.CgoCPPFLAGS = append(di.CgoCPPFLAGS, args...) + case "CXXFLAGS": + di.CgoCXXFLAGS = append(di.CgoCXXFLAGS, args...) + case "FFLAGS": + di.CgoFFLAGS = append(di.CgoFFLAGS, args...) + case "LDFLAGS": + di.CgoLDFLAGS = append(di.CgoLDFLAGS, args...) + case "pkg-config": + di.CgoPkgConfig = append(di.CgoPkgConfig, args...) + default: + return fmt.Errorf("%s: invalid #cgo verb: %s", filename, orig) + } + } + return nil +} + +// expandSrcDir expands any occurrence of ${SRCDIR}, making sure +// the result is safe for the shell. +func expandSrcDir(str string, srcdir string) (string, bool) { + // "\" delimited paths cause safeCgoName to fail + // so convert native paths with a different delimiter + // to "/" before starting (eg: on windows). + srcdir = filepath.ToSlash(srcdir) + + chunks := strings.Split(str, "${SRCDIR}") + if len(chunks) < 2 { + return str, safeCgoName(str) + } + ok := true + for _, chunk := range chunks { + ok = ok && (chunk == "" || safeCgoName(chunk)) + } + ok = ok && (srcdir == "" || safeCgoName(srcdir)) + res := strings.Join(chunks, srcdir) + return res, ok && res != "" +} + +// makePathsAbsolute looks for compiler options that take paths and +// makes them absolute. We do this because through the 1.8 release we +// ran the compiler in the package directory, so any relative -I or -L +// options would be relative to that directory. In 1.9 we changed to +// running the compiler in the build directory, to get consistent +// build results (issue #19964). To keep builds working, we change any +// relative -I or -L options to be absolute. +// +// Using filepath.IsAbs and filepath.Join here means the results will be +// different on different systems, but that's OK: -I and -L options are +// inherently system-dependent. +func (ctxt *Context) makePathsAbsolute(args []string, srcDir string) { + nextPath := false + for i, arg := range args { + if nextPath { + if !filepath.IsAbs(arg) { + args[i] = filepath.Join(srcDir, arg) + } + nextPath = false + } else if strings.HasPrefix(arg, "-I") || strings.HasPrefix(arg, "-L") { + if len(arg) == 2 { + nextPath = true + } else { + if !filepath.IsAbs(arg[2:]) { + args[i] = arg[:2] + filepath.Join(srcDir, arg[2:]) + } + } + } + } +} + +// NOTE: $ is not safe for the shell, but it is allowed here because of linker options like -Wl,$ORIGIN. +// We never pass these arguments to a shell (just to programs we construct argv for), so this should be okay. +// See golang.org/issue/6038. +// The @ is for OS X. See golang.org/issue/13720. +// The % is for Jenkins. See golang.org/issue/16959. +// The ! is because module paths may use them. See golang.org/issue/26716. +// The ~ and ^ are for sr.ht. See golang.org/issue/32260. +const safeString = "+-.,/0123456789=ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz:$@%! ~^" + +func safeCgoName(s string) bool { + if s == "" { + return false + } + for i := 0; i < len(s); i++ { + if c := s[i]; c < utf8.RuneSelf && strings.IndexByte(safeString, c) < 0 { + return false + } + } + return true +} + +// splitQuoted splits the string s around each instance of one or more consecutive +// white space characters while taking into account quotes and escaping, and +// returns an array of substrings of s or an empty list if s contains only white space. +// Single quotes and double quotes are recognized to prevent splitting within the +// quoted region, and are removed from the resulting substrings. If a quote in s +// isn't closed err will be set and r will have the unclosed argument as the +// last element. The backslash is used for escaping. +// +// For example, the following string: +// +// a b:"c d" 'e''f' "g\"" +// +// Would be parsed as: +// +// []string{"a", "b:c d", "ef", `g"`} +func splitQuoted(s string) (r []string, err error) { + var args []string + arg := make([]rune, len(s)) + escaped := false + quoted := false + quote := '\x00' + i := 0 + for _, rune := range s { + switch { + case escaped: + escaped = false + case rune == '\\': + escaped = true + continue + case quote != '\x00': + if rune == quote { + quote = '\x00' + continue + } + case rune == '"' || rune == '\'': + quoted = true + quote = rune + continue + case unicode.IsSpace(rune): + if quoted || i > 0 { + quoted = false + args = append(args, string(arg[:i])) + i = 0 + } + continue + } + arg[i] = rune + i++ + } + if quoted || i > 0 { + args = append(args, string(arg[:i])) + } + if quote != 0 { + err = errors.New("unclosed quote") + } else if escaped { + err = errors.New("unfinished escaping") + } + return args, err +} + +// matchAuto interprets text as either a +build or //go:build expression (whichever works), +// reporting whether the expression matches the build context. +// +// matchAuto is only used for testing of tag evaluation +// and in #cgo lines, which accept either syntax. +func (ctxt *Context) matchAuto(text string, allTags map[string]bool) bool { + if strings.ContainsAny(text, "&|()") { + text = "//go:build " + text + } else { + text = "// +build " + text + } + x, err := constraint.Parse(text) + if err != nil { + return false + } + return ctxt.eval(x, allTags) +} + +func (ctxt *Context) eval(x constraint.Expr, allTags map[string]bool) bool { + return x.Eval(func(tag string) bool { return ctxt.matchTag(tag, allTags) }) +} + +// matchTag reports whether the name is one of: +// +// cgo (if cgo is enabled) +// $GOOS +// $GOARCH +// ctxt.Compiler +// linux (if GOOS = android) +// solaris (if GOOS = illumos) +// darwin (if GOOS = ios) +// unix (if this is a Unix GOOS) +// boringcrypto (if GOEXPERIMENT=boringcrypto is enabled) +// tag (if tag is listed in ctxt.BuildTags, ctxt.ToolTags, or ctxt.ReleaseTags) +// +// It records all consulted tags in allTags. +func (ctxt *Context) matchTag(name string, allTags map[string]bool) bool { + if allTags != nil { + allTags[name] = true + } + + // special tags + if ctxt.CgoEnabled && name == "cgo" { + return true + } + if name == ctxt.GOOS || name == ctxt.GOARCH || name == ctxt.Compiler { + return true + } + if ctxt.GOOS == "android" && name == "linux" { + return true + } + if ctxt.GOOS == "illumos" && name == "solaris" { + return true + } + if ctxt.GOOS == "ios" && name == "darwin" { + return true + } + if name == "unix" && unixOS[ctxt.GOOS] { + return true + } + if name == "boringcrypto" { + name = "goexperiment.boringcrypto" // boringcrypto is an old name for goexperiment.boringcrypto + } + + // other tags + for _, tag := range ctxt.BuildTags { + if tag == name { + return true + } + } + for _, tag := range ctxt.ToolTags { + if tag == name { + return true + } + } + for _, tag := range ctxt.ReleaseTags { + if tag == name { + return true + } + } + + return false +} + +// goodOSArchFile returns false if the name contains a $GOOS or $GOARCH +// suffix which does not match the current system. +// The recognized name formats are: +// +// name_$(GOOS).* +// name_$(GOARCH).* +// name_$(GOOS)_$(GOARCH).* +// name_$(GOOS)_test.* +// name_$(GOARCH)_test.* +// name_$(GOOS)_$(GOARCH)_test.* +// +// Exceptions: +// if GOOS=android, then files with GOOS=linux are also matched. +// if GOOS=illumos, then files with GOOS=solaris are also matched. +// if GOOS=ios, then files with GOOS=darwin are also matched. +func (ctxt *Context) goodOSArchFile(name string, allTags map[string]bool) bool { + name, _, _ = strings.Cut(name, ".") + + // Before Go 1.4, a file called "linux.go" would be equivalent to having a + // build tag "linux" in that file. For Go 1.4 and beyond, we require this + // auto-tagging to apply only to files with a non-empty prefix, so + // "foo_linux.go" is tagged but "linux.go" is not. This allows new operating + // systems, such as android, to arrive without breaking existing code with + // innocuous source code in "android.go". The easiest fix: cut everything + // in the name before the initial _. + i := strings.Index(name, "_") + if i < 0 { + return true + } + name = name[i:] // ignore everything before first _ + + l := strings.Split(name, "_") + if n := len(l); n > 0 && l[n-1] == "test" { + l = l[:n-1] + } + n := len(l) + if n >= 2 && knownOS[l[n-2]] && knownArch[l[n-1]] { + if allTags != nil { + // In case we short-circuit on l[n-1]. + allTags[l[n-2]] = true + } + return ctxt.matchTag(l[n-1], allTags) && ctxt.matchTag(l[n-2], allTags) + } + if n >= 1 && (knownOS[l[n-1]] || knownArch[l[n-1]]) { + return ctxt.matchTag(l[n-1], allTags) + } + return true +} + +// ToolDir is the directory containing build tools. +var ToolDir = getToolDir() + +// IsLocalImport reports whether the import path is +// a local import path, like ".", "..", "./foo", or "../foo". +func IsLocalImport(path string) bool { + return path == "." || path == ".." || + strings.HasPrefix(path, "./") || strings.HasPrefix(path, "../") +} + +// ArchChar returns "?" and an error. +// In earlier versions of Go, the returned string was used to derive +// the compiler and linker tool names, the default object file suffix, +// and the default linker output name. As of Go 1.5, those strings +// no longer vary by architecture; they are compile, link, .o, and a.out, respectively. +func ArchChar(goarch string) (string, error) { + return "?", errors.New("architecture letter no longer used") +} diff --git a/src/go/build/build_test.go b/src/go/build/build_test.go new file mode 100644 index 0000000..2e60ecc --- /dev/null +++ b/src/go/build/build_test.go @@ -0,0 +1,803 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "internal/testenv" + "io" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +func TestMain(m *testing.M) { + Default.GOROOT = testenv.GOROOT(nil) + os.Exit(m.Run()) +} + +func TestMatch(t *testing.T) { + ctxt := Default + what := "default" + match := func(tag string, want map[string]bool) { + t.Helper() + m := make(map[string]bool) + if !ctxt.matchAuto(tag, m) { + t.Errorf("%s context should match %s, does not", what, tag) + } + if !reflect.DeepEqual(m, want) { + t.Errorf("%s tags = %v, want %v", tag, m, want) + } + } + nomatch := func(tag string, want map[string]bool) { + t.Helper() + m := make(map[string]bool) + if ctxt.matchAuto(tag, m) { + t.Errorf("%s context should NOT match %s, does", what, tag) + } + if !reflect.DeepEqual(m, want) { + t.Errorf("%s tags = %v, want %v", tag, m, want) + } + } + + match(runtime.GOOS+","+runtime.GOARCH, map[string]bool{runtime.GOOS: true, runtime.GOARCH: true}) + match(runtime.GOOS+","+runtime.GOARCH+",!foo", map[string]bool{runtime.GOOS: true, runtime.GOARCH: true, "foo": true}) + nomatch(runtime.GOOS+","+runtime.GOARCH+",foo", map[string]bool{runtime.GOOS: true, runtime.GOARCH: true, "foo": true}) + + what = "modified" + ctxt.BuildTags = []string{"foo"} + match(runtime.GOOS+","+runtime.GOARCH, map[string]bool{runtime.GOOS: true, runtime.GOARCH: true}) + match(runtime.GOOS+","+runtime.GOARCH+",foo", map[string]bool{runtime.GOOS: true, runtime.GOARCH: true, "foo": true}) + nomatch(runtime.GOOS+","+runtime.GOARCH+",!foo", map[string]bool{runtime.GOOS: true, runtime.GOARCH: true, "foo": true}) + match(runtime.GOOS+","+runtime.GOARCH+",!bar", map[string]bool{runtime.GOOS: true, runtime.GOARCH: true, "bar": true}) + nomatch(runtime.GOOS+","+runtime.GOARCH+",bar", map[string]bool{runtime.GOOS: true, runtime.GOARCH: true, "bar": true}) +} + +func TestDotSlashImport(t *testing.T) { + p, err := ImportDir("testdata/other", 0) + if err != nil { + t.Fatal(err) + } + if len(p.Imports) != 1 || p.Imports[0] != "./file" { + t.Fatalf("testdata/other: Imports=%v, want [./file]", p.Imports) + } + + p1, err := Import("./file", "testdata/other", 0) + if err != nil { + t.Fatal(err) + } + if p1.Name != "file" { + t.Fatalf("./file: Name=%q, want %q", p1.Name, "file") + } + dir := filepath.Clean("testdata/other/file") // Clean to use \ on Windows + if p1.Dir != dir { + t.Fatalf("./file: Dir=%q, want %q", p1.Name, dir) + } +} + +func TestEmptyImport(t *testing.T) { + p, err := Import("", testenv.GOROOT(t), FindOnly) + if err == nil { + t.Fatal(`Import("") returned nil error.`) + } + if p == nil { + t.Fatal(`Import("") returned nil package.`) + } + if p.ImportPath != "" { + t.Fatalf("ImportPath=%q, want %q.", p.ImportPath, "") + } +} + +func TestEmptyFolderImport(t *testing.T) { + _, err := Import(".", "testdata/empty", 0) + if _, ok := err.(*NoGoError); !ok { + t.Fatal(`Import("testdata/empty") did not return NoGoError.`) + } +} + +func TestMultiplePackageImport(t *testing.T) { + pkg, err := Import(".", "testdata/multi", 0) + + mpe, ok := err.(*MultiplePackageError) + if !ok { + t.Fatal(`Import("testdata/multi") did not return MultiplePackageError.`) + } + want := &MultiplePackageError{ + Dir: filepath.FromSlash("testdata/multi"), + Packages: []string{"main", "test_package"}, + Files: []string{"file.go", "file_appengine.go"}, + } + if !reflect.DeepEqual(mpe, want) { + t.Errorf("err = %#v; want %#v", mpe, want) + } + + // TODO(#45999): Since the name is ambiguous, pkg.Name should be left empty. + if wantName := "main"; pkg.Name != wantName { + t.Errorf("pkg.Name = %q; want %q", pkg.Name, wantName) + } + + if wantGoFiles := []string{"file.go", "file_appengine.go"}; !reflect.DeepEqual(pkg.GoFiles, wantGoFiles) { + t.Errorf("pkg.GoFiles = %q; want %q", pkg.GoFiles, wantGoFiles) + } + + if wantInvalidFiles := []string{"file_appengine.go"}; !reflect.DeepEqual(pkg.InvalidGoFiles, wantInvalidFiles) { + t.Errorf("pkg.InvalidGoFiles = %q; want %q", pkg.InvalidGoFiles, wantInvalidFiles) + } +} + +func TestLocalDirectory(t *testing.T) { + if runtime.GOOS == "ios" { + t.Skipf("skipping on %s/%s, no valid GOROOT", runtime.GOOS, runtime.GOARCH) + } + + cwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + p, err := ImportDir(cwd, 0) + if err != nil { + t.Fatal(err) + } + if p.ImportPath != "go/build" { + t.Fatalf("ImportPath=%q, want %q", p.ImportPath, "go/build") + } +} + +var shouldBuildTests = []struct { + name string + content string + tags map[string]bool + binaryOnly bool + shouldBuild bool + err error +}{ + { + name: "Yes", + content: "// +build yes\n\n" + + "package main\n", + tags: map[string]bool{"yes": true}, + shouldBuild: true, + }, + { + name: "Yes2", + content: "//go:build yes\n" + + "package main\n", + tags: map[string]bool{"yes": true}, + shouldBuild: true, + }, + { + name: "Or", + content: "// +build no yes\n\n" + + "package main\n", + tags: map[string]bool{"yes": true, "no": true}, + shouldBuild: true, + }, + { + name: "Or2", + content: "//go:build no || yes\n" + + "package main\n", + tags: map[string]bool{"yes": true, "no": true}, + shouldBuild: true, + }, + { + name: "And", + content: "// +build no,yes\n\n" + + "package main\n", + tags: map[string]bool{"yes": true, "no": true}, + shouldBuild: false, + }, + { + name: "And2", + content: "//go:build no && yes\n" + + "package main\n", + tags: map[string]bool{"yes": true, "no": true}, + shouldBuild: false, + }, + { + name: "Cgo", + content: "// +build cgo\n\n" + + "// Copyright The Go Authors.\n\n" + + "// This package implements parsing of tags like\n" + + "// +build tag1\n" + + "package build", + tags: map[string]bool{"cgo": true}, + shouldBuild: false, + }, + { + name: "Cgo2", + content: "//go:build cgo\n" + + "// Copyright The Go Authors.\n\n" + + "// This package implements parsing of tags like\n" + + "// +build tag1\n" + + "package build", + tags: map[string]bool{"cgo": true}, + shouldBuild: false, + }, + { + name: "AfterPackage", + content: "// Copyright The Go Authors.\n\n" + + "package build\n\n" + + "// shouldBuild checks tags given by lines of the form\n" + + "// +build tag\n" + + "//go:build tag\n" + + "func shouldBuild(content []byte)\n", + tags: map[string]bool{}, + shouldBuild: true, + }, + { + name: "TooClose", + content: "// +build yes\n" + + "package main\n", + tags: map[string]bool{}, + shouldBuild: true, + }, + { + name: "TooClose2", + content: "//go:build yes\n" + + "package main\n", + tags: map[string]bool{"yes": true}, + shouldBuild: true, + }, + { + name: "TooCloseNo", + content: "// +build no\n" + + "package main\n", + tags: map[string]bool{}, + shouldBuild: true, + }, + { + name: "TooCloseNo2", + content: "//go:build no\n" + + "package main\n", + tags: map[string]bool{"no": true}, + shouldBuild: false, + }, + { + name: "BinaryOnly", + content: "//go:binary-only-package\n" + + "// +build yes\n" + + "package main\n", + tags: map[string]bool{}, + binaryOnly: true, + shouldBuild: true, + }, + { + name: "BinaryOnly2", + content: "//go:binary-only-package\n" + + "//go:build no\n" + + "package main\n", + tags: map[string]bool{"no": true}, + binaryOnly: true, + shouldBuild: false, + }, + { + name: "ValidGoBuild", + content: "// +build yes\n\n" + + "//go:build no\n" + + "package main\n", + tags: map[string]bool{"no": true}, + shouldBuild: false, + }, + { + name: "MissingBuild2", + content: "/* */\n" + + "// +build yes\n\n" + + "//go:build no\n" + + "package main\n", + tags: map[string]bool{"no": true}, + shouldBuild: false, + }, + { + name: "Comment1", + content: "/*\n" + + "//go:build no\n" + + "*/\n\n" + + "package main\n", + tags: map[string]bool{}, + shouldBuild: true, + }, + { + name: "Comment2", + content: "/*\n" + + "text\n" + + "*/\n\n" + + "//go:build no\n" + + "package main\n", + tags: map[string]bool{"no": true}, + shouldBuild: false, + }, + { + name: "Comment3", + content: "/*/*/ /* hi *//* \n" + + "text\n" + + "*/\n\n" + + "//go:build no\n" + + "package main\n", + tags: map[string]bool{"no": true}, + shouldBuild: false, + }, + { + name: "Comment4", + content: "/**///go:build no\n" + + "package main\n", + tags: map[string]bool{}, + shouldBuild: true, + }, + { + name: "Comment5", + content: "/**/\n" + + "//go:build no\n" + + "package main\n", + tags: map[string]bool{"no": true}, + shouldBuild: false, + }, +} + +func TestShouldBuild(t *testing.T) { + for _, tt := range shouldBuildTests { + t.Run(tt.name, func(t *testing.T) { + ctx := &Context{BuildTags: []string{"yes"}} + tags := map[string]bool{} + shouldBuild, binaryOnly, err := ctx.shouldBuild([]byte(tt.content), tags) + if shouldBuild != tt.shouldBuild || binaryOnly != tt.binaryOnly || !reflect.DeepEqual(tags, tt.tags) || err != tt.err { + t.Errorf("mismatch:\n"+ + "have shouldBuild=%v, binaryOnly=%v, tags=%v, err=%v\n"+ + "want shouldBuild=%v, binaryOnly=%v, tags=%v, err=%v", + shouldBuild, binaryOnly, tags, err, + tt.shouldBuild, tt.binaryOnly, tt.tags, tt.err) + } + }) + } +} + +func TestGoodOSArchFile(t *testing.T) { + ctx := &Context{BuildTags: []string{"linux"}, GOOS: "darwin"} + m := map[string]bool{} + want := map[string]bool{"linux": true} + if !ctx.goodOSArchFile("hello_linux.go", m) { + t.Errorf("goodOSArchFile(hello_linux.go) = false, want true") + } + if !reflect.DeepEqual(m, want) { + t.Errorf("goodOSArchFile(hello_linux.go) tags = %v, want %v", m, want) + } +} + +type readNopCloser struct { + io.Reader +} + +func (r readNopCloser) Close() error { + return nil +} + +var ( + ctxtP9 = Context{GOARCH: "arm", GOOS: "plan9"} + ctxtAndroid = Context{GOARCH: "arm", GOOS: "android"} +) + +var matchFileTests = []struct { + ctxt Context + name string + data string + match bool +}{ + {ctxtP9, "foo_arm.go", "", true}, + {ctxtP9, "foo1_arm.go", "// +build linux\n\npackage main\n", false}, + {ctxtP9, "foo_darwin.go", "", false}, + {ctxtP9, "foo.go", "", true}, + {ctxtP9, "foo1.go", "// +build linux\n\npackage main\n", false}, + {ctxtP9, "foo.badsuffix", "", false}, + {ctxtAndroid, "foo_linux.go", "", true}, + {ctxtAndroid, "foo_android.go", "", true}, + {ctxtAndroid, "foo_plan9.go", "", false}, + {ctxtAndroid, "android.go", "", true}, + {ctxtAndroid, "plan9.go", "", true}, + {ctxtAndroid, "plan9_test.go", "", true}, + {ctxtAndroid, "arm.s", "", true}, + {ctxtAndroid, "amd64.s", "", true}, +} + +func TestMatchFile(t *testing.T) { + for _, tt := range matchFileTests { + ctxt := tt.ctxt + ctxt.OpenFile = func(path string) (r io.ReadCloser, err error) { + if path != "x+"+tt.name { + t.Fatalf("OpenFile asked for %q, expected %q", path, "x+"+tt.name) + } + return &readNopCloser{strings.NewReader(tt.data)}, nil + } + ctxt.JoinPath = func(elem ...string) string { + return strings.Join(elem, "+") + } + match, err := ctxt.MatchFile("x", tt.name) + if match != tt.match || err != nil { + t.Fatalf("MatchFile(%q) = %v, %v, want %v, nil", tt.name, match, err, tt.match) + } + } +} + +func TestImportCmd(t *testing.T) { + if runtime.GOOS == "ios" { + t.Skipf("skipping on %s/%s, no valid GOROOT", runtime.GOOS, runtime.GOARCH) + } + + p, err := Import("cmd/internal/objfile", "", 0) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(filepath.ToSlash(p.Dir), "src/cmd/internal/objfile") { + t.Fatalf("Import cmd/internal/objfile returned Dir=%q, want %q", filepath.ToSlash(p.Dir), ".../src/cmd/internal/objfile") + } +} + +var ( + expandSrcDirPath = filepath.Join(string(filepath.Separator)+"projects", "src", "add") +) + +var expandSrcDirTests = []struct { + input, expected string +}{ + {"-L ${SRCDIR}/libs -ladd", "-L /projects/src/add/libs -ladd"}, + {"${SRCDIR}/add_linux_386.a -pthread -lstdc++", "/projects/src/add/add_linux_386.a -pthread -lstdc++"}, + {"Nothing to expand here!", "Nothing to expand here!"}, + {"$", "$"}, + {"$$", "$$"}, + {"${", "${"}, + {"$}", "$}"}, + {"$FOO ${BAR}", "$FOO ${BAR}"}, + {"Find me the $SRCDIRECTORY.", "Find me the $SRCDIRECTORY."}, + {"$SRCDIR is missing braces", "$SRCDIR is missing braces"}, +} + +func TestExpandSrcDir(t *testing.T) { + for _, test := range expandSrcDirTests { + output, _ := expandSrcDir(test.input, expandSrcDirPath) + if output != test.expected { + t.Errorf("%q expands to %q with SRCDIR=%q when %q is expected", test.input, output, expandSrcDirPath, test.expected) + } else { + t.Logf("%q expands to %q with SRCDIR=%q", test.input, output, expandSrcDirPath) + } + } +} + +func TestShellSafety(t *testing.T) { + tests := []struct { + input, srcdir, expected string + result bool + }{ + {"-I${SRCDIR}/../include", "/projects/src/issue 11868", "-I/projects/src/issue 11868/../include", true}, + {"-I${SRCDIR}", "~wtf$@%^", "-I~wtf$@%^", true}, + {"-X${SRCDIR}/1,${SRCDIR}/2", "/projects/src/issue 11868", "-X/projects/src/issue 11868/1,/projects/src/issue 11868/2", true}, + {"-I/tmp -I/tmp", "/tmp2", "-I/tmp -I/tmp", true}, + {"-I/tmp", "/tmp/[0]", "-I/tmp", true}, + {"-I${SRCDIR}/dir", "/tmp/[0]", "-I/tmp/[0]/dir", false}, + {"-I${SRCDIR}/dir", "/tmp/go go", "-I/tmp/go go/dir", true}, + {"-I${SRCDIR}/dir dir", "/tmp/go", "-I/tmp/go/dir dir", true}, + } + for _, test := range tests { + output, ok := expandSrcDir(test.input, test.srcdir) + if ok != test.result { + t.Errorf("Expected %t while %q expands to %q with SRCDIR=%q; got %t", test.result, test.input, output, test.srcdir, ok) + } + if output != test.expected { + t.Errorf("Expected %q while %q expands with SRCDIR=%q; got %q", test.expected, test.input, test.srcdir, output) + } + } +} + +// Want to get a "cannot find package" error when directory for package does not exist. +// There should be valid partial information in the returned non-nil *Package. +func TestImportDirNotExist(t *testing.T) { + testenv.MustHaveGoBuild(t) // really must just have source + ctxt := Default + + emptyDir := t.TempDir() + + ctxt.GOPATH = emptyDir + ctxt.Dir = emptyDir + + tests := []struct { + label string + path, srcDir string + mode ImportMode + }{ + {"Import(full, 0)", "go/build/doesnotexist", "", 0}, + {"Import(local, 0)", "./doesnotexist", filepath.Join(ctxt.GOROOT, "src/go/build"), 0}, + {"Import(full, FindOnly)", "go/build/doesnotexist", "", FindOnly}, + {"Import(local, FindOnly)", "./doesnotexist", filepath.Join(ctxt.GOROOT, "src/go/build"), FindOnly}, + } + + defer os.Setenv("GO111MODULE", os.Getenv("GO111MODULE")) + + for _, GO111MODULE := range []string{"off", "on"} { + t.Run("GO111MODULE="+GO111MODULE, func(t *testing.T) { + os.Setenv("GO111MODULE", GO111MODULE) + + for _, test := range tests { + p, err := ctxt.Import(test.path, test.srcDir, test.mode) + + errOk := (err != nil && strings.HasPrefix(err.Error(), "cannot find package")) + wantErr := `"cannot find package" error` + if test.srcDir == "" { + if err != nil && strings.Contains(err.Error(), "is not in GOROOT") { + errOk = true + } + wantErr = `"cannot find package" or "is not in GOROOT" error` + } + if !errOk { + t.Errorf("%s got error: %q, want %s", test.label, err, wantErr) + } + // If an error occurs, build.Import is documented to return + // a non-nil *Package containing partial information. + if p == nil { + t.Fatalf(`%s got nil p, want non-nil *Package`, test.label) + } + // Verify partial information in p. + if p.ImportPath != "go/build/doesnotexist" { + t.Errorf(`%s got p.ImportPath: %q, want "go/build/doesnotexist"`, test.label, p.ImportPath) + } + } + }) + } +} + +func TestImportVendor(t *testing.T) { + testenv.MustHaveGoBuild(t) // really must just have source + + t.Setenv("GO111MODULE", "off") + + ctxt := Default + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + ctxt.GOPATH = filepath.Join(wd, "testdata/withvendor") + p, err := ctxt.Import("c/d", filepath.Join(ctxt.GOPATH, "src/a/b"), 0) + if err != nil { + t.Fatalf("cannot find vendored c/d from testdata src/a/b directory: %v", err) + } + want := "a/vendor/c/d" + if p.ImportPath != want { + t.Fatalf("Import succeeded but found %q, want %q", p.ImportPath, want) + } +} + +func BenchmarkImportVendor(b *testing.B) { + testenv.MustHaveGoBuild(b) // really must just have source + + b.Setenv("GO111MODULE", "off") + + ctxt := Default + wd, err := os.Getwd() + if err != nil { + b.Fatal(err) + } + ctxt.GOPATH = filepath.Join(wd, "testdata/withvendor") + dir := filepath.Join(ctxt.GOPATH, "src/a/b") + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := ctxt.Import("c/d", dir, 0) + if err != nil { + b.Fatalf("cannot find vendored c/d from testdata src/a/b directory: %v", err) + } + } +} + +func TestImportVendorFailure(t *testing.T) { + testenv.MustHaveGoBuild(t) // really must just have source + + t.Setenv("GO111MODULE", "off") + + ctxt := Default + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + ctxt.GOPATH = filepath.Join(wd, "testdata/withvendor") + p, err := ctxt.Import("x.com/y/z", filepath.Join(ctxt.GOPATH, "src/a/b"), 0) + if err == nil { + t.Fatalf("found made-up package x.com/y/z in %s", p.Dir) + } + + e := err.Error() + if !strings.Contains(e, " (vendor tree)") { + t.Fatalf("error on failed import does not mention GOROOT/src/vendor directory:\n%s", e) + } +} + +func TestImportVendorParentFailure(t *testing.T) { + testenv.MustHaveGoBuild(t) // really must just have source + + t.Setenv("GO111MODULE", "off") + + ctxt := Default + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + ctxt.GOPATH = filepath.Join(wd, "testdata/withvendor") + // This import should fail because the vendor/c directory has no source code. + p, err := ctxt.Import("c", filepath.Join(ctxt.GOPATH, "src/a/b"), 0) + if err == nil { + t.Fatalf("found empty parent in %s", p.Dir) + } + if p != nil && p.Dir != "" { + t.Fatalf("decided to use %s", p.Dir) + } + e := err.Error() + if !strings.Contains(e, " (vendor tree)") { + t.Fatalf("error on failed import does not mention GOROOT/src/vendor directory:\n%s", e) + } +} + +// Check that a package is loaded in module mode if GO111MODULE=on, even when +// no go.mod file is present. It should fail to resolve packages outside std. +// Verifies golang.org/issue/34669. +func TestImportPackageOutsideModule(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // Disable module fetching for this test so that 'go list' fails quickly + // without trying to find the latest version of a module. + t.Setenv("GOPROXY", "off") + + // Create a GOPATH in a temporary directory. We don't use testdata + // because it's in GOROOT, which interferes with the module heuristic. + gopath := t.TempDir() + if err := os.MkdirAll(filepath.Join(gopath, "src/example.com/p"), 0777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(gopath, "src/example.com/p/p.go"), []byte("package p"), 0666); err != nil { + t.Fatal(err) + } + + t.Setenv("GO111MODULE", "on") + t.Setenv("GOPATH", gopath) + ctxt := Default + ctxt.GOPATH = gopath + ctxt.Dir = filepath.Join(gopath, "src/example.com/p") + + want := "go.mod file not found in current directory or any parent directory" + if _, err := ctxt.Import("example.com/p", gopath, FindOnly); err == nil { + t.Fatal("importing package when no go.mod is present succeeded unexpectedly") + } else if errStr := err.Error(); !strings.Contains(errStr, want) { + t.Fatalf("error when importing package when no go.mod is present: got %q; want %q", errStr, want) + } else { + t.Logf(`ctxt.Import("example.com/p", _, FindOnly): %v`, err) + } +} + +// TestIssue23594 prevents go/build from regressing and populating Package.Doc +// from comments in test files. +func TestIssue23594(t *testing.T) { + // Package testdata/doc contains regular and external test files + // with comments attached to their package declarations. The names of the files + // ensure that we see the comments from the test files first. + p, err := ImportDir("testdata/doc", 0) + if err != nil { + t.Fatalf("could not import testdata: %v", err) + } + + if p.Doc != "Correct" { + t.Fatalf("incorrectly set .Doc to %q", p.Doc) + } +} + +// TestIssue56509 tests that go/build does not add non-go files to InvalidGoFiles +// when they have unparsable comments. +func TestIssue56509(t *testing.T) { + // The directory testdata/bads contains a .s file that has an unparsable + // comment. (go/build parses initial comments in non-go files looking for + // //go:build or //+go build comments). + p, err := ImportDir("testdata/bads", 0) + if err == nil { + t.Fatalf("could not import testdata/bads: %v", err) + } + + if len(p.InvalidGoFiles) != 0 { + t.Fatalf("incorrectly added non-go file to InvalidGoFiles") + } +} + +// TestMissingImportErrorRepetition checks that when an unknown package is +// imported, the package path is only shown once in the error. +// Verifies golang.org/issue/34752. +func TestMissingImportErrorRepetition(t *testing.T) { + testenv.MustHaveGoBuild(t) // need 'go list' internally + tmp := t.TempDir() + if err := os.WriteFile(filepath.Join(tmp, "go.mod"), []byte("module m"), 0666); err != nil { + t.Fatal(err) + } + t.Setenv("GO111MODULE", "on") + t.Setenv("GOPROXY", "off") + t.Setenv("GONOPROXY", "none") + + ctxt := Default + ctxt.Dir = tmp + + pkgPath := "example.com/hello" + _, err := ctxt.Import(pkgPath, tmp, FindOnly) + if err == nil { + t.Fatal("unexpected success") + } + + // Don't count the package path with a URL like https://...?go-get=1. + // See golang.org/issue/35986. + errStr := strings.ReplaceAll(err.Error(), "://"+pkgPath+"?go-get=1", "://...?go-get=1") + + // Also don't count instances in suggested "go get" or similar commands + // (see https://golang.org/issue/41576). The suggested command typically + // follows a semicolon. + errStr, _, _ = strings.Cut(errStr, ";") + + if n := strings.Count(errStr, pkgPath); n != 1 { + t.Fatalf("package path %q appears in error %d times; should appear once\nerror: %v", pkgPath, n, err) + } +} + +// TestCgoImportsIgnored checks that imports in cgo files are not included +// in the imports list when cgo is disabled. +// Verifies golang.org/issue/35946. +func TestCgoImportsIgnored(t *testing.T) { + ctxt := Default + ctxt.CgoEnabled = false + p, err := ctxt.ImportDir("testdata/cgo_disabled", 0) + if err != nil { + t.Fatal(err) + } + for _, path := range p.Imports { + if path == "should/be/ignored" { + t.Errorf("found import %q in ignored cgo file", path) + } + } +} + +// Issue #52053. Check that if there is a file x_GOOS_GOARCH.go that both +// GOOS and GOARCH show up in the Package.AllTags field. We test both the +// case where the file matches and where the file does not match. +// The latter case used to fail, incorrectly omitting GOOS. +func TestAllTags(t *testing.T) { + ctxt := Default + ctxt.GOARCH = "arm" + ctxt.GOOS = "netbsd" + p, err := ctxt.ImportDir("testdata/alltags", 0) + if err != nil { + t.Fatal(err) + } + want := []string{"arm", "netbsd"} + if !reflect.DeepEqual(p.AllTags, want) { + t.Errorf("AllTags = %v, want %v", p.AllTags, want) + } + wantFiles := []string{"alltags.go", "x_netbsd_arm.go"} + if !reflect.DeepEqual(p.GoFiles, wantFiles) { + t.Errorf("GoFiles = %v, want %v", p.GoFiles, wantFiles) + } + + ctxt.GOARCH = "amd64" + ctxt.GOOS = "linux" + p, err = ctxt.ImportDir("testdata/alltags", 0) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(p.AllTags, want) { + t.Errorf("AllTags = %v, want %v", p.AllTags, want) + } + wantFiles = []string{"alltags.go"} + if !reflect.DeepEqual(p.GoFiles, wantFiles) { + t.Errorf("GoFiles = %v, want %v", p.GoFiles, wantFiles) + } +} + +func TestAllTagsNonSourceFile(t *testing.T) { + p, err := Default.ImportDir("testdata/non_source_tags", 0) + if err != nil { + t.Fatal(err) + } + if len(p.AllTags) > 0 { + t.Errorf("AllTags = %v, want empty", p.AllTags) + } +} diff --git a/src/go/build/constraint/expr.go b/src/go/build/constraint/expr.go new file mode 100644 index 0000000..505cbff --- /dev/null +++ b/src/go/build/constraint/expr.go @@ -0,0 +1,574 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package constraint implements parsing and evaluation of build constraint lines. +// See https://golang.org/cmd/go/#hdr-Build_constraints for documentation about build constraints themselves. +// +// This package parses both the original ā€œ// +buildā€ syntax and the ā€œ//go:buildā€ syntax that was added in Go 1.17. +// See https://golang.org/design/draft-gobuild for details about the ā€œ//go:buildā€ syntax. +package constraint + +import ( + "errors" + "strings" + "unicode" + "unicode/utf8" +) + +// An Expr is a build tag constraint expression. +// The underlying concrete type is *AndExpr, *OrExpr, *NotExpr, or *TagExpr. +type Expr interface { + // String returns the string form of the expression, + // using the boolean syntax used in //go:build lines. + String() string + + // Eval reports whether the expression evaluates to true. + // It calls ok(tag) as needed to find out whether a given build tag + // is satisfied by the current build configuration. + Eval(ok func(tag string) bool) bool + + // The presence of an isExpr method explicitly marks the type as an Expr. + // Only implementations in this package should be used as Exprs. + isExpr() +} + +// A TagExpr is an Expr for the single tag Tag. +type TagExpr struct { + Tag string // for example, ā€œlinuxā€ or ā€œcgoā€ +} + +func (x *TagExpr) isExpr() {} + +func (x *TagExpr) Eval(ok func(tag string) bool) bool { + return ok(x.Tag) +} + +func (x *TagExpr) String() string { + return x.Tag +} + +func tag(tag string) Expr { return &TagExpr{tag} } + +// A NotExpr represents the expression !X (the negation of X). +type NotExpr struct { + X Expr +} + +func (x *NotExpr) isExpr() {} + +func (x *NotExpr) Eval(ok func(tag string) bool) bool { + return !x.X.Eval(ok) +} + +func (x *NotExpr) String() string { + s := x.X.String() + switch x.X.(type) { + case *AndExpr, *OrExpr: + s = "(" + s + ")" + } + return "!" + s +} + +func not(x Expr) Expr { return &NotExpr{x} } + +// An AndExpr represents the expression X && Y. +type AndExpr struct { + X, Y Expr +} + +func (x *AndExpr) isExpr() {} + +func (x *AndExpr) Eval(ok func(tag string) bool) bool { + // Note: Eval both, to make sure ok func observes all tags. + xok := x.X.Eval(ok) + yok := x.Y.Eval(ok) + return xok && yok +} + +func (x *AndExpr) String() string { + return andArg(x.X) + " && " + andArg(x.Y) +} + +func andArg(x Expr) string { + s := x.String() + if _, ok := x.(*OrExpr); ok { + s = "(" + s + ")" + } + return s +} + +func and(x, y Expr) Expr { + return &AndExpr{x, y} +} + +// An OrExpr represents the expression X || Y. +type OrExpr struct { + X, Y Expr +} + +func (x *OrExpr) isExpr() {} + +func (x *OrExpr) Eval(ok func(tag string) bool) bool { + // Note: Eval both, to make sure ok func observes all tags. + xok := x.X.Eval(ok) + yok := x.Y.Eval(ok) + return xok || yok +} + +func (x *OrExpr) String() string { + return orArg(x.X) + " || " + orArg(x.Y) +} + +func orArg(x Expr) string { + s := x.String() + if _, ok := x.(*AndExpr); ok { + s = "(" + s + ")" + } + return s +} + +func or(x, y Expr) Expr { + return &OrExpr{x, y} +} + +// A SyntaxError reports a syntax error in a parsed build expression. +type SyntaxError struct { + Offset int // byte offset in input where error was detected + Err string // description of error +} + +func (e *SyntaxError) Error() string { + return e.Err +} + +var errNotConstraint = errors.New("not a build constraint") + +// Parse parses a single build constraint line of the form ā€œ//go:build ...ā€ or ā€œ// +build ...ā€ +// and returns the corresponding boolean expression. +func Parse(line string) (Expr, error) { + if text, ok := splitGoBuild(line); ok { + return parseExpr(text) + } + if text, ok := splitPlusBuild(line); ok { + return parsePlusBuildExpr(text), nil + } + return nil, errNotConstraint +} + +// IsGoBuild reports whether the line of text is a ā€œ//go:buildā€ constraint. +// It only checks the prefix of the text, not that the expression itself parses. +func IsGoBuild(line string) bool { + _, ok := splitGoBuild(line) + return ok +} + +// splitGoBuild splits apart the leading //go:build prefix in line from the build expression itself. +// It returns "", false if the input is not a //go:build line or if the input contains multiple lines. +func splitGoBuild(line string) (expr string, ok bool) { + // A single trailing newline is OK; otherwise multiple lines are not. + if len(line) > 0 && line[len(line)-1] == '\n' { + line = line[:len(line)-1] + } + if strings.Contains(line, "\n") { + return "", false + } + + if !strings.HasPrefix(line, "//go:build") { + return "", false + } + + line = strings.TrimSpace(line) + line = line[len("//go:build"):] + + // If strings.TrimSpace finds more to trim after removing the //go:build prefix, + // it means that the prefix was followed by a space, making this a //go:build line + // (as opposed to a //go:buildsomethingelse line). + // If line is empty, we had "//go:build" by itself, which also counts. + trim := strings.TrimSpace(line) + if len(line) == len(trim) && line != "" { + return "", false + } + + return trim, true +} + +// An exprParser holds state for parsing a build expression. +type exprParser struct { + s string // input string + i int // next read location in s + + tok string // last token read + isTag bool + pos int // position (start) of last token +} + +// parseExpr parses a boolean build tag expression. +func parseExpr(text string) (x Expr, err error) { + defer func() { + if e := recover(); e != nil { + if e, ok := e.(*SyntaxError); ok { + err = e + return + } + panic(e) // unreachable unless parser has a bug + } + }() + + p := &exprParser{s: text} + x = p.or() + if p.tok != "" { + panic(&SyntaxError{Offset: p.pos, Err: "unexpected token " + p.tok}) + } + return x, nil +} + +// or parses a sequence of || expressions. +// On entry, the next input token has not yet been lexed. +// On exit, the next input token has been lexed and is in p.tok. +func (p *exprParser) or() Expr { + x := p.and() + for p.tok == "||" { + x = or(x, p.and()) + } + return x +} + +// and parses a sequence of && expressions. +// On entry, the next input token has not yet been lexed. +// On exit, the next input token has been lexed and is in p.tok. +func (p *exprParser) and() Expr { + x := p.not() + for p.tok == "&&" { + x = and(x, p.not()) + } + return x +} + +// not parses a ! expression. +// On entry, the next input token has not yet been lexed. +// On exit, the next input token has been lexed and is in p.tok. +func (p *exprParser) not() Expr { + p.lex() + if p.tok == "!" { + p.lex() + if p.tok == "!" { + panic(&SyntaxError{Offset: p.pos, Err: "double negation not allowed"}) + } + return not(p.atom()) + } + return p.atom() +} + +// atom parses a tag or a parenthesized expression. +// On entry, the next input token HAS been lexed. +// On exit, the next input token has been lexed and is in p.tok. +func (p *exprParser) atom() Expr { + // first token already in p.tok + if p.tok == "(" { + pos := p.pos + defer func() { + if e := recover(); e != nil { + if e, ok := e.(*SyntaxError); ok && e.Err == "unexpected end of expression" { + e.Err = "missing close paren" + } + panic(e) + } + }() + x := p.or() + if p.tok != ")" { + panic(&SyntaxError{Offset: pos, Err: "missing close paren"}) + } + p.lex() + return x + } + + if !p.isTag { + if p.tok == "" { + panic(&SyntaxError{Offset: p.pos, Err: "unexpected end of expression"}) + } + panic(&SyntaxError{Offset: p.pos, Err: "unexpected token " + p.tok}) + } + tok := p.tok + p.lex() + return tag(tok) +} + +// lex finds and consumes the next token in the input stream. +// On return, p.tok is set to the token text, +// p.isTag reports whether the token was a tag, +// and p.pos records the byte offset of the start of the token in the input stream. +// If lex reaches the end of the input, p.tok is set to the empty string. +// For any other syntax error, lex panics with a SyntaxError. +func (p *exprParser) lex() { + p.isTag = false + for p.i < len(p.s) && (p.s[p.i] == ' ' || p.s[p.i] == '\t') { + p.i++ + } + if p.i >= len(p.s) { + p.tok = "" + p.pos = p.i + return + } + switch p.s[p.i] { + case '(', ')', '!': + p.pos = p.i + p.i++ + p.tok = p.s[p.pos:p.i] + return + + case '&', '|': + if p.i+1 >= len(p.s) || p.s[p.i+1] != p.s[p.i] { + panic(&SyntaxError{Offset: p.i, Err: "invalid syntax at " + string(rune(p.s[p.i]))}) + } + p.pos = p.i + p.i += 2 + p.tok = p.s[p.pos:p.i] + return + } + + tag := p.s[p.i:] + for i, c := range tag { + if !unicode.IsLetter(c) && !unicode.IsDigit(c) && c != '_' && c != '.' { + tag = tag[:i] + break + } + } + if tag == "" { + c, _ := utf8.DecodeRuneInString(p.s[p.i:]) + panic(&SyntaxError{Offset: p.i, Err: "invalid syntax at " + string(c)}) + } + + p.pos = p.i + p.i += len(tag) + p.tok = p.s[p.pos:p.i] + p.isTag = true +} + +// IsPlusBuild reports whether the line of text is a ā€œ// +buildā€ constraint. +// It only checks the prefix of the text, not that the expression itself parses. +func IsPlusBuild(line string) bool { + _, ok := splitPlusBuild(line) + return ok +} + +// splitPlusBuild splits apart the leading // +build prefix in line from the build expression itself. +// It returns "", false if the input is not a // +build line or if the input contains multiple lines. +func splitPlusBuild(line string) (expr string, ok bool) { + // A single trailing newline is OK; otherwise multiple lines are not. + if len(line) > 0 && line[len(line)-1] == '\n' { + line = line[:len(line)-1] + } + if strings.Contains(line, "\n") { + return "", false + } + + if !strings.HasPrefix(line, "//") { + return "", false + } + line = line[len("//"):] + // Note the space is optional; "//+build" is recognized too. + line = strings.TrimSpace(line) + + if !strings.HasPrefix(line, "+build") { + return "", false + } + line = line[len("+build"):] + + // If strings.TrimSpace finds more to trim after removing the +build prefix, + // it means that the prefix was followed by a space, making this a +build line + // (as opposed to a +buildsomethingelse line). + // If line is empty, we had "// +build" by itself, which also counts. + trim := strings.TrimSpace(line) + if len(line) == len(trim) && line != "" { + return "", false + } + + return trim, true +} + +// parsePlusBuildExpr parses a legacy build tag expression (as used with ā€œ// +buildā€). +func parsePlusBuildExpr(text string) Expr { + var x Expr + for _, clause := range strings.Fields(text) { + var y Expr + for _, lit := range strings.Split(clause, ",") { + var z Expr + var neg bool + if strings.HasPrefix(lit, "!!") || lit == "!" { + z = tag("ignore") + } else { + if strings.HasPrefix(lit, "!") { + neg = true + lit = lit[len("!"):] + } + if isValidTag(lit) { + z = tag(lit) + } else { + z = tag("ignore") + } + if neg { + z = not(z) + } + } + if y == nil { + y = z + } else { + y = and(y, z) + } + } + if x == nil { + x = y + } else { + x = or(x, y) + } + } + if x == nil { + x = tag("ignore") + } + return x +} + +// isValidTag reports whether the word is a valid build tag. +// Tags must be letters, digits, underscores or dots. +// Unlike in Go identifiers, all digits are fine (e.g., "386"). +func isValidTag(word string) bool { + if word == "" { + return false + } + for _, c := range word { + if !unicode.IsLetter(c) && !unicode.IsDigit(c) && c != '_' && c != '.' { + return false + } + } + return true +} + +var errComplex = errors.New("expression too complex for // +build lines") + +// PlusBuildLines returns a sequence of ā€œ// +buildā€ lines that evaluate to the build expression x. +// If the expression is too complex to convert directly to ā€œ// +buildā€ lines, PlusBuildLines returns an error. +func PlusBuildLines(x Expr) ([]string, error) { + // Push all NOTs to the expression leaves, so that //go:build !(x && y) can be treated as !x || !y. + // This rewrite is both efficient and commonly needed, so it's worth doing. + // Essentially all other possible rewrites are too expensive and too rarely needed. + x = pushNot(x, false) + + // Split into AND of ORs of ANDs of literals (tag or NOT tag). + var split [][][]Expr + for _, or := range appendSplitAnd(nil, x) { + var ands [][]Expr + for _, and := range appendSplitOr(nil, or) { + var lits []Expr + for _, lit := range appendSplitAnd(nil, and) { + switch lit.(type) { + case *TagExpr, *NotExpr: + lits = append(lits, lit) + default: + return nil, errComplex + } + } + ands = append(ands, lits) + } + split = append(split, ands) + } + + // If all the ORs have length 1 (no actual OR'ing going on), + // push the top-level ANDs to the bottom level, so that we get + // one // +build line instead of many. + maxOr := 0 + for _, or := range split { + if maxOr < len(or) { + maxOr = len(or) + } + } + if maxOr == 1 { + var lits []Expr + for _, or := range split { + lits = append(lits, or[0]...) + } + split = [][][]Expr{{lits}} + } + + // Prepare the +build lines. + var lines []string + for _, or := range split { + line := "// +build" + for _, and := range or { + clause := "" + for i, lit := range and { + if i > 0 { + clause += "," + } + clause += lit.String() + } + line += " " + clause + } + lines = append(lines, line) + } + + return lines, nil +} + +// pushNot applies DeMorgan's law to push negations down the expression, +// so that only tags are negated in the result. +// (It applies the rewrites !(X && Y) => (!X || !Y) and !(X || Y) => (!X && !Y).) +func pushNot(x Expr, not bool) Expr { + switch x := x.(type) { + default: + // unreachable + return x + case *NotExpr: + if _, ok := x.X.(*TagExpr); ok && !not { + return x + } + return pushNot(x.X, !not) + case *TagExpr: + if not { + return &NotExpr{X: x} + } + return x + case *AndExpr: + x1 := pushNot(x.X, not) + y1 := pushNot(x.Y, not) + if not { + return or(x1, y1) + } + if x1 == x.X && y1 == x.Y { + return x + } + return and(x1, y1) + case *OrExpr: + x1 := pushNot(x.X, not) + y1 := pushNot(x.Y, not) + if not { + return and(x1, y1) + } + if x1 == x.X && y1 == x.Y { + return x + } + return or(x1, y1) + } +} + +// appendSplitAnd appends x to list while splitting apart any top-level && expressions. +// For example, appendSplitAnd({W}, X && Y && Z) = {W, X, Y, Z}. +func appendSplitAnd(list []Expr, x Expr) []Expr { + if x, ok := x.(*AndExpr); ok { + list = appendSplitAnd(list, x.X) + list = appendSplitAnd(list, x.Y) + return list + } + return append(list, x) +} + +// appendSplitOr appends x to list while splitting apart any top-level || expressions. +// For example, appendSplitOr({W}, X || Y || Z) = {W, X, Y, Z}. +func appendSplitOr(list []Expr, x Expr) []Expr { + if x, ok := x.(*OrExpr); ok { + list = appendSplitOr(list, x.X) + list = appendSplitOr(list, x.Y) + return list + } + return append(list, x) +} diff --git a/src/go/build/constraint/expr_test.go b/src/go/build/constraint/expr_test.go new file mode 100644 index 0000000..15d1890 --- /dev/null +++ b/src/go/build/constraint/expr_test.go @@ -0,0 +1,321 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package constraint + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +var exprStringTests = []struct { + x Expr + out string +}{ + { + x: tag("abc"), + out: "abc", + }, + { + x: not(tag("abc")), + out: "!abc", + }, + { + x: not(and(tag("abc"), tag("def"))), + out: "!(abc && def)", + }, + { + x: and(tag("abc"), or(tag("def"), tag("ghi"))), + out: "abc && (def || ghi)", + }, + { + x: or(and(tag("abc"), tag("def")), tag("ghi")), + out: "(abc && def) || ghi", + }, +} + +func TestExprString(t *testing.T) { + for i, tt := range exprStringTests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + s := tt.x.String() + if s != tt.out { + t.Errorf("String() mismatch:\nhave %s\nwant %s", s, tt.out) + } + }) + } +} + +var lexTests = []struct { + in string + out string +}{ + {"", ""}, + {"x", "x"}, + {"x.y", "x.y"}, + {"x_y", "x_y"}, + {"Ī±x", "Ī±x"}, + {"Ī±xĀ²", "Ī±x err: invalid syntax at Ā²"}, + {"go1.2", "go1.2"}, + {"x y", "x y"}, + {"x!y", "x ! y"}, + {"&&||!()xy yx ", "&& || ! ( ) xy yx"}, + {"x~", "x err: invalid syntax at ~"}, + {"x ~", "x err: invalid syntax at ~"}, + {"x &", "x err: invalid syntax at &"}, + {"x &y", "x err: invalid syntax at &"}, +} + +func TestLex(t *testing.T) { + for i, tt := range lexTests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + p := &exprParser{s: tt.in} + out := "" + for { + tok, err := lexHelp(p) + if tok == "" && err == nil { + break + } + if out != "" { + out += " " + } + if err != nil { + out += "err: " + err.Error() + break + } + out += tok + } + if out != tt.out { + t.Errorf("lex(%q):\nhave %s\nwant %s", tt.in, out, tt.out) + } + }) + } +} + +func lexHelp(p *exprParser) (tok string, err error) { + defer func() { + if e := recover(); e != nil { + if e, ok := e.(*SyntaxError); ok { + err = e + return + } + panic(e) + } + }() + + p.lex() + return p.tok, nil +} + +var parseExprTests = []struct { + in string + x Expr +}{ + {"x", tag("x")}, + {"x&&y", and(tag("x"), tag("y"))}, + {"x||y", or(tag("x"), tag("y"))}, + {"(x)", tag("x")}, + {"x||y&&z", or(tag("x"), and(tag("y"), tag("z")))}, + {"x&&y||z", or(and(tag("x"), tag("y")), tag("z"))}, + {"x&&(y||z)", and(tag("x"), or(tag("y"), tag("z")))}, + {"(x||y)&&z", and(or(tag("x"), tag("y")), tag("z"))}, + {"!(x&&y)", not(and(tag("x"), tag("y")))}, +} + +func TestParseExpr(t *testing.T) { + for i, tt := range parseExprTests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + x, err := parseExpr(tt.in) + if err != nil { + t.Fatal(err) + } + if x.String() != tt.x.String() { + t.Errorf("parseExpr(%q):\nhave %s\nwant %s", tt.in, x, tt.x) + } + }) + } +} + +var parseExprErrorTests = []struct { + in string + err error +}{ + {"x && ", &SyntaxError{Offset: 5, Err: "unexpected end of expression"}}, + {"x && (", &SyntaxError{Offset: 6, Err: "missing close paren"}}, + {"x && ||", &SyntaxError{Offset: 5, Err: "unexpected token ||"}}, + {"x && !", &SyntaxError{Offset: 6, Err: "unexpected end of expression"}}, + {"x && !!", &SyntaxError{Offset: 6, Err: "double negation not allowed"}}, + {"x !", &SyntaxError{Offset: 2, Err: "unexpected token !"}}, + {"x && (y", &SyntaxError{Offset: 5, Err: "missing close paren"}}, +} + +func TestParseError(t *testing.T) { + for i, tt := range parseExprErrorTests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + x, err := parseExpr(tt.in) + if err == nil { + t.Fatalf("parseExpr(%q) = %v, want error", tt.in, x) + } + if !reflect.DeepEqual(err, tt.err) { + t.Fatalf("parseExpr(%q): wrong error:\nhave %#v\nwant %#v", tt.in, err, tt.err) + } + }) + } +} + +var exprEvalTests = []struct { + in string + ok bool + tags string +}{ + {"x", false, "x"}, + {"x && y", false, "x y"}, + {"x || y", false, "x y"}, + {"!x && yes", true, "x yes"}, + {"yes || y", true, "y yes"}, +} + +func TestExprEval(t *testing.T) { + for i, tt := range exprEvalTests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + x, err := parseExpr(tt.in) + if err != nil { + t.Fatal(err) + } + tags := make(map[string]bool) + wantTags := make(map[string]bool) + for _, tag := range strings.Fields(tt.tags) { + wantTags[tag] = true + } + hasTag := func(tag string) bool { + tags[tag] = true + return tag == "yes" + } + ok := x.Eval(hasTag) + if ok != tt.ok || !reflect.DeepEqual(tags, wantTags) { + t.Errorf("Eval(%#q):\nhave ok=%v, tags=%v\nwant ok=%v, tags=%v", + tt.in, ok, tags, tt.ok, wantTags) + } + }) + } +} + +var parsePlusBuildExprTests = []struct { + in string + x Expr +}{ + {"x", tag("x")}, + {"x,y", and(tag("x"), tag("y"))}, + {"x y", or(tag("x"), tag("y"))}, + {"x y,z", or(tag("x"), and(tag("y"), tag("z")))}, + {"x,y z", or(and(tag("x"), tag("y")), tag("z"))}, + {"x,!y !z", or(and(tag("x"), not(tag("y"))), not(tag("z")))}, + {"!! x", or(tag("ignore"), tag("x"))}, + {"!!x", tag("ignore")}, + {"!x", not(tag("x"))}, + {"!", tag("ignore")}, + {"", tag("ignore")}, +} + +func TestParsePlusBuildExpr(t *testing.T) { + for i, tt := range parsePlusBuildExprTests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + x := parsePlusBuildExpr(tt.in) + if x.String() != tt.x.String() { + t.Errorf("parsePlusBuildExpr(%q):\nhave %v\nwant %v", tt.in, x, tt.x) + } + }) + } +} + +var constraintTests = []struct { + in string + x Expr + err string +}{ + {"//+build !", tag("ignore"), ""}, + {"//+build", tag("ignore"), ""}, + {"//+build x y", or(tag("x"), tag("y")), ""}, + {"// +build x y \n", or(tag("x"), tag("y")), ""}, + {"// +build x y \n ", nil, "not a build constraint"}, + {"// +build x y \nmore", nil, "not a build constraint"}, + {" //+build x y", nil, "not a build constraint"}, + + {"//go:build x && y", and(tag("x"), tag("y")), ""}, + {"//go:build x && y\n", and(tag("x"), tag("y")), ""}, + {"//go:build x && y\n ", nil, "not a build constraint"}, + {"//go:build x && y\nmore", nil, "not a build constraint"}, + {" //go:build x && y", nil, "not a build constraint"}, + {"//go:build\n", nil, "unexpected end of expression"}, +} + +func TestParse(t *testing.T) { + for i, tt := range constraintTests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + x, err := Parse(tt.in) + if err != nil { + if tt.err == "" { + t.Errorf("Constraint(%q): unexpected error: %v", tt.in, err) + } else if !strings.Contains(err.Error(), tt.err) { + t.Errorf("Constraint(%q): error %v, want %v", tt.in, err, tt.err) + } + return + } + if tt.err != "" { + t.Errorf("Constraint(%q) = %v, want error %v", tt.in, x, tt.err) + return + } + if x.String() != tt.x.String() { + t.Errorf("Constraint(%q):\nhave %v\nwant %v", tt.in, x, tt.x) + } + }) + } +} + +var plusBuildLinesTests = []struct { + in string + out []string + err error +}{ + {"x", []string{"x"}, nil}, + {"x && !y", []string{"x,!y"}, nil}, + {"x || y", []string{"x y"}, nil}, + {"x && (y || z)", []string{"x", "y z"}, nil}, + {"!(x && y)", []string{"!x !y"}, nil}, + {"x || (y && z)", []string{"x y,z"}, nil}, + {"w && (x || (y && z))", []string{"w", "x y,z"}, nil}, + {"v || (w && (x || (y && z)))", nil, errComplex}, +} + +func TestPlusBuildLines(t *testing.T) { + for i, tt := range plusBuildLinesTests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + x, err := parseExpr(tt.in) + if err != nil { + t.Fatal(err) + } + lines, err := PlusBuildLines(x) + if err != nil { + if tt.err == nil { + t.Errorf("PlusBuildLines(%q): unexpected error: %v", tt.in, err) + } else if tt.err != err { + t.Errorf("PlusBuildLines(%q): error %v, want %v", tt.in, err, tt.err) + } + return + } + if tt.err != nil { + t.Errorf("PlusBuildLines(%q) = %v, want error %v", tt.in, lines, tt.err) + return + } + var want []string + for _, line := range tt.out { + want = append(want, "// +build "+line) + } + if !reflect.DeepEqual(lines, want) { + t.Errorf("PlusBuildLines(%q):\nhave %q\nwant %q", tt.in, lines, want) + } + }) + } +} diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go new file mode 100644 index 0000000..1ec6ee7 --- /dev/null +++ b/src/go/build/deps_test.go @@ -0,0 +1,774 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file exercises the import parser but also checks that +// some low-level packages do not have new dependencies added. + +package build + +import ( + "bytes" + "fmt" + "go/token" + "internal/dag" + "internal/testenv" + "io/fs" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" +) + +// depsRules defines the expected dependencies between packages in +// the Go source tree. It is a statement of policy. +// +// DO NOT CHANGE THIS DATA TO FIX BUILDS. +// Existing packages should not have their constraints relaxed +// without prior discussion. +// Negative assertions should almost never be removed. +// +// "a < b" means package b can import package a. +// +// See `go doc internal/dag' for the full syntax. +// +// All-caps names are pseudo-names for specific points +// in the dependency lattice. +var depsRules = ` + # No dependencies allowed for any of these packages. + NONE + < constraints, container/list, container/ring, + internal/cfg, internal/coverage, internal/coverage/rtcov, + internal/coverage/uleb128, internal/coverage/calloc, + internal/cpu, internal/goarch, + internal/goexperiment, internal/goos, + internal/goversion, internal/nettrace, internal/platform, + unicode/utf8, unicode/utf16, unicode, + unsafe; + + # These packages depend only on internal/goarch and unsafe. + internal/goarch, unsafe + < internal/abi; + + # RUNTIME is the core runtime group of packages, all of them very light-weight. + internal/abi, internal/cpu, internal/goarch, + internal/coverage/rtcov, internal/goexperiment, + internal/goos, unsafe + < internal/bytealg + < internal/itoa + < internal/unsafeheader + < runtime/internal/sys + < runtime/internal/syscall + < runtime/internal/atomic + < runtime/internal/math + < runtime + < sync/atomic + < internal/race + < sync + < internal/godebug + < internal/reflectlite + < errors + < internal/oserror, math/bits + < RUNTIME; + + RUNTIME + < sort + < container/heap; + + RUNTIME + < io; + + RUNTIME + < arena; + + syscall !< io; + reflect !< sort; + + RUNTIME, unicode/utf8 + < path; + + unicode !< path; + + # SYSCALL is RUNTIME plus the packages necessary for basic system calls. + RUNTIME, unicode/utf8, unicode/utf16 + < internal/syscall/windows/sysdll, syscall/js + < syscall + < internal/syscall/unix, internal/syscall/windows, internal/syscall/windows/registry + < internal/syscall/execenv + < SYSCALL; + + # TIME is SYSCALL plus the core packages about time, including context. + SYSCALL + < time/tzdata + < time + < context + < TIME; + + TIME, io, path, sort + < io/fs; + + # MATH is RUNTIME plus the basic math packages. + RUNTIME + < math + < MATH; + + unicode !< math; + + MATH + < math/cmplx; + + MATH + < math/rand; + + MATH + < runtime/metrics; + + MATH, unicode/utf8 + < strconv; + + unicode !< strconv; + + # STR is basic string and buffer manipulation. + RUNTIME, io, unicode/utf8, unicode/utf16, unicode + < bytes, strings + < bufio; + + bufio, path, strconv + < STR; + + # OS is basic OS access, including helpers (path/filepath, os/exec, etc). + # OS includes string routines, but those must be layered above package os. + # OS does not include reflection. + io/fs + < internal/testlog + < internal/poll + < internal/safefilepath + < os + < os/signal; + + io/fs + < embed; + + unicode, fmt !< net, os, os/signal; + + os/signal, internal/safefilepath, STR + < path/filepath + < io/ioutil; + + path/filepath, internal/godebug < os/exec; + + io/ioutil, os/exec, os/signal + < OS; + + reflect !< OS; + + OS + < golang.org/x/sys/cpu; + + # FMT is OS (which includes string routines) plus reflect and fmt. + # It does not include package log, which should be avoided in core packages. + arena, strconv, unicode + < reflect; + + os, reflect + < internal/fmtsort + < fmt; + + OS, fmt + < FMT; + + log !< FMT; + + # Misc packages needing only FMT. + FMT + < html, + internal/dag, + internal/goroot, + internal/types/errors, + mime/quotedprintable, + net/internal/socktest, + net/url, + runtime/trace, + text/scanner, + text/tabwriter; + + io, reflect + < internal/saferio; + + # encodings + # core ones do not use fmt. + io, strconv + < encoding; + + encoding, reflect + < encoding/binary + < encoding/base32, encoding/base64; + + FMT, encoding < flag; + + fmt !< encoding/base32, encoding/base64; + + FMT, encoding/base32, encoding/base64, internal/saferio + < encoding/ascii85, encoding/csv, encoding/gob, encoding/hex, + encoding/json, encoding/pem, encoding/xml, mime; + + # hashes + io + < hash + < hash/adler32, hash/crc32, hash/crc64, hash/fnv, hash/maphash; + + # math/big + FMT, encoding/binary, math/rand + < math/big; + + # compression + FMT, encoding/binary, hash/adler32, hash/crc32 + < compress/bzip2, compress/flate, compress/lzw + < archive/zip, compress/gzip, compress/zlib; + + # templates + FMT + < text/template/parse; + + net/url, text/template/parse + < text/template + < internal/lazytemplate; + + # regexp + FMT + < regexp/syntax + < regexp + < internal/lazyregexp; + + encoding/json, html, text/template, regexp + < html/template; + + # suffix array + encoding/binary, regexp + < index/suffixarray; + + # executable parsing + FMT, encoding/binary, compress/zlib, internal/saferio + < runtime/debug + < debug/dwarf + < debug/elf, debug/gosym, debug/macho, debug/pe, debug/plan9obj, internal/xcoff + < debug/buildinfo + < DEBUG; + + # go parser and friends. + FMT + < go/token + < go/scanner + < go/ast + < go/internal/typeparams + < go/parser; + + FMT + < go/build/constraint, go/doc/comment; + + go/build/constraint, go/doc/comment, go/parser, text/tabwriter + < go/printer + < go/format; + + go/doc/comment, go/parser, internal/lazyregexp, text/template + < go/doc; + + math/big, go/token + < go/constant; + + container/heap, go/constant, go/parser, internal/types/errors, regexp + < go/types; + + FMT, internal/goexperiment + < internal/buildcfg; + + go/build/constraint, go/doc, go/parser, internal/buildcfg, internal/goroot, internal/goversion + < go/build; + + # databases + FMT + < database/sql/internal + < database/sql/driver + < database/sql; + + # images + FMT, compress/lzw, compress/zlib + < image/color + < image, image/color/palette + < image/internal/imageutil + < image/draw + < image/gif, image/jpeg, image/png; + + # cgo, delayed as long as possible. + # If you add a dependency on CGO, you must add the package + # to cgoPackages in cmd/dist/test.go as well. + RUNTIME + < C + < runtime/cgo + < CGO + < runtime/msan, runtime/asan; + + # runtime/race + NONE < runtime/race/internal/amd64v1; + NONE < runtime/race/internal/amd64v3; + CGO, runtime/race/internal/amd64v1, runtime/race/internal/amd64v3 < runtime/race; + + # Bulk of the standard library must not use cgo. + # The prohibition stops at net and os/user. + C !< fmt, go/types, CRYPTO-MATH; + + CGO, OS + < plugin; + + CGO, FMT + < os/user + < archive/tar; + + sync + < internal/singleflight; + + os + < golang.org/x/net/dns/dnsmessage, + golang.org/x/net/lif, + golang.org/x/net/route; + + os, runtime, strconv, sync, unsafe, + internal/godebug + < internal/intern; + + internal/bytealg, internal/intern, internal/itoa, math/bits, sort, strconv + < net/netip; + + # net is unavoidable when doing any networking, + # so large dependencies must be kept out. + # This is a long-looking list but most of these + # are small with few dependencies. + CGO, + golang.org/x/net/dns/dnsmessage, + golang.org/x/net/lif, + golang.org/x/net/route, + internal/godebug, + internal/nettrace, + internal/poll, + internal/singleflight, + internal/race, + net/netip, + os + < net; + + fmt, unicode !< net; + math/rand !< net; # net uses runtime instead + + # NET is net plus net-helper packages. + FMT, net + < net/textproto; + + mime, net/textproto, net/url + < NET; + + # logging - most packages should not import; http and up is allowed + FMT + < log; + + log !< crypto/tls, database/sql, go/importer, testing; + + FMT, log, net + < log/syslog; + + NET, log + < net/mail; + + NONE < crypto/internal/boring/sig, crypto/internal/boring/syso; + sync/atomic < crypto/internal/boring/bcache, crypto/internal/boring/fipstls; + crypto/internal/boring/sig, crypto/internal/boring/fipstls < crypto/tls/fipsonly; + + # CRYPTO is core crypto algorithms - no cgo, fmt, net. + # Unfortunately, stuck with reflect via encoding/binary. + crypto/internal/boring/sig, + crypto/internal/boring/syso, + encoding/binary, + golang.org/x/sys/cpu, + hash, embed + < crypto + < crypto/subtle + < crypto/internal/alias + < crypto/cipher; + + crypto/cipher, + crypto/internal/boring/bcache + < crypto/internal/boring + < crypto/boring; + + crypto/internal/alias + < crypto/internal/randutil + < crypto/internal/nistec/fiat + < crypto/internal/nistec + < crypto/internal/edwards25519/field + < crypto/internal/edwards25519; + + crypto/boring + < crypto/aes, crypto/des, crypto/hmac, crypto/md5, crypto/rc4, + crypto/sha1, crypto/sha256, crypto/sha512; + + crypto/boring, crypto/internal/edwards25519/field + < crypto/ecdh; + + crypto/aes, + crypto/des, + crypto/ecdh, + crypto/hmac, + crypto/internal/edwards25519, + crypto/md5, + crypto/rc4, + crypto/sha1, + crypto/sha256, + crypto/sha512 + < CRYPTO; + + CGO, fmt, net !< CRYPTO; + + # CRYPTO-MATH is core bignum-based crypto - no cgo, net; fmt now ok. + CRYPTO, FMT, math/big + < crypto/internal/boring/bbig + < crypto/rand + < crypto/ed25519 + < encoding/asn1 + < golang.org/x/crypto/cryptobyte/asn1 + < golang.org/x/crypto/cryptobyte + < crypto/internal/bigmod + < crypto/dsa, crypto/elliptic, crypto/rsa + < crypto/ecdsa + < CRYPTO-MATH; + + CGO, net !< CRYPTO-MATH; + + # TLS, Prince of Dependencies. + CRYPTO-MATH, NET, container/list, encoding/hex, encoding/pem + < golang.org/x/crypto/internal/alias + < golang.org/x/crypto/internal/subtle + < golang.org/x/crypto/chacha20 + < golang.org/x/crypto/internal/poly1305 + < golang.org/x/crypto/chacha20poly1305 + < golang.org/x/crypto/hkdf + < crypto/x509/internal/macos + < crypto/x509/pkix; + + crypto/internal/boring/fipstls, crypto/x509/pkix + < crypto/x509 + < crypto/tls; + + # crypto-aware packages + + DEBUG, go/build, go/types, text/scanner, crypto/md5 + < internal/pkgbits + < go/internal/gcimporter, go/internal/gccgoimporter, go/internal/srcimporter + < go/importer; + + NET, crypto/rand, mime/quotedprintable + < mime/multipart; + + crypto/tls + < net/smtp; + + # HTTP, King of Dependencies. + + FMT + < golang.org/x/net/http2/hpack + < net/http/internal, net/http/internal/ascii, net/http/internal/testcert; + + FMT, NET, container/list, encoding/binary, log + < golang.org/x/text/transform + < golang.org/x/text/unicode/norm + < golang.org/x/text/unicode/bidi + < golang.org/x/text/secure/bidirule + < golang.org/x/net/idna + < golang.org/x/net/http/httpguts, golang.org/x/net/http/httpproxy; + + NET, crypto/tls + < net/http/httptrace; + + compress/gzip, + golang.org/x/net/http/httpguts, + golang.org/x/net/http/httpproxy, + golang.org/x/net/http2/hpack, + net/http/internal, + net/http/internal/ascii, + net/http/internal/testcert, + net/http/httptrace, + mime/multipart, + log + < net/http; + + # HTTP-aware packages + + encoding/json, net/http + < expvar; + + net/http, net/http/internal/ascii + < net/http/cookiejar, net/http/httputil; + + net/http, flag + < net/http/httptest; + + net/http, regexp + < net/http/cgi + < net/http/fcgi; + + # Profiling + FMT, compress/gzip, encoding/binary, text/tabwriter + < runtime/pprof; + + OS, compress/gzip, regexp + < internal/profile; + + html, internal/profile, net/http, runtime/pprof, runtime/trace + < net/http/pprof; + + # RPC + encoding/gob, encoding/json, go/token, html/template, net/http + < net/rpc + < net/rpc/jsonrpc; + + # System Information + internal/cpu, sync + < internal/sysinfo; + + # Test-only + log + < testing/iotest + < testing/fstest; + + FMT, flag, math/rand + < testing/quick; + + FMT, DEBUG, flag, runtime/trace, internal/sysinfo, math/rand + < testing; + + FMT, crypto/sha256, encoding/json, go/ast, go/parser, go/token, + internal/godebug, math/rand, encoding/hex, crypto/sha256 + < internal/fuzz; + + internal/fuzz, internal/testlog, runtime/pprof, regexp + < testing/internal/testdeps; + + OS, flag, testing, internal/cfg, internal/platform, internal/goroot + < internal/testenv; + + OS, encoding/base64 + < internal/obscuretestdata; + + CGO, OS, fmt + < internal/testpty; + + NET, testing, math/rand + < golang.org/x/net/nettest; + + syscall + < os/exec/internal/fdtest; + + FMT, container/heap, math/rand + < internal/trace; + + FMT + < internal/diff, internal/txtar; + + FMT, crypto/md5, encoding/binary, regexp, sort, text/tabwriter, unsafe, + internal/coverage, internal/coverage/uleb128 + < internal/coverage/cmerge, + internal/coverage/pods, + internal/coverage/slicereader, + internal/coverage/slicewriter; + + internal/coverage/slicereader, internal/coverage/slicewriter + < internal/coverage/stringtab + < internal/coverage/decodecounter, internal/coverage/decodemeta, + internal/coverage/encodecounter, internal/coverage/encodemeta; + + internal/coverage/cmerge + < internal/coverage/cformat; + + runtime/debug, + internal/coverage/calloc, + internal/coverage/cformat, + internal/coverage/decodecounter, internal/coverage/decodemeta, + internal/coverage/encodecounter, internal/coverage/encodemeta, + internal/coverage/pods + < runtime/coverage; +` + +// listStdPkgs returns the same list of packages as "go list std". +func listStdPkgs(goroot string) ([]string, error) { + // Based on cmd/go's matchPackages function. + var pkgs []string + + src := filepath.Join(goroot, "src") + string(filepath.Separator) + walkFn := func(path string, d fs.DirEntry, err error) error { + if err != nil || !d.IsDir() || path == src { + return nil + } + + base := filepath.Base(path) + if strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") || base == "testdata" { + return filepath.SkipDir + } + + name := filepath.ToSlash(path[len(src):]) + if name == "builtin" || name == "cmd" { + return filepath.SkipDir + } + + pkgs = append(pkgs, strings.TrimPrefix(name, "vendor/")) + return nil + } + if err := filepath.WalkDir(src, walkFn); err != nil { + return nil, err + } + return pkgs, nil +} + +func TestDependencies(t *testing.T) { + if !testenv.HasSrc() { + // Tests run in a limited file system and we do not + // provide access to every source file. + t.Skipf("skipping on %s/%s, missing full GOROOT", runtime.GOOS, runtime.GOARCH) + } + + ctxt := Default + all, err := listStdPkgs(ctxt.GOROOT) + if err != nil { + t.Fatal(err) + } + sort.Strings(all) + + sawImport := map[string]map[string]bool{} // from package => to package => true + policy := depsPolicy(t) + + for _, pkg := range all { + imports, err := findImports(pkg) + if err != nil { + t.Error(err) + continue + } + if sawImport[pkg] == nil { + sawImport[pkg] = map[string]bool{} + } + var bad []string + for _, imp := range imports { + sawImport[pkg][imp] = true + if !policy.HasEdge(pkg, imp) { + bad = append(bad, imp) + } + } + if bad != nil { + t.Errorf("unexpected dependency: %s imports %v", pkg, bad) + } + } +} + +var buildIgnore = []byte("\n//go:build ignore") + +func findImports(pkg string) ([]string, error) { + vpkg := pkg + if strings.HasPrefix(pkg, "golang.org") { + vpkg = "vendor/" + pkg + } + dir := filepath.Join(Default.GOROOT, "src", vpkg) + files, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + var imports []string + var haveImport = map[string]bool{} + if pkg == "crypto/internal/boring" { + haveImport["C"] = true // kludge: prevent C from appearing in crypto/internal/boring imports + } + fset := token.NewFileSet() + for _, file := range files { + name := file.Name() + if name == "slice_go14.go" || name == "slice_go18.go" { + // These files are for compiler bootstrap with older versions of Go and not built in the standard build. + continue + } + if !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + info := fileInfo{ + name: filepath.Join(dir, name), + fset: fset, + } + f, err := os.Open(info.name) + if err != nil { + return nil, err + } + err = readGoInfo(f, &info) + f.Close() + if err != nil { + return nil, fmt.Errorf("reading %v: %v", name, err) + } + if info.parsed.Name.Name == "main" { + continue + } + if bytes.Contains(info.header, buildIgnore) { + continue + } + for _, imp := range info.imports { + path := imp.path + if !haveImport[path] { + haveImport[path] = true + imports = append(imports, path) + } + } + } + sort.Strings(imports) + return imports, nil +} + +// depsPolicy returns a map m such that m[p][d] == true when p can import d. +func depsPolicy(t *testing.T) *dag.Graph { + g, err := dag.Parse(depsRules) + if err != nil { + t.Fatal(err) + } + return g +} + +// TestStdlibLowercase tests that all standard library package names are +// lowercase. See Issue 40065. +func TestStdlibLowercase(t *testing.T) { + if !testenv.HasSrc() { + t.Skipf("skipping on %s/%s, missing full GOROOT", runtime.GOOS, runtime.GOARCH) + } + + ctxt := Default + all, err := listStdPkgs(ctxt.GOROOT) + if err != nil { + t.Fatal(err) + } + + for _, pkgname := range all { + if strings.ToLower(pkgname) != pkgname { + t.Errorf("package %q should not use upper-case path", pkgname) + } + } +} + +// TestFindImports tests that findImports works. See #43249. +func TestFindImports(t *testing.T) { + imports, err := findImports("go/build") + if err != nil { + t.Fatal(err) + } + t.Logf("go/build imports %q", imports) + want := []string{"bytes", "os", "path/filepath", "strings"} +wantLoop: + for _, w := range want { + for _, imp := range imports { + if imp == w { + continue wantLoop + } + } + t.Errorf("expected to find %q in import list", w) + } +} diff --git a/src/go/build/doc.go b/src/go/build/doc.go new file mode 100644 index 0000000..cd1d3fd --- /dev/null +++ b/src/go/build/doc.go @@ -0,0 +1,98 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package build gathers information about Go packages. +// +// # Go Path +// +// The Go path is a list of directory trees containing Go source code. +// It is consulted to resolve imports that cannot be found in the standard +// Go tree. The default path is the value of the GOPATH environment +// variable, interpreted as a path list appropriate to the operating system +// (on Unix, the variable is a colon-separated string; +// on Windows, a semicolon-separated string; +// on Plan 9, a list). +// +// Each directory listed in the Go path must have a prescribed structure: +// +// The src/ directory holds source code. The path below 'src' determines +// the import path or executable name. +// +// The pkg/ directory holds installed package objects. +// As in the Go tree, each target operating system and +// architecture pair has its own subdirectory of pkg +// (pkg/GOOS_GOARCH). +// +// If DIR is a directory listed in the Go path, a package with +// source in DIR/src/foo/bar can be imported as "foo/bar" and +// has its compiled form installed to "DIR/pkg/GOOS_GOARCH/foo/bar.a" +// (or, for gccgo, "DIR/pkg/gccgo/foo/libbar.a"). +// +// The bin/ directory holds compiled commands. +// Each command is named for its source directory, but only +// using the final element, not the entire path. That is, the +// command with source in DIR/src/foo/quux is installed into +// DIR/bin/quux, not DIR/bin/foo/quux. The foo/ is stripped +// so that you can add DIR/bin to your PATH to get at the +// installed commands. +// +// Here's an example directory layout: +// +// GOPATH=/home/user/gocode +// +// /home/user/gocode/ +// src/ +// foo/ +// bar/ (go code in package bar) +// x.go +// quux/ (go code in package main) +// y.go +// bin/ +// quux (installed command) +// pkg/ +// linux_amd64/ +// foo/ +// bar.a (installed package object) +// +// # Build Constraints +// +// A build constraint, also known as a build tag, is a condition under which a +// file should be included in the package. Build constraints are given by a +// line comment that begins +// +// //go:build +// +// Build constraints may also be part of a file's name +// (for example, source_windows.go will only be included if the target +// operating system is windows). +// +// See 'go help buildconstraint' +// (https://golang.org/cmd/go/#hdr-Build_constraints) for details. +// +// # Binary-Only Packages +// +// In Go 1.12 and earlier, it was possible to distribute packages in binary +// form without including the source code used for compiling the package. +// The package was distributed with a source file not excluded by build +// constraints and containing a "//go:binary-only-package" comment. Like a +// build constraint, this comment appeared at the top of a file, preceded +// only by blank lines and other line comments and with a blank line +// following the comment, to separate it from the package documentation. +// Unlike build constraints, this comment is only recognized in non-test +// Go source files. +// +// The minimal source code for a binary-only package was therefore: +// +// //go:binary-only-package +// +// package mypkg +// +// The source code could include additional Go code. That code was never +// compiled but would be processed by tools like godoc and might be useful +// as end-user documentation. +// +// "go build" and other commands no longer support binary-only-packages. +// Import and ImportDir will still set the BinaryOnly flag in packages +// containing these comments for use in tools and error messages. +package build diff --git a/src/go/build/gc.go b/src/go/build/gc.go new file mode 100644 index 0000000..434991f --- /dev/null +++ b/src/go/build/gc.go @@ -0,0 +1,17 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gc + +package build + +import ( + "path/filepath" + "runtime" +) + +// getToolDir returns the default value of ToolDir. +func getToolDir() string { + return filepath.Join(runtime.GOROOT(), "pkg/tool/"+runtime.GOOS+"_"+runtime.GOARCH) +} diff --git a/src/go/build/gccgo.go b/src/go/build/gccgo.go new file mode 100644 index 0000000..f806729 --- /dev/null +++ b/src/go/build/gccgo.go @@ -0,0 +1,14 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build gccgo + +package build + +import "runtime" + +// getToolDir returns the default value of ToolDir. +func getToolDir() string { + return envOr("GCCGOTOOLDIR", runtime.GCCGOTOOLDIR) +} diff --git a/src/go/build/read.go b/src/go/build/read.go new file mode 100644 index 0000000..adcf82f --- /dev/null +++ b/src/go/build/read.go @@ -0,0 +1,600 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "go/ast" + "go/parser" + "go/scanner" + "go/token" + "io" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +type importReader struct { + b *bufio.Reader + buf []byte + peek byte + err error + eof bool + nerr int + pos token.Position +} + +var bom = []byte{0xef, 0xbb, 0xbf} + +func newImportReader(name string, r io.Reader) *importReader { + b := bufio.NewReader(r) + // Remove leading UTF-8 BOM. + // Per https://golang.org/ref/spec#Source_code_representation: + // a compiler may ignore a UTF-8-encoded byte order mark (U+FEFF) + // if it is the first Unicode code point in the source text. + if leadingBytes, err := b.Peek(3); err == nil && bytes.Equal(leadingBytes, bom) { + b.Discard(3) + } + return &importReader{ + b: b, + pos: token.Position{ + Filename: name, + Line: 1, + Column: 1, + }, + } +} + +func isIdent(c byte) bool { + return 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' || c == '_' || c >= utf8.RuneSelf +} + +var ( + errSyntax = errors.New("syntax error") + errNUL = errors.New("unexpected NUL in input") +) + +// syntaxError records a syntax error, but only if an I/O error has not already been recorded. +func (r *importReader) syntaxError() { + if r.err == nil { + r.err = errSyntax + } +} + +// readByte reads the next byte from the input, saves it in buf, and returns it. +// If an error occurs, readByte records the error in r.err and returns 0. +func (r *importReader) readByte() byte { + c, err := r.b.ReadByte() + if err == nil { + r.buf = append(r.buf, c) + if c == 0 { + err = errNUL + } + } + if err != nil { + if err == io.EOF { + r.eof = true + } else if r.err == nil { + r.err = err + } + c = 0 + } + return c +} + +// readByteNoBuf is like readByte but doesn't buffer the byte. +// It exhausts r.buf before reading from r.b. +func (r *importReader) readByteNoBuf() byte { + var c byte + var err error + if len(r.buf) > 0 { + c = r.buf[0] + r.buf = r.buf[1:] + } else { + c, err = r.b.ReadByte() + if err == nil && c == 0 { + err = errNUL + } + } + + if err != nil { + if err == io.EOF { + r.eof = true + } else if r.err == nil { + r.err = err + } + return 0 + } + r.pos.Offset++ + if c == '\n' { + r.pos.Line++ + r.pos.Column = 1 + } else { + r.pos.Column++ + } + return c +} + +// peekByte returns the next byte from the input reader but does not advance beyond it. +// If skipSpace is set, peekByte skips leading spaces and comments. +func (r *importReader) peekByte(skipSpace bool) byte { + if r.err != nil { + if r.nerr++; r.nerr > 10000 { + panic("go/build: import reader looping") + } + return 0 + } + + // Use r.peek as first input byte. + // Don't just return r.peek here: it might have been left by peekByte(false) + // and this might be peekByte(true). + c := r.peek + if c == 0 { + c = r.readByte() + } + for r.err == nil && !r.eof { + if skipSpace { + // For the purposes of this reader, semicolons are never necessary to + // understand the input and are treated as spaces. + switch c { + case ' ', '\f', '\t', '\r', '\n', ';': + c = r.readByte() + continue + + case '/': + c = r.readByte() + if c == '/' { + for c != '\n' && r.err == nil && !r.eof { + c = r.readByte() + } + } else if c == '*' { + var c1 byte + for (c != '*' || c1 != '/') && r.err == nil { + if r.eof { + r.syntaxError() + } + c, c1 = c1, r.readByte() + } + } else { + r.syntaxError() + } + c = r.readByte() + continue + } + } + break + } + r.peek = c + return r.peek +} + +// nextByte is like peekByte but advances beyond the returned byte. +func (r *importReader) nextByte(skipSpace bool) byte { + c := r.peekByte(skipSpace) + r.peek = 0 + return c +} + +var goEmbed = []byte("go:embed") + +// findEmbed advances the input reader to the next //go:embed comment. +// It reports whether it found a comment. +// (Otherwise it found an error or EOF.) +func (r *importReader) findEmbed(first bool) bool { + // The import block scan stopped after a non-space character, + // so the reader is not at the start of a line on the first call. + // After that, each //go:embed extraction leaves the reader + // at the end of a line. + startLine := !first + var c byte + for r.err == nil && !r.eof { + c = r.readByteNoBuf() + Reswitch: + switch c { + default: + startLine = false + + case '\n': + startLine = true + + case ' ', '\t': + // leave startLine alone + + case '"': + startLine = false + for r.err == nil { + if r.eof { + r.syntaxError() + } + c = r.readByteNoBuf() + if c == '\\' { + r.readByteNoBuf() + if r.err != nil { + r.syntaxError() + return false + } + continue + } + if c == '"' { + c = r.readByteNoBuf() + goto Reswitch + } + } + goto Reswitch + + case '`': + startLine = false + for r.err == nil { + if r.eof { + r.syntaxError() + } + c = r.readByteNoBuf() + if c == '`' { + c = r.readByteNoBuf() + goto Reswitch + } + } + + case '\'': + startLine = false + for r.err == nil { + if r.eof { + r.syntaxError() + } + c = r.readByteNoBuf() + if c == '\\' { + r.readByteNoBuf() + if r.err != nil { + r.syntaxError() + return false + } + continue + } + if c == '\'' { + c = r.readByteNoBuf() + goto Reswitch + } + } + + case '/': + c = r.readByteNoBuf() + switch c { + default: + startLine = false + goto Reswitch + + case '*': + var c1 byte + for (c != '*' || c1 != '/') && r.err == nil { + if r.eof { + r.syntaxError() + } + c, c1 = c1, r.readByteNoBuf() + } + startLine = false + + case '/': + if startLine { + // Try to read this as a //go:embed comment. + for i := range goEmbed { + c = r.readByteNoBuf() + if c != goEmbed[i] { + goto SkipSlashSlash + } + } + c = r.readByteNoBuf() + if c == ' ' || c == '\t' { + // Found one! + return true + } + } + SkipSlashSlash: + for c != '\n' && r.err == nil && !r.eof { + c = r.readByteNoBuf() + } + startLine = true + } + } + } + return false +} + +// readKeyword reads the given keyword from the input. +// If the keyword is not present, readKeyword records a syntax error. +func (r *importReader) readKeyword(kw string) { + r.peekByte(true) + for i := 0; i < len(kw); i++ { + if r.nextByte(false) != kw[i] { + r.syntaxError() + return + } + } + if isIdent(r.peekByte(false)) { + r.syntaxError() + } +} + +// readIdent reads an identifier from the input. +// If an identifier is not present, readIdent records a syntax error. +func (r *importReader) readIdent() { + c := r.peekByte(true) + if !isIdent(c) { + r.syntaxError() + return + } + for isIdent(r.peekByte(false)) { + r.peek = 0 + } +} + +// readString reads a quoted string literal from the input. +// If an identifier is not present, readString records a syntax error. +func (r *importReader) readString() { + switch r.nextByte(true) { + case '`': + for r.err == nil { + if r.nextByte(false) == '`' { + break + } + if r.eof { + r.syntaxError() + } + } + case '"': + for r.err == nil { + c := r.nextByte(false) + if c == '"' { + break + } + if r.eof || c == '\n' { + r.syntaxError() + } + if c == '\\' { + r.nextByte(false) + } + } + default: + r.syntaxError() + } +} + +// readImport reads an import clause - optional identifier followed by quoted string - +// from the input. +func (r *importReader) readImport() { + c := r.peekByte(true) + if c == '.' { + r.peek = 0 + } else if isIdent(c) { + r.readIdent() + } + r.readString() +} + +// readComments is like io.ReadAll, except that it only reads the leading +// block of comments in the file. +func readComments(f io.Reader) ([]byte, error) { + r := newImportReader("", f) + r.peekByte(true) + if r.err == nil && !r.eof { + // Didn't reach EOF, so must have found a non-space byte. Remove it. + r.buf = r.buf[:len(r.buf)-1] + } + return r.buf, r.err +} + +// readGoInfo expects a Go file as input and reads the file up to and including the import section. +// It records what it learned in *info. +// If info.fset is non-nil, readGoInfo parses the file and sets info.parsed, info.parseErr, +// info.imports and info.embeds. +// +// It only returns an error if there are problems reading the file, +// not for syntax errors in the file itself. +func readGoInfo(f io.Reader, info *fileInfo) error { + r := newImportReader(info.name, f) + + r.readKeyword("package") + r.readIdent() + for r.peekByte(true) == 'i' { + r.readKeyword("import") + if r.peekByte(true) == '(' { + r.nextByte(false) + for r.peekByte(true) != ')' && r.err == nil { + r.readImport() + } + r.nextByte(false) + } else { + r.readImport() + } + } + + info.header = r.buf + + // If we stopped successfully before EOF, we read a byte that told us we were done. + // Return all but that last byte, which would cause a syntax error if we let it through. + if r.err == nil && !r.eof { + info.header = r.buf[:len(r.buf)-1] + } + + // If we stopped for a syntax error, consume the whole file so that + // we are sure we don't change the errors that go/parser returns. + if r.err == errSyntax { + r.err = nil + for r.err == nil && !r.eof { + r.readByte() + } + info.header = r.buf + } + if r.err != nil { + return r.err + } + + if info.fset == nil { + return nil + } + + // Parse file header & record imports. + info.parsed, info.parseErr = parser.ParseFile(info.fset, info.name, info.header, parser.ImportsOnly|parser.ParseComments) + if info.parseErr != nil { + return nil + } + + hasEmbed := false + for _, decl := range info.parsed.Decls { + d, ok := decl.(*ast.GenDecl) + if !ok { + continue + } + for _, dspec := range d.Specs { + spec, ok := dspec.(*ast.ImportSpec) + if !ok { + continue + } + quoted := spec.Path.Value + path, err := strconv.Unquote(quoted) + if err != nil { + return fmt.Errorf("parser returned invalid quoted string: <%s>", quoted) + } + if !isValidImport(path) { + // The parser used to return a parse error for invalid import paths, but + // no longer does, so check for and create the error here instead. + info.parseErr = scanner.Error{Pos: info.fset.Position(spec.Pos()), Msg: "invalid import path: " + path} + info.imports = nil + return nil + } + if path == "embed" { + hasEmbed = true + } + + doc := spec.Doc + if doc == nil && len(d.Specs) == 1 { + doc = d.Doc + } + info.imports = append(info.imports, fileImport{path, spec.Pos(), doc}) + } + } + + // If the file imports "embed", + // we have to look for //go:embed comments + // in the remainder of the file. + // The compiler will enforce the mapping of comments to + // declared variables. We just need to know the patterns. + // If there were //go:embed comments earlier in the file + // (near the package statement or imports), the compiler + // will reject them. They can be (and have already been) ignored. + if hasEmbed { + var line []byte + for first := true; r.findEmbed(first); first = false { + line = line[:0] + pos := r.pos + for { + c := r.readByteNoBuf() + if c == '\n' || r.err != nil || r.eof { + break + } + line = append(line, c) + } + // Add args if line is well-formed. + // Ignore badly-formed lines - the compiler will report them when it finds them, + // and we can pretend they are not there to help go list succeed with what it knows. + embs, err := parseGoEmbed(string(line), pos) + if err == nil { + info.embeds = append(info.embeds, embs...) + } + } + } + + return nil +} + +// isValidImport checks if the import is a valid import using the more strict +// checks allowed by the implementation restriction in https://go.dev/ref/spec#Import_declarations. +// It was ported from the function of the same name that was removed from the +// parser in CL 424855, when the parser stopped doing these checks. +func isValidImport(s string) bool { + const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" + for _, r := range s { + if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { + return false + } + } + return s != "" +} + +// parseGoEmbed parses the text following "//go:embed" to extract the glob patterns. +// It accepts unquoted space-separated patterns as well as double-quoted and back-quoted Go strings. +// This is based on a similar function in cmd/compile/internal/gc/noder.go; +// this version calculates position information as well. +func parseGoEmbed(args string, pos token.Position) ([]fileEmbed, error) { + trimBytes := func(n int) { + pos.Offset += n + pos.Column += utf8.RuneCountInString(args[:n]) + args = args[n:] + } + trimSpace := func() { + trim := strings.TrimLeftFunc(args, unicode.IsSpace) + trimBytes(len(args) - len(trim)) + } + + var list []fileEmbed + for trimSpace(); args != ""; trimSpace() { + var path string + pathPos := pos + Switch: + switch args[0] { + default: + i := len(args) + for j, c := range args { + if unicode.IsSpace(c) { + i = j + break + } + } + path = args[:i] + trimBytes(i) + + case '`': + var ok bool + path, _, ok = strings.Cut(args[1:], "`") + if !ok { + return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args) + } + trimBytes(1 + len(path) + 1) + + case '"': + i := 1 + for ; i < len(args); i++ { + if args[i] == '\\' { + i++ + continue + } + if args[i] == '"' { + q, err := strconv.Unquote(args[:i+1]) + if err != nil { + return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args[:i+1]) + } + path = q + trimBytes(i + 1) + break Switch + } + } + if i >= len(args) { + return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args) + } + } + + if args != "" { + r, _ := utf8.DecodeRuneInString(args) + if !unicode.IsSpace(r) { + return nil, fmt.Errorf("invalid quoted string in //go:embed: %s", args) + } + } + list = append(list, fileEmbed{path, pathPos}) + } + return list, nil +} diff --git a/src/go/build/read_test.go b/src/go/build/read_test.go new file mode 100644 index 0000000..6851e6b --- /dev/null +++ b/src/go/build/read_test.go @@ -0,0 +1,352 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "fmt" + "go/token" + "io" + "strings" + "testing" +) + +const quote = "`" + +type readTest struct { + // Test input contains ā„™ where readGoInfo should stop. + in string + err string +} + +var readGoInfoTests = []readTest{ + { + `package p`, + "", + }, + { + `package p; import "x"`, + "", + }, + { + `package p; import . "x"`, + "", + }, + { + `package p; import "x";ā„™var x = 1`, + "", + }, + { + `package p + + // comment + + import "x" + import _ "x" + import a "x" + + /* comment */ + + import ( + "x" /* comment */ + _ "x" + a "x" // comment + ` + quote + `x` + quote + ` + _ /*comment*/ ` + quote + `x` + quote + ` + a ` + quote + `x` + quote + ` + ) + import ( + ) + import () + import()import()import() + import();import();import() + + ā„™var x = 1 + `, + "", + }, + { + "\ufeffš”»" + `package p; import "x";ā„™var x = 1`, + "", + }, +} + +var readCommentsTests = []readTest{ + { + `ā„™package p`, + "", + }, + { + `ā„™package p; import "x"`, + "", + }, + { + `ā„™package p; import . "x"`, + "", + }, + { + "\ufeffš”»" + `ā„™package p; import . "x"`, + "", + }, + { + `// foo + + /* bar */ + + /* quux */ // baz + + /*/ zot */ + + // asdf + ā„™Hello, world`, + "", + }, + { + "\ufeffš”»" + `// foo + + /* bar */ + + /* quux */ // baz + + /*/ zot */ + + // asdf + ā„™Hello, world`, + "", + }, +} + +func testRead(t *testing.T, tests []readTest, read func(io.Reader) ([]byte, error)) { + for i, tt := range tests { + beforeP, afterP, _ := strings.Cut(tt.in, "ā„™") + in := beforeP + afterP + testOut := beforeP + + if beforeD, afterD, ok := strings.Cut(beforeP, "š”»"); ok { + in = beforeD + afterD + afterP + testOut = afterD + } + + r := strings.NewReader(in) + buf, err := read(r) + if err != nil { + if tt.err == "" { + t.Errorf("#%d: err=%q, expected success (%q)", i, err, string(buf)) + } else if !strings.Contains(err.Error(), tt.err) { + t.Errorf("#%d: err=%q, expected %q", i, err, tt.err) + } + continue + } + if tt.err != "" { + t.Errorf("#%d: success, expected %q", i, tt.err) + continue + } + + out := string(buf) + if out != testOut { + t.Errorf("#%d: wrong output:\nhave %q\nwant %q\n", i, out, testOut) + } + } +} + +func TestReadGoInfo(t *testing.T) { + testRead(t, readGoInfoTests, func(r io.Reader) ([]byte, error) { + var info fileInfo + err := readGoInfo(r, &info) + return info.header, err + }) +} + +func TestReadComments(t *testing.T) { + testRead(t, readCommentsTests, readComments) +} + +var readFailuresTests = []readTest{ + { + `package`, + "syntax error", + }, + { + "package p\n\x00\nimport `math`\n", + "unexpected NUL in input", + }, + { + `package p; import`, + "syntax error", + }, + { + `package p; import "`, + "syntax error", + }, + { + "package p; import ` \n\n", + "syntax error", + }, + { + `package p; import "x`, + "syntax error", + }, + { + `package p; import _`, + "syntax error", + }, + { + `package p; import _ "`, + "syntax error", + }, + { + `package p; import _ "x`, + "syntax error", + }, + { + `package p; import .`, + "syntax error", + }, + { + `package p; import . "`, + "syntax error", + }, + { + `package p; import . "x`, + "syntax error", + }, + { + `package p; import (`, + "syntax error", + }, + { + `package p; import ("`, + "syntax error", + }, + { + `package p; import ("x`, + "syntax error", + }, + { + `package p; import ("x"`, + "syntax error", + }, +} + +func TestReadFailuresIgnored(t *testing.T) { + // Syntax errors should not be reported (false arg to readImports). + // Instead, entire file should be the output and no error. + // Convert tests not to return syntax errors. + tests := make([]readTest, len(readFailuresTests)) + copy(tests, readFailuresTests) + for i := range tests { + tt := &tests[i] + if !strings.Contains(tt.err, "NUL") { + tt.err = "" + } + } + testRead(t, tests, func(r io.Reader) ([]byte, error) { + var info fileInfo + err := readGoInfo(r, &info) + return info.header, err + }) +} + +var readEmbedTests = []struct { + in, out string +}{ + { + "package p\n", + "", + }, + { + "package p\nimport \"embed\"\nvar i int\n//go:embed x y z\nvar files embed.FS", + `test:4:12:x + test:4:14:y + test:4:16:z`, + }, + { + "package p\nimport \"embed\"\nvar i int\n//go:embed x \"\\x79\" `z`\nvar files embed.FS", + `test:4:12:x + test:4:14:y + test:4:21:z`, + }, + { + "package p\nimport \"embed\"\nvar i int\n//go:embed x y\n//go:embed z\nvar files embed.FS", + `test:4:12:x + test:4:14:y + test:5:12:z`, + }, + { + "package p\nimport \"embed\"\nvar i int\n\t //go:embed x y\n\t //go:embed z\n\t var files embed.FS", + `test:4:14:x + test:4:16:y + test:5:14:z`, + }, + { + "package p\nimport \"embed\"\n//go:embed x y z\nvar files embed.FS", + `test:3:12:x + test:3:14:y + test:3:16:z`, + }, + { + "\ufeffpackage p\nimport \"embed\"\n//go:embed x y z\nvar files embed.FS", + `test:3:12:x + test:3:14:y + test:3:16:z`, + }, + { + "package p\nimport \"embed\"\nvar s = \"/*\"\n//go:embed x\nvar files embed.FS", + `test:4:12:x`, + }, + { + `package p + import "embed" + var s = "\"\\\\" + //go:embed x + var files embed.FS`, + `test:4:15:x`, + }, + { + "package p\nimport \"embed\"\nvar s = `/*`\n//go:embed x\nvar files embed.FS", + `test:4:12:x`, + }, + { + "package p\nimport \"embed\"\nvar s = z/ *y\n//go:embed pointer\nvar pointer embed.FS", + "test:4:12:pointer", + }, + { + "package p\n//go:embed x y z\n", // no import, no scan + "", + }, + { + "package p\n//go:embed x y z\nvar files embed.FS", // no import, no scan + "", + }, + { + "\ufeffpackage p\n//go:embed x y z\nvar files embed.FS", // no import, no scan + "", + }, +} + +func TestReadEmbed(t *testing.T) { + fset := token.NewFileSet() + for i, tt := range readEmbedTests { + info := fileInfo{ + name: "test", + fset: fset, + } + err := readGoInfo(strings.NewReader(tt.in), &info) + if err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + b := &strings.Builder{} + sep := "" + for _, emb := range info.embeds { + fmt.Fprintf(b, "%s%v:%s", sep, emb.pos, emb.pattern) + sep = "\n" + } + got := b.String() + want := strings.Join(strings.Fields(tt.out), "\n") + if got != want { + t.Errorf("#%d: embeds:\n%s\nwant:\n%s", i, got, want) + } + } +} diff --git a/src/go/build/syslist.go b/src/go/build/syslist.go new file mode 100644 index 0000000..78ca565 --- /dev/null +++ b/src/go/build/syslist.go @@ -0,0 +1,80 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +// Note that this file is read by internal/goarch/gengoarch.go and by +// internal/goos/gengoos.go. If you change this file, look at those +// files as well. + +// knownOS is the list of past, present, and future known GOOS values. +// Do not remove from this list, as it is used for filename matching. +// If you add an entry to this list, look at unixOS, below. +var knownOS = map[string]bool{ + "aix": true, + "android": true, + "darwin": true, + "dragonfly": true, + "freebsd": true, + "hurd": true, + "illumos": true, + "ios": true, + "js": true, + "linux": true, + "nacl": true, + "netbsd": true, + "openbsd": true, + "plan9": true, + "solaris": true, + "windows": true, + "zos": true, +} + +// unixOS is the set of GOOS values matched by the "unix" build tag. +// This is not used for filename matching. +// This list also appears in cmd/dist/build.go and +// cmd/go/internal/imports/build.go. +var unixOS = map[string]bool{ + "aix": true, + "android": true, + "darwin": true, + "dragonfly": true, + "freebsd": true, + "hurd": true, + "illumos": true, + "ios": true, + "linux": true, + "netbsd": true, + "openbsd": true, + "solaris": true, +} + +// knownArch is the list of past, present, and future known GOARCH values. +// Do not remove from this list, as it is used for filename matching. +var knownArch = map[string]bool{ + "386": true, + "amd64": true, + "amd64p32": true, + "arm": true, + "armbe": true, + "arm64": true, + "arm64be": true, + "loong64": true, + "mips": true, + "mipsle": true, + "mips64": true, + "mips64le": true, + "mips64p32": true, + "mips64p32le": true, + "ppc": true, + "ppc64": true, + "ppc64le": true, + "riscv": true, + "riscv64": true, + "s390": true, + "s390x": true, + "sparc": true, + "sparc64": true, + "wasm": true, +} diff --git a/src/go/build/syslist_test.go b/src/go/build/syslist_test.go new file mode 100644 index 0000000..2b7b4c7 --- /dev/null +++ b/src/go/build/syslist_test.go @@ -0,0 +1,62 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package build + +import ( + "runtime" + "testing" +) + +var ( + thisOS = runtime.GOOS + thisArch = runtime.GOARCH + otherOS = anotherOS() + otherArch = anotherArch() +) + +func anotherOS() string { + if thisOS != "darwin" && thisOS != "ios" { + return "darwin" + } + return "linux" +} + +func anotherArch() string { + if thisArch != "amd64" { + return "amd64" + } + return "386" +} + +type GoodFileTest struct { + name string + result bool +} + +var tests = []GoodFileTest{ + {"file.go", true}, + {"file.c", true}, + {"file_foo.go", true}, + {"file_" + thisArch + ".go", true}, + {"file_" + otherArch + ".go", false}, + {"file_" + thisOS + ".go", true}, + {"file_" + otherOS + ".go", false}, + {"file_" + thisOS + "_" + thisArch + ".go", true}, + {"file_" + otherOS + "_" + thisArch + ".go", false}, + {"file_" + thisOS + "_" + otherArch + ".go", false}, + {"file_" + otherOS + "_" + otherArch + ".go", false}, + {"file_foo_" + thisArch + ".go", true}, + {"file_foo_" + otherArch + ".go", false}, + {"file_" + thisOS + ".c", true}, + {"file_" + otherOS + ".c", false}, +} + +func TestGoodOSArch(t *testing.T) { + for _, test := range tests { + if Default.goodOSArchFile(test.name, make(map[string]bool)) != test.result { + t.Fatalf("goodOSArchFile(%q) != %v", test.name, test.result) + } + } +} diff --git a/src/go/build/testdata/alltags/alltags.go b/src/go/build/testdata/alltags/alltags.go new file mode 100644 index 0000000..5d30855 --- /dev/null +++ b/src/go/build/testdata/alltags/alltags.go @@ -0,0 +1,5 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package alltags diff --git a/src/go/build/testdata/alltags/x_netbsd_arm.go b/src/go/build/testdata/alltags/x_netbsd_arm.go new file mode 100644 index 0000000..5d30855 --- /dev/null +++ b/src/go/build/testdata/alltags/x_netbsd_arm.go @@ -0,0 +1,5 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package alltags diff --git a/src/go/build/testdata/bads/bad.s b/src/go/build/testdata/bads/bad.s new file mode 100644 index 0000000..b670f82 --- /dev/null +++ b/src/go/build/testdata/bads/bad.s @@ -0,0 +1 @@ +;/ diff --git a/src/go/build/testdata/cgo_disabled/cgo_disabled.go b/src/go/build/testdata/cgo_disabled/cgo_disabled.go new file mode 100644 index 0000000..d1edb99 --- /dev/null +++ b/src/go/build/testdata/cgo_disabled/cgo_disabled.go @@ -0,0 +1,5 @@ +package cgo_disabled + +import "C" + +import _ "should/be/ignored" diff --git a/src/go/build/testdata/cgo_disabled/empty.go b/src/go/build/testdata/cgo_disabled/empty.go new file mode 100644 index 0000000..63afe42 --- /dev/null +++ b/src/go/build/testdata/cgo_disabled/empty.go @@ -0,0 +1 @@ +package cgo_disabled diff --git a/src/go/build/testdata/doc/a_test.go b/src/go/build/testdata/doc/a_test.go new file mode 100644 index 0000000..1c07b56 --- /dev/null +++ b/src/go/build/testdata/doc/a_test.go @@ -0,0 +1,2 @@ +// Doc from xtests +package doc_test diff --git a/src/go/build/testdata/doc/b_test.go b/src/go/build/testdata/doc/b_test.go new file mode 100644 index 0000000..0cf1605 --- /dev/null +++ b/src/go/build/testdata/doc/b_test.go @@ -0,0 +1 @@ +package doc_test diff --git a/src/go/build/testdata/doc/c_test.go b/src/go/build/testdata/doc/c_test.go new file mode 100644 index 0000000..1025707 --- /dev/null +++ b/src/go/build/testdata/doc/c_test.go @@ -0,0 +1 @@ +package doc diff --git a/src/go/build/testdata/doc/d_test.go b/src/go/build/testdata/doc/d_test.go new file mode 100644 index 0000000..ec19564 --- /dev/null +++ b/src/go/build/testdata/doc/d_test.go @@ -0,0 +1,2 @@ +// Doc from regular tests. +package doc diff --git a/src/go/build/testdata/doc/e.go b/src/go/build/testdata/doc/e.go new file mode 100644 index 0000000..1025707 --- /dev/null +++ b/src/go/build/testdata/doc/e.go @@ -0,0 +1 @@ +package doc diff --git a/src/go/build/testdata/doc/f.go b/src/go/build/testdata/doc/f.go new file mode 100644 index 0000000..ab1d0bc --- /dev/null +++ b/src/go/build/testdata/doc/f.go @@ -0,0 +1,2 @@ +// Correct +package doc diff --git a/src/go/build/testdata/empty/dummy b/src/go/build/testdata/empty/dummy new file mode 100644 index 0000000..e69de29 diff --git a/src/go/build/testdata/multi/file.go b/src/go/build/testdata/multi/file.go new file mode 100644 index 0000000..ee946eb --- /dev/null +++ b/src/go/build/testdata/multi/file.go @@ -0,0 +1,5 @@ +// Test data - not compiled. + +package main + +func main() {} diff --git a/src/go/build/testdata/multi/file_appengine.go b/src/go/build/testdata/multi/file_appengine.go new file mode 100644 index 0000000..4ea31e7 --- /dev/null +++ b/src/go/build/testdata/multi/file_appengine.go @@ -0,0 +1,5 @@ +// Test data - not compiled. + +package test_package + +func init() {} diff --git a/src/go/build/testdata/non_source_tags/non_source_tags.go b/src/go/build/testdata/non_source_tags/non_source_tags.go new file mode 100644 index 0000000..068acc4 --- /dev/null +++ b/src/go/build/testdata/non_source_tags/non_source_tags.go @@ -0,0 +1,5 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package non_source_tags diff --git a/src/go/build/testdata/non_source_tags/x_arm.go.ignore b/src/go/build/testdata/non_source_tags/x_arm.go.ignore new file mode 100644 index 0000000..068acc4 --- /dev/null +++ b/src/go/build/testdata/non_source_tags/x_arm.go.ignore @@ -0,0 +1,5 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package non_source_tags diff --git a/src/go/build/testdata/other/file/file.go b/src/go/build/testdata/other/file/file.go new file mode 100644 index 0000000..bbfd3e9 --- /dev/null +++ b/src/go/build/testdata/other/file/file.go @@ -0,0 +1,5 @@ +// Test data - not compiled. + +package file + +func F() {} diff --git a/src/go/build/testdata/other/main.go b/src/go/build/testdata/other/main.go new file mode 100644 index 0000000..e090435 --- /dev/null +++ b/src/go/build/testdata/other/main.go @@ -0,0 +1,11 @@ +// Test data - not compiled. + +package main + +import ( + "./file" +) + +func main() { + file.F() +} diff --git a/src/go/build/testdata/withvendor/src/a/b/b.go b/src/go/build/testdata/withvendor/src/a/b/b.go new file mode 100644 index 0000000..4405d54 --- /dev/null +++ b/src/go/build/testdata/withvendor/src/a/b/b.go @@ -0,0 +1,3 @@ +package b + +import _ "c/d" diff --git a/src/go/build/testdata/withvendor/src/a/vendor/c/d/d.go b/src/go/build/testdata/withvendor/src/a/vendor/c/d/d.go new file mode 100644 index 0000000..142fb42 --- /dev/null +++ b/src/go/build/testdata/withvendor/src/a/vendor/c/d/d.go @@ -0,0 +1 @@ +package d diff --git a/src/go/constant/example_test.go b/src/go/constant/example_test.go new file mode 100644 index 0000000..6443ee6 --- /dev/null +++ b/src/go/constant/example_test.go @@ -0,0 +1,180 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package constant_test + +import ( + "fmt" + "go/constant" + "go/token" + "math" + "sort" +) + +func Example_complexNumbers() { + // Create the complex number 2.3 + 5i. + ar := constant.MakeFloat64(2.3) + ai := constant.MakeImag(constant.MakeInt64(5)) + a := constant.BinaryOp(ar, token.ADD, ai) + + // Compute (2.3 + 5i) * 11. + b := constant.MakeUint64(11) + c := constant.BinaryOp(a, token.MUL, b) + + // Convert c into a complex128. + Ar, exact := constant.Float64Val(constant.Real(c)) + if !exact { + fmt.Printf("Could not represent real part %s exactly as float64\n", constant.Real(c)) + } + Ai, exact := constant.Float64Val(constant.Imag(c)) + if !exact { + fmt.Printf("Could not represent imaginary part %s as exactly as float64\n", constant.Imag(c)) + } + C := complex(Ar, Ai) + + fmt.Println("literal", 25.3+55i) + fmt.Println("go/constant", c) + fmt.Println("complex128", C) + + // Output: + // + // Could not represent real part 25.3 exactly as float64 + // literal (25.3+55i) + // go/constant (25.3 + 55i) + // complex128 (25.299999999999997+55i) +} + +func ExampleBinaryOp() { + // 11 / 0.5 + a := constant.MakeUint64(11) + b := constant.MakeFloat64(0.5) + c := constant.BinaryOp(a, token.QUO, b) + fmt.Println(c) + + // Output: 22 +} + +func ExampleUnaryOp() { + vs := []constant.Value{ + constant.MakeBool(true), + constant.MakeFloat64(2.7), + constant.MakeUint64(42), + } + + for i, v := range vs { + switch v.Kind() { + case constant.Bool: + vs[i] = constant.UnaryOp(token.NOT, v, 0) + + case constant.Float: + vs[i] = constant.UnaryOp(token.SUB, v, 0) + + case constant.Int: + // Use 16-bit precision. + // This would be equivalent to ^uint16(v). + vs[i] = constant.UnaryOp(token.XOR, v, 16) + } + } + + for _, v := range vs { + fmt.Println(v) + } + + // Output: + // + // false + // -2.7 + // 65493 +} + +func ExampleCompare() { + vs := []constant.Value{ + constant.MakeString("Z"), + constant.MakeString("bacon"), + constant.MakeString("go"), + constant.MakeString("Frame"), + constant.MakeString("defer"), + constant.MakeFromLiteral(`"a"`, token.STRING, 0), + } + + sort.Slice(vs, func(i, j int) bool { + // Equivalent to vs[i] <= vs[j]. + return constant.Compare(vs[i], token.LEQ, vs[j]) + }) + + for _, v := range vs { + fmt.Println(constant.StringVal(v)) + } + + // Output: + // + // Frame + // Z + // a + // bacon + // defer + // go +} + +func ExampleSign() { + zero := constant.MakeInt64(0) + one := constant.MakeInt64(1) + negOne := constant.MakeInt64(-1) + + mkComplex := func(a, b constant.Value) constant.Value { + b = constant.MakeImag(b) + return constant.BinaryOp(a, token.ADD, b) + } + + vs := []constant.Value{ + negOne, + mkComplex(zero, negOne), + mkComplex(one, negOne), + mkComplex(negOne, one), + mkComplex(negOne, negOne), + zero, + mkComplex(zero, zero), + one, + mkComplex(zero, one), + mkComplex(one, one), + } + + for _, v := range vs { + fmt.Printf("% d %s\n", constant.Sign(v), v) + } + + // Output: + // + // -1 -1 + // -1 (0 + -1i) + // -1 (1 + -1i) + // -1 (-1 + 1i) + // -1 (-1 + -1i) + // 0 0 + // 0 (0 + 0i) + // 1 1 + // 1 (0 + 1i) + // 1 (1 + 1i) +} + +func ExampleVal() { + maxint := constant.MakeInt64(math.MaxInt64) + fmt.Printf("%v\n", constant.Val(maxint)) + + e := constant.MakeFloat64(math.E) + fmt.Printf("%v\n", constant.Val(e)) + + b := constant.MakeBool(true) + fmt.Printf("%v\n", constant.Val(b)) + + b = constant.Make(false) + fmt.Printf("%v\n", constant.Val(b)) + + // Output: + // + // 9223372036854775807 + // 6121026514868073/2251799813685248 + // true + // false +} diff --git a/src/go/constant/kind_string.go b/src/go/constant/kind_string.go new file mode 100644 index 0000000..7003325 --- /dev/null +++ b/src/go/constant/kind_string.go @@ -0,0 +1,28 @@ +// Code generated by "stringer -type Kind"; DO NOT EDIT. + +package constant + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Unknown-0] + _ = x[Bool-1] + _ = x[String-2] + _ = x[Int-3] + _ = x[Float-4] + _ = x[Complex-5] +} + +const _Kind_name = "UnknownBoolStringIntFloatComplex" + +var _Kind_index = [...]uint8{0, 7, 11, 17, 20, 25, 32} + +func (i Kind) String() string { + if i < 0 || i >= Kind(len(_Kind_index)-1) { + return "Kind(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Kind_name[_Kind_index[i]:_Kind_index[i+1]] +} diff --git a/src/go/constant/value.go b/src/go/constant/value.go new file mode 100644 index 0000000..ae300c7 --- /dev/null +++ b/src/go/constant/value.go @@ -0,0 +1,1410 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package constant implements Values representing untyped +// Go constants and their corresponding operations. +// +// A special Unknown value may be used when a value +// is unknown due to an error. Operations on unknown +// values produce unknown values unless specified +// otherwise. +package constant + +import ( + "fmt" + "go/token" + "math" + "math/big" + "math/bits" + "strconv" + "strings" + "sync" + "unicode/utf8" +) + +//go:generate stringer -type Kind + +// Kind specifies the kind of value represented by a Value. +type Kind int + +const ( + // unknown values + Unknown Kind = iota + + // non-numeric values + Bool + String + + // numeric values + Int + Float + Complex +) + +// A Value represents the value of a Go constant. +type Value interface { + // Kind returns the value kind. + Kind() Kind + + // String returns a short, quoted (human-readable) form of the value. + // For numeric values, the result may be an approximation; + // for String values the result may be a shortened string. + // Use ExactString for a string representing a value exactly. + String() string + + // ExactString returns an exact, quoted (human-readable) form of the value. + // If the Value is of Kind String, use StringVal to obtain the unquoted string. + ExactString() string + + // Prevent external implementations. + implementsValue() +} + +// ---------------------------------------------------------------------------- +// Implementations + +// Maximum supported mantissa precision. +// The spec requires at least 256 bits; typical implementations use 512 bits. +const prec = 512 + +// TODO(gri) Consider storing "error" information in an unknownVal so clients +// can provide better error messages. For instance, if a number is +// too large (incl. infinity), that could be recorded in unknownVal. +// See also #20583 and #42695 for use cases. + +// Representation of values: +// +// Values of Int and Float Kind have two different representations each: int64Val +// and intVal, and ratVal and floatVal. When possible, the "smaller", respectively +// more precise (for Floats) representation is chosen. However, once a Float value +// is represented as a floatVal, any subsequent results remain floatVals (unless +// explicitly converted); i.e., no attempt is made to convert a floatVal back into +// a ratVal. The reasoning is that all representations but floatVal are mathematically +// exact, but once that precision is lost (by moving to floatVal), moving back to +// a different representation implies a precision that's not actually there. + +type ( + unknownVal struct{} + boolVal bool + stringVal struct { + // Lazy value: either a string (l,r==nil) or an addition (l,r!=nil). + mu sync.Mutex + s string + l, r *stringVal + } + int64Val int64 // Int values representable as an int64 + intVal struct{ val *big.Int } // Int values not representable as an int64 + ratVal struct{ val *big.Rat } // Float values representable as a fraction + floatVal struct{ val *big.Float } // Float values not representable as a fraction + complexVal struct{ re, im Value } +) + +func (unknownVal) Kind() Kind { return Unknown } +func (boolVal) Kind() Kind { return Bool } +func (*stringVal) Kind() Kind { return String } +func (int64Val) Kind() Kind { return Int } +func (intVal) Kind() Kind { return Int } +func (ratVal) Kind() Kind { return Float } +func (floatVal) Kind() Kind { return Float } +func (complexVal) Kind() Kind { return Complex } + +func (unknownVal) String() string { return "unknown" } +func (x boolVal) String() string { return strconv.FormatBool(bool(x)) } + +// String returns a possibly shortened quoted form of the String value. +func (x *stringVal) String() string { + const maxLen = 72 // a reasonable length + s := strconv.Quote(x.string()) + if utf8.RuneCountInString(s) > maxLen { + // The string without the enclosing quotes is greater than maxLen-2 runes + // long. Remove the last 3 runes (including the closing '"') by keeping + // only the first maxLen-3 runes; then add "...". + i := 0 + for n := 0; n < maxLen-3; n++ { + _, size := utf8.DecodeRuneInString(s[i:]) + i += size + } + s = s[:i] + "..." + } + return s +} + +// string constructs and returns the actual string literal value. +// If x represents an addition, then it rewrites x to be a single +// string, to speed future calls. This lazy construction avoids +// building different string values for all subpieces of a large +// concatenation. See golang.org/issue/23348. +func (x *stringVal) string() string { + x.mu.Lock() + if x.l != nil { + x.s = strings.Join(reverse(x.appendReverse(nil)), "") + x.l = nil + x.r = nil + } + s := x.s + x.mu.Unlock() + + return s +} + +// reverse reverses x in place and returns it. +func reverse(x []string) []string { + n := len(x) + for i := 0; i+i < n; i++ { + x[i], x[n-1-i] = x[n-1-i], x[i] + } + return x +} + +// appendReverse appends to list all of x's subpieces, but in reverse, +// and returns the result. Appending the reversal allows processing +// the right side in a recursive call and the left side in a loop. +// Because a chain like a + b + c + d + e is actually represented +// as ((((a + b) + c) + d) + e), the left-side loop avoids deep recursion. +// x must be locked. +func (x *stringVal) appendReverse(list []string) []string { + y := x + for y.r != nil { + y.r.mu.Lock() + list = y.r.appendReverse(list) + y.r.mu.Unlock() + + l := y.l + if y != x { + y.mu.Unlock() + } + l.mu.Lock() + y = l + } + s := y.s + if y != x { + y.mu.Unlock() + } + return append(list, s) +} + +func (x int64Val) String() string { return strconv.FormatInt(int64(x), 10) } +func (x intVal) String() string { return x.val.String() } +func (x ratVal) String() string { return rtof(x).String() } + +// String returns a decimal approximation of the Float value. +func (x floatVal) String() string { + f := x.val + + // Don't try to convert infinities (will not terminate). + if f.IsInf() { + return f.String() + } + + // Use exact fmt formatting if in float64 range (common case): + // proceed if f doesn't underflow to 0 or overflow to inf. + if x, _ := f.Float64(); f.Sign() == 0 == (x == 0) && !math.IsInf(x, 0) { + s := fmt.Sprintf("%.6g", x) + if !f.IsInt() && strings.IndexByte(s, '.') < 0 { + // f is not an integer, but its string representation + // doesn't reflect that. Use more digits. See issue 56220. + s = fmt.Sprintf("%g", x) + } + return s + } + + // Out of float64 range. Do approximate manual to decimal + // conversion to avoid precise but possibly slow Float + // formatting. + // f = mant * 2**exp + var mant big.Float + exp := f.MantExp(&mant) // 0.5 <= |mant| < 1.0 + + // approximate float64 mantissa m and decimal exponent d + // f ~ m * 10**d + m, _ := mant.Float64() // 0.5 <= |m| < 1.0 + d := float64(exp) * (math.Ln2 / math.Ln10) // log_10(2) + + // adjust m for truncated (integer) decimal exponent e + e := int64(d) + m *= math.Pow(10, d-float64(e)) + + // ensure 1 <= |m| < 10 + switch am := math.Abs(m); { + case am < 1-0.5e-6: + // The %.6g format below rounds m to 5 digits after the + // decimal point. Make sure that m*10 < 10 even after + // rounding up: m*10 + 0.5e-5 < 10 => m < 1 - 0.5e6. + m *= 10 + e-- + case am >= 10: + m /= 10 + e++ + } + + return fmt.Sprintf("%.6ge%+d", m, e) +} + +func (x complexVal) String() string { return fmt.Sprintf("(%s + %si)", x.re, x.im) } + +func (x unknownVal) ExactString() string { return x.String() } +func (x boolVal) ExactString() string { return x.String() } +func (x *stringVal) ExactString() string { return strconv.Quote(x.string()) } +func (x int64Val) ExactString() string { return x.String() } +func (x intVal) ExactString() string { return x.String() } + +func (x ratVal) ExactString() string { + r := x.val + if r.IsInt() { + return r.Num().String() + } + return r.String() +} + +func (x floatVal) ExactString() string { return x.val.Text('p', 0) } + +func (x complexVal) ExactString() string { + return fmt.Sprintf("(%s + %si)", x.re.ExactString(), x.im.ExactString()) +} + +func (unknownVal) implementsValue() {} +func (boolVal) implementsValue() {} +func (*stringVal) implementsValue() {} +func (int64Val) implementsValue() {} +func (ratVal) implementsValue() {} +func (intVal) implementsValue() {} +func (floatVal) implementsValue() {} +func (complexVal) implementsValue() {} + +func newInt() *big.Int { return new(big.Int) } +func newRat() *big.Rat { return new(big.Rat) } +func newFloat() *big.Float { return new(big.Float).SetPrec(prec) } + +func i64toi(x int64Val) intVal { return intVal{newInt().SetInt64(int64(x))} } +func i64tor(x int64Val) ratVal { return ratVal{newRat().SetInt64(int64(x))} } +func i64tof(x int64Val) floatVal { return floatVal{newFloat().SetInt64(int64(x))} } +func itor(x intVal) ratVal { return ratVal{newRat().SetInt(x.val)} } +func itof(x intVal) floatVal { return floatVal{newFloat().SetInt(x.val)} } +func rtof(x ratVal) floatVal { return floatVal{newFloat().SetRat(x.val)} } +func vtoc(x Value) complexVal { return complexVal{x, int64Val(0)} } + +func makeInt(x *big.Int) Value { + if x.IsInt64() { + return int64Val(x.Int64()) + } + return intVal{x} +} + +func makeRat(x *big.Rat) Value { + a := x.Num() + b := x.Denom() + if smallInt(a) && smallInt(b) { + // ok to remain fraction + return ratVal{x} + } + // components too large => switch to float + return floatVal{newFloat().SetRat(x)} +} + +var floatVal0 = floatVal{newFloat()} + +func makeFloat(x *big.Float) Value { + // convert -0 + if x.Sign() == 0 { + return floatVal0 + } + if x.IsInf() { + return unknownVal{} + } + // No attempt is made to "go back" to ratVal, even if possible, + // to avoid providing the illusion of a mathematically exact + // representation. + return floatVal{x} +} + +func makeComplex(re, im Value) Value { + if re.Kind() == Unknown || im.Kind() == Unknown { + return unknownVal{} + } + return complexVal{re, im} +} + +func makeFloatFromLiteral(lit string) Value { + if f, ok := newFloat().SetString(lit); ok { + if smallFloat(f) { + // ok to use rationals + if f.Sign() == 0 { + // Issue 20228: If the float underflowed to zero, parse just "0". + // Otherwise, lit might contain a value with a large negative exponent, + // such as -6e-1886451601. As a float, that will underflow to 0, + // but it'll take forever to parse as a Rat. + lit = "0" + } + if r, ok := newRat().SetString(lit); ok { + return ratVal{r} + } + } + // otherwise use floats + return makeFloat(f) + } + return nil +} + +// Permit fractions with component sizes up to maxExp +// before switching to using floating-point numbers. +const maxExp = 4 << 10 + +// smallInt reports whether x would lead to "reasonably"-sized fraction +// if converted to a *big.Rat. +func smallInt(x *big.Int) bool { + return x.BitLen() < maxExp +} + +// smallFloat64 reports whether x would lead to "reasonably"-sized fraction +// if converted to a *big.Rat. +func smallFloat64(x float64) bool { + if math.IsInf(x, 0) { + return false + } + _, e := math.Frexp(x) + return -maxExp < e && e < maxExp +} + +// smallFloat reports whether x would lead to "reasonably"-sized fraction +// if converted to a *big.Rat. +func smallFloat(x *big.Float) bool { + if x.IsInf() { + return false + } + e := x.MantExp(nil) + return -maxExp < e && e < maxExp +} + +// ---------------------------------------------------------------------------- +// Factories + +// MakeUnknown returns the Unknown value. +func MakeUnknown() Value { return unknownVal{} } + +// MakeBool returns the Bool value for b. +func MakeBool(b bool) Value { return boolVal(b) } + +// MakeString returns the String value for s. +func MakeString(s string) Value { + if s == "" { + return &emptyString // common case + } + return &stringVal{s: s} +} + +var emptyString stringVal + +// MakeInt64 returns the Int value for x. +func MakeInt64(x int64) Value { return int64Val(x) } + +// MakeUint64 returns the Int value for x. +func MakeUint64(x uint64) Value { + if x < 1<<63 { + return int64Val(int64(x)) + } + return intVal{newInt().SetUint64(x)} +} + +// MakeFloat64 returns the Float value for x. +// If x is -0.0, the result is 0.0. +// If x is not finite, the result is an Unknown. +func MakeFloat64(x float64) Value { + if math.IsInf(x, 0) || math.IsNaN(x) { + return unknownVal{} + } + if smallFloat64(x) { + return ratVal{newRat().SetFloat64(x + 0)} // convert -0 to 0 + } + return floatVal{newFloat().SetFloat64(x + 0)} +} + +// MakeFromLiteral returns the corresponding integer, floating-point, +// imaginary, character, or string value for a Go literal string. The +// tok value must be one of token.INT, token.FLOAT, token.IMAG, +// token.CHAR, or token.STRING. The final argument must be zero. +// If the literal string syntax is invalid, the result is an Unknown. +func MakeFromLiteral(lit string, tok token.Token, zero uint) Value { + if zero != 0 { + panic("MakeFromLiteral called with non-zero last argument") + } + + switch tok { + case token.INT: + if x, err := strconv.ParseInt(lit, 0, 64); err == nil { + return int64Val(x) + } + if x, ok := newInt().SetString(lit, 0); ok { + return intVal{x} + } + + case token.FLOAT: + if x := makeFloatFromLiteral(lit); x != nil { + return x + } + + case token.IMAG: + if n := len(lit); n > 0 && lit[n-1] == 'i' { + if im := makeFloatFromLiteral(lit[:n-1]); im != nil { + return makeComplex(int64Val(0), im) + } + } + + case token.CHAR: + if n := len(lit); n >= 2 { + if code, _, _, err := strconv.UnquoteChar(lit[1:n-1], '\''); err == nil { + return MakeInt64(int64(code)) + } + } + + case token.STRING: + if s, err := strconv.Unquote(lit); err == nil { + return MakeString(s) + } + + default: + panic(fmt.Sprintf("%v is not a valid token", tok)) + } + + return unknownVal{} +} + +// ---------------------------------------------------------------------------- +// Accessors +// +// For unknown arguments the result is the zero value for the respective +// accessor type, except for Sign, where the result is 1. + +// BoolVal returns the Go boolean value of x, which must be a Bool or an Unknown. +// If x is Unknown, the result is false. +func BoolVal(x Value) bool { + switch x := x.(type) { + case boolVal: + return bool(x) + case unknownVal: + return false + default: + panic(fmt.Sprintf("%v not a Bool", x)) + } +} + +// StringVal returns the Go string value of x, which must be a String or an Unknown. +// If x is Unknown, the result is "". +func StringVal(x Value) string { + switch x := x.(type) { + case *stringVal: + return x.string() + case unknownVal: + return "" + default: + panic(fmt.Sprintf("%v not a String", x)) + } +} + +// Int64Val returns the Go int64 value of x and whether the result is exact; +// x must be an Int or an Unknown. If the result is not exact, its value is undefined. +// If x is Unknown, the result is (0, false). +func Int64Val(x Value) (int64, bool) { + switch x := x.(type) { + case int64Val: + return int64(x), true + case intVal: + return x.val.Int64(), false // not an int64Val and thus not exact + case unknownVal: + return 0, false + default: + panic(fmt.Sprintf("%v not an Int", x)) + } +} + +// Uint64Val returns the Go uint64 value of x and whether the result is exact; +// x must be an Int or an Unknown. If the result is not exact, its value is undefined. +// If x is Unknown, the result is (0, false). +func Uint64Val(x Value) (uint64, bool) { + switch x := x.(type) { + case int64Val: + return uint64(x), x >= 0 + case intVal: + return x.val.Uint64(), x.val.IsUint64() + case unknownVal: + return 0, false + default: + panic(fmt.Sprintf("%v not an Int", x)) + } +} + +// Float32Val is like Float64Val but for float32 instead of float64. +func Float32Val(x Value) (float32, bool) { + switch x := x.(type) { + case int64Val: + f := float32(x) + return f, int64Val(f) == x + case intVal: + f, acc := newFloat().SetInt(x.val).Float32() + return f, acc == big.Exact + case ratVal: + return x.val.Float32() + case floatVal: + f, acc := x.val.Float32() + return f, acc == big.Exact + case unknownVal: + return 0, false + default: + panic(fmt.Sprintf("%v not a Float", x)) + } +} + +// Float64Val returns the nearest Go float64 value of x and whether the result is exact; +// x must be numeric or an Unknown, but not Complex. For values too small (too close to 0) +// to represent as float64, Float64Val silently underflows to 0. The result sign always +// matches the sign of x, even for 0. +// If x is Unknown, the result is (0, false). +func Float64Val(x Value) (float64, bool) { + switch x := x.(type) { + case int64Val: + f := float64(int64(x)) + return f, int64Val(f) == x + case intVal: + f, acc := newFloat().SetInt(x.val).Float64() + return f, acc == big.Exact + case ratVal: + return x.val.Float64() + case floatVal: + f, acc := x.val.Float64() + return f, acc == big.Exact + case unknownVal: + return 0, false + default: + panic(fmt.Sprintf("%v not a Float", x)) + } +} + +// Val returns the underlying value for a given constant. Since it returns an +// interface, it is up to the caller to type assert the result to the expected +// type. The possible dynamic return types are: +// +// x Kind type of result +// ----------------------------------------- +// Bool bool +// String string +// Int int64 or *big.Int +// Float *big.Float or *big.Rat +// everything else nil +func Val(x Value) any { + switch x := x.(type) { + case boolVal: + return bool(x) + case *stringVal: + return x.string() + case int64Val: + return int64(x) + case intVal: + return x.val + case ratVal: + return x.val + case floatVal: + return x.val + default: + return nil + } +} + +// Make returns the Value for x. +// +// type of x result Kind +// ---------------------------- +// bool Bool +// string String +// int64 Int +// *big.Int Int +// *big.Float Float +// *big.Rat Float +// anything else Unknown +func Make(x any) Value { + switch x := x.(type) { + case bool: + return boolVal(x) + case string: + return &stringVal{s: x} + case int64: + return int64Val(x) + case *big.Int: + return makeInt(x) + case *big.Rat: + return makeRat(x) + case *big.Float: + return makeFloat(x) + default: + return unknownVal{} + } +} + +// BitLen returns the number of bits required to represent +// the absolute value x in binary representation; x must be an Int or an Unknown. +// If x is Unknown, the result is 0. +func BitLen(x Value) int { + switch x := x.(type) { + case int64Val: + u := uint64(x) + if x < 0 { + u = uint64(-x) + } + return 64 - bits.LeadingZeros64(u) + case intVal: + return x.val.BitLen() + case unknownVal: + return 0 + default: + panic(fmt.Sprintf("%v not an Int", x)) + } +} + +// Sign returns -1, 0, or 1 depending on whether x < 0, x == 0, or x > 0; +// x must be numeric or Unknown. For complex values x, the sign is 0 if x == 0, +// otherwise it is != 0. If x is Unknown, the result is 1. +func Sign(x Value) int { + switch x := x.(type) { + case int64Val: + switch { + case x < 0: + return -1 + case x > 0: + return 1 + } + return 0 + case intVal: + return x.val.Sign() + case ratVal: + return x.val.Sign() + case floatVal: + return x.val.Sign() + case complexVal: + return Sign(x.re) | Sign(x.im) + case unknownVal: + return 1 // avoid spurious division by zero errors + default: + panic(fmt.Sprintf("%v not numeric", x)) + } +} + +// ---------------------------------------------------------------------------- +// Support for assembling/disassembling numeric values + +const ( + // Compute the size of a Word in bytes. + _m = ^big.Word(0) + _log = _m>>8&1 + _m>>16&1 + _m>>32&1 + wordSize = 1 << _log +) + +// Bytes returns the bytes for the absolute value of x in little- +// endian binary representation; x must be an Int. +func Bytes(x Value) []byte { + var t intVal + switch x := x.(type) { + case int64Val: + t = i64toi(x) + case intVal: + t = x + default: + panic(fmt.Sprintf("%v not an Int", x)) + } + + words := t.val.Bits() + bytes := make([]byte, len(words)*wordSize) + + i := 0 + for _, w := range words { + for j := 0; j < wordSize; j++ { + bytes[i] = byte(w) + w >>= 8 + i++ + } + } + // remove leading 0's + for i > 0 && bytes[i-1] == 0 { + i-- + } + + return bytes[:i] +} + +// MakeFromBytes returns the Int value given the bytes of its little-endian +// binary representation. An empty byte slice argument represents 0. +func MakeFromBytes(bytes []byte) Value { + words := make([]big.Word, (len(bytes)+(wordSize-1))/wordSize) + + i := 0 + var w big.Word + var s uint + for _, b := range bytes { + w |= big.Word(b) << s + if s += 8; s == wordSize*8 { + words[i] = w + i++ + w = 0 + s = 0 + } + } + // store last word + if i < len(words) { + words[i] = w + i++ + } + // remove leading 0's + for i > 0 && words[i-1] == 0 { + i-- + } + + return makeInt(newInt().SetBits(words[:i])) +} + +// Num returns the numerator of x; x must be Int, Float, or Unknown. +// If x is Unknown, or if it is too large or small to represent as a +// fraction, the result is Unknown. Otherwise the result is an Int +// with the same sign as x. +func Num(x Value) Value { + switch x := x.(type) { + case int64Val, intVal: + return x + case ratVal: + return makeInt(x.val.Num()) + case floatVal: + if smallFloat(x.val) { + r, _ := x.val.Rat(nil) + return makeInt(r.Num()) + } + case unknownVal: + break + default: + panic(fmt.Sprintf("%v not Int or Float", x)) + } + return unknownVal{} +} + +// Denom returns the denominator of x; x must be Int, Float, or Unknown. +// If x is Unknown, or if it is too large or small to represent as a +// fraction, the result is Unknown. Otherwise the result is an Int >= 1. +func Denom(x Value) Value { + switch x := x.(type) { + case int64Val, intVal: + return int64Val(1) + case ratVal: + return makeInt(x.val.Denom()) + case floatVal: + if smallFloat(x.val) { + r, _ := x.val.Rat(nil) + return makeInt(r.Denom()) + } + case unknownVal: + break + default: + panic(fmt.Sprintf("%v not Int or Float", x)) + } + return unknownVal{} +} + +// MakeImag returns the Complex value x*i; +// x must be Int, Float, or Unknown. +// If x is Unknown, the result is Unknown. +func MakeImag(x Value) Value { + switch x.(type) { + case unknownVal: + return x + case int64Val, intVal, ratVal, floatVal: + return makeComplex(int64Val(0), x) + default: + panic(fmt.Sprintf("%v not Int or Float", x)) + } +} + +// Real returns the real part of x, which must be a numeric or unknown value. +// If x is Unknown, the result is Unknown. +func Real(x Value) Value { + switch x := x.(type) { + case unknownVal, int64Val, intVal, ratVal, floatVal: + return x + case complexVal: + return x.re + default: + panic(fmt.Sprintf("%v not numeric", x)) + } +} + +// Imag returns the imaginary part of x, which must be a numeric or unknown value. +// If x is Unknown, the result is Unknown. +func Imag(x Value) Value { + switch x := x.(type) { + case unknownVal: + return x + case int64Val, intVal, ratVal, floatVal: + return int64Val(0) + case complexVal: + return x.im + default: + panic(fmt.Sprintf("%v not numeric", x)) + } +} + +// ---------------------------------------------------------------------------- +// Numeric conversions + +// ToInt converts x to an Int value if x is representable as an Int. +// Otherwise it returns an Unknown. +func ToInt(x Value) Value { + switch x := x.(type) { + case int64Val, intVal: + return x + + case ratVal: + if x.val.IsInt() { + return makeInt(x.val.Num()) + } + + case floatVal: + // avoid creation of huge integers + // (Existing tests require permitting exponents of at least 1024; + // allow any value that would also be permissible as a fraction.) + if smallFloat(x.val) { + i := newInt() + if _, acc := x.val.Int(i); acc == big.Exact { + return makeInt(i) + } + + // If we can get an integer by rounding up or down, + // assume x is not an integer because of rounding + // errors in prior computations. + + const delta = 4 // a small number of bits > 0 + var t big.Float + t.SetPrec(prec - delta) + + // try rounding down a little + t.SetMode(big.ToZero) + t.Set(x.val) + if _, acc := t.Int(i); acc == big.Exact { + return makeInt(i) + } + + // try rounding up a little + t.SetMode(big.AwayFromZero) + t.Set(x.val) + if _, acc := t.Int(i); acc == big.Exact { + return makeInt(i) + } + } + + case complexVal: + if re := ToFloat(x); re.Kind() == Float { + return ToInt(re) + } + } + + return unknownVal{} +} + +// ToFloat converts x to a Float value if x is representable as a Float. +// Otherwise it returns an Unknown. +func ToFloat(x Value) Value { + switch x := x.(type) { + case int64Val: + return i64tor(x) // x is always a small int + case intVal: + if smallInt(x.val) { + return itor(x) + } + return itof(x) + case ratVal, floatVal: + return x + case complexVal: + if Sign(x.im) == 0 { + return ToFloat(x.re) + } + } + return unknownVal{} +} + +// ToComplex converts x to a Complex value if x is representable as a Complex. +// Otherwise it returns an Unknown. +func ToComplex(x Value) Value { + switch x := x.(type) { + case int64Val, intVal, ratVal, floatVal: + return vtoc(x) + case complexVal: + return x + } + return unknownVal{} +} + +// ---------------------------------------------------------------------------- +// Operations + +// is32bit reports whether x can be represented using 32 bits. +func is32bit(x int64) bool { + const s = 32 + return -1<<(s-1) <= x && x <= 1<<(s-1)-1 +} + +// is63bit reports whether x can be represented using 63 bits. +func is63bit(x int64) bool { + const s = 63 + return -1<<(s-1) <= x && x <= 1<<(s-1)-1 +} + +// UnaryOp returns the result of the unary expression op y. +// The operation must be defined for the operand. +// If prec > 0 it specifies the ^ (xor) result size in bits. +// If y is Unknown, the result is Unknown. +func UnaryOp(op token.Token, y Value, prec uint) Value { + switch op { + case token.ADD: + switch y.(type) { + case unknownVal, int64Val, intVal, ratVal, floatVal, complexVal: + return y + } + + case token.SUB: + switch y := y.(type) { + case unknownVal: + return y + case int64Val: + if z := -y; z != y { + return z // no overflow + } + return makeInt(newInt().Neg(big.NewInt(int64(y)))) + case intVal: + return makeInt(newInt().Neg(y.val)) + case ratVal: + return makeRat(newRat().Neg(y.val)) + case floatVal: + return makeFloat(newFloat().Neg(y.val)) + case complexVal: + re := UnaryOp(token.SUB, y.re, 0) + im := UnaryOp(token.SUB, y.im, 0) + return makeComplex(re, im) + } + + case token.XOR: + z := newInt() + switch y := y.(type) { + case unknownVal: + return y + case int64Val: + z.Not(big.NewInt(int64(y))) + case intVal: + z.Not(y.val) + default: + goto Error + } + // For unsigned types, the result will be negative and + // thus "too large": We must limit the result precision + // to the type's precision. + if prec > 0 { + z.AndNot(z, newInt().Lsh(big.NewInt(-1), prec)) // z &^= (-1)< oy: + y, x = match0(y, x) + } + return x, y +} + +// match0 must only be called by match. +// Invariant: ord(x) < ord(y) +func match0(x, y Value) (_, _ Value) { + // Prefer to return the original x and y arguments when possible, + // to avoid unnecessary heap allocations. + + switch y.(type) { + case intVal: + switch x1 := x.(type) { + case int64Val: + return i64toi(x1), y + } + case ratVal: + switch x1 := x.(type) { + case int64Val: + return i64tor(x1), y + case intVal: + return itor(x1), y + } + case floatVal: + switch x1 := x.(type) { + case int64Val: + return i64tof(x1), y + case intVal: + return itof(x1), y + case ratVal: + return rtof(x1), y + } + case complexVal: + return vtoc(x), y + } + + // force unknown and invalid values into "x position" in callers of match + // (don't panic here so that callers can provide a better error message) + return x, x +} + +// BinaryOp returns the result of the binary expression x op y. +// The operation must be defined for the operands. If one of the +// operands is Unknown, the result is Unknown. +// BinaryOp doesn't handle comparisons or shifts; use Compare +// or Shift instead. +// +// To force integer division of Int operands, use op == token.QUO_ASSIGN +// instead of token.QUO; the result is guaranteed to be Int in this case. +// Division by zero leads to a run-time panic. +func BinaryOp(x_ Value, op token.Token, y_ Value) Value { + x, y := match(x_, y_) + + switch x := x.(type) { + case unknownVal: + return x + + case boolVal: + y := y.(boolVal) + switch op { + case token.LAND: + return x && y + case token.LOR: + return x || y + } + + case int64Val: + a := int64(x) + b := int64(y.(int64Val)) + var c int64 + switch op { + case token.ADD: + if !is63bit(a) || !is63bit(b) { + return makeInt(newInt().Add(big.NewInt(a), big.NewInt(b))) + } + c = a + b + case token.SUB: + if !is63bit(a) || !is63bit(b) { + return makeInt(newInt().Sub(big.NewInt(a), big.NewInt(b))) + } + c = a - b + case token.MUL: + if !is32bit(a) || !is32bit(b) { + return makeInt(newInt().Mul(big.NewInt(a), big.NewInt(b))) + } + c = a * b + case token.QUO: + return makeRat(big.NewRat(a, b)) + case token.QUO_ASSIGN: // force integer division + c = a / b + case token.REM: + c = a % b + case token.AND: + c = a & b + case token.OR: + c = a | b + case token.XOR: + c = a ^ b + case token.AND_NOT: + c = a &^ b + default: + goto Error + } + return int64Val(c) + + case intVal: + a := x.val + b := y.(intVal).val + c := newInt() + switch op { + case token.ADD: + c.Add(a, b) + case token.SUB: + c.Sub(a, b) + case token.MUL: + c.Mul(a, b) + case token.QUO: + return makeRat(newRat().SetFrac(a, b)) + case token.QUO_ASSIGN: // force integer division + c.Quo(a, b) + case token.REM: + c.Rem(a, b) + case token.AND: + c.And(a, b) + case token.OR: + c.Or(a, b) + case token.XOR: + c.Xor(a, b) + case token.AND_NOT: + c.AndNot(a, b) + default: + goto Error + } + return makeInt(c) + + case ratVal: + a := x.val + b := y.(ratVal).val + c := newRat() + switch op { + case token.ADD: + c.Add(a, b) + case token.SUB: + c.Sub(a, b) + case token.MUL: + c.Mul(a, b) + case token.QUO: + c.Quo(a, b) + default: + goto Error + } + return makeRat(c) + + case floatVal: + a := x.val + b := y.(floatVal).val + c := newFloat() + switch op { + case token.ADD: + c.Add(a, b) + case token.SUB: + c.Sub(a, b) + case token.MUL: + c.Mul(a, b) + case token.QUO: + c.Quo(a, b) + default: + goto Error + } + return makeFloat(c) + + case complexVal: + y := y.(complexVal) + a, b := x.re, x.im + c, d := y.re, y.im + var re, im Value + switch op { + case token.ADD: + // (a+c) + i(b+d) + re = add(a, c) + im = add(b, d) + case token.SUB: + // (a-c) + i(b-d) + re = sub(a, c) + im = sub(b, d) + case token.MUL: + // (ac-bd) + i(bc+ad) + ac := mul(a, c) + bd := mul(b, d) + bc := mul(b, c) + ad := mul(a, d) + re = sub(ac, bd) + im = add(bc, ad) + case token.QUO: + // (ac+bd)/s + i(bc-ad)/s, with s = cc + dd + ac := mul(a, c) + bd := mul(b, d) + bc := mul(b, c) + ad := mul(a, d) + cc := mul(c, c) + dd := mul(d, d) + s := add(cc, dd) + re = add(ac, bd) + re = quo(re, s) + im = sub(bc, ad) + im = quo(im, s) + default: + goto Error + } + return makeComplex(re, im) + + case *stringVal: + if op == token.ADD { + return &stringVal{l: x, r: y.(*stringVal)} + } + } + +Error: + panic(fmt.Sprintf("invalid binary operation %v %s %v", x_, op, y_)) +} + +func add(x, y Value) Value { return BinaryOp(x, token.ADD, y) } +func sub(x, y Value) Value { return BinaryOp(x, token.SUB, y) } +func mul(x, y Value) Value { return BinaryOp(x, token.MUL, y) } +func quo(x, y Value) Value { return BinaryOp(x, token.QUO, y) } + +// Shift returns the result of the shift expression x op s +// with op == token.SHL or token.SHR (<< or >>). x must be +// an Int or an Unknown. If x is Unknown, the result is x. +func Shift(x Value, op token.Token, s uint) Value { + switch x := x.(type) { + case unknownVal: + return x + + case int64Val: + if s == 0 { + return x + } + switch op { + case token.SHL: + z := i64toi(x).val + return makeInt(z.Lsh(z, s)) + case token.SHR: + return x >> s + } + + case intVal: + if s == 0 { + return x + } + z := newInt() + switch op { + case token.SHL: + return makeInt(z.Lsh(x.val, s)) + case token.SHR: + return makeInt(z.Rsh(x.val, s)) + } + } + + panic(fmt.Sprintf("invalid shift %v %s %d", x, op, s)) +} + +func cmpZero(x int, op token.Token) bool { + switch op { + case token.EQL: + return x == 0 + case token.NEQ: + return x != 0 + case token.LSS: + return x < 0 + case token.LEQ: + return x <= 0 + case token.GTR: + return x > 0 + case token.GEQ: + return x >= 0 + } + panic(fmt.Sprintf("invalid comparison %v %s 0", x, op)) +} + +// Compare returns the result of the comparison x op y. +// The comparison must be defined for the operands. +// If one of the operands is Unknown, the result is +// false. +func Compare(x_ Value, op token.Token, y_ Value) bool { + x, y := match(x_, y_) + + switch x := x.(type) { + case unknownVal: + return false + + case boolVal: + y := y.(boolVal) + switch op { + case token.EQL: + return x == y + case token.NEQ: + return x != y + } + + case int64Val: + y := y.(int64Val) + switch op { + case token.EQL: + return x == y + case token.NEQ: + return x != y + case token.LSS: + return x < y + case token.LEQ: + return x <= y + case token.GTR: + return x > y + case token.GEQ: + return x >= y + } + + case intVal: + return cmpZero(x.val.Cmp(y.(intVal).val), op) + + case ratVal: + return cmpZero(x.val.Cmp(y.(ratVal).val), op) + + case floatVal: + return cmpZero(x.val.Cmp(y.(floatVal).val), op) + + case complexVal: + y := y.(complexVal) + re := Compare(x.re, token.EQL, y.re) + im := Compare(x.im, token.EQL, y.im) + switch op { + case token.EQL: + return re && im + case token.NEQ: + return !re || !im + } + + case *stringVal: + xs := x.string() + ys := y.(*stringVal).string() + switch op { + case token.EQL: + return xs == ys + case token.NEQ: + return xs != ys + case token.LSS: + return xs < ys + case token.LEQ: + return xs <= ys + case token.GTR: + return xs > ys + case token.GEQ: + return xs >= ys + } + } + + panic(fmt.Sprintf("invalid comparison %v %s %v", x_, op, y_)) +} diff --git a/src/go/constant/value_test.go b/src/go/constant/value_test.go new file mode 100644 index 0000000..e41315e --- /dev/null +++ b/src/go/constant/value_test.go @@ -0,0 +1,729 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package constant + +import ( + "fmt" + "go/token" + "math" + "math/big" + "strings" + "testing" +) + +var intTests = []string{ + // 0-octals + `0_123 = 0123`, + `0123_456 = 0123456`, + + // decimals + `1_234 = 1234`, + `1_234_567 = 1234567`, + + // hexadecimals + `0X_0 = 0`, + `0X_1234 = 0x1234`, + `0X_CAFE_f00d = 0xcafef00d`, + + // octals + `0o0 = 0`, + `0o1234 = 01234`, + `0o01234567 = 01234567`, + + `0O0 = 0`, + `0O1234 = 01234`, + `0O01234567 = 01234567`, + + `0o_0 = 0`, + `0o_1234 = 01234`, + `0o0123_4567 = 01234567`, + + `0O_0 = 0`, + `0O_1234 = 01234`, + `0O0123_4567 = 01234567`, + + // binaries + `0b0 = 0`, + `0b1011 = 0xb`, + `0b00101101 = 0x2d`, + + `0B0 = 0`, + `0B1011 = 0xb`, + `0B00101101 = 0x2d`, + + `0b_0 = 0`, + `0b10_11 = 0xb`, + `0b_0010_1101 = 0x2d`, +} + +// The RHS operand may be a floating-point quotient n/d of two integer values n and d. +var floatTests = []string{ + // decimal floats + `1_2_3. = 123.`, + `0_123. = 123.`, + + `0_0e0 = 0.`, + `1_2_3e0 = 123.`, + `0_123e0 = 123.`, + + `0e-0_0 = 0.`, + `1_2_3E+0 = 123.`, + `0123E1_2_3 = 123e123`, + + `0.e+1 = 0.`, + `123.E-1_0 = 123e-10`, + `01_23.e123 = 123e123`, + + `.0e-1 = .0`, + `.123E+10 = .123e10`, + `.0123E123 = .0123e123`, + + `1_2_3.123 = 123.123`, + `0123.01_23 = 123.0123`, + + `1e-1000000000 = 0`, + `1e+1000000000 = ?`, + `6e5518446744 = ?`, + `-6e5518446744 = ?`, + + // hexadecimal floats + `0x0.p+0 = 0.`, + `0Xdeadcafe.p-10 = 0xdeadcafe/1024`, + `0x1234.P84 = 0x1234000000000000000000000`, + + `0x.1p-0 = 1/16`, + `0X.deadcafep4 = 0xdeadcafe/0x10000000`, + `0x.1234P+12 = 0x1234/0x10`, + + `0x0p0 = 0.`, + `0Xdeadcafep+1 = 0x1bd5b95fc`, + `0x1234P-10 = 0x1234/1024`, + + `0x0.0p0 = 0.`, + `0Xdead.cafep+1 = 0x1bd5b95fc/0x10000`, + `0x12.34P-10 = 0x1234/0x40000`, + + `0Xdead_cafep+1 = 0xdeadcafep+1`, + `0x_1234P-10 = 0x1234p-10`, + + `0X_dead_cafe.p-10 = 0xdeadcafe.p-10`, + `0x12_34.P1_2_3 = 0x1234.p123`, +} + +var imagTests = []string{ + `1_234i = 1234i`, + `1_234_567i = 1234567i`, + + `0.i = 0i`, + `123.i = 123i`, + `0123.i = 123i`, + + `0.e+1i = 0i`, + `123.E-1_0i = 123e-10i`, + `01_23.e123i = 123e123i`, + + `1e-1000000000i = 0i`, + `1e+1000000000i = ?`, + `6e5518446744i = ?`, + `-6e5518446744i = ?`, +} + +func testNumbers(t *testing.T, kind token.Token, tests []string) { + for _, test := range tests { + a := strings.Split(test, " = ") + if len(a) != 2 { + t.Errorf("invalid test case: %s", test) + continue + } + + x := MakeFromLiteral(a[0], kind, 0) + var y Value + if a[1] == "?" { + y = MakeUnknown() + } else { + if ns, ds, ok := strings.Cut(a[1], "/"); ok && kind == token.FLOAT { + n := MakeFromLiteral(ns, token.INT, 0) + d := MakeFromLiteral(ds, token.INT, 0) + y = BinaryOp(n, token.QUO, d) + } else { + y = MakeFromLiteral(a[1], kind, 0) + } + if y.Kind() == Unknown { + panic(fmt.Sprintf("invalid test case: %s %d", test, y.Kind())) + } + } + + xk := x.Kind() + yk := y.Kind() + if xk != yk { + t.Errorf("%s: got kind %d != %d", test, xk, yk) + continue + } + + if yk == Unknown { + continue + } + + if !Compare(x, token.EQL, y) { + t.Errorf("%s: %s != %s", test, x, y) + } + } +} + +// TestNumbers verifies that differently written literals +// representing the same number do have the same value. +func TestNumbers(t *testing.T) { + testNumbers(t, token.INT, intTests) + testNumbers(t, token.FLOAT, floatTests) + testNumbers(t, token.IMAG, imagTests) +} + +var opTests = []string{ + // unary operations + `+ 0 = 0`, + `+ ? = ?`, + `- 1 = -1`, + `- ? = ?`, + `^ 0 = -1`, + `^ ? = ?`, + + `! true = false`, + `! false = true`, + `! ? = ?`, + + // etc. + + // binary operations + `"" + "" = ""`, + `"foo" + "" = "foo"`, + `"" + "bar" = "bar"`, + `"foo" + "bar" = "foobar"`, + + `0 + 0 = 0`, + `0 + 0.1 = 0.1`, + `0 + 0.1i = 0.1i`, + `0.1 + 0.9 = 1`, + `1e100 + 1e100 = 2e100`, + `? + 0 = ?`, + `0 + ? = ?`, + + `0 - 0 = 0`, + `0 - 0.1 = -0.1`, + `0 - 0.1i = -0.1i`, + `1e100 - 1e100 = 0`, + `? - 0 = ?`, + `0 - ? = ?`, + + `0 * 0 = 0`, + `1 * 0.1 = 0.1`, + `1 * 0.1i = 0.1i`, + `1i * 1i = -1`, + `? * 0 = ?`, + `0 * ? = ?`, + `0 * 1e+1000000000 = ?`, + + `0 / 0 = "division_by_zero"`, + `10 / 2 = 5`, + `5 / 3 = 5/3`, + `5i / 3i = 5/3`, + `? / 0 = ?`, + `0 / ? = ?`, + `0 * 1e+1000000000i = ?`, + + `0 % 0 = "runtime_error:_integer_divide_by_zero"`, // TODO(gri) should be the same as for / + `10 % 3 = 1`, + `? % 0 = ?`, + `0 % ? = ?`, + + `0 & 0 = 0`, + `12345 & 0 = 0`, + `0xff & 0xf = 0xf`, + `? & 0 = ?`, + `0 & ? = ?`, + + `0 | 0 = 0`, + `12345 | 0 = 12345`, + `0xb | 0xa0 = 0xab`, + `? | 0 = ?`, + `0 | ? = ?`, + + `0 ^ 0 = 0`, + `1 ^ -1 = -2`, + `? ^ 0 = ?`, + `0 ^ ? = ?`, + + `0 &^ 0 = 0`, + `0xf &^ 1 = 0xe`, + `1 &^ 0xf = 0`, + // etc. + + // shifts + `0 << 0 = 0`, + `1 << 10 = 1024`, + `0 >> 0 = 0`, + `1024 >> 10 == 1`, + `? << 0 == ?`, + `? >> 10 == ?`, + // etc. + + // comparisons + `false == false = true`, + `false == true = false`, + `true == false = false`, + `true == true = true`, + + `false != false = false`, + `false != true = true`, + `true != false = true`, + `true != true = false`, + + `"foo" == "bar" = false`, + `"foo" != "bar" = true`, + `"foo" < "bar" = false`, + `"foo" <= "bar" = false`, + `"foo" > "bar" = true`, + `"foo" >= "bar" = true`, + + `0 == 0 = true`, + `0 != 0 = false`, + `0 < 10 = true`, + `10 <= 10 = true`, + `0 > 10 = false`, + `10 >= 10 = true`, + + `1/123456789 == 1/123456789 == true`, + `1/123456789 != 1/123456789 == false`, + `1/123456789 < 1/123456788 == true`, + `1/123456788 <= 1/123456789 == false`, + `0.11 > 0.11 = false`, + `0.11 >= 0.11 = true`, + + `? == 0 = false`, + `? != 0 = false`, + `? < 10 = false`, + `? <= 10 = false`, + `? > 10 = false`, + `? >= 10 = false`, + + `0 == ? = false`, + `0 != ? = false`, + `0 < ? = false`, + `10 <= ? = false`, + `0 > ? = false`, + `10 >= ? = false`, + + // etc. +} + +func TestOps(t *testing.T) { + for _, test := range opTests { + a := strings.Split(test, " ") + i := 0 // operator index + + var x, x0 Value + switch len(a) { + case 4: + // unary operation + case 5: + // binary operation + x, x0 = val(a[0]), val(a[0]) + i = 1 + default: + t.Errorf("invalid test case: %s", test) + continue + } + + op, ok := optab[a[i]] + if !ok { + panic("missing optab entry for " + a[i]) + } + + y, y0 := val(a[i+1]), val(a[i+1]) + + got := doOp(x, op, y) + want := val(a[i+3]) + if !eql(got, want) { + t.Errorf("%s: got %s; want %s", test, got, want) + continue + } + + if x0 != nil && !eql(x, x0) { + t.Errorf("%s: x changed to %s", test, x) + continue + } + + if !eql(y, y0) { + t.Errorf("%s: y changed to %s", test, y) + continue + } + } +} + +func eql(x, y Value) bool { + _, ux := x.(unknownVal) + _, uy := y.(unknownVal) + if ux || uy { + return ux == uy + } + return Compare(x, token.EQL, y) +} + +// ---------------------------------------------------------------------------- +// String tests + +var xxx = strings.Repeat("x", 68) +var issue14262 = `"ŲØŁ…ŁˆŲ¬ŲØ Ų§Ł„Ų“Ų±ŁˆŲ· Ų§Ł„ŲŖŲ§Ł„ŁŠŲ© Ł†Ų³ŲØ Ų§Ł„Ł…ŲµŁ†Ł ā€” ŁŠŲ¬ŲØ Ų¹Ł„ŁŠŁƒ Ų£Ł† ŲŖŁ†Ų³ŲØ Ų§Ł„Ų¹Ł…Ł„ ŲØŲ§Ł„Ų·Ų±ŁŠŁ‚Ų© Ų§Ł„ŲŖŁŠ ŲŖŲ­ŲÆŲÆŁ‡Ų§ Ų§Ł„Ł…Ų¤Ł„Ł Ų£Łˆ Ų§Ł„Ł…Ų±Ų®Ųµ (ŁˆŁ„ŁƒŁ† Ł„ŁŠŲ³ ŲØŲ£ŁŠ Ų­Ų§Ł„ Ł…Ł† Ų§Ł„Ų£Ų­ŁˆŲ§Ł„ Ų£Ł† ŲŖŁˆŲ­ŁŠ ŁˆŲŖŁ‚ŲŖŲ±Ų­ ŲØŲŖŲ­ŁˆŁ„ Ų£Łˆ Ų§Ų³ŲŖŲ®ŲÆŲ§Ł…Łƒ Ł„Ł„Ų¹Ł…Ł„). Ų§Ł„Ł…Ų“Ų§Ų±ŁƒŲ© Ų¹Ł„Ł‰ Ł‚ŲÆŁ… Ų§Ł„Ł…Ų³Ų§ŁˆŲ§Ų© ā€” Ų„Ų°Ų§ ŁƒŁ†ŲŖ ŁŠŲ¹ŲÆŁ„ ŲŒ ŁˆŲ§Ł„ŲŖŲŗŁŠŁŠŲ± ŲŒ Ų£Łˆ Ų§Ł„Ų§Ų³ŲŖŁŲ§ŲÆŲ© Ł…Ł† Ł‡Ų°Ų§ Ų§Ł„Ų¹Ł…Ł„ ŲŒ Ł‚ŲÆ ŁŠŁ†ŲŖŲ¬ Ų¹Ł† ŲŖŁˆŲ²ŁŠŲ¹ Ų§Ł„Ų¹Ł…Ł„ Ų„Ł„Ų§ ŁŁŠ ŲøŁ„ ŲŖŲ“Ų§ŲØŁ‡ Ų§Łˆ ŲŖŲ·Ų§ŲØŁ‚ ŁŁ‰ ŁˆŲ§Ų­ŲÆ Ł„Ł‡Ų°Ų§ Ų§Ł„ŲŖŲ±Ų®ŁŠŲµ."` + +var stringTests = []struct { + input, short, exact string +}{ + // Unknown + {"", "unknown", "unknown"}, + {"0x", "unknown", "unknown"}, + {"'", "unknown", "unknown"}, + {"1f0", "unknown", "unknown"}, + {"unknown", "unknown", "unknown"}, + + // Bool + {"true", "true", "true"}, + {"false", "false", "false"}, + + // String + {`""`, `""`, `""`}, + {`"foo"`, `"foo"`, `"foo"`}, + {`"` + xxx + `xx"`, `"` + xxx + `xx"`, `"` + xxx + `xx"`}, + {`"` + xxx + `xxx"`, `"` + xxx + `...`, `"` + xxx + `xxx"`}, + {`"` + xxx + xxx + `xxx"`, `"` + xxx + `...`, `"` + xxx + xxx + `xxx"`}, + {issue14262, `"ŲØŁ…ŁˆŲ¬ŲØ Ų§Ł„Ų“Ų±ŁˆŲ· Ų§Ł„ŲŖŲ§Ł„ŁŠŲ© Ł†Ų³ŲØ Ų§Ł„Ł…ŲµŁ†Ł ā€” ŁŠŲ¬ŲØ Ų¹Ł„ŁŠŁƒ Ų£Ł† ŲŖŁ†Ų³ŲØ Ų§Ł„Ų¹Ł…Ł„ ŲØŲ§Ł„Ų·Ų±ŁŠŁ‚Ų© Ų§Ł„...`, issue14262}, + + // Int + {"0", "0", "0"}, + {"-1", "-1", "-1"}, + {"12345", "12345", "12345"}, + {"-12345678901234567890", "-12345678901234567890", "-12345678901234567890"}, + {"12345678901234567890", "12345678901234567890", "12345678901234567890"}, + + // Float + {"0.", "0", "0"}, + {"-0.0", "0", "0"}, + {"10.0", "10", "10"}, + {"2.1", "2.1", "21/10"}, + {"-2.1", "-2.1", "-21/10"}, + {"1e9999", "1e+9999", "0x.f8d4a9da224650a8cb2959e10d985ad92adbd44c62917e608b1f24c0e1b76b6f61edffeb15c135a4b601637315f7662f325f82325422b244286a07663c9415d2p+33216"}, + {"1e-9999", "1e-9999", "0x.83b01ba6d8c0425eec1b21e96f7742d63c2653ed0a024cf8a2f9686df578d7b07d7a83d84df6a2ec70a921d1f6cd5574893a7eda4d28ee719e13a5dce2700759p-33215"}, + {"2.71828182845904523536028747135266249775724709369995957496696763", "2.71828", "271828182845904523536028747135266249775724709369995957496696763/100000000000000000000000000000000000000000000000000000000000000"}, + {"0e9999999999", "0", "0"}, // issue #16176 + {"-6e-1886451601", "0", "0"}, // issue #20228 + + // Complex + {"0i", "(0 + 0i)", "(0 + 0i)"}, + {"-0i", "(0 + 0i)", "(0 + 0i)"}, + {"10i", "(0 + 10i)", "(0 + 10i)"}, + {"-10i", "(0 + -10i)", "(0 + -10i)"}, + {"1e9999i", "(0 + 1e+9999i)", "(0 + 0x.f8d4a9da224650a8cb2959e10d985ad92adbd44c62917e608b1f24c0e1b76b6f61edffeb15c135a4b601637315f7662f325f82325422b244286a07663c9415d2p+33216i)"}, +} + +func TestString(t *testing.T) { + for _, test := range stringTests { + x := val(test.input) + if got := x.String(); got != test.short { + t.Errorf("%s: got %q; want %q as short string", test.input, got, test.short) + } + if got := x.ExactString(); got != test.exact { + t.Errorf("%s: got %q; want %q as exact string", test.input, got, test.exact) + } + } +} + +// ---------------------------------------------------------------------------- +// Support functions + +func val(lit string) Value { + if len(lit) == 0 { + return MakeUnknown() + } + + switch lit { + case "?": + return MakeUnknown() + case "true": + return MakeBool(true) + case "false": + return MakeBool(false) + } + + if as, bs, ok := strings.Cut(lit, "/"); ok { + // assume fraction + a := MakeFromLiteral(as, token.INT, 0) + b := MakeFromLiteral(bs, token.INT, 0) + return BinaryOp(a, token.QUO, b) + } + + tok := token.INT + switch first, last := lit[0], lit[len(lit)-1]; { + case first == '"' || first == '`': + tok = token.STRING + lit = strings.ReplaceAll(lit, "_", " ") + case first == '\'': + tok = token.CHAR + case last == 'i': + tok = token.IMAG + default: + if !strings.HasPrefix(lit, "0x") && strings.ContainsAny(lit, "./Ee") { + tok = token.FLOAT + } + } + + return MakeFromLiteral(lit, tok, 0) +} + +var optab = map[string]token.Token{ + "!": token.NOT, + + "+": token.ADD, + "-": token.SUB, + "*": token.MUL, + "/": token.QUO, + "%": token.REM, + + "<<": token.SHL, + ">>": token.SHR, + + "&": token.AND, + "|": token.OR, + "^": token.XOR, + "&^": token.AND_NOT, + + "==": token.EQL, + "!=": token.NEQ, + "<": token.LSS, + "<=": token.LEQ, + ">": token.GTR, + ">=": token.GEQ, +} + +func panicHandler(v *Value) { + switch p := recover().(type) { + case nil: + // nothing to do + case string: + *v = MakeString(p) + case error: + *v = MakeString(p.Error()) + default: + panic(p) + } +} + +func doOp(x Value, op token.Token, y Value) (z Value) { + defer panicHandler(&z) + + if x == nil { + return UnaryOp(op, y, 0) + } + + switch op { + case token.EQL, token.NEQ, token.LSS, token.LEQ, token.GTR, token.GEQ: + return MakeBool(Compare(x, op, y)) + case token.SHL, token.SHR: + s, _ := Int64Val(y) + return Shift(x, op, uint(s)) + default: + return BinaryOp(x, op, y) + } +} + +// ---------------------------------------------------------------------------- +// Other tests + +var fracTests = []string{ + "0", + "1", + "-1", + "1.2", + "-0.991", + "2.718281828", + "3.14159265358979323e-10", + "1e100", + "1e1000", +} + +func TestFractions(t *testing.T) { + for _, test := range fracTests { + x := val(test) + // We don't check the actual numerator and denominator because they + // are unlikely to be 100% correct due to floatVal rounding errors. + // Instead, we compute the fraction again and compare the rounded + // result. + q := BinaryOp(Num(x), token.QUO, Denom(x)) + got := q.String() + want := x.String() + if got != want { + t.Errorf("%s: got quotient %s, want %s", x, got, want) + } + } +} + +var bytesTests = []string{ + "0", + "1", + "123456789", + "123456789012345678901234567890123456789012345678901234567890", +} + +func TestBytes(t *testing.T) { + for _, test := range bytesTests { + x := val(test) + bytes := Bytes(x) + + // special case 0 + if Sign(x) == 0 && len(bytes) != 0 { + t.Errorf("%s: got %v; want empty byte slice", test, bytes) + } + + if n := len(bytes); n > 0 && bytes[n-1] == 0 { + t.Errorf("%s: got %v; want no leading 0 byte", test, bytes) + } + + if got := MakeFromBytes(bytes); !eql(got, x) { + t.Errorf("%s: got %s; want %s (bytes = %v)", test, got, x, bytes) + } + } +} + +func TestUnknown(t *testing.T) { + u := MakeUnknown() + var values = []Value{ + u, + MakeBool(false), // token.ADD ok below, operation is never considered + MakeString(""), + MakeInt64(1), + MakeFromLiteral("''", token.CHAR, 0), + MakeFromLiteral("-1234567890123456789012345678901234567890", token.INT, 0), + MakeFloat64(1.2), + MakeImag(MakeFloat64(1.2)), + } + for _, val := range values { + x, y := val, u + for i := range [2]int{} { + if i == 1 { + x, y = y, x + } + if got := BinaryOp(x, token.ADD, y); got.Kind() != Unknown { + t.Errorf("%s + %s: got %s; want %s", x, y, got, u) + } + if got := Compare(x, token.EQL, y); got { + t.Errorf("%s == %s: got true; want false", x, y) + } + } + } +} + +func TestMakeFloat64(t *testing.T) { + var zero float64 + for _, arg := range []float64{ + -math.MaxFloat32, + -10, + -0.5, + -zero, + zero, + 1, + 10, + 123456789.87654321e-23, + 1e10, + math.MaxFloat64, + } { + val := MakeFloat64(arg) + if val.Kind() != Float { + t.Errorf("%v: got kind = %d; want %d", arg, val.Kind(), Float) + } + + // -0.0 is mapped to 0.0 + got, exact := Float64Val(val) + if !exact || math.Float64bits(got) != math.Float64bits(arg+0) { + t.Errorf("%v: got %v (exact = %v)", arg, got, exact) + } + } + + // infinity + for sign := range []int{-1, 1} { + arg := math.Inf(sign) + val := MakeFloat64(arg) + if val.Kind() != Unknown { + t.Errorf("%v: got kind = %d; want %d", arg, val.Kind(), Unknown) + } + } +} + +type makeTestCase struct { + kind Kind + arg, want any +} + +func dup(k Kind, x any) makeTestCase { return makeTestCase{k, x, x} } + +func TestMake(t *testing.T) { + for _, test := range []makeTestCase{ + {Bool, false, false}, + {String, "hello", "hello"}, + + {Int, int64(1), int64(1)}, + {Int, big.NewInt(10), int64(10)}, + {Int, new(big.Int).Lsh(big.NewInt(1), 62), int64(1 << 62)}, + dup(Int, new(big.Int).Lsh(big.NewInt(1), 63)), + + {Float, big.NewFloat(0), floatVal0.val}, + dup(Float, big.NewFloat(2.0)), + dup(Float, big.NewRat(1, 3)), + } { + val := Make(test.arg) + got := Val(val) + if val.Kind() != test.kind || got != test.want { + t.Errorf("got %v (%T, kind = %d); want %v (%T, kind = %d)", + got, got, val.Kind(), test.want, test.want, test.kind) + } + } +} + +func BenchmarkStringAdd(b *testing.B) { + for size := 1; size <= 65536; size *= 4 { + b.Run(fmt.Sprint(size), func(b *testing.B) { + b.ReportAllocs() + n := int64(0) + for i := 0; i < b.N; i++ { + x := MakeString(strings.Repeat("x", 100)) + y := x + for j := 0; j < size-1; j++ { + y = BinaryOp(y, token.ADD, x) + } + n += int64(len(StringVal(y))) + } + if n != int64(b.N)*int64(size)*100 { + b.Fatalf("bad string %d != %d", n, int64(b.N)*int64(size)*100) + } + }) + } +} + +var bitLenTests = []struct { + val int64 + want int +}{ + {0, 0}, + {1, 1}, + {-16, 5}, + {1 << 61, 62}, + {1 << 62, 63}, + {-1 << 62, 63}, + {-1 << 63, 64}, +} + +func TestBitLen(t *testing.T) { + for _, test := range bitLenTests { + if got := BitLen(MakeInt64(test.val)); got != test.want { + t.Errorf("%v: got %v, want %v", test.val, got, test.want) + } + } +} diff --git a/src/go/doc/Makefile b/src/go/doc/Makefile new file mode 100644 index 0000000..ca4948f --- /dev/null +++ b/src/go/doc/Makefile @@ -0,0 +1,7 @@ +# Copyright 2009 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Script to test heading detection heuristic +headscan: headscan.go + go build headscan.go diff --git a/src/go/doc/comment.go b/src/go/doc/comment.go new file mode 100644 index 0000000..4f73664 --- /dev/null +++ b/src/go/doc/comment.go @@ -0,0 +1,71 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "go/doc/comment" + "io" +) + +// ToHTML converts comment text to formatted HTML. +// +// Deprecated: ToHTML cannot identify documentation links +// in the doc comment, because they depend on knowing what +// package the text came from, which is not included in this API. +// +// Given the *[doc.Package] p where text was found, +// ToHTML(w, text, nil) can be replaced by: +// +// w.Write(p.HTML(text)) +// +// which is in turn shorthand for: +// +// w.Write(p.Printer().HTML(p.Parser().Parse(text))) +// +// If words may be non-nil, the longer replacement is: +// +// parser := p.Parser() +// parser.Words = words +// w.Write(p.Printer().HTML(parser.Parse(d))) +func ToHTML(w io.Writer, text string, words map[string]string) { + p := new(Package).Parser() + p.Words = words + d := p.Parse(text) + pr := new(comment.Printer) + w.Write(pr.HTML(d)) +} + +// ToText converts comment text to formatted text. +// +// Deprecated: ToText cannot identify documentation links +// in the doc comment, because they depend on knowing what +// package the text came from, which is not included in this API. +// +// Given the *[doc.Package] p where text was found, +// ToText(w, text, "", "\t", 80) can be replaced by: +// +// w.Write(p.Text(text)) +// +// In the general case, ToText(w, text, prefix, codePrefix, width) +// can be replaced by: +// +// d := p.Parser().Parse(text) +// pr := p.Printer() +// pr.TextPrefix = prefix +// pr.TextCodePrefix = codePrefix +// pr.TextWidth = width +// w.Write(pr.Text(d)) +// +// See the documentation for [Package.Text] and [comment.Printer.Text] +// for more details. +func ToText(w io.Writer, text string, prefix, codePrefix string, width int) { + d := new(Package).Parser().Parse(text) + pr := &comment.Printer{ + TextPrefix: prefix, + TextCodePrefix: codePrefix, + TextWidth: width, + } + w.Write(pr.Text(d)) +} diff --git a/src/go/doc/comment/doc.go b/src/go/doc/comment/doc.go new file mode 100644 index 0000000..45a476a --- /dev/null +++ b/src/go/doc/comment/doc.go @@ -0,0 +1,36 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package comment implements parsing and reformatting of Go doc comments, +(documentation comments), which are comments that immediately precede +a top-level declaration of a package, const, func, type, or var. + +Go doc comment syntax is a simplified subset of Markdown that supports +links, headings, paragraphs, lists (without nesting), and preformatted text blocks. +The details of the syntax are documented at https://go.dev/doc/comment. + +To parse the text associated with a doc comment (after removing comment markers), +use a [Parser]: + + var p comment.Parser + doc := p.Parse(text) + +The result is a [*Doc]. +To reformat it as a doc comment, HTML, Markdown, or plain text, +use a [Printer]: + + var pr comment.Printer + os.Stdout.Write(pr.Text(doc)) + +The [Parser] and [Printer] types are structs whose fields can be +modified to customize the operations. +For details, see the documentation for those types. + +Use cases that need additional control over reformatting can +implement their own logic by inspecting the parsed syntax itself. +See the documentation for [Doc], [Block], [Text] for an overview +and links to additional types. +*/ +package comment diff --git a/src/go/doc/comment/html.go b/src/go/doc/comment/html.go new file mode 100644 index 0000000..bc076f6 --- /dev/null +++ b/src/go/doc/comment/html.go @@ -0,0 +1,169 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package comment + +import ( + "bytes" + "fmt" + "strconv" +) + +// An htmlPrinter holds the state needed for printing a Doc as HTML. +type htmlPrinter struct { + *Printer + tight bool +} + +// HTML returns an HTML formatting of the Doc. +// See the [Printer] documentation for ways to customize the HTML output. +func (p *Printer) HTML(d *Doc) []byte { + hp := &htmlPrinter{Printer: p} + var out bytes.Buffer + for _, x := range d.Content { + hp.block(&out, x) + } + return out.Bytes() +} + +// block prints the block x to out. +func (p *htmlPrinter) block(out *bytes.Buffer, x Block) { + switch x := x.(type) { + default: + fmt.Fprintf(out, "?%T", x) + + case *Paragraph: + if !p.tight { + out.WriteString("

") + } + p.text(out, x.Text) + out.WriteString("\n") + + case *Heading: + out.WriteString("") + p.text(out, x.Text) + out.WriteString("\n") + + case *Code: + out.WriteString("

")
+		p.escape(out, x.Text)
+		out.WriteString("
\n") + + case *List: + kind := "ol>\n" + if x.Items[0].Number == "" { + kind = "ul>\n" + } + out.WriteString("<") + out.WriteString(kind) + next := "1" + for _, item := range x.Items { + out.WriteString("") + p.tight = !x.BlankBetween() + for _, blk := range item.Content { + p.block(out, blk) + } + p.tight = false + } + out.WriteString("= 0; i-- { + if b[i] < '9' { + b[i]++ + return string(b) + } + b[i] = '0' + } + return "1" + string(b) +} + +// text prints the text sequence x to out. +func (p *htmlPrinter) text(out *bytes.Buffer, x []Text) { + for _, t := range x { + switch t := t.(type) { + case Plain: + p.escape(out, string(t)) + case Italic: + out.WriteString("") + p.escape(out, string(t)) + out.WriteString("") + case *Link: + out.WriteString(``) + p.text(out, t.Text) + out.WriteString("") + case *DocLink: + url := p.docLinkURL(t) + if url != "" { + out.WriteString(``) + } + p.text(out, t.Text) + if url != "" { + out.WriteString("") + } + } + } +} + +// escape prints s to out as plain text, +// escaping < & " ' and > to avoid being misinterpreted +// in larger HTML constructs. +func (p *htmlPrinter) escape(out *bytes.Buffer, s string) { + start := 0 + for i := 0; i < len(s); i++ { + switch s[i] { + case '<': + out.WriteString(s[start:i]) + out.WriteString("<") + start = i + 1 + case '&': + out.WriteString(s[start:i]) + out.WriteString("&") + start = i + 1 + case '"': + out.WriteString(s[start:i]) + out.WriteString(""") + start = i + 1 + case '\'': + out.WriteString(s[start:i]) + out.WriteString("'") + start = i + 1 + case '>': + out.WriteString(s[start:i]) + out.WriteString(">") + start = i + 1 + } + } + out.WriteString(s[start:]) +} diff --git a/src/go/doc/comment/markdown.go b/src/go/doc/comment/markdown.go new file mode 100644 index 0000000..d8550f2 --- /dev/null +++ b/src/go/doc/comment/markdown.go @@ -0,0 +1,188 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package comment + +import ( + "bytes" + "fmt" + "strings" +) + +// An mdPrinter holds the state needed for printing a Doc as Markdown. +type mdPrinter struct { + *Printer + headingPrefix string + raw bytes.Buffer +} + +// Markdown returns a Markdown formatting of the Doc. +// See the [Printer] documentation for ways to customize the Markdown output. +func (p *Printer) Markdown(d *Doc) []byte { + mp := &mdPrinter{ + Printer: p, + headingPrefix: strings.Repeat("#", p.headingLevel()) + " ", + } + + var out bytes.Buffer + for i, x := range d.Content { + if i > 0 { + out.WriteByte('\n') + } + mp.block(&out, x) + } + return out.Bytes() +} + +// block prints the block x to out. +func (p *mdPrinter) block(out *bytes.Buffer, x Block) { + switch x := x.(type) { + default: + fmt.Fprintf(out, "?%T", x) + + case *Paragraph: + p.text(out, x.Text) + out.WriteString("\n") + + case *Heading: + out.WriteString(p.headingPrefix) + p.text(out, x.Text) + if id := p.headingID(x); id != "" { + out.WriteString(" {#") + out.WriteString(id) + out.WriteString("}") + } + out.WriteString("\n") + + case *Code: + md := x.Text + for md != "" { + var line string + line, md, _ = strings.Cut(md, "\n") + if line != "" { + out.WriteString("\t") + out.WriteString(line) + } + out.WriteString("\n") + } + + case *List: + loose := x.BlankBetween() + for i, item := range x.Items { + if i > 0 && loose { + out.WriteString("\n") + } + if n := item.Number; n != "" { + out.WriteString(" ") + out.WriteString(n) + out.WriteString(". ") + } else { + out.WriteString(" - ") // SP SP - SP + } + for i, blk := range item.Content { + const fourSpace = " " + if i > 0 { + out.WriteString("\n" + fourSpace) + } + p.text(out, blk.(*Paragraph).Text) + out.WriteString("\n") + } + } + } +} + +// text prints the text sequence x to out. +func (p *mdPrinter) text(out *bytes.Buffer, x []Text) { + p.raw.Reset() + p.rawText(&p.raw, x) + line := bytes.TrimSpace(p.raw.Bytes()) + if len(line) == 0 { + return + } + switch line[0] { + case '+', '-', '*', '#': + // Escape what would be the start of an unordered list or heading. + out.WriteByte('\\') + case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': + i := 1 + for i < len(line) && '0' <= line[i] && line[i] <= '9' { + i++ + } + if i < len(line) && (line[i] == '.' || line[i] == ')') { + // Escape what would be the start of an ordered list. + out.Write(line[:i]) + out.WriteByte('\\') + line = line[i:] + } + } + out.Write(line) +} + +// rawText prints the text sequence x to out, +// without worrying about escaping characters +// that have special meaning at the start of a Markdown line. +func (p *mdPrinter) rawText(out *bytes.Buffer, x []Text) { + for _, t := range x { + switch t := t.(type) { + case Plain: + p.escape(out, string(t)) + case Italic: + out.WriteString("*") + p.escape(out, string(t)) + out.WriteString("*") + case *Link: + out.WriteString("[") + p.rawText(out, t.Text) + out.WriteString("](") + out.WriteString(t.URL) + out.WriteString(")") + case *DocLink: + url := p.docLinkURL(t) + if url != "" { + out.WriteString("[") + } + p.rawText(out, t.Text) + if url != "" { + out.WriteString("](") + url = strings.ReplaceAll(url, "(", "%28") + url = strings.ReplaceAll(url, ")", "%29") + out.WriteString(url) + out.WriteString(")") + } + } + } +} + +// escape prints s to out as plain text, +// escaping special characters to avoid being misinterpreted +// as Markdown markup sequences. +func (p *mdPrinter) escape(out *bytes.Buffer, s string) { + start := 0 + for i := 0; i < len(s); i++ { + switch s[i] { + case '\n': + // Turn all \n into spaces, for a few reasons: + // - Avoid introducing paragraph breaks accidentally. + // - Avoid the need to reindent after the newline. + // - Avoid problems with Markdown renderers treating + // every mid-paragraph newline as a
. + out.WriteString(s[start:i]) + out.WriteByte(' ') + start = i + 1 + continue + case '`', '_', '*', '[', '<', '\\': + // Not all of these need to be escaped all the time, + // but is valid and easy to do so. + // We assume the Markdown is being passed to a + // Markdown renderer, not edited by a person, + // so it's fine to have escapes that are not strictly + // necessary in some cases. + out.WriteString(s[start:i]) + out.WriteByte('\\') + out.WriteByte(s[i]) + start = i + 1 + } + } + out.WriteString(s[start:]) +} diff --git a/src/go/doc/comment/mkstd.sh b/src/go/doc/comment/mkstd.sh new file mode 100755 index 0000000..c9dee8c --- /dev/null +++ b/src/go/doc/comment/mkstd.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2022 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This could be a good use for embed but go/doc/comment +# is built into the bootstrap go command, so it can't use embed. +# Also not using embed lets us emit a string array directly +# and avoid init-time work. + +( +echo "// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated by 'go generate' DO NOT EDIT. +//go:generate ./mkstd.sh + +package comment + +var stdPkgs = []string{" +go list std | grep -v / | sort | sed 's/.*/"&",/' +echo "}" +) | gofmt >std.go.tmp && mv std.go.tmp std.go diff --git a/src/go/doc/comment/old_test.go b/src/go/doc/comment/old_test.go new file mode 100644 index 0000000..944f94d --- /dev/null +++ b/src/go/doc/comment/old_test.go @@ -0,0 +1,80 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// These tests are carried forward from the old go/doc implementation. + +package comment + +import "testing" + +var oldHeadingTests = []struct { + line string + ok bool +}{ + {"Section", true}, + {"A typical usage", true}, + {"Ī”Ī›Īž is Greek", true}, + {"Foo 42", true}, + {"", false}, + {"section", false}, + {"A typical usage:", false}, + {"This code:", false}, + {"Ī“ is Greek", false}, + {"Foo Ā§", false}, + {"Fermat's Last Sentence", true}, + {"Fermat's", true}, + {"'sX", false}, + {"Ted 'Too' Bar", false}, + {"Use n+m", false}, + {"Scanning:", false}, + {"N:M", false}, +} + +func TestIsOldHeading(t *testing.T) { + for _, tt := range oldHeadingTests { + if isOldHeading(tt.line, []string{"Text.", "", tt.line, "", "Text."}, 2) != tt.ok { + t.Errorf("isOldHeading(%q) = %v, want %v", tt.line, !tt.ok, tt.ok) + } + } +} + +var autoURLTests = []struct { + in, out string +}{ + {"", ""}, + {"http://[::1]:8080/foo.txt", "http://[::1]:8080/foo.txt"}, + {"https://www.google.com) after", "https://www.google.com"}, + {"https://www.google.com:30/x/y/z:b::c. After", "https://www.google.com:30/x/y/z:b::c"}, + {"http://www.google.com/path/:;!-/?query=%34b#093124", "http://www.google.com/path/:;!-/?query=%34b#093124"}, + {"http://www.google.com/path/:;!-/?query=%34bar#093124", "http://www.google.com/path/:;!-/?query=%34bar#093124"}, + {"http://www.google.com/index.html! After", "http://www.google.com/index.html"}, + {"http://www.google.com/", "http://www.google.com/"}, + {"https://www.google.com/", "https://www.google.com/"}, + {"http://www.google.com/path.", "http://www.google.com/path"}, + {"http://en.wikipedia.org/wiki/Camellia_(cipher)", "http://en.wikipedia.org/wiki/Camellia_(cipher)"}, + {"http://www.google.com/)", "http://www.google.com/"}, + {"http://gmail.com)", "http://gmail.com"}, + {"http://gmail.com))", "http://gmail.com"}, + {"http://gmail.com ((http://gmail.com)) ()", "http://gmail.com"}, + {"http://example.com/ quux!", "http://example.com/"}, + {"http://example.com/%2f/ /world.", "http://example.com/%2f/"}, + {"http: ipsum //host/path", ""}, + {"javascript://is/not/linked", ""}, + {"http://foo", "http://foo"}, + {"https://www.example.com/person/][Person Name]]", "https://www.example.com/person/"}, + {"http://golang.org/)", "http://golang.org/"}, + {"http://golang.org/hello())", "http://golang.org/hello()"}, + {"http://git.qemu.org/?p=qemu.git;a=blob;f=qapi-schema.json;hb=HEAD", "http://git.qemu.org/?p=qemu.git;a=blob;f=qapi-schema.json;hb=HEAD"}, + {"https://foo.bar/bal/x(])", "https://foo.bar/bal/x"}, // inner ] causes (]) to be cut off from URL + {"http://bar(])", "http://bar"}, // same +} + +func TestAutoURL(t *testing.T) { + for _, tt := range autoURLTests { + url, ok := autoURL(tt.in) + if url != tt.out || ok != (tt.out != "") { + t.Errorf("autoURL(%q) = %q, %v, want %q, %v", tt.in, url, ok, tt.out, tt.out != "") + } + } +} diff --git a/src/go/doc/comment/parse.go b/src/go/doc/comment/parse.go new file mode 100644 index 0000000..62a0f8f --- /dev/null +++ b/src/go/doc/comment/parse.go @@ -0,0 +1,1262 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package comment + +import ( + "sort" + "strings" + "unicode" + "unicode/utf8" +) + +// A Doc is a parsed Go doc comment. +type Doc struct { + // Content is the sequence of content blocks in the comment. + Content []Block + + // Links is the link definitions in the comment. + Links []*LinkDef +} + +// A LinkDef is a single link definition. +type LinkDef struct { + Text string // the link text + URL string // the link URL + Used bool // whether the comment uses the definition +} + +// A Block is block-level content in a doc comment, +// one of [*Code], [*Heading], [*List], or [*Paragraph]. +type Block interface { + block() +} + +// A Heading is a doc comment heading. +type Heading struct { + Text []Text // the heading text +} + +func (*Heading) block() {} + +// A List is a numbered or bullet list. +// Lists are always non-empty: len(Items) > 0. +// In a numbered list, every Items[i].Number is a non-empty string. +// In a bullet list, every Items[i].Number is an empty string. +type List struct { + // Items is the list items. + Items []*ListItem + + // ForceBlankBefore indicates that the list must be + // preceded by a blank line when reformatting the comment, + // overriding the usual conditions. See the BlankBefore method. + // + // The comment parser sets ForceBlankBefore for any list + // that is preceded by a blank line, to make sure + // the blank line is preserved when printing. + ForceBlankBefore bool + + // ForceBlankBetween indicates that list items must be + // separated by blank lines when reformatting the comment, + // overriding the usual conditions. See the BlankBetween method. + // + // The comment parser sets ForceBlankBetween for any list + // that has a blank line between any two of its items, to make sure + // the blank lines are preserved when printing. + ForceBlankBetween bool +} + +func (*List) block() {} + +// BlankBefore reports whether a reformatting of the comment +// should include a blank line before the list. +// The default rule is the same as for [BlankBetween]: +// if the list item content contains any blank lines +// (meaning at least one item has multiple paragraphs) +// then the list itself must be preceded by a blank line. +// A preceding blank line can be forced by setting [List].ForceBlankBefore. +func (l *List) BlankBefore() bool { + return l.ForceBlankBefore || l.BlankBetween() +} + +// BlankBetween reports whether a reformatting of the comment +// should include a blank line between each pair of list items. +// The default rule is that if the list item content contains any blank lines +// (meaning at least one item has multiple paragraphs) +// then list items must themselves be separated by blank lines. +// Blank line separators can be forced by setting [List].ForceBlankBetween. +func (l *List) BlankBetween() bool { + if l.ForceBlankBetween { + return true + } + for _, item := range l.Items { + if len(item.Content) != 1 { + // Unreachable for parsed comments today, + // since the only way to get multiple item.Content + // is multiple paragraphs, which must have been + // separated by a blank line. + return true + } + } + return false +} + +// A ListItem is a single item in a numbered or bullet list. +type ListItem struct { + // Number is a decimal string in a numbered list + // or an empty string in a bullet list. + Number string // "1", "2", ...; "" for bullet list + + // Content is the list content. + // Currently, restrictions in the parser and printer + // require every element of Content to be a *Paragraph. + Content []Block // Content of this item. +} + +// A Paragraph is a paragraph of text. +type Paragraph struct { + Text []Text +} + +func (*Paragraph) block() {} + +// A Code is a preformatted code block. +type Code struct { + // Text is the preformatted text, ending with a newline character. + // It may be multiple lines, each of which ends with a newline character. + // It is never empty, nor does it start or end with a blank line. + Text string +} + +func (*Code) block() {} + +// A Text is text-level content in a doc comment, +// one of [Plain], [Italic], [*Link], or [*DocLink]. +type Text interface { + text() +} + +// A Plain is a string rendered as plain text (not italicized). +type Plain string + +func (Plain) text() {} + +// An Italic is a string rendered as italicized text. +type Italic string + +func (Italic) text() {} + +// A Link is a link to a specific URL. +type Link struct { + Auto bool // is this an automatic (implicit) link of a literal URL? + Text []Text // text of link + URL string // target URL of link +} + +func (*Link) text() {} + +// A DocLink is a link to documentation for a Go package or symbol. +type DocLink struct { + Text []Text // text of link + + // ImportPath, Recv, and Name identify the Go package or symbol + // that is the link target. The potential combinations of + // non-empty fields are: + // - ImportPath: a link to another package + // - ImportPath, Name: a link to a const, func, type, or var in another package + // - ImportPath, Recv, Name: a link to a method in another package + // - Name: a link to a const, func, type, or var in this package + // - Recv, Name: a link to a method in this package + ImportPath string // import path + Recv string // receiver type, without any pointer star, for methods + Name string // const, func, type, var, or method name +} + +func (*DocLink) text() {} + +// A Parser is a doc comment parser. +// The fields in the struct can be filled in before calling Parse +// in order to customize the details of the parsing process. +type Parser struct { + // Words is a map of Go identifier words that + // should be italicized and potentially linked. + // If Words[w] is the empty string, then the word w + // is only italicized. Otherwise it is linked, using + // Words[w] as the link target. + // Words corresponds to the [go/doc.ToHTML] words parameter. + Words map[string]string + + // LookupPackage resolves a package name to an import path. + // + // If LookupPackage(name) returns ok == true, then [name] + // (or [name.Sym] or [name.Sym.Method]) + // is considered a documentation link to importPath's package docs. + // It is valid to return "", true, in which case name is considered + // to refer to the current package. + // + // If LookupPackage(name) returns ok == false, + // then [name] (or [name.Sym] or [name.Sym.Method]) + // will not be considered a documentation link, + // except in the case where name is the full (but single-element) import path + // of a package in the standard library, such as in [math] or [io.Reader]. + // LookupPackage is still called for such names, + // in order to permit references to imports of other packages + // with the same package names. + // + // Setting LookupPackage to nil is equivalent to setting it to + // a function that always returns "", false. + LookupPackage func(name string) (importPath string, ok bool) + + // LookupSym reports whether a symbol name or method name + // exists in the current package. + // + // If LookupSym("", "Name") returns true, then [Name] + // is considered a documentation link for a const, func, type, or var. + // + // Similarly, if LookupSym("Recv", "Name") returns true, + // then [Recv.Name] is considered a documentation link for + // type Recv's method Name. + // + // Setting LookupSym to nil is equivalent to setting it to a function + // that always returns false. + LookupSym func(recv, name string) (ok bool) +} + +// parseDoc is parsing state for a single doc comment. +type parseDoc struct { + *Parser + *Doc + links map[string]*LinkDef + lines []string + lookupSym func(recv, name string) bool +} + +// lookupPkg is called to look up the pkg in [pkg], [pkg.Name], and [pkg.Name.Recv]. +// If pkg has a slash, it is assumed to be the full import path and is returned with ok = true. +// +// Otherwise, pkg is probably a simple package name like "rand" (not "crypto/rand" or "math/rand"). +// d.LookupPackage provides a way for the caller to allow resolving such names with reference +// to the imports in the surrounding package. +// +// There is one collision between these two cases: single-element standard library names +// like "math" are full import paths but don't contain slashes. We let d.LookupPackage have +// the first chance to resolve it, in case there's a different package imported as math, +// and otherwise we refer to a built-in list of single-element standard library package names. +func (d *parseDoc) lookupPkg(pkg string) (importPath string, ok bool) { + if strings.Contains(pkg, "/") { // assume a full import path + if validImportPath(pkg) { + return pkg, true + } + return "", false + } + if d.LookupPackage != nil { + // Give LookupPackage a chance. + if path, ok := d.LookupPackage(pkg); ok { + return path, true + } + } + return DefaultLookupPackage(pkg) +} + +func isStdPkg(path string) bool { + // TODO(rsc): Use sort.Find once we don't have to worry about + // copying this code into older Go environments. + i := sort.Search(len(stdPkgs), func(i int) bool { return stdPkgs[i] >= path }) + return i < len(stdPkgs) && stdPkgs[i] == path +} + +// DefaultLookupPackage is the default package lookup +// function, used when [Parser].LookupPackage is nil. +// It recognizes names of the packages from the standard +// library with single-element import paths, such as math, +// which would otherwise be impossible to name. +// +// Note that the go/doc package provides a more sophisticated +// lookup based on the imports used in the current package. +func DefaultLookupPackage(name string) (importPath string, ok bool) { + if isStdPkg(name) { + return name, true + } + return "", false +} + +// Parse parses the doc comment text and returns the *Doc form. +// Comment markers (/* // and */) in the text must have already been removed. +func (p *Parser) Parse(text string) *Doc { + lines := unindent(strings.Split(text, "\n")) + d := &parseDoc{ + Parser: p, + Doc: new(Doc), + links: make(map[string]*LinkDef), + lines: lines, + lookupSym: func(recv, name string) bool { return false }, + } + if p.LookupSym != nil { + d.lookupSym = p.LookupSym + } + + // First pass: break into block structure and collect known links. + // The text is all recorded as Plain for now. + var prev span + for _, s := range parseSpans(lines) { + var b Block + switch s.kind { + default: + panic("go/doc/comment: internal error: unknown span kind") + case spanList: + b = d.list(lines[s.start:s.end], prev.end < s.start) + case spanCode: + b = d.code(lines[s.start:s.end]) + case spanOldHeading: + b = d.oldHeading(lines[s.start]) + case spanHeading: + b = d.heading(lines[s.start]) + case spanPara: + b = d.paragraph(lines[s.start:s.end]) + } + if b != nil { + d.Content = append(d.Content, b) + } + prev = s + } + + // Second pass: interpret all the Plain text now that we know the links. + for _, b := range d.Content { + switch b := b.(type) { + case *Paragraph: + b.Text = d.parseLinkedText(string(b.Text[0].(Plain))) + case *List: + for _, i := range b.Items { + for _, c := range i.Content { + p := c.(*Paragraph) + p.Text = d.parseLinkedText(string(p.Text[0].(Plain))) + } + } + } + } + + return d.Doc +} + +// A span represents a single span of comment lines (lines[start:end]) +// of an identified kind (code, heading, paragraph, and so on). +type span struct { + start int + end int + kind spanKind +} + +// A spanKind describes the kind of span. +type spanKind int + +const ( + _ spanKind = iota + spanCode + spanHeading + spanList + spanOldHeading + spanPara +) + +func parseSpans(lines []string) []span { + var spans []span + + // The loop may process a line twice: once as unindented + // and again forced indented. So the maximum expected + // number of iterations is 2*len(lines). The repeating logic + // can be subtle, though, and to protect against introduction + // of infinite loops in future changes, we watch to see that + // we are not looping too much. A panic is better than a + // quiet infinite loop. + watchdog := 2 * len(lines) + + i := 0 + forceIndent := 0 +Spans: + for { + // Skip blank lines. + for i < len(lines) && lines[i] == "" { + i++ + } + if i >= len(lines) { + break + } + if watchdog--; watchdog < 0 { + panic("go/doc/comment: internal error: not making progress") + } + + var kind spanKind + start := i + end := i + if i < forceIndent || indented(lines[i]) { + // Indented (or force indented). + // Ends before next unindented. (Blank lines are OK.) + // If this is an unindented list that we are heuristically treating as indented, + // then accept unindented list item lines up to the first blank lines. + // The heuristic is disabled at blank lines to contain its effect + // to non-gofmt'ed sections of the comment. + unindentedListOK := isList(lines[i]) && i < forceIndent + i++ + for i < len(lines) && (lines[i] == "" || i < forceIndent || indented(lines[i]) || (unindentedListOK && isList(lines[i]))) { + if lines[i] == "" { + unindentedListOK = false + } + i++ + } + + // Drop trailing blank lines. + end = i + for end > start && lines[end-1] == "" { + end-- + } + + // If indented lines are followed (without a blank line) + // by an unindented line ending in a brace, + // take that one line too. This fixes the common mistake + // of pasting in something like + // + // func main() { + // fmt.Println("hello, world") + // } + // + // and forgetting to indent it. + // The heuristic will never trigger on a gofmt'ed comment, + // because any gofmt'ed code block or list would be + // followed by a blank line or end of comment. + if end < len(lines) && strings.HasPrefix(lines[end], "}") { + end++ + } + + if isList(lines[start]) { + kind = spanList + } else { + kind = spanCode + } + } else { + // Unindented. Ends at next blank or indented line. + i++ + for i < len(lines) && lines[i] != "" && !indented(lines[i]) { + i++ + } + end = i + + // If unindented lines are followed (without a blank line) + // by an indented line that would start a code block, + // check whether the final unindented lines + // should be left for the indented section. + // This can happen for the common mistakes of + // unindented code or unindented lists. + // The heuristic will never trigger on a gofmt'ed comment, + // because any gofmt'ed code block would have a blank line + // preceding it after the unindented lines. + if i < len(lines) && lines[i] != "" && !isList(lines[i]) { + switch { + case isList(lines[i-1]): + // If the final unindented line looks like a list item, + // this may be the first indented line wrap of + // a mistakenly unindented list. + // Leave all the unindented list items. + forceIndent = end + end-- + for end > start && isList(lines[end-1]) { + end-- + } + + case strings.HasSuffix(lines[i-1], "{") || strings.HasSuffix(lines[i-1], `\`): + // If the final unindented line ended in { or \ + // it is probably the start of a misindented code block. + // Give the user a single line fix. + // Often that's enough; if not, the user can fix the others themselves. + forceIndent = end + end-- + } + + if start == end && forceIndent > start { + i = start + continue Spans + } + } + + // Span is either paragraph or heading. + if end-start == 1 && isHeading(lines[start]) { + kind = spanHeading + } else if end-start == 1 && isOldHeading(lines[start], lines, start) { + kind = spanOldHeading + } else { + kind = spanPara + } + } + + spans = append(spans, span{start, end, kind}) + i = end + } + + return spans +} + +// indented reports whether line is indented +// (starts with a leading space or tab). +func indented(line string) bool { + return line != "" && (line[0] == ' ' || line[0] == '\t') +} + +// unindent removes any common space/tab prefix +// from each line in lines, returning a copy of lines in which +// those prefixes have been trimmed from each line. +// It also replaces any lines containing only spaces with blank lines (empty strings). +func unindent(lines []string) []string { + // Trim leading and trailing blank lines. + for len(lines) > 0 && isBlank(lines[0]) { + lines = lines[1:] + } + for len(lines) > 0 && isBlank(lines[len(lines)-1]) { + lines = lines[:len(lines)-1] + } + if len(lines) == 0 { + return nil + } + + // Compute and remove common indentation. + prefix := leadingSpace(lines[0]) + for _, line := range lines[1:] { + if !isBlank(line) { + prefix = commonPrefix(prefix, leadingSpace(line)) + } + } + + out := make([]string, len(lines)) + for i, line := range lines { + line = strings.TrimPrefix(line, prefix) + if strings.TrimSpace(line) == "" { + line = "" + } + out[i] = line + } + for len(out) > 0 && out[0] == "" { + out = out[1:] + } + for len(out) > 0 && out[len(out)-1] == "" { + out = out[:len(out)-1] + } + return out +} + +// isBlank reports whether s is a blank line. +func isBlank(s string) bool { + return len(s) == 0 || (len(s) == 1 && s[0] == '\n') +} + +// commonPrefix returns the longest common prefix of a and b. +func commonPrefix(a, b string) string { + i := 0 + for i < len(a) && i < len(b) && a[i] == b[i] { + i++ + } + return a[0:i] +} + +// leadingSpace returns the longest prefix of s consisting of spaces and tabs. +func leadingSpace(s string) string { + i := 0 + for i < len(s) && (s[i] == ' ' || s[i] == '\t') { + i++ + } + return s[:i] +} + +// isOldHeading reports whether line is an old-style section heading. +// line is all[off]. +func isOldHeading(line string, all []string, off int) bool { + if off <= 0 || all[off-1] != "" || off+2 >= len(all) || all[off+1] != "" || leadingSpace(all[off+2]) != "" { + return false + } + + line = strings.TrimSpace(line) + + // a heading must start with an uppercase letter + r, _ := utf8.DecodeRuneInString(line) + if !unicode.IsLetter(r) || !unicode.IsUpper(r) { + return false + } + + // it must end in a letter or digit: + r, _ = utf8.DecodeLastRuneInString(line) + if !unicode.IsLetter(r) && !unicode.IsDigit(r) { + return false + } + + // exclude lines with illegal characters. we allow "()," + if strings.ContainsAny(line, ";:!?+*/=[]{}_^Ā°&Ā§~%#@<\">\\") { + return false + } + + // allow "'" for possessive "'s" only + for b := line; ; { + var ok bool + if _, b, ok = strings.Cut(b, "'"); !ok { + break + } + if b != "s" && !strings.HasPrefix(b, "s ") { + return false // ' not followed by s and then end-of-word + } + } + + // allow "." when followed by non-space + for b := line; ; { + var ok bool + if _, b, ok = strings.Cut(b, "."); !ok { + break + } + if b == "" || strings.HasPrefix(b, " ") { + return false // not followed by non-space + } + } + + return true +} + +// oldHeading returns the *Heading for the given old-style section heading line. +func (d *parseDoc) oldHeading(line string) Block { + return &Heading{Text: []Text{Plain(strings.TrimSpace(line))}} +} + +// isHeading reports whether line is a new-style section heading. +func isHeading(line string) bool { + return len(line) >= 2 && + line[0] == '#' && + (line[1] == ' ' || line[1] == '\t') && + strings.TrimSpace(line) != "#" +} + +// heading returns the *Heading for the given new-style section heading line. +func (d *parseDoc) heading(line string) Block { + return &Heading{Text: []Text{Plain(strings.TrimSpace(line[1:]))}} +} + +// code returns a code block built from the lines. +func (d *parseDoc) code(lines []string) *Code { + body := unindent(lines) + body = append(body, "") // to get final \n from Join + return &Code{Text: strings.Join(body, "\n")} +} + +// paragraph returns a paragraph block built from the lines. +// If the lines are link definitions, paragraph adds them to d and returns nil. +func (d *parseDoc) paragraph(lines []string) Block { + // Is this a block of known links? Handle. + var defs []*LinkDef + for _, line := range lines { + def, ok := parseLink(line) + if !ok { + goto NoDefs + } + defs = append(defs, def) + } + for _, def := range defs { + d.Links = append(d.Links, def) + if d.links[def.Text] == nil { + d.links[def.Text] = def + } + } + return nil +NoDefs: + + return &Paragraph{Text: []Text{Plain(strings.Join(lines, "\n"))}} +} + +// parseLink parses a single link definition line: +// +// [text]: url +// +// It returns the link definition and whether the line was well formed. +func parseLink(line string) (*LinkDef, bool) { + if line == "" || line[0] != '[' { + return nil, false + } + i := strings.Index(line, "]:") + if i < 0 || i+3 >= len(line) || (line[i+2] != ' ' && line[i+2] != '\t') { + return nil, false + } + + text := line[1:i] + url := strings.TrimSpace(line[i+3:]) + j := strings.Index(url, "://") + if j < 0 || !isScheme(url[:j]) { + return nil, false + } + + // Line has right form and has valid scheme://. + // That's good enough for us - we are not as picky + // about the characters beyond the :// as we are + // when extracting inline URLs from text. + return &LinkDef{Text: text, URL: url}, true +} + +// list returns a list built from the indented lines, +// using forceBlankBefore as the value of the List's ForceBlankBefore field. +func (d *parseDoc) list(lines []string, forceBlankBefore bool) *List { + num, _, _ := listMarker(lines[0]) + var ( + list *List = &List{ForceBlankBefore: forceBlankBefore} + item *ListItem + text []string + ) + flush := func() { + if item != nil { + if para := d.paragraph(text); para != nil { + item.Content = append(item.Content, para) + } + } + text = nil + } + + for _, line := range lines { + if n, after, ok := listMarker(line); ok && (n != "") == (num != "") { + // start new list item + flush() + + item = &ListItem{Number: n} + list.Items = append(list.Items, item) + line = after + } + line = strings.TrimSpace(line) + if line == "" { + list.ForceBlankBetween = true + flush() + continue + } + text = append(text, strings.TrimSpace(line)) + } + flush() + return list +} + +// listMarker parses the line as beginning with a list marker. +// If it can do that, it returns the numeric marker ("" for a bullet list), +// the rest of the line, and ok == true. +// Otherwise, it returns "", "", false. +func listMarker(line string) (num, rest string, ok bool) { + line = strings.TrimSpace(line) + if line == "" { + return "", "", false + } + + // Can we find a marker? + if r, n := utf8.DecodeRuneInString(line); r == 'ā€¢' || r == '*' || r == '+' || r == '-' { + num, rest = "", line[n:] + } else if '0' <= line[0] && line[0] <= '9' { + n := 1 + for n < len(line) && '0' <= line[n] && line[n] <= '9' { + n++ + } + if n >= len(line) || (line[n] != '.' && line[n] != ')') { + return "", "", false + } + num, rest = line[:n], line[n+1:] + } else { + return "", "", false + } + + if !indented(rest) || strings.TrimSpace(rest) == "" { + return "", "", false + } + + return num, rest, true +} + +// isList reports whether the line is the first line of a list, +// meaning starts with a list marker after any indentation. +// (The caller is responsible for checking the line is indented, as appropriate.) +func isList(line string) bool { + _, _, ok := listMarker(line) + return ok +} + +// parseLinkedText parses text that is allowed to contain explicit links, +// such as [math.Sin] or [Go home page], into a slice of Text items. +// +// A ā€œpkgā€ is only assumed to be a full import path if it starts with +// a domain name (a path element with a dot) or is one of the packages +// from the standard library (ā€œ[os]ā€, ā€œ[encoding/json]ā€, and so on). +// To avoid problems with maps, generics, and array types, doc links +// must be both preceded and followed by punctuation, spaces, tabs, +// or the start or end of a line. An example problem would be treating +// map[ast.Expr]TypeAndValue as containing a link. +func (d *parseDoc) parseLinkedText(text string) []Text { + var out []Text + wrote := 0 + flush := func(i int) { + if wrote < i { + out = d.parseText(out, text[wrote:i], true) + wrote = i + } + } + + start := -1 + var buf []byte + for i := 0; i < len(text); i++ { + c := text[i] + if c == '\n' || c == '\t' { + c = ' ' + } + switch c { + case '[': + start = i + case ']': + if start >= 0 { + if def, ok := d.links[string(buf)]; ok { + def.Used = true + flush(start) + out = append(out, &Link{ + Text: d.parseText(nil, text[start+1:i], false), + URL: def.URL, + }) + wrote = i + 1 + } else if link, ok := d.docLink(text[start+1:i], text[:start], text[i+1:]); ok { + flush(start) + link.Text = d.parseText(nil, text[start+1:i], false) + out = append(out, link) + wrote = i + 1 + } + } + start = -1 + buf = buf[:0] + } + if start >= 0 && i != start { + buf = append(buf, c) + } + } + + flush(len(text)) + return out +} + +// docLink parses text, which was found inside [ ] brackets, +// as a doc link if possible, returning the DocLink and ok == true +// or else nil, false. +// The before and after strings are the text before the [ and after the ] +// on the same line. Doc links must be preceded and followed by +// punctuation, spaces, tabs, or the start or end of a line. +func (d *parseDoc) docLink(text, before, after string) (link *DocLink, ok bool) { + if before != "" { + r, _ := utf8.DecodeLastRuneInString(before) + if !unicode.IsPunct(r) && r != ' ' && r != '\t' && r != '\n' { + return nil, false + } + } + if after != "" { + r, _ := utf8.DecodeRuneInString(after) + if !unicode.IsPunct(r) && r != ' ' && r != '\t' && r != '\n' { + return nil, false + } + } + text = strings.TrimPrefix(text, "*") + pkg, name, ok := splitDocName(text) + var recv string + if ok { + pkg, recv, _ = splitDocName(pkg) + } + if pkg != "" { + if pkg, ok = d.lookupPkg(pkg); !ok { + return nil, false + } + } else { + if ok = d.lookupSym(recv, name); !ok { + return nil, false + } + } + link = &DocLink{ + ImportPath: pkg, + Recv: recv, + Name: name, + } + return link, true +} + +// If text is of the form before.Name, where Name is a capitalized Go identifier, +// then splitDocName returns before, name, true. +// Otherwise it returns text, "", false. +func splitDocName(text string) (before, name string, foundDot bool) { + i := strings.LastIndex(text, ".") + name = text[i+1:] + if !isName(name) { + return text, "", false + } + if i >= 0 { + before = text[:i] + } + return before, name, true +} + +// parseText parses s as text and returns the result of appending +// those parsed Text elements to out. +// parseText does not handle explicit links like [math.Sin] or [Go home page]: +// those are handled by parseLinkedText. +// If autoLink is true, then parseText recognizes URLs and words from d.Words +// and converts those to links as appropriate. +func (d *parseDoc) parseText(out []Text, s string, autoLink bool) []Text { + var w strings.Builder + wrote := 0 + writeUntil := func(i int) { + w.WriteString(s[wrote:i]) + wrote = i + } + flush := func(i int) { + writeUntil(i) + if w.Len() > 0 { + out = append(out, Plain(w.String())) + w.Reset() + } + } + for i := 0; i < len(s); { + t := s[i:] + if autoLink { + if url, ok := autoURL(t); ok { + flush(i) + // Note: The old comment parser would look up the URL in words + // and replace the target with words[URL] if it was non-empty. + // That would allow creating links that display as one URL but + // when clicked go to a different URL. Not sure what the point + // of that is, so we're not doing that lookup here. + out = append(out, &Link{Auto: true, Text: []Text{Plain(url)}, URL: url}) + i += len(url) + wrote = i + continue + } + if id, ok := ident(t); ok { + url, italics := d.Words[id] + if !italics { + i += len(id) + continue + } + flush(i) + if url == "" { + out = append(out, Italic(id)) + } else { + out = append(out, &Link{Auto: true, Text: []Text{Italic(id)}, URL: url}) + } + i += len(id) + wrote = i + continue + } + } + switch { + case strings.HasPrefix(t, "``"): + if len(t) >= 3 && t[2] == '`' { + // Do not convert `` inside ```, in case people are mistakenly writing Markdown. + i += 3 + for i < len(t) && t[i] == '`' { + i++ + } + break + } + writeUntil(i) + w.WriteRune('ā€œ') + i += 2 + wrote = i + case strings.HasPrefix(t, "''"): + writeUntil(i) + w.WriteRune('ā€') + i += 2 + wrote = i + default: + i++ + } + } + flush(len(s)) + return out +} + +// autoURL checks whether s begins with a URL that should be hyperlinked. +// If so, it returns the URL, which is a prefix of s, and ok == true. +// Otherwise it returns "", false. +// The caller should skip over the first len(url) bytes of s +// before further processing. +func autoURL(s string) (url string, ok bool) { + // Find the ://. Fast path to pick off non-URL, + // since we call this at every position in the string. + // The shortest possible URL is ftp://x, 7 bytes. + var i int + switch { + case len(s) < 7: + return "", false + case s[3] == ':': + i = 3 + case s[4] == ':': + i = 4 + case s[5] == ':': + i = 5 + case s[6] == ':': + i = 6 + default: + return "", false + } + if i+3 > len(s) || s[i:i+3] != "://" { + return "", false + } + + // Check valid scheme. + if !isScheme(s[:i]) { + return "", false + } + + // Scan host part. Must have at least one byte, + // and must start and end in non-punctuation. + i += 3 + if i >= len(s) || !isHost(s[i]) || isPunct(s[i]) { + return "", false + } + i++ + end := i + for i < len(s) && isHost(s[i]) { + if !isPunct(s[i]) { + end = i + 1 + } + i++ + } + i = end + + // At this point we are definitely returning a URL (scheme://host). + // We just have to find the longest path we can add to it. + // Heuristics abound. + // We allow parens, braces, and brackets, + // but only if they match (#5043, #22285). + // We allow .,:;?! in the path but not at the end, + // to avoid end-of-sentence punctuation (#18139, #16565). + stk := []byte{} + end = i +Path: + for ; i < len(s); i++ { + if isPunct(s[i]) { + continue + } + if !isPath(s[i]) { + break + } + switch s[i] { + case '(': + stk = append(stk, ')') + case '{': + stk = append(stk, '}') + case '[': + stk = append(stk, ']') + case ')', '}', ']': + if len(stk) == 0 || stk[len(stk)-1] != s[i] { + break Path + } + stk = stk[:len(stk)-1] + } + if len(stk) == 0 { + end = i + 1 + } + } + + return s[:end], true +} + +// isScheme reports whether s is a recognized URL scheme. +// Note that if strings of new length (beyond 3-7) +// are added here, the fast path at the top of autoURL will need updating. +func isScheme(s string) bool { + switch s { + case "file", + "ftp", + "gopher", + "http", + "https", + "mailto", + "nntp": + return true + } + return false +} + +// isHost reports whether c is a byte that can appear in a URL host, +// like www.example.com or user@[::1]:8080 +func isHost(c byte) bool { + // mask is a 128-bit bitmap with 1s for allowed bytes, + // so that the byte c can be tested with a shift and an and. + // If c > 128, then 1<>64)) != 0 +} + +// isPunct reports whether c is a punctuation byte that can appear +// inside a path but not at the end. +func isPunct(c byte) bool { + // mask is a 128-bit bitmap with 1s for allowed bytes, + // so that the byte c can be tested with a shift and an and. + // If c > 128, then 1<>64)) != 0 +} + +// isPath reports whether c is a (non-punctuation) path byte. +func isPath(c byte) bool { + // mask is a 128-bit bitmap with 1s for allowed bytes, + // so that the byte c can be tested with a shift and an and. + // If c > 128, then 1<>64)) != 0 +} + +// isName reports whether s is a capitalized Go identifier (like Name). +func isName(s string) bool { + t, ok := ident(s) + if !ok || t != s { + return false + } + r, _ := utf8.DecodeRuneInString(s) + return unicode.IsUpper(r) +} + +// ident checks whether s begins with a Go identifier. +// If so, it returns the identifier, which is a prefix of s, and ok == true. +// Otherwise it returns "", false. +// The caller should skip over the first len(id) bytes of s +// before further processing. +func ident(s string) (id string, ok bool) { + // Scan [\pL_][\pL_0-9]* + n := 0 + for n < len(s) { + if c := s[n]; c < utf8.RuneSelf { + if isIdentASCII(c) && (n > 0 || c < '0' || c > '9') { + n++ + continue + } + break + } + r, nr := utf8.DecodeRuneInString(s[n:]) + if unicode.IsLetter(r) { + n += nr + continue + } + break + } + return s[:n], n > 0 +} + +// isIdentASCII reports whether c is an ASCII identifier byte. +func isIdentASCII(c byte) bool { + // mask is a 128-bit bitmap with 1s for allowed bytes, + // so that the byte c can be tested with a shift and an and. + // If c > 128, then 1<>64)) != 0 +} + +// validImportPath reports whether path is a valid import path. +// It is a lightly edited copy of golang.org/x/mod/module.CheckImportPath. +func validImportPath(path string) bool { + if !utf8.ValidString(path) { + return false + } + if path == "" { + return false + } + if path[0] == '-' { + return false + } + if strings.Contains(path, "//") { + return false + } + if path[len(path)-1] == '/' { + return false + } + elemStart := 0 + for i, r := range path { + if r == '/' { + if !validImportPathElem(path[elemStart:i]) { + return false + } + elemStart = i + 1 + } + } + return validImportPathElem(path[elemStart:]) +} + +func validImportPathElem(elem string) bool { + if elem == "" || elem[0] == '.' || elem[len(elem)-1] == '.' { + return false + } + for i := 0; i < len(elem); i++ { + if !importPathOK(elem[i]) { + return false + } + } + return true +} + +func importPathOK(c byte) bool { + // mask is a 128-bit bitmap with 1s for allowed bytes, + // so that the byte c can be tested with a shift and an and. + // If c > 128, then 1<>64)) != 0 +} diff --git a/src/go/doc/comment/parse_test.go b/src/go/doc/comment/parse_test.go new file mode 100644 index 0000000..bce733e --- /dev/null +++ b/src/go/doc/comment/parse_test.go @@ -0,0 +1,12 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package comment + +import "testing" + +// See https://golang.org/issue/52353 +func Test52353(t *testing.T) { + ident("š«•ļÆÆ") +} diff --git a/src/go/doc/comment/print.go b/src/go/doc/comment/print.go new file mode 100644 index 0000000..4e9da3d --- /dev/null +++ b/src/go/doc/comment/print.go @@ -0,0 +1,290 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package comment + +import ( + "bytes" + "fmt" + "strings" +) + +// A Printer is a doc comment printer. +// The fields in the struct can be filled in before calling +// any of the printing methods +// in order to customize the details of the printing process. +type Printer struct { + // HeadingLevel is the nesting level used for + // HTML and Markdown headings. + // If HeadingLevel is zero, it defaults to level 3, + // meaning to use

and ###. + HeadingLevel int + + // HeadingID is a function that computes the heading ID + // (anchor tag) to use for the heading h when generating + // HTML and Markdown. If HeadingID returns an empty string, + // then the heading ID is omitted. + // If HeadingID is nil, h.DefaultID is used. + HeadingID func(h *Heading) string + + // DocLinkURL is a function that computes the URL for the given DocLink. + // If DocLinkURL is nil, then link.DefaultURL(p.DocLinkBaseURL) is used. + DocLinkURL func(link *DocLink) string + + // DocLinkBaseURL is used when DocLinkURL is nil, + // passed to [DocLink.DefaultURL] to construct a DocLink's URL. + // See that method's documentation for details. + DocLinkBaseURL string + + // TextPrefix is a prefix to print at the start of every line + // when generating text output using the Text method. + TextPrefix string + + // TextCodePrefix is the prefix to print at the start of each + // preformatted (code block) line when generating text output, + // instead of (not in addition to) TextPrefix. + // If TextCodePrefix is the empty string, it defaults to TextPrefix+"\t". + TextCodePrefix string + + // TextWidth is the maximum width text line to generate, + // measured in Unicode code points, + // excluding TextPrefix and the newline character. + // If TextWidth is zero, it defaults to 80 minus the number of code points in TextPrefix. + // If TextWidth is negative, there is no limit. + TextWidth int +} + +func (p *Printer) headingLevel() int { + if p.HeadingLevel <= 0 { + return 3 + } + return p.HeadingLevel +} + +func (p *Printer) headingID(h *Heading) string { + if p.HeadingID == nil { + return h.DefaultID() + } + return p.HeadingID(h) +} + +func (p *Printer) docLinkURL(link *DocLink) string { + if p.DocLinkURL != nil { + return p.DocLinkURL(link) + } + return link.DefaultURL(p.DocLinkBaseURL) +} + +// DefaultURL constructs and returns the documentation URL for l, +// using baseURL as a prefix for links to other packages. +// +// The possible forms returned by DefaultURL are: +// - baseURL/ImportPath, for a link to another package +// - baseURL/ImportPath#Name, for a link to a const, func, type, or var in another package +// - baseURL/ImportPath#Recv.Name, for a link to a method in another package +// - #Name, for a link to a const, func, type, or var in this package +// - #Recv.Name, for a link to a method in this package +// +// If baseURL ends in a trailing slash, then DefaultURL inserts +// a slash between ImportPath and # in the anchored forms. +// For example, here are some baseURL values and URLs they can generate: +// +// "/pkg/" ā†’ "/pkg/math/#Sqrt" +// "/pkg" ā†’ "/pkg/math#Sqrt" +// "/" ā†’ "/math/#Sqrt" +// "" ā†’ "/math#Sqrt" +func (l *DocLink) DefaultURL(baseURL string) string { + if l.ImportPath != "" { + slash := "" + if strings.HasSuffix(baseURL, "/") { + slash = "/" + } else { + baseURL += "/" + } + switch { + case l.Name == "": + return baseURL + l.ImportPath + slash + case l.Recv != "": + return baseURL + l.ImportPath + slash + "#" + l.Recv + "." + l.Name + default: + return baseURL + l.ImportPath + slash + "#" + l.Name + } + } + if l.Recv != "" { + return "#" + l.Recv + "." + l.Name + } + return "#" + l.Name +} + +// DefaultID returns the default anchor ID for the heading h. +// +// The default anchor ID is constructed by converting every +// rune that is not alphanumeric ASCII to an underscore +// and then adding the prefix ā€œhdr-ā€. +// For example, if the heading text is ā€œGo Doc Commentsā€, +// the default ID is ā€œhdr-Go_Doc_Commentsā€. +func (h *Heading) DefaultID() string { + // Note: The ā€œhdr-ā€ prefix is important to avoid DOM clobbering attacks. + // See https://pkg.go.dev/github.com/google/safehtml#Identifier. + var out strings.Builder + var p textPrinter + p.oneLongLine(&out, h.Text) + s := strings.TrimSpace(out.String()) + if s == "" { + return "" + } + out.Reset() + out.WriteString("hdr-") + for _, r := range s { + if r < 0x80 && isIdentASCII(byte(r)) { + out.WriteByte(byte(r)) + } else { + out.WriteByte('_') + } + } + return out.String() +} + +type commentPrinter struct { + *Printer + headingPrefix string + needDoc map[string]bool +} + +// Comment returns the standard Go formatting of the Doc, +// without any comment markers. +func (p *Printer) Comment(d *Doc) []byte { + cp := &commentPrinter{Printer: p} + var out bytes.Buffer + for i, x := range d.Content { + if i > 0 && blankBefore(x) { + out.WriteString("\n") + } + cp.block(&out, x) + } + + // Print one block containing all the link definitions that were used, + // and then a second block containing all the unused ones. + // This makes it easy to clean up the unused ones: gofmt and + // delete the final block. And it's a nice visual signal without + // affecting the way the comment formats for users. + for i := 0; i < 2; i++ { + used := i == 0 + first := true + for _, def := range d.Links { + if def.Used == used { + if first { + out.WriteString("\n") + first = false + } + out.WriteString("[") + out.WriteString(def.Text) + out.WriteString("]: ") + out.WriteString(def.URL) + out.WriteString("\n") + } + } + } + + return out.Bytes() +} + +// blankBefore reports whether the block x requires a blank line before it. +// All blocks do, except for Lists that return false from x.BlankBefore(). +func blankBefore(x Block) bool { + if x, ok := x.(*List); ok { + return x.BlankBefore() + } + return true +} + +// block prints the block x to out. +func (p *commentPrinter) block(out *bytes.Buffer, x Block) { + switch x := x.(type) { + default: + fmt.Fprintf(out, "?%T", x) + + case *Paragraph: + p.text(out, "", x.Text) + out.WriteString("\n") + + case *Heading: + out.WriteString("# ") + p.text(out, "", x.Text) + out.WriteString("\n") + + case *Code: + md := x.Text + for md != "" { + var line string + line, md, _ = strings.Cut(md, "\n") + if line != "" { + out.WriteString("\t") + out.WriteString(line) + } + out.WriteString("\n") + } + + case *List: + loose := x.BlankBetween() + for i, item := range x.Items { + if i > 0 && loose { + out.WriteString("\n") + } + out.WriteString(" ") + if item.Number == "" { + out.WriteString(" - ") + } else { + out.WriteString(item.Number) + out.WriteString(". ") + } + for i, blk := range item.Content { + const fourSpace = " " + if i > 0 { + out.WriteString("\n" + fourSpace) + } + p.text(out, fourSpace, blk.(*Paragraph).Text) + out.WriteString("\n") + } + } + } +} + +// text prints the text sequence x to out. +func (p *commentPrinter) text(out *bytes.Buffer, indent string, x []Text) { + for _, t := range x { + switch t := t.(type) { + case Plain: + p.indent(out, indent, string(t)) + case Italic: + p.indent(out, indent, string(t)) + case *Link: + if t.Auto { + p.text(out, indent, t.Text) + } else { + out.WriteString("[") + p.text(out, indent, t.Text) + out.WriteString("]") + } + case *DocLink: + out.WriteString("[") + p.text(out, indent, t.Text) + out.WriteString("]") + } + } +} + +// indent prints s to out, indenting with the indent string +// after each newline in s. +func (p *commentPrinter) indent(out *bytes.Buffer, indent, s string) { + for s != "" { + line, rest, ok := strings.Cut(s, "\n") + out.WriteString(line) + if ok { + out.WriteString("\n") + out.WriteString(indent) + } + s = rest + } +} diff --git a/src/go/doc/comment/std.go b/src/go/doc/comment/std.go new file mode 100644 index 0000000..71f15f4 --- /dev/null +++ b/src/go/doc/comment/std.go @@ -0,0 +1,44 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Code generated by 'go generate' DO NOT EDIT. +//go:generate ./mkstd.sh + +package comment + +var stdPkgs = []string{ + "bufio", + "bytes", + "context", + "crypto", + "embed", + "encoding", + "errors", + "expvar", + "flag", + "fmt", + "hash", + "html", + "image", + "io", + "log", + "math", + "mime", + "net", + "os", + "path", + "plugin", + "reflect", + "regexp", + "runtime", + "sort", + "strconv", + "strings", + "sync", + "syscall", + "testing", + "time", + "unicode", + "unsafe", +} diff --git a/src/go/doc/comment/std_test.go b/src/go/doc/comment/std_test.go new file mode 100644 index 0000000..89206e6 --- /dev/null +++ b/src/go/doc/comment/std_test.go @@ -0,0 +1,34 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package comment + +import ( + "internal/diff" + "internal/testenv" + "sort" + "strings" + "testing" +) + +func TestStd(t *testing.T) { + out, err := testenv.Command(t, testenv.GoToolPath(t), "list", "std").CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + + var list []string + for _, pkg := range strings.Fields(string(out)) { + if !strings.Contains(pkg, "/") { + list = append(list, pkg) + } + } + sort.Strings(list) + + have := strings.Join(stdPkgs, "\n") + "\n" + want := strings.Join(list, "\n") + "\n" + if have != want { + t.Errorf("stdPkgs is out of date: regenerate with 'go generate'\n%s", diff.Diff("stdPkgs", []byte(have), "want", []byte(want))) + } +} diff --git a/src/go/doc/comment/testdata/README.md b/src/go/doc/comment/testdata/README.md new file mode 100644 index 0000000..d6f2c54 --- /dev/null +++ b/src/go/doc/comment/testdata/README.md @@ -0,0 +1,42 @@ +This directory contains test files (*.txt) for the comment parser. + +The files are in [txtar format](https://pkg.go.dev/golang.org/x/tools/txtar). +Consider this example: + + -- input -- + Hello. + -- gofmt -- + Hello. + -- html -- +

Hello. + -- markdown -- + Hello. + -- text -- + Hello. + +Each `-- name --` line introduces a new file with the given name. +The file named ā€œinputā€ must be first and contains the input to +[comment.Parser](https://pkg.go.dev/go/doc/comment/#Parser). + +The remaining files contain the expected output for the named format generated by +[comment.Printer](https://pkg.go.dev/go/doc/comment/#Printer): +ā€œgofmtā€ for Printer.Comment (Go comment format, as used by gofmt), +ā€œhtmlā€ for Printer.HTML, ā€œmarkdownā€ for Printer.Markdown, and ā€œtextā€ for Printer.Text. +The format can also be ā€œdumpā€ for a textual dump of the raw data structures. + +The text before the `-- input --` line, if present, is JSON to be unmarshalled +to initialize a comment.Printer. For example, this test case sets the Printer's +TextWidth field to 20: + + {"TextWidth": 20} + -- input -- + Package gob manages streams of gobs - binary values exchanged between an + Encoder (transmitter) and a Decoder (receiver). + -- text -- + Package gob + manages streams + of gobs - binary + values exchanged + between an Encoder + (transmitter) and a + Decoder (receiver). diff --git a/src/go/doc/comment/testdata/blank.txt b/src/go/doc/comment/testdata/blank.txt new file mode 100644 index 0000000..9049fde --- /dev/null +++ b/src/go/doc/comment/testdata/blank.txt @@ -0,0 +1,12 @@ +-- input -- + $ + Blank line at start and end. + $ +-- gofmt -- +Blank line at start and end. +-- text -- +Blank line at start and end. +-- markdown -- +Blank line at start and end. +-- html -- +

Blank line at start and end. diff --git a/src/go/doc/comment/testdata/code.txt b/src/go/doc/comment/testdata/code.txt new file mode 100644 index 0000000..06b1519 --- /dev/null +++ b/src/go/doc/comment/testdata/code.txt @@ -0,0 +1,94 @@ +-- input -- +Text. + A tab-indented + (no, not eight-space indented) + code block and haiku. +More text. + One space + is + enough + to + start + a + block. +More text. + + Blocks + can + + have + blank + lines. +-- gofmt -- +Text. + + A tab-indented + (no, not eight-space indented) + code block and haiku. + +More text. + + One space + is + enough + to + start + a + block. + +More text. + + Blocks + can + + have + blank + lines. +-- markdown -- +Text. + + A tab-indented + (no, not eight-space indented) + code block and haiku. + +More text. + + One space + is + enough + to + start + a + block. + +More text. + + Blocks + can + + have + blank + lines. +-- html -- +

Text. +

A tab-indented
+(no, not eight-space indented)
+code block and haiku.
+
+

More text. +

One space
+ is
+  enough
+   to
+    start
+     a
+      block.
+
+

More text. +

    Blocks
+  can
+
+have
+  blank
+    lines.
+
diff --git a/src/go/doc/comment/testdata/code2.txt b/src/go/doc/comment/testdata/code2.txt new file mode 100644 index 0000000..0810bed --- /dev/null +++ b/src/go/doc/comment/testdata/code2.txt @@ -0,0 +1,31 @@ +-- input -- +Text. + + A tab-indented + (no, not eight-space indented) + code block and haiku. + +More text. +-- gofmt -- +Text. + + A tab-indented + (no, not eight-space indented) + code block and haiku. + +More text. +-- markdown -- +Text. + + A tab-indented + (no, not eight-space indented) + code block and haiku. + +More text. +-- html -- +

Text. +

A tab-indented
+(no, not eight-space indented)
+code block and haiku.
+
+

More text. diff --git a/src/go/doc/comment/testdata/code3.txt b/src/go/doc/comment/testdata/code3.txt new file mode 100644 index 0000000..4a96a0e --- /dev/null +++ b/src/go/doc/comment/testdata/code3.txt @@ -0,0 +1,33 @@ +-- input -- +Text. + + $ + A tab-indented + (surrounded by more blank lines) + code block and haiku. + $ + +More text. +-- gofmt -- +Text. + + A tab-indented + (surrounded by more blank lines) + code block and haiku. + +More text. +-- markdown -- +Text. + + A tab-indented + (surrounded by more blank lines) + code block and haiku. + +More text. +-- html -- +

Text. +

A tab-indented
+(surrounded by more blank lines)
+code block and haiku.
+
+

More text. diff --git a/src/go/doc/comment/testdata/code4.txt b/src/go/doc/comment/testdata/code4.txt new file mode 100644 index 0000000..f128c9a --- /dev/null +++ b/src/go/doc/comment/testdata/code4.txt @@ -0,0 +1,38 @@ +-- input -- +To test, run this command: + go test -more + +Or, to test specific things, run this command: + +go test -more \ + -pkg first/package \ + -pkg second/package \ + -pkg third/package + +Happy testing! +-- gofmt -- +To test, run this command: + + go test -more + +Or, to test specific things, run this command: + + go test -more \ + -pkg first/package \ + -pkg second/package \ + -pkg third/package + +Happy testing! +-- markdown -- +To test, run this command: + + go test -more + +Or, to test specific things, run this command: + + go test -more \ + -pkg first/package \ + -pkg second/package \ + -pkg third/package + +Happy testing! diff --git a/src/go/doc/comment/testdata/code5.txt b/src/go/doc/comment/testdata/code5.txt new file mode 100644 index 0000000..0e340dd --- /dev/null +++ b/src/go/doc/comment/testdata/code5.txt @@ -0,0 +1,21 @@ +-- input -- +L1 +L2 +L3 +L4 +L5 +- L6 { + L7 +} +L8 +-- gofmt -- +L1 +L2 +L3 +L4 +L5 + - L6 { + L7 + } + +L8 diff --git a/src/go/doc/comment/testdata/code6.txt b/src/go/doc/comment/testdata/code6.txt new file mode 100644 index 0000000..d2915d1 --- /dev/null +++ b/src/go/doc/comment/testdata/code6.txt @@ -0,0 +1,24 @@ +-- input -- +Run this program: + +func main() { + fmt.Println("hello, world") +} + +Or this: + +go func() { + fmt.Println("hello, world") +}() +-- gofmt -- +Run this program: + + func main() { + fmt.Println("hello, world") + } + +Or this: + + go func() { + fmt.Println("hello, world") + }() diff --git a/src/go/doc/comment/testdata/crash1.txt b/src/go/doc/comment/testdata/crash1.txt new file mode 100644 index 0000000..6bb2f6f --- /dev/null +++ b/src/go/doc/comment/testdata/crash1.txt @@ -0,0 +1,16 @@ +-- input -- +[] + +[]: http:// +-- gofmt -- +[] + +[]: http:// +-- html -- +

+-- markdown -- +[](http://) +-- text -- + + +[]: http:// diff --git a/src/go/doc/comment/testdata/doclink.txt b/src/go/doc/comment/testdata/doclink.txt new file mode 100644 index 0000000..a932347 --- /dev/null +++ b/src/go/doc/comment/testdata/doclink.txt @@ -0,0 +1,21 @@ +-- input -- +In this package, see [Doc] and [Parser.Parse]. +There is no [Undef] or [Undef.Method]. +See also the [comment] package, +especially [comment.Doc] and [comment.Parser.Parse]. +-- gofmt -- +In this package, see [Doc] and [Parser.Parse]. +There is no [Undef] or [Undef.Method]. +See also the [comment] package, +especially [comment.Doc] and [comment.Parser.Parse]. +-- text -- +In this package, see Doc and Parser.Parse. There is no [Undef] or +[Undef.Method]. See also the comment package, especially comment.Doc and +comment.Parser.Parse. +-- markdown -- +In this package, see [Doc](#Doc) and [Parser.Parse](#Parser.Parse). There is no \[Undef] or \[Undef.Method]. See also the [comment](/go/doc/comment) package, especially [comment.Doc](/go/doc/comment#Doc) and [comment.Parser.Parse](/go/doc/comment#Parser.Parse). +-- html -- +

In this package, see Doc and Parser.Parse. +There is no [Undef] or [Undef.Method]. +See also the comment package, +especially comment.Doc and comment.Parser.Parse. diff --git a/src/go/doc/comment/testdata/doclink2.txt b/src/go/doc/comment/testdata/doclink2.txt new file mode 100644 index 0000000..ecd8e4e --- /dev/null +++ b/src/go/doc/comment/testdata/doclink2.txt @@ -0,0 +1,8 @@ +-- input -- +We use [io.Reader] a lot, and also a few map[io.Reader]string. + +Never [io.Reader]int or Slice[io.Reader] though. +-- markdown -- +We use [io.Reader](/io#Reader) a lot, and also a few map\[io.Reader]string. + +Never \[io.Reader]int or Slice\[io.Reader] though. diff --git a/src/go/doc/comment/testdata/doclink3.txt b/src/go/doc/comment/testdata/doclink3.txt new file mode 100644 index 0000000..0ccfb3d --- /dev/null +++ b/src/go/doc/comment/testdata/doclink3.txt @@ -0,0 +1,8 @@ +-- input -- +[encoding/json.Marshal] is a doc link. + +[rot13.Marshal] is not. +-- markdown -- +[encoding/json.Marshal](/encoding/json#Marshal) is a doc link. + +\[rot13.Marshal] is not. diff --git a/src/go/doc/comment/testdata/doclink4.txt b/src/go/doc/comment/testdata/doclink4.txt new file mode 100644 index 0000000..c709527 --- /dev/null +++ b/src/go/doc/comment/testdata/doclink4.txt @@ -0,0 +1,7 @@ +-- input -- +[io] at start of comment. +[io] at start of line. +At end of line: [io] +At end of comment: [io] +-- markdown -- +[io](/io) at start of comment. [io](/io) at start of line. At end of line: [io](/io) At end of comment: [io](/io) diff --git a/src/go/doc/comment/testdata/doclink5.txt b/src/go/doc/comment/testdata/doclink5.txt new file mode 100644 index 0000000..ac7b3ae --- /dev/null +++ b/src/go/doc/comment/testdata/doclink5.txt @@ -0,0 +1,5 @@ +{"DocLinkBaseURL": "https://pkg.go.dev"} +-- input -- +[encoding/json.Marshal] is a doc link. +-- markdown -- +[encoding/json.Marshal](https://pkg.go.dev/encoding/json#Marshal) is a doc link. diff --git a/src/go/doc/comment/testdata/doclink6.txt b/src/go/doc/comment/testdata/doclink6.txt new file mode 100644 index 0000000..1acd03b --- /dev/null +++ b/src/go/doc/comment/testdata/doclink6.txt @@ -0,0 +1,5 @@ +{"DocLinkBaseURL": "https://go.dev/pkg/"} +-- input -- +[encoding/json.Marshal] is a doc link, and so is [rsc.io/quote.NonExist]. +-- markdown -- +[encoding/json.Marshal](https://go.dev/pkg/encoding/json/#Marshal) is a doc link, and so is [rsc.io/quote.NonExist](https://go.dev/pkg/rsc.io/quote/#NonExist). diff --git a/src/go/doc/comment/testdata/doclink7.txt b/src/go/doc/comment/testdata/doclink7.txt new file mode 100644 index 0000000..d34979a --- /dev/null +++ b/src/go/doc/comment/testdata/doclink7.txt @@ -0,0 +1,4 @@ +-- input -- +You see more [*bytes.Buffer] than [bytes.Buffer]. +-- markdown -- +You see more [\*bytes.Buffer](/bytes#Buffer) than [bytes.Buffer](/bytes#Buffer). diff --git a/src/go/doc/comment/testdata/escape.txt b/src/go/doc/comment/testdata/escape.txt new file mode 100644 index 0000000..f54663f --- /dev/null +++ b/src/go/doc/comment/testdata/escape.txt @@ -0,0 +1,55 @@ +-- input -- +What the ~!@#$%^&*()_+-=`{}|[]\:";',./<>? + ++ Line + +- Line + +* Line + +999. Line + +## Line +-- gofmt -- +What the ~!@#$%^&*()_+-=`{}|[]\:";',./<>? + ++ Line + +- Line + +* Line + +999. Line + +## Line +-- text -- +What the ~!@#$%^&*()_+-=`{}|[]\:";',./<>? + ++ Line + +- Line + +* Line + +999. Line + +## Line +-- markdown -- +What the ~!@#$%^&\*()\_+-=\`{}|\[]\\:";',./\<>? + +\+ Line + +\- Line + +\* Line + +999\. Line + +\## Line +-- html -- +

What the ~!@#$%^&*()_+-=`{}|[]\:";',./<>? +

+ Line +

- Line +

* Line +

999. Line +

## Line diff --git a/src/go/doc/comment/testdata/head.txt b/src/go/doc/comment/testdata/head.txt new file mode 100644 index 0000000..b99a8c5 --- /dev/null +++ b/src/go/doc/comment/testdata/head.txt @@ -0,0 +1,92 @@ +-- input -- +Some text. + +An Old Heading + +Not An Old Heading. + +And some text. + +# A New Heading. + +And some more text. + +# Not a heading, +because text follows it. + +Because text precedes it, +# not a heading. + +## Not a heading either. + +-- gofmt -- +Some text. + +# An Old Heading + +Not An Old Heading. + +And some text. + +# A New Heading. + +And some more text. + +# Not a heading, +because text follows it. + +Because text precedes it, +# not a heading. + +## Not a heading either. + +-- text -- +Some text. + +# An Old Heading + +Not An Old Heading. + +And some text. + +# A New Heading. + +And some more text. + +# Not a heading, because text follows it. + +Because text precedes it, # not a heading. + +## Not a heading either. + +-- markdown -- +Some text. + +### An Old Heading {#hdr-An_Old_Heading} + +Not An Old Heading. + +And some text. + +### A New Heading. {#hdr-A_New_Heading_} + +And some more text. + +\# Not a heading, because text follows it. + +Because text precedes it, # not a heading. + +\## Not a heading either. + +-- html -- +

Some text. +

An Old Heading

+

Not An Old Heading. +

And some text. +

A New Heading.

+

And some more text. +

# Not a heading, +because text follows it. +

Because text precedes it, +# not a heading. +

## Not a heading either. diff --git a/src/go/doc/comment/testdata/head2.txt b/src/go/doc/comment/testdata/head2.txt new file mode 100644 index 0000000..d357632 --- /dev/null +++ b/src/go/doc/comment/testdata/head2.txt @@ -0,0 +1,36 @@ +-- input -- +āœ¦ + +Almost a+heading + +āœ¦ + +Don't be a heading + +āœ¦ + +A.b is a heading + +āœ¦ + +A. b is not a heading + +āœ¦ +-- gofmt -- +āœ¦ + +Almost a+heading + +āœ¦ + +Don't be a heading + +āœ¦ + +# A.b is a heading + +āœ¦ + +A. b is not a heading + +āœ¦ diff --git a/src/go/doc/comment/testdata/head3.txt b/src/go/doc/comment/testdata/head3.txt new file mode 100644 index 0000000..dbb7cb3 --- /dev/null +++ b/src/go/doc/comment/testdata/head3.txt @@ -0,0 +1,7 @@ +{"HeadingLevel": 5} +-- input -- +# Heading +-- markdown -- +##### Heading {#hdr-Heading} +-- html -- +

Heading
diff --git a/src/go/doc/comment/testdata/hello.txt b/src/go/doc/comment/testdata/hello.txt new file mode 100644 index 0000000..fb07f1e --- /dev/null +++ b/src/go/doc/comment/testdata/hello.txt @@ -0,0 +1,35 @@ +-- input -- + Hello, + world + + This is + a test. +-- dump -- +Doc + Paragraph + Plain + "Hello,\n" + "world" + Paragraph + Plain + "This is\n" + "a test." +-- gofmt -- +Hello, +world + +This is +a test. +-- html -- +

Hello, +world +

This is +a test. +-- markdown -- +Hello, world + +This is a test. +-- text -- +Hello, world + +This is a test. diff --git a/src/go/doc/comment/testdata/link.txt b/src/go/doc/comment/testdata/link.txt new file mode 100644 index 0000000..551e306 --- /dev/null +++ b/src/go/doc/comment/testdata/link.txt @@ -0,0 +1,17 @@ +-- input -- +The Go home page is https://go.dev/. +It used to be https://golang.org. + +-- gofmt -- +The Go home page is https://go.dev/. +It used to be https://golang.org. + +-- text -- +The Go home page is https://go.dev/. It used to be https://golang.org. + +-- markdown -- +The Go home page is [https://go.dev/](https://go.dev/). It used to be [https://golang.org](https://golang.org). + +-- html -- +

The Go home page is https://go.dev/. +It used to be https://golang.org. diff --git a/src/go/doc/comment/testdata/link2.txt b/src/go/doc/comment/testdata/link2.txt new file mode 100644 index 0000000..8637a32 --- /dev/null +++ b/src/go/doc/comment/testdata/link2.txt @@ -0,0 +1,31 @@ +-- input -- +The Go home page is https://go.dev/. +It used to be https://golang.org. +https:// is not a link. +Nor is https:// +https://ā˜ŗ is not a link. +https://:80 is not a link. + +-- gofmt -- +The Go home page is https://go.dev/. +It used to be https://golang.org. +https:// is not a link. +Nor is https:// +https://ā˜ŗ is not a link. +https://:80 is not a link. + +-- text -- +The Go home page is https://go.dev/. It used to be https://golang.org. https:// +is not a link. Nor is https:// https://ā˜ŗ is not a link. https://:80 is not a +link. + +-- markdown -- +The Go home page is [https://go.dev/](https://go.dev/). It used to be [https://golang.org](https://golang.org). https:// is not a link. Nor is https:// https://ā˜ŗ is not a link. https://:80 is not a link. + +-- html -- +

The Go home page is https://go.dev/. +It used to be https://golang.org. +https:// is not a link. +Nor is https:// +https://ā˜ŗ is not a link. +https://:80 is not a link. diff --git a/src/go/doc/comment/testdata/link3.txt b/src/go/doc/comment/testdata/link3.txt new file mode 100644 index 0000000..5a115b5 --- /dev/null +++ b/src/go/doc/comment/testdata/link3.txt @@ -0,0 +1,14 @@ +-- input -- +Doc text. + +[Go home page]: https://go.dev +-- gofmt -- +Doc text. + +[Go home page]: https://go.dev +-- text -- +Doc text. +-- markdown -- +Doc text. +-- html -- +

Doc text. diff --git a/src/go/doc/comment/testdata/link4.txt b/src/go/doc/comment/testdata/link4.txt new file mode 100644 index 0000000..75f194c --- /dev/null +++ b/src/go/doc/comment/testdata/link4.txt @@ -0,0 +1,77 @@ +-- input -- +These are not links. + +[x + +[x]: + +[x]:https://go.dev + +[x]https://go.dev + +[x]: surprise://go.dev + +[x]: surprise! + +But this is, with a tab (although it's unused). + +[z]: https://go.dev +-- gofmt -- +These are not links. + +[x + +[x]: + +[x]:https://go.dev + +[x]https://go.dev + +[x]: surprise://go.dev + +[x]: surprise! + +But this is, with a tab (although it's unused). + +[z]: https://go.dev +-- text -- +These are not links. + +[x + +[x]: + +[x]:https://go.dev + +[x]https://go.dev + +[x]: surprise://go.dev + +[x]: surprise! + +But this is, with a tab (although it's unused). +-- markdown -- +These are not links. + +\[x + +\[x]: + +\[x]:[https://go.dev](https://go.dev) + +\[x][https://go.dev](https://go.dev) + +\[x]: surprise://go.dev + +\[x]: surprise! + +But this is, with a tab (although it's unused). +-- html -- +

These are not links. +

[x +

[x]: +

[x]:https://go.dev +

[x]https://go.dev +

[x]: surprise://go.dev +

[x]: surprise! +

But this is, with a tab (although it's unused). diff --git a/src/go/doc/comment/testdata/link5.txt b/src/go/doc/comment/testdata/link5.txt new file mode 100644 index 0000000..b4fb588 --- /dev/null +++ b/src/go/doc/comment/testdata/link5.txt @@ -0,0 +1,36 @@ +-- input -- +See the [Go home page] and the [pkg +site]. + +[Go home page]: https://go.dev/ +[pkg site]: https://pkg.go.dev +[Go home page]: https://duplicate.ignored + +They're really great! + +-- gofmt -- +See the [Go home page] and the [pkg +site]. + +They're really great! + +[Go home page]: https://go.dev/ +[pkg site]: https://pkg.go.dev + +[Go home page]: https://duplicate.ignored + +-- text -- +See the Go home page and the pkg site. + +They're really great! + +[Go home page]: https://go.dev/ +[pkg site]: https://pkg.go.dev +-- markdown -- +See the [Go home page](https://go.dev/) and the [pkg site](https://pkg.go.dev). + +They're really great! +-- html -- +

See the Go home page and the pkg +site. +

They're really great! diff --git a/src/go/doc/comment/testdata/link6.txt b/src/go/doc/comment/testdata/link6.txt new file mode 100644 index 0000000..ff629b4 --- /dev/null +++ b/src/go/doc/comment/testdata/link6.txt @@ -0,0 +1,50 @@ +-- input -- +URLs with punctuation are hard. +We don't want to consume the end-of-sentence punctuation. + +For example, https://en.wikipedia.org/wiki/John_Adams_(miniseries). +And https://example.com/[foo]/bar{. +And https://example.com/(foo)/bar! +And https://example.com/{foo}/bar{. +And https://example.com/)baz{foo}. + +[And https://example.com/]. + +-- gofmt -- +URLs with punctuation are hard. +We don't want to consume the end-of-sentence punctuation. + +For example, https://en.wikipedia.org/wiki/John_Adams_(miniseries). +And https://example.com/[foo]/bar{. +And https://example.com/(foo)/bar! +And https://example.com/{foo}/bar{. +And https://example.com/)baz{foo}. + +[And https://example.com/]. + +-- text -- +URLs with punctuation are hard. We don't want to consume the end-of-sentence +punctuation. + +For example, https://en.wikipedia.org/wiki/John_Adams_(miniseries). +And https://example.com/[foo]/bar{. And https://example.com/(foo)/bar! And +https://example.com/{foo}/bar{. And https://example.com/)baz{foo}. + +[And https://example.com/]. + +-- markdown -- +URLs with punctuation are hard. We don't want to consume the end-of-sentence punctuation. + +For example, [https://en.wikipedia.org/wiki/John\_Adams\_(miniseries)](https://en.wikipedia.org/wiki/John_Adams_(miniseries)). And [https://example.com/\[foo]/bar](https://example.com/[foo]/bar){. And [https://example.com/(foo)/bar](https://example.com/(foo)/bar)! And [https://example.com/{foo}/bar](https://example.com/{foo}/bar){. And [https://example.com/](https://example.com/))baz{foo}. + +\[And [https://example.com/](https://example.com/)]. + +-- html -- +

URLs with punctuation are hard. +We don't want to consume the end-of-sentence punctuation. +

For example, https://en.wikipedia.org/wiki/John_Adams_(miniseries). +And https://example.com/[foo]/bar{. +And https://example.com/(foo)/bar! +And https://example.com/{foo}/bar{. +And https://example.com/)baz{foo}. +

[And https://example.com/]. diff --git a/src/go/doc/comment/testdata/link7.txt b/src/go/doc/comment/testdata/link7.txt new file mode 100644 index 0000000..89a8b31 --- /dev/null +++ b/src/go/doc/comment/testdata/link7.txt @@ -0,0 +1,25 @@ +-- input -- +[math] is a package but this is not a doc link. + +[io] is a doc link. + +[math]: https://example.com +-- gofmt -- +[math] is a package but this is not a doc link. + +[io] is a doc link. + +[math]: https://example.com +-- text -- +math is a package but this is not a doc link. + +io is a doc link. + +[math]: https://example.com +-- markdown -- +[math](https://example.com) is a package but this is not a doc link. + +[io](/io) is a doc link. +-- html -- +

math is a package but this is not a doc link. +

io is a doc link. diff --git a/src/go/doc/comment/testdata/linklist.txt b/src/go/doc/comment/testdata/linklist.txt new file mode 100644 index 0000000..baf4062 --- /dev/null +++ b/src/go/doc/comment/testdata/linklist.txt @@ -0,0 +1,18 @@ +{"DocLinkBaseURL": "https://pkg.go.dev"} +-- input -- +Did you know? + + - [encoding/json.Marshal] is a doc link. So is [encoding/json.Unmarshal]. +-- text -- +Did you know? + + - encoding/json.Marshal is a doc link. So is encoding/json.Unmarshal. +-- markdown -- +Did you know? + + - [encoding/json.Marshal](https://pkg.go.dev/encoding/json#Marshal) is a doc link. So is [encoding/json.Unmarshal](https://pkg.go.dev/encoding/json#Unmarshal). +-- html -- +

Did you know? +

diff --git a/src/go/doc/comment/testdata/linklist2.txt b/src/go/doc/comment/testdata/linklist2.txt new file mode 100644 index 0000000..81b3061 --- /dev/null +++ b/src/go/doc/comment/testdata/linklist2.txt @@ -0,0 +1,39 @@ +{"DocLinkBaseURL": "https://pkg.go.dev"} +-- input -- +Did you know? + + - [testing.T] is one doc link. + - So is [testing.M]. + - So is [testing.B]. + This is the same list paragraph. + + There is [testing.PB] in this list item, too! +-- text -- +Did you know? + + - testing.T is one doc link. + + - So is testing.M. + + - So is testing.B. This is the same list paragraph. + + There is testing.PB in this list item, too! +-- markdown -- +Did you know? + + - [testing.T](https://pkg.go.dev/testing#T) is one doc link. + + - So is [testing.M](https://pkg.go.dev/testing#M). + + - So is [testing.B](https://pkg.go.dev/testing#B). This is the same list paragraph. + + There is [testing.PB](https://pkg.go.dev/testing#PB) in this list item, too! +-- html -- +

Did you know? +

diff --git a/src/go/doc/comment/testdata/linklist3.txt b/src/go/doc/comment/testdata/linklist3.txt new file mode 100644 index 0000000..701a54e --- /dev/null +++ b/src/go/doc/comment/testdata/linklist3.txt @@ -0,0 +1,31 @@ +{"DocLinkBaseURL": "https://pkg.go.dev"} +-- input -- +Cool things: + + - Foo + - [Go] + - Bar + +[Go]: https://go.dev/ +-- text -- +Cool things: + + - Foo + - Go + - Bar + +[Go]: https://go.dev/ +-- markdown -- +Cool things: + + - Foo + - [Go](https://go.dev/) + - Bar + +-- html -- +

Cool things: +

    +
  • Foo +
  • Go +
  • Bar +
diff --git a/src/go/doc/comment/testdata/linklist4.txt b/src/go/doc/comment/testdata/linklist4.txt new file mode 100644 index 0000000..db39ec4 --- /dev/null +++ b/src/go/doc/comment/testdata/linklist4.txt @@ -0,0 +1,36 @@ +{"DocLinkBaseURL": "https://pkg.go.dev"} +-- input -- +Cool things: + + - Foo + - [Go] is great + + [Go]: https://go.dev/ + - Bar + +-- text -- +Cool things: + + - Foo + + - Go is great + + - Bar + +[Go]: https://go.dev/ +-- markdown -- +Cool things: + + - Foo + + - [Go](https://go.dev/) is great + + - Bar + +-- html -- +

Cool things: +

    +

  • Foo +

  • Go is great +

  • Bar +

diff --git a/src/go/doc/comment/testdata/list.txt b/src/go/doc/comment/testdata/list.txt new file mode 100644 index 0000000..455782f --- /dev/null +++ b/src/go/doc/comment/testdata/list.txt @@ -0,0 +1,48 @@ +-- input -- +Text. +- Not a list. + - Here is the list. + ā€¢ Using multiple bullets. + * Indentation does not matter. + + Lots of bullets. +More text. + +-- gofmt -- +Text. +- Not a list. + - Here is the list. + - Using multiple bullets. + - Indentation does not matter. + - Lots of bullets. + +More text. + +-- text -- +Text. - Not a list. + - Here is the list. + - Using multiple bullets. + - Indentation does not matter. + - Lots of bullets. + +More text. + +-- markdown -- +Text. - Not a list. + + - Here is the list. + - Using multiple bullets. + - Indentation does not matter. + - Lots of bullets. + +More text. + +-- html -- +

Text. +- Not a list. +

    +
  • Here is the list. +
  • Using multiple bullets. +
  • Indentation does not matter. +
  • Lots of bullets. +
+

More text. diff --git a/src/go/doc/comment/testdata/list10.txt b/src/go/doc/comment/testdata/list10.txt new file mode 100644 index 0000000..9c49083 --- /dev/null +++ b/src/go/doc/comment/testdata/list10.txt @@ -0,0 +1,13 @@ +-- input -- + + 1. This list + 2. Starts the comment + 3. And also has a blank line before it. + +All of which is a little weird. +-- gofmt -- + 1. This list + 2. Starts the comment + 3. And also has a blank line before it. + +All of which is a little weird. diff --git a/src/go/doc/comment/testdata/list2.txt b/src/go/doc/comment/testdata/list2.txt new file mode 100644 index 0000000..c390b3d --- /dev/null +++ b/src/go/doc/comment/testdata/list2.txt @@ -0,0 +1,57 @@ +-- input -- +Text. + 1. Uno + 2) Dos + 3. Tres + 5. Cinco + 7. Siete + 11. Once + 12. Doce + 13. Trece. + +-- gofmt -- +Text. + 1. Uno + 2. Dos + 3. Tres + 5. Cinco + 7. Siete + 11. Once + 12. Doce + 13. Trece. + +-- text -- +Text. + 1. Uno + 2. Dos + 3. Tres + 5. Cinco + 7. Siete + 11. Once + 12. Doce + 13. Trece. + +-- markdown -- +Text. + + 1. Uno + 2. Dos + 3. Tres + 5. Cinco + 7. Siete + 11. Once + 12. Doce + 13. Trece. + +-- html -- +

Text. +

    +
  1. Uno +
  2. Dos +
  3. Tres +
  4. Cinco +
  5. Siete +
  6. Once +
  7. Doce +
  8. Trece. +
diff --git a/src/go/doc/comment/testdata/list3.txt b/src/go/doc/comment/testdata/list3.txt new file mode 100644 index 0000000..d7d345d --- /dev/null +++ b/src/go/doc/comment/testdata/list3.txt @@ -0,0 +1,32 @@ +-- input -- +Text. + + 1. Uno + 1. Dos + 1. Tres + 1. Quatro + +-- gofmt -- +Text. + + 1. Uno + 1. Dos + 1. Tres + 1. Quatro + +-- markdown -- +Text. + + 1. Uno + 1. Dos + 1. Tres + 1. Quatro + +-- html -- +

Text. +

    +
  1. Uno +
  2. Dos +
  3. Tres +
  4. Quatro +
diff --git a/src/go/doc/comment/testdata/list4.txt b/src/go/doc/comment/testdata/list4.txt new file mode 100644 index 0000000..9c28d65 --- /dev/null +++ b/src/go/doc/comment/testdata/list4.txt @@ -0,0 +1,38 @@ +-- input -- +Text. + 1. List +2. Not indented, not a list. + 3. Another list. + +-- gofmt -- +Text. + 1. List + +2. Not indented, not a list. + 3. Another list. + +-- text -- +Text. + 1. List + +2. Not indented, not a list. + 3. Another list. + +-- markdown -- +Text. + + 1. List + +2\. Not indented, not a list. + + 3. Another list. + +-- html -- +

Text. +

    +
  1. List +
+

2. Not indented, not a list. +

    +
  1. Another list. +
diff --git a/src/go/doc/comment/testdata/list5.txt b/src/go/doc/comment/testdata/list5.txt new file mode 100644 index 0000000..a5128e5 --- /dev/null +++ b/src/go/doc/comment/testdata/list5.txt @@ -0,0 +1,40 @@ +-- input -- +Text. + + 1. One + 999999999999999999999. Big + 1000000000000000000000. Bigger + 1000000000000000000001. Biggest + +-- gofmt -- +Text. + + 1. One + 999999999999999999999. Big + 1000000000000000000000. Bigger + 1000000000000000000001. Biggest + +-- text -- +Text. + + 1. One + 999999999999999999999. Big + 1000000000000000000000. Bigger + 1000000000000000000001. Biggest + +-- markdown -- +Text. + + 1. One + 999999999999999999999. Big + 1000000000000000000000. Bigger + 1000000000000000000001. Biggest + +-- html -- +

Text. +

    +
  1. One +
  2. Big +
  3. Bigger +
  4. Biggest +
diff --git a/src/go/doc/comment/testdata/list6.txt b/src/go/doc/comment/testdata/list6.txt new file mode 100644 index 0000000..ffc0122 --- /dev/null +++ b/src/go/doc/comment/testdata/list6.txt @@ -0,0 +1,129 @@ +-- input -- +Text. + - List immediately after. + - Another. + +More text. + + - List after blank line. + - Another. + +Even more text. + - List immediately after. + + - Blank line between items. + +Yet more text. + + - Another list after blank line. + + - Blank line between items. + +Still more text. + - One list item. + + Multiple paragraphs. +-- dump -- +Doc + Paragraph + Plain "Text." + List ForceBlankBefore=false ForceBlankBetween=false + Item Number="" + Paragraph + Plain "List immediately after." + Item Number="" + Paragraph + Plain "Another." + Paragraph + Plain "More text." + List ForceBlankBefore=true ForceBlankBetween=false + Item Number="" + Paragraph + Plain "List after blank line." + Item Number="" + Paragraph + Plain "Another." + Paragraph + Plain "Even more text." + List ForceBlankBefore=false ForceBlankBetween=true + Item Number="" + Paragraph + Plain "List immediately after." + Item Number="" + Paragraph + Plain "Blank line between items." + Paragraph + Plain "Yet more text." + List ForceBlankBefore=true ForceBlankBetween=true + Item Number="" + Paragraph + Plain "Another list after blank line." + Item Number="" + Paragraph + Plain "Blank line between items." + Paragraph + Plain "Still more text." + List ForceBlankBefore=false ForceBlankBetween=true + Item Number="" + Paragraph + Plain "One list item." + Paragraph + Plain "Multiple paragraphs." + +-- gofmt -- +Text. + - List immediately after. + - Another. + +More text. + + - List after blank line. + - Another. + +Even more text. + + - List immediately after. + + - Blank line between items. + +Yet more text. + + - Another list after blank line. + + - Blank line between items. + +Still more text. + + - One list item. + + Multiple paragraphs. + +-- markdown -- +Text. + + - List immediately after. + - Another. + +More text. + + - List after blank line. + - Another. + +Even more text. + + - List immediately after. + + - Blank line between items. + +Yet more text. + + - Another list after blank line. + + - Blank line between items. + +Still more text. + + - One list item. + + Multiple paragraphs. + diff --git a/src/go/doc/comment/testdata/list7.txt b/src/go/doc/comment/testdata/list7.txt new file mode 100644 index 0000000..4466050 --- /dev/null +++ b/src/go/doc/comment/testdata/list7.txt @@ -0,0 +1,98 @@ +-- input -- +Almost list markers (but not quite): + + - + +ā¦ + + - $ + +ā¦ + + - $ + +ā¦ + + $ + $ + +ā¦ + + 1! List. + +ā¦ +-- gofmt -- +Almost list markers (but not quite): + + - + +ā¦ + + - $ + +ā¦ + + - $ + +ā¦ + +ā¦ + + 1! List. + +ā¦ +-- text -- +Almost list markers (but not quite): + + - + +ā¦ + + - + +ā¦ + + - + +ā¦ + +ā¦ + + 1! List. + +ā¦ +-- markdown -- +Almost list markers (but not quite): + + - + +ā¦ + + - $ + +ā¦ + + - $ + +ā¦ + +ā¦ + + 1! List. + +ā¦ +-- html -- +

Almost list markers (but not quite): +

-
+
+

ā¦ +

- $
+
+

ā¦ +

- $
+
+

ā¦ +

ā¦ +

1! List.
+
+

ā¦ diff --git a/src/go/doc/comment/testdata/list8.txt b/src/go/doc/comment/testdata/list8.txt new file mode 100644 index 0000000..fc46b0d --- /dev/null +++ b/src/go/doc/comment/testdata/list8.txt @@ -0,0 +1,56 @@ +-- input -- +Loose lists. + - A + + B + - C + D + - E + - F +-- gofmt -- +Loose lists. + + - A + + B + + - C + D + + - E + + - F +-- text -- +Loose lists. + + - A + + B + + - C D + + - E + + - F +-- markdown -- +Loose lists. + + - A + + B + + - C D + + - E + + - F +-- html -- +

Loose lists. +

    +

  • A +

    B +

  • C +D +

  • E +

  • F +

diff --git a/src/go/doc/comment/testdata/list9.txt b/src/go/doc/comment/testdata/list9.txt new file mode 100644 index 0000000..48e4673 --- /dev/null +++ b/src/go/doc/comment/testdata/list9.txt @@ -0,0 +1,30 @@ +-- input -- +Text. + +1. Not a list +2. because it is +3. unindented. + +4. This one + is a list + because of the indented text. +5. More wrapped + items. +6. And unwrapped. + +7. The blank line stops the heuristic. +-- gofmt -- +Text. + +1. Not a list +2. because it is +3. unindented. + + 4. This one + is a list + because of the indented text. + 5. More wrapped + items. + 6. And unwrapped. + +7. The blank line stops the heuristic. diff --git a/src/go/doc/comment/testdata/para.txt b/src/go/doc/comment/testdata/para.txt new file mode 100644 index 0000000..2355fa8 --- /dev/null +++ b/src/go/doc/comment/testdata/para.txt @@ -0,0 +1,17 @@ +-- input -- +Hello, world. +This is a paragraph. + +-- gofmt -- +Hello, world. +This is a paragraph. + +-- text -- +Hello, world. This is a paragraph. + +-- markdown -- +Hello, world. This is a paragraph. + +-- html -- +

Hello, world. +This is a paragraph. diff --git a/src/go/doc/comment/testdata/quote.txt b/src/go/doc/comment/testdata/quote.txt new file mode 100644 index 0000000..b64adae --- /dev/null +++ b/src/go/doc/comment/testdata/quote.txt @@ -0,0 +1,15 @@ +-- input -- +Doubled single quotes like `` and '' turn into Unicode double quotes, +but single quotes ` and ' do not. +Misplaced markdown fences ``` do not either. +-- gofmt -- +Doubled single quotes like ā€œ and ā€ turn into Unicode double quotes, +but single quotes ` and ' do not. +Misplaced markdown fences ``` do not either. +-- text -- +Doubled single quotes like ā€œ and ā€ turn into Unicode double quotes, but single +quotes ` and ' do not. Misplaced markdown fences ``` do not either. +-- html -- +

Doubled single quotes like ā€œ and ā€ turn into Unicode double quotes, +but single quotes ` and ' do not. +Misplaced markdown fences ``` do not either. diff --git a/src/go/doc/comment/testdata/text.txt b/src/go/doc/comment/testdata/text.txt new file mode 100644 index 0000000..c4de6e2 --- /dev/null +++ b/src/go/doc/comment/testdata/text.txt @@ -0,0 +1,62 @@ +{"TextPrefix":"|", "TextCodePrefix": "@"} +-- input -- +Hello, world + Code block here. +More text. +Tight list + - one + - two + - three +Loose list + - one + + - two + + - three + +# Heading + +More text. +-- gofmt -- +Hello, world + + Code block here. + +More text. +Tight list + - one + - two + - three + +Loose list + + - one + + - two + + - three + +# Heading + +More text. +-- text -- +|Hello, world +| +@Code block here. +| +|More text. Tight list +| - one +| - two +| - three +| +|Loose list +| +| - one +| +| - two +| +| - three +| +|# Heading +| +|More text. diff --git a/src/go/doc/comment/testdata/text2.txt b/src/go/doc/comment/testdata/text2.txt new file mode 100644 index 0000000..a099d0b --- /dev/null +++ b/src/go/doc/comment/testdata/text2.txt @@ -0,0 +1,14 @@ +{"TextWidth": -1} +-- input -- +Package gob manages streams of gobs - binary values exchanged between an +Encoder (transmitter) and a Decoder (receiver). A typical use is +transporting arguments and results of remote procedure calls (RPCs) such as +those provided by package "net/rpc". + +The implementation compiles a custom codec for each data type in the stream +and is most efficient when a single Encoder is used to transmit a stream of +values, amortizing the cost of compilation. +-- text -- +Package gob manages streams of gobs - binary values exchanged between an Encoder (transmitter) and a Decoder (receiver). A typical use is transporting arguments and results of remote procedure calls (RPCs) such as those provided by package "net/rpc". + +The implementation compiles a custom codec for each data type in the stream and is most efficient when a single Encoder is used to transmit a stream of values, amortizing the cost of compilation. diff --git a/src/go/doc/comment/testdata/text3.txt b/src/go/doc/comment/testdata/text3.txt new file mode 100644 index 0000000..75d2c37 --- /dev/null +++ b/src/go/doc/comment/testdata/text3.txt @@ -0,0 +1,28 @@ +{"TextWidth": 30} +-- input -- +Package gob manages streams of gobs - binary values exchanged between an +Encoder (transmitter) and a Decoder (receiver). A typical use is +transporting arguments and results of remote procedure calls (RPCs) such as +those provided by package "net/rpc". + +The implementation compiles a custom codec for each data type in the stream +and is most efficient when a single Encoder is used to transmit a stream of +values, amortizing the cost of compilation. +-- text -- +Package gob manages streams +of gobs - binary values +exchanged between an Encoder +(transmitter) and a Decoder +(receiver). A typical use is +transporting arguments and +results of remote procedure +calls (RPCs) such as those +provided by package "net/rpc". + +The implementation compiles +a custom codec for each data +type in the stream and is +most efficient when a single +Encoder is used to transmit a +stream of values, amortizing +the cost of compilation. diff --git a/src/go/doc/comment/testdata/text4.txt b/src/go/doc/comment/testdata/text4.txt new file mode 100644 index 0000000..e429985 --- /dev/null +++ b/src/go/doc/comment/testdata/text4.txt @@ -0,0 +1,29 @@ +{"TextWidth": 29} +-- input -- +Package gob manages streams of gobs - binary values exchanged between an +Encoder (transmitter) and a Decoder (receiver). A typical use is +transporting arguments and results of remote procedure calls (RPCs) such as +those provided by package "net/rpc". + +The implementation compiles a custom codec for each data type in the stream +and is most efficient when a single Encoder is used to transmit a stream of +values, amortizing the cost of compilation. +-- text -- +Package gob manages streams +of gobs - binary values +exchanged between an Encoder +(transmitter) and a Decoder +(receiver). A typical use +is transporting arguments +and results of remote +procedure calls (RPCs) such +as those provided by package +"net/rpc". + +The implementation compiles +a custom codec for each data +type in the stream and is +most efficient when a single +Encoder is used to transmit a +stream of values, amortizing +the cost of compilation. diff --git a/src/go/doc/comment/testdata/text5.txt b/src/go/doc/comment/testdata/text5.txt new file mode 100644 index 0000000..2408fc5 --- /dev/null +++ b/src/go/doc/comment/testdata/text5.txt @@ -0,0 +1,38 @@ +{"TextWidth": 20} +-- input -- +Package gob manages streams of gobs - binary values exchanged between an +Encoder (transmitter) and a Decoder (receiver). A typical use is +transporting arguments and results of remote procedure calls (RPCs) such as +those provided by package "net/rpc". + +The implementation compiles a custom codec for each data type in the stream +and is most efficient when a single Encoder is used to transmit a stream of +values, amortizing the cost of compilation. +-- text -- +Package gob +manages streams +of gobs - binary +values exchanged +between an Encoder +(transmitter) and a +Decoder (receiver). +A typical use +is transporting +arguments and +results of remote +procedure calls +(RPCs) such as those +provided by package +"net/rpc". + +The implementation +compiles a custom +codec for each +data type in the +stream and is most +efficient when a +single Encoder is +used to transmit a +stream of values, +amortizing the cost +of compilation. diff --git a/src/go/doc/comment/testdata/text6.txt b/src/go/doc/comment/testdata/text6.txt new file mode 100644 index 0000000..d6deff5 --- /dev/null +++ b/src/go/doc/comment/testdata/text6.txt @@ -0,0 +1,18 @@ +-- input -- +Package gob manages streams of gobs - binary values exchanged between an +Encoder (transmitter) and a Decoder (receiver). A typical use is +transporting arguments and results of remote procedure calls (RPCs) such as +those provided by package "net/rpc". + +The implementation compiles a custom codec for each data type in the stream +and is most efficient when a single Encoder is used to transmit a stream of +values, amortizing the cost of compilation. +-- text -- +Package gob manages streams of gobs - binary values exchanged between an Encoder +(transmitter) and a Decoder (receiver). A typical use is transporting arguments +and results of remote procedure calls (RPCs) such as those provided by package +"net/rpc". + +The implementation compiles a custom codec for each data type in the stream and +is most efficient when a single Encoder is used to transmit a stream of values, +amortizing the cost of compilation. diff --git a/src/go/doc/comment/testdata/text7.txt b/src/go/doc/comment/testdata/text7.txt new file mode 100644 index 0000000..c9fb6d3 --- /dev/null +++ b/src/go/doc/comment/testdata/text7.txt @@ -0,0 +1,21 @@ +{"TextPrefix": " "} +-- input -- +Package gob manages streams of gobs - binary values exchanged between an +Encoder (transmitter) and a Decoder (receiver). A typical use is +transporting arguments and results of remote procedure calls (RPCs) such as +those provided by package "net/rpc". + +The implementation compiles a custom codec for each data type in the stream +and is most efficient when a single Encoder is used to transmit a stream of +values, amortizing the cost of compilation. +-- text -- + Package gob manages streams of gobs - binary values + exchanged between an Encoder (transmitter) and a Decoder + (receiver). A typical use is transporting arguments and + results of remote procedure calls (RPCs) such as those + provided by package "net/rpc". + + The implementation compiles a custom codec for each data + type in the stream and is most efficient when a single + Encoder is used to transmit a stream of values, amortizing + the cost of compilation. diff --git a/src/go/doc/comment/testdata/text8.txt b/src/go/doc/comment/testdata/text8.txt new file mode 100644 index 0000000..560ac95 --- /dev/null +++ b/src/go/doc/comment/testdata/text8.txt @@ -0,0 +1,94 @@ +{"TextWidth": 40} +-- input -- +If the arguments have version suffixes (like @latest or @v1.0.0), "go install" +builds packages in module-aware mode, ignoring the go.mod file in the current +directory or any parent directory, if there is one. This is useful for +installing executables without affecting the dependencies of the main module. +To eliminate ambiguity about which module versions are used in the build, the +arguments must satisfy the following constraints: + + - Arguments must be package paths or package patterns (with "..." wildcards). + They must not be standard packages (like fmt), meta-patterns (std, cmd, + all), or relative or absolute file paths. + + - All arguments must have the same version suffix. Different queries are not + allowed, even if they refer to the same version. + + - All arguments must refer to packages in the same module at the same version. + + - Package path arguments must refer to main packages. Pattern arguments + will only match main packages. + + - No module is considered the "main" module. If the module containing + packages named on the command line has a go.mod file, it must not contain + directives (replace and exclude) that would cause it to be interpreted + differently than if it were the main module. The module must not require + a higher version of itself. + + - Vendor directories are not used in any module. (Vendor directories are not + included in the module zip files downloaded by 'go install'.) + +If the arguments don't have version suffixes, "go install" may run in +module-aware mode or GOPATH mode, depending on the GO111MODULE environment +variable and the presence of a go.mod file. See 'go help modules' for details. +If module-aware mode is enabled, "go install" runs in the context of the main +module. +-- text -- +If the arguments have version suffixes +(like @latest or @v1.0.0), "go install" +builds packages in module-aware mode, +ignoring the go.mod file in the current +directory or any parent directory, +if there is one. This is useful for +installing executables without affecting +the dependencies of the main module. +To eliminate ambiguity about which +module versions are used in the build, +the arguments must satisfy the following +constraints: + + - Arguments must be package paths + or package patterns (with "..." + wildcards). They must not be + standard packages (like fmt), + meta-patterns (std, cmd, all), + or relative or absolute file paths. + + - All arguments must have the same + version suffix. Different queries + are not allowed, even if they refer + to the same version. + + - All arguments must refer to packages + in the same module at the same + version. + + - Package path arguments must refer + to main packages. Pattern arguments + will only match main packages. + + - No module is considered the "main" + module. If the module containing + packages named on the command line + has a go.mod file, it must not + contain directives (replace and + exclude) that would cause it to be + interpreted differently than if it + were the main module. The module + must not require a higher version of + itself. + + - Vendor directories are not used in + any module. (Vendor directories are + not included in the module zip files + downloaded by 'go install'.) + +If the arguments don't have version +suffixes, "go install" may run in +module-aware mode or GOPATH mode, +depending on the GO111MODULE environment +variable and the presence of a go.mod +file. See 'go help modules' for details. +If module-aware mode is enabled, +"go install" runs in the context of the +main module. diff --git a/src/go/doc/comment/testdata/text9.txt b/src/go/doc/comment/testdata/text9.txt new file mode 100644 index 0000000..07a64aa --- /dev/null +++ b/src/go/doc/comment/testdata/text9.txt @@ -0,0 +1,12 @@ +{"TextPrefix":"|", "TextCodePrefix": "@"} +-- input -- +Hello, world + Code block here. +-- gofmt -- +Hello, world + + Code block here. +-- text -- +|Hello, world +| +@Code block here. diff --git a/src/go/doc/comment/testdata/words.txt b/src/go/doc/comment/testdata/words.txt new file mode 100644 index 0000000..63c7e1a --- /dev/null +++ b/src/go/doc/comment/testdata/words.txt @@ -0,0 +1,10 @@ +-- input -- +This is an italicword and a linkedword and Unicƶde. +-- gofmt -- +This is an italicword and a linkedword and Unicƶde. +-- text -- +This is an italicword and a linkedword and Unicƶde. +-- markdown -- +This is an *italicword* and a [*linkedword*](https://example.com/linkedword) and Unicƶde. +-- html -- +

This is an italicword and a linkedword and Unicƶde. diff --git a/src/go/doc/comment/testdata_test.go b/src/go/doc/comment/testdata_test.go new file mode 100644 index 0000000..0676d86 --- /dev/null +++ b/src/go/doc/comment/testdata_test.go @@ -0,0 +1,202 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package comment + +import ( + "bytes" + "encoding/json" + "fmt" + "internal/diff" + "internal/txtar" + "path/filepath" + "strings" + "testing" +) + +func TestTestdata(t *testing.T) { + files, _ := filepath.Glob("testdata/*.txt") + if len(files) == 0 { + t.Fatalf("no testdata") + } + var p Parser + p.Words = map[string]string{ + "italicword": "", + "linkedword": "https://example.com/linkedword", + } + p.LookupPackage = func(name string) (importPath string, ok bool) { + if name == "comment" { + return "go/doc/comment", true + } + return DefaultLookupPackage(name) + } + p.LookupSym = func(recv, name string) (ok bool) { + if recv == "Parser" && name == "Parse" || + recv == "" && name == "Doc" || + recv == "" && name == "NoURL" { + return true + } + return false + } + + stripDollars := func(b []byte) []byte { + // Remove trailing $ on lines. + // They make it easier to see lines with trailing spaces, + // as well as turning them into lines without trailing spaces, + // in case editors remove trailing spaces. + return bytes.ReplaceAll(b, []byte("$\n"), []byte("\n")) + } + for _, file := range files { + t.Run(filepath.Base(file), func(t *testing.T) { + var pr Printer + a, err := txtar.ParseFile(file) + if err != nil { + t.Fatal(err) + } + if len(a.Comment) > 0 { + err := json.Unmarshal(a.Comment, &pr) + if err != nil { + t.Fatalf("unmarshalling top json: %v", err) + } + } + if len(a.Files) < 1 || a.Files[0].Name != "input" { + t.Fatalf("first file is not %q", "input") + } + d := p.Parse(string(stripDollars(a.Files[0].Data))) + for _, f := range a.Files[1:] { + want := stripDollars(f.Data) + for len(want) >= 2 && want[len(want)-1] == '\n' && want[len(want)-2] == '\n' { + want = want[:len(want)-1] + } + var out []byte + switch f.Name { + default: + t.Fatalf("unknown output file %q", f.Name) + case "dump": + out = dump(d) + case "gofmt": + out = pr.Comment(d) + case "html": + out = pr.HTML(d) + case "markdown": + out = pr.Markdown(d) + case "text": + out = pr.Text(d) + } + if string(out) != string(want) { + t.Errorf("%s: %s", file, diff.Diff(f.Name, want, "have", out)) + } + } + }) + } +} + +func dump(d *Doc) []byte { + var out bytes.Buffer + dumpTo(&out, 0, d) + return out.Bytes() +} + +func dumpTo(out *bytes.Buffer, indent int, x any) { + switch x := x.(type) { + default: + fmt.Fprintf(out, "?%T", x) + + case *Doc: + fmt.Fprintf(out, "Doc") + dumpTo(out, indent+1, x.Content) + if len(x.Links) > 0 { + dumpNL(out, indent+1) + fmt.Fprintf(out, "Links") + dumpTo(out, indent+2, x.Links) + } + fmt.Fprintf(out, "\n") + + case []*LinkDef: + for _, def := range x { + dumpNL(out, indent) + dumpTo(out, indent, def) + } + + case *LinkDef: + fmt.Fprintf(out, "LinkDef Used:%v Text:%q URL:%s", x.Used, x.Text, x.URL) + + case []Block: + for _, blk := range x { + dumpNL(out, indent) + dumpTo(out, indent, blk) + } + + case *Heading: + fmt.Fprintf(out, "Heading") + dumpTo(out, indent+1, x.Text) + + case *List: + fmt.Fprintf(out, "List ForceBlankBefore=%v ForceBlankBetween=%v", x.ForceBlankBefore, x.ForceBlankBetween) + dumpTo(out, indent+1, x.Items) + + case []*ListItem: + for _, item := range x { + dumpNL(out, indent) + dumpTo(out, indent, item) + } + + case *ListItem: + fmt.Fprintf(out, "Item Number=%q", x.Number) + dumpTo(out, indent+1, x.Content) + + case *Paragraph: + fmt.Fprintf(out, "Paragraph") + dumpTo(out, indent+1, x.Text) + + case *Code: + fmt.Fprintf(out, "Code") + dumpTo(out, indent+1, x.Text) + + case []Text: + for _, t := range x { + dumpNL(out, indent) + dumpTo(out, indent, t) + } + + case Plain: + if !strings.Contains(string(x), "\n") { + fmt.Fprintf(out, "Plain %q", string(x)) + } else { + fmt.Fprintf(out, "Plain") + dumpTo(out, indent+1, string(x)) + } + + case Italic: + if !strings.Contains(string(x), "\n") { + fmt.Fprintf(out, "Italic %q", string(x)) + } else { + fmt.Fprintf(out, "Italic") + dumpTo(out, indent+1, string(x)) + } + + case string: + for _, line := range strings.SplitAfter(x, "\n") { + if line != "" { + dumpNL(out, indent) + fmt.Fprintf(out, "%q", line) + } + } + + case *Link: + fmt.Fprintf(out, "Link %q", x.URL) + dumpTo(out, indent+1, x.Text) + + case *DocLink: + fmt.Fprintf(out, "DocLink pkg:%q, recv:%q, name:%q", x.ImportPath, x.Recv, x.Name) + dumpTo(out, indent+1, x.Text) + } +} + +func dumpNL(out *bytes.Buffer, n int) { + out.WriteByte('\n') + for i := 0; i < n; i++ { + out.WriteByte('\t') + } +} diff --git a/src/go/doc/comment/text.go b/src/go/doc/comment/text.go new file mode 100644 index 0000000..6f9c2e2 --- /dev/null +++ b/src/go/doc/comment/text.go @@ -0,0 +1,337 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package comment + +import ( + "bytes" + "fmt" + "sort" + "strings" + "unicode/utf8" +) + +// A textPrinter holds the state needed for printing a Doc as plain text. +type textPrinter struct { + *Printer + long strings.Builder + prefix string + codePrefix string + width int +} + +// Text returns a textual formatting of the Doc. +// See the [Printer] documentation for ways to customize the text output. +func (p *Printer) Text(d *Doc) []byte { + tp := &textPrinter{ + Printer: p, + prefix: p.TextPrefix, + codePrefix: p.TextCodePrefix, + width: p.TextWidth, + } + if tp.codePrefix == "" { + tp.codePrefix = p.TextPrefix + "\t" + } + if tp.width == 0 { + tp.width = 80 - utf8.RuneCountInString(tp.prefix) + } + + var out bytes.Buffer + for i, x := range d.Content { + if i > 0 && blankBefore(x) { + out.WriteString(tp.prefix) + writeNL(&out) + } + tp.block(&out, x) + } + anyUsed := false + for _, def := range d.Links { + if def.Used { + anyUsed = true + break + } + } + if anyUsed { + writeNL(&out) + for _, def := range d.Links { + if def.Used { + fmt.Fprintf(&out, "[%s]: %s\n", def.Text, def.URL) + } + } + } + return out.Bytes() +} + +// writeNL calls out.WriteByte('\n') +// but first trims trailing spaces on the previous line. +func writeNL(out *bytes.Buffer) { + // Trim trailing spaces. + data := out.Bytes() + n := 0 + for n < len(data) && (data[len(data)-n-1] == ' ' || data[len(data)-n-1] == '\t') { + n++ + } + if n > 0 { + out.Truncate(len(data) - n) + } + out.WriteByte('\n') +} + +// block prints the block x to out. +func (p *textPrinter) block(out *bytes.Buffer, x Block) { + switch x := x.(type) { + default: + fmt.Fprintf(out, "?%T\n", x) + + case *Paragraph: + out.WriteString(p.prefix) + p.text(out, "", x.Text) + + case *Heading: + out.WriteString(p.prefix) + out.WriteString("# ") + p.text(out, "", x.Text) + + case *Code: + text := x.Text + for text != "" { + var line string + line, text, _ = strings.Cut(text, "\n") + if line != "" { + out.WriteString(p.codePrefix) + out.WriteString(line) + } + writeNL(out) + } + + case *List: + loose := x.BlankBetween() + for i, item := range x.Items { + if i > 0 && loose { + out.WriteString(p.prefix) + writeNL(out) + } + out.WriteString(p.prefix) + out.WriteString(" ") + if item.Number == "" { + out.WriteString(" - ") + } else { + out.WriteString(item.Number) + out.WriteString(". ") + } + for i, blk := range item.Content { + const fourSpace = " " + if i > 0 { + writeNL(out) + out.WriteString(p.prefix) + out.WriteString(fourSpace) + } + p.text(out, fourSpace, blk.(*Paragraph).Text) + } + } + } +} + +// text prints the text sequence x to out. +func (p *textPrinter) text(out *bytes.Buffer, indent string, x []Text) { + p.oneLongLine(&p.long, x) + words := strings.Fields(p.long.String()) + p.long.Reset() + + var seq []int + if p.width < 0 || len(words) == 0 { + seq = []int{0, len(words)} // one long line + } else { + seq = wrap(words, p.width-utf8.RuneCountInString(indent)) + } + for i := 0; i+1 < len(seq); i++ { + if i > 0 { + out.WriteString(p.prefix) + out.WriteString(indent) + } + for j, w := range words[seq[i]:seq[i+1]] { + if j > 0 { + out.WriteString(" ") + } + out.WriteString(w) + } + writeNL(out) + } +} + +// oneLongLine prints the text sequence x to out as one long line, +// without worrying about line wrapping. +// Explicit links have the [ ] dropped to improve readability. +func (p *textPrinter) oneLongLine(out *strings.Builder, x []Text) { + for _, t := range x { + switch t := t.(type) { + case Plain: + out.WriteString(string(t)) + case Italic: + out.WriteString(string(t)) + case *Link: + p.oneLongLine(out, t.Text) + case *DocLink: + p.oneLongLine(out, t.Text) + } + } +} + +// wrap wraps words into lines of at most max runes, +// minimizing the sum of the squares of the leftover lengths +// at the end of each line (except the last, of course), +// with a preference for ending lines at punctuation (.,:;). +// +// The returned slice gives the indexes of the first words +// on each line in the wrapped text with a final entry of len(words). +// Thus the lines are words[seq[0]:seq[1]], words[seq[1]:seq[2]], +// ..., words[seq[len(seq)-2]:seq[len(seq)-1]]. +// +// The implementation runs in O(n log n) time, where n = len(words), +// using the algorithm described in D. S. Hirschberg and L. L. Larmore, +// ā€œ[The least weight subsequence problem],ā€ FOCS 1985, pp. 137-143. +// +// [The least weight subsequence problem]: https://doi.org/10.1109/SFCS.1985.60 +func wrap(words []string, max int) (seq []int) { + // The algorithm requires that our scoring function be concave, + // meaning that for all iā‚€ ā‰¤ iā‚ < jā‚€ ā‰¤ jā‚, + // weight(iā‚€, jā‚€) + weight(iā‚, jā‚) ā‰¤ weight(iā‚€, jā‚) + weight(iā‚, jā‚€). + // + // Our weights are two-element pairs [hi, lo] + // ordered by elementwise comparison. + // The hi entry counts the weight for lines that are longer than max, + // and the lo entry counts the weight for lines that are not. + // This forces the algorithm to first minimize the number of lines + // that are longer than max, which correspond to lines with + // single very long words. Having done that, it can move on to + // minimizing the lo score, which is more interesting. + // + // The lo score is the sum for each line of the square of the + // number of spaces remaining at the end of the line and a + // penalty of 64 given out for not ending the line in a + // punctuation character (.,:;). + // The penalty is somewhat arbitrarily chosen by trying + // different amounts and judging how nice the wrapped text looks. + // Roughly speaking, using 64 means that we are willing to + // end a line with eight blank spaces in order to end at a + // punctuation character, even if the next word would fit in + // those spaces. + // + // We care about ending in punctuation characters because + // it makes the text easier to skim if not too many sentences + // or phrases begin with a single word on the previous line. + + // A score is the score (also called weight) for a given line. + // add and cmp add and compare scores. + type score struct { + hi int64 + lo int64 + } + add := func(s, t score) score { return score{s.hi + t.hi, s.lo + t.lo} } + cmp := func(s, t score) int { + switch { + case s.hi < t.hi: + return -1 + case s.hi > t.hi: + return +1 + case s.lo < t.lo: + return -1 + case s.lo > t.lo: + return +1 + } + return 0 + } + + // total[j] is the total number of runes + // (including separating spaces) in words[:j]. + total := make([]int, len(words)+1) + total[0] = 0 + for i, s := range words { + total[1+i] = total[i] + utf8.RuneCountInString(s) + 1 + } + + // weight returns weight(i, j). + weight := func(i, j int) score { + // On the last line, there is zero weight for being too short. + n := total[j] - 1 - total[i] + if j == len(words) && n <= max { + return score{0, 0} + } + + // Otherwise the weight is the penalty plus the square of the number of + // characters remaining on the line or by which the line goes over. + // In the latter case, that value goes in the hi part of the score. + // (See note above.) + p := wrapPenalty(words[j-1]) + v := int64(max-n) * int64(max-n) + if n > max { + return score{v, p} + } + return score{0, v + p} + } + + // The rest of this function is ā€œThe Basic Algorithmā€ from + // Hirschberg and Larmore's conference paper, + // using the same names as in the paper. + f := []score{{0, 0}} + g := func(i, j int) score { return add(f[i], weight(i, j)) } + + bridge := func(a, b, c int) bool { + k := c + sort.Search(len(words)+1-c, func(k int) bool { + k += c + return cmp(g(a, k), g(b, k)) > 0 + }) + if k > len(words) { + return true + } + return cmp(g(c, k), g(b, k)) <= 0 + } + + // d is a one-ended deque implemented as a slice. + d := make([]int, 1, len(words)) + d[0] = 0 + bestleft := make([]int, 1, len(words)) + bestleft[0] = -1 + for m := 1; m < len(words); m++ { + f = append(f, g(d[0], m)) + bestleft = append(bestleft, d[0]) + for len(d) > 1 && cmp(g(d[1], m+1), g(d[0], m+1)) <= 0 { + d = d[1:] // ā€œRetireā€ + } + for len(d) > 1 && bridge(d[len(d)-2], d[len(d)-1], m) { + d = d[:len(d)-1] // ā€œFireā€ + } + if cmp(g(m, len(words)), g(d[len(d)-1], len(words))) < 0 { + d = append(d, m) // ā€œHireā€ + // The next few lines are not in the paper but are necessary + // to handle two-word inputs correctly. It appears to be + // just a bug in the paper's pseudocode. + if len(d) == 2 && cmp(g(d[1], m+1), g(d[0], m+1)) <= 0 { + d = d[1:] + } + } + } + bestleft = append(bestleft, d[0]) + + // Recover least weight sequence from bestleft. + n := 1 + for m := len(words); m > 0; m = bestleft[m] { + n++ + } + seq = make([]int, n) + for m := len(words); m > 0; m = bestleft[m] { + n-- + seq[n] = m + } + return seq +} + +// wrapPenalty is the penalty for inserting a line break after word s. +func wrapPenalty(s string) int64 { + switch s[len(s)-1] { + case '.', ',', ':', ';': + return 0 + } + return 64 +} diff --git a/src/go/doc/comment/wrap_test.go b/src/go/doc/comment/wrap_test.go new file mode 100644 index 0000000..f9802c9 --- /dev/null +++ b/src/go/doc/comment/wrap_test.go @@ -0,0 +1,141 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package comment + +import ( + "flag" + "fmt" + "math/rand" + "testing" + "time" + "unicode/utf8" +) + +var wrapSeed = flag.Int64("wrapseed", 0, "use `seed` for wrap test (default auto-seeds)") + +func TestWrap(t *testing.T) { + if *wrapSeed == 0 { + *wrapSeed = time.Now().UnixNano() + } + t.Logf("-wrapseed=%#x\n", *wrapSeed) + r := rand.New(rand.NewSource(*wrapSeed)) + + // Generate words of random length. + s := "1234567890Ī±Ī²cdefghijklmnopqrstuvwxyz" + sN := utf8.RuneCountInString(s) + var words []string + for i := 0; i < 100; i++ { + n := 1 + r.Intn(sN-1) + if n >= 12 { + n++ // extra byte for Ī² + } + if n >= 11 { + n++ // extra byte for Ī± + } + words = append(words, s[:n]) + } + + for n := 1; n <= len(words) && !t.Failed(); n++ { + t.Run(fmt.Sprint("n=", n), func(t *testing.T) { + words := words[:n] + t.Logf("words: %v", words) + for max := 1; max < 100 && !t.Failed(); max++ { + t.Run(fmt.Sprint("max=", max), func(t *testing.T) { + seq := wrap(words, max) + + // Compute score for seq. + start := 0 + score := int64(0) + if len(seq) == 0 { + t.Fatalf("wrap seq is empty") + } + if seq[0] != 0 { + t.Fatalf("wrap seq does not start with 0") + } + for _, n := range seq[1:] { + if n <= start { + t.Fatalf("wrap seq is non-increasing: %v", seq) + } + if n > len(words) { + t.Fatalf("wrap seq contains %d > %d: %v", n, len(words), seq) + } + size := -1 + for _, s := range words[start:n] { + size += 1 + utf8.RuneCountInString(s) + } + if n-start == 1 && size >= max { + // no score + } else if size > max { + t.Fatalf("wrap used overlong line %d:%d: %v", start, n, words[start:n]) + } else if n != len(words) { + score += int64(max-size)*int64(max-size) + wrapPenalty(words[n-1]) + } + start = n + } + if start != len(words) { + t.Fatalf("wrap seq does not use all words (%d < %d): %v", start, len(words), seq) + } + + // Check that score matches slow reference implementation. + slowSeq, slowScore := wrapSlow(words, max) + if score != slowScore { + t.Fatalf("wrap score = %d != wrapSlow score %d\nwrap: %v\nslow: %v", score, slowScore, seq, slowSeq) + } + }) + } + }) + } +} + +// wrapSlow is an O(nĀ²) reference implementation for wrap. +// It returns a minimal-score sequence along with the score. +// It is OK if wrap returns a different sequence as long as that +// sequence has the same score. +func wrapSlow(words []string, max int) (seq []int, score int64) { + // Quadratic dynamic programming algorithm for line wrapping problem. + // best[i] tracks the best score possible for words[:i], + // assuming that for i < len(words) the line breaks after those words. + // bestleft[i] tracks the previous line break for best[i]. + best := make([]int64, len(words)+1) + bestleft := make([]int, len(words)+1) + best[0] = 0 + for i, w := range words { + if utf8.RuneCountInString(w) >= max { + // Overlong word must appear on line by itself. No effect on score. + best[i+1] = best[i] + continue + } + best[i+1] = 1e18 + p := wrapPenalty(w) + n := -1 + for j := i; j >= 0; j-- { + n += 1 + utf8.RuneCountInString(words[j]) + if n > max { + break + } + line := int64(n-max)*int64(n-max) + p + if i == len(words)-1 { + line = 0 // no score for final line being too short + } + s := best[j] + line + if best[i+1] > s { + best[i+1] = s + bestleft[i+1] = j + } + } + } + + // Recover least weight sequence from bestleft. + n := 1 + for m := len(words); m > 0; m = bestleft[m] { + n++ + } + seq = make([]int, n) + for m := len(words); m > 0; m = bestleft[m] { + n-- + seq[n] = m + } + return seq, best[len(words)] +} diff --git a/src/go/doc/comment_test.go b/src/go/doc/comment_test.go new file mode 100644 index 0000000..004ae9d --- /dev/null +++ b/src/go/doc/comment_test.go @@ -0,0 +1,67 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "bytes" + "go/parser" + "go/token" + "internal/diff" + "testing" +) + +func TestComment(t *testing.T) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, "testdata/pkgdoc", nil, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + if pkgs["pkgdoc"] == nil { + t.Fatal("missing package pkgdoc") + } + pkg := New(pkgs["pkgdoc"], "testdata/pkgdoc", 0) + + var ( + input = "[T] and [U] are types, and [T.M] is a method, but [V] is a broken link. [rand.Int] and [crand.Reader] are things. [G.M1] and [G.M2] are generic methods.\n" + wantHTML = `

T and U are types, and T.M is a method, but [V] is a broken link. rand.Int and crand.Reader are things. G.M1 and G.M2 are generic methods.` + "\n" + wantOldHTML = "

[T] and [U] are types, and [T.M] is a method, but [V] is a broken link. [rand.Int] and [crand.Reader] are things. [G.M1] and [G.M2] are generic methods.\n" + wantMarkdown = "[T](#T) and [U](#U) are types, and [T.M](#T.M) is a method, but \\[V] is a broken link. [rand.Int](/math/rand#Int) and [crand.Reader](/crypto/rand#Reader) are things. [G.M1](#G.M1) and [G.M2](#G.M2) are generic methods.\n" + wantText = "T and U are types, and T.M is a method, but [V] is a broken link. rand.Int and\ncrand.Reader are things. G.M1 and G.M2 are generic methods.\n" + wantOldText = "[T] and [U] are types, and [T.M] is a method, but [V] is a broken link.\n[rand.Int] and [crand.Reader] are things. [G.M1] and [G.M2] are generic methods.\n" + wantSynopsis = "T and U are types, and T.M is a method, but [V] is a broken link." + wantOldSynopsis = "[T] and [U] are types, and [T.M] is a method, but [V] is a broken link." + ) + + if b := pkg.HTML(input); string(b) != wantHTML { + t.Errorf("%s", diff.Diff("pkg.HTML", b, "want", []byte(wantHTML))) + } + if b := pkg.Markdown(input); string(b) != wantMarkdown { + t.Errorf("%s", diff.Diff("pkg.Markdown", b, "want", []byte(wantMarkdown))) + } + if b := pkg.Text(input); string(b) != wantText { + t.Errorf("%s", diff.Diff("pkg.Text", b, "want", []byte(wantText))) + } + if b := pkg.Synopsis(input); b != wantSynopsis { + t.Errorf("%s", diff.Diff("pkg.Synopsis", []byte(b), "want", []byte(wantText))) + } + + var buf bytes.Buffer + + buf.Reset() + ToHTML(&buf, input, map[string]string{"types": ""}) + if b := buf.Bytes(); string(b) != wantOldHTML { + t.Errorf("%s", diff.Diff("ToHTML", b, "want", []byte(wantOldHTML))) + } + + buf.Reset() + ToText(&buf, input, "", "\t", 80) + if b := buf.Bytes(); string(b) != wantOldText { + t.Errorf("%s", diff.Diff("ToText", b, "want", []byte(wantOldText))) + } + + if b := Synopsis(input); b != wantOldSynopsis { + t.Errorf("%s", diff.Diff("Synopsis", []byte(b), "want", []byte(wantOldText))) + } +} diff --git a/src/go/doc/doc.go b/src/go/doc/doc.go new file mode 100644 index 0000000..eefadfa --- /dev/null +++ b/src/go/doc/doc.go @@ -0,0 +1,354 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package doc extracts source code documentation from a Go AST. +package doc + +import ( + "fmt" + "go/ast" + "go/doc/comment" + "go/token" + "strings" +) + +// Package is the documentation for an entire package. +type Package struct { + Doc string + Name string + ImportPath string + Imports []string + Filenames []string + Notes map[string][]*Note + + // Deprecated: For backward compatibility Bugs is still populated, + // but all new code should use Notes instead. + Bugs []string + + // declarations + Consts []*Value + Types []*Type + Vars []*Value + Funcs []*Func + + // Examples is a sorted list of examples associated with + // the package. Examples are extracted from _test.go files + // provided to NewFromFiles. + Examples []*Example + + importByName map[string]string + syms map[string]bool +} + +// Value is the documentation for a (possibly grouped) var or const declaration. +type Value struct { + Doc string + Names []string // var or const names in declaration order + Decl *ast.GenDecl + + order int +} + +// Type is the documentation for a type declaration. +type Type struct { + Doc string + Name string + Decl *ast.GenDecl + + // associated declarations + Consts []*Value // sorted list of constants of (mostly) this type + Vars []*Value // sorted list of variables of (mostly) this type + Funcs []*Func // sorted list of functions returning this type + Methods []*Func // sorted list of methods (including embedded ones) of this type + + // Examples is a sorted list of examples associated with + // this type. Examples are extracted from _test.go files + // provided to NewFromFiles. + Examples []*Example +} + +// Func is the documentation for a func declaration. +type Func struct { + Doc string + Name string + Decl *ast.FuncDecl + + // methods + // (for functions, these fields have the respective zero value) + Recv string // actual receiver "T" or "*T" possibly followed by type parameters [P1, ..., Pn] + Orig string // original receiver "T" or "*T" + Level int // embedding level; 0 means not embedded + + // Examples is a sorted list of examples associated with this + // function or method. Examples are extracted from _test.go files + // provided to NewFromFiles. + Examples []*Example +} + +// A Note represents a marked comment starting with "MARKER(uid): note body". +// Any note with a marker of 2 or more upper case [A-Z] letters and a uid of +// at least one character is recognized. The ":" following the uid is optional. +// Notes are collected in the Package.Notes map indexed by the notes marker. +type Note struct { + Pos, End token.Pos // position range of the comment containing the marker + UID string // uid found with the marker + Body string // note body text +} + +// Mode values control the operation of New and NewFromFiles. +type Mode int + +const ( + // AllDecls says to extract documentation for all package-level + // declarations, not just exported ones. + AllDecls Mode = 1 << iota + + // AllMethods says to show all embedded methods, not just the ones of + // invisible (unexported) anonymous fields. + AllMethods + + // PreserveAST says to leave the AST unmodified. Originally, pieces of + // the AST such as function bodies were nil-ed out to save memory in + // godoc, but not all programs want that behavior. + PreserveAST +) + +// New computes the package documentation for the given package AST. +// New takes ownership of the AST pkg and may edit or overwrite it. +// To have the Examples fields populated, use NewFromFiles and include +// the package's _test.go files. +func New(pkg *ast.Package, importPath string, mode Mode) *Package { + var r reader + r.readPackage(pkg, mode) + r.computeMethodSets() + r.cleanupTypes() + p := &Package{ + Doc: r.doc, + Name: pkg.Name, + ImportPath: importPath, + Imports: sortedKeys(r.imports), + Filenames: r.filenames, + Notes: r.notes, + Bugs: noteBodies(r.notes["BUG"]), + Consts: sortedValues(r.values, token.CONST), + Types: sortedTypes(r.types, mode&AllMethods != 0), + Vars: sortedValues(r.values, token.VAR), + Funcs: sortedFuncs(r.funcs, true), + + importByName: r.importByName, + syms: make(map[string]bool), + } + + p.collectValues(p.Consts) + p.collectValues(p.Vars) + p.collectTypes(p.Types) + p.collectFuncs(p.Funcs) + + return p +} + +func (p *Package) collectValues(values []*Value) { + for _, v := range values { + for _, name := range v.Names { + p.syms[name] = true + } + } +} + +func (p *Package) collectTypes(types []*Type) { + for _, t := range types { + if p.syms[t.Name] { + // Shouldn't be any cycles but stop just in case. + continue + } + p.syms[t.Name] = true + p.collectValues(t.Consts) + p.collectValues(t.Vars) + p.collectFuncs(t.Funcs) + p.collectFuncs(t.Methods) + } +} + +func (p *Package) collectFuncs(funcs []*Func) { + for _, f := range funcs { + if f.Recv != "" { + r := strings.TrimPrefix(f.Recv, "*") + if i := strings.IndexByte(r, '['); i >= 0 { + r = r[:i] // remove type parameters + } + p.syms[r+"."+f.Name] = true + } else { + p.syms[f.Name] = true + } + } +} + +// NewFromFiles computes documentation for a package. +// +// The package is specified by a list of *ast.Files and corresponding +// file set, which must not be nil. +// NewFromFiles uses all provided files when computing documentation, +// so it is the caller's responsibility to provide only the files that +// match the desired build context. "go/build".Context.MatchFile can +// be used for determining whether a file matches a build context with +// the desired GOOS and GOARCH values, and other build constraints. +// The import path of the package is specified by importPath. +// +// Examples found in _test.go files are associated with the corresponding +// type, function, method, or the package, based on their name. +// If the example has a suffix in its name, it is set in the +// Example.Suffix field. Examples with malformed names are skipped. +// +// Optionally, a single extra argument of type Mode can be provided to +// control low-level aspects of the documentation extraction behavior. +// +// NewFromFiles takes ownership of the AST files and may edit them, +// unless the PreserveAST Mode bit is on. +func NewFromFiles(fset *token.FileSet, files []*ast.File, importPath string, opts ...any) (*Package, error) { + // Check for invalid API usage. + if fset == nil { + panic(fmt.Errorf("doc.NewFromFiles: no token.FileSet provided (fset == nil)")) + } + var mode Mode + switch len(opts) { // There can only be 0 or 1 options, so a simple switch works for now. + case 0: + // Nothing to do. + case 1: + m, ok := opts[0].(Mode) + if !ok { + panic(fmt.Errorf("doc.NewFromFiles: option argument type must be doc.Mode")) + } + mode = m + default: + panic(fmt.Errorf("doc.NewFromFiles: there must not be more than 1 option argument")) + } + + // Collect .go and _test.go files. + var ( + goFiles = make(map[string]*ast.File) + testGoFiles []*ast.File + ) + for i := range files { + f := fset.File(files[i].Pos()) + if f == nil { + return nil, fmt.Errorf("file files[%d] is not found in the provided file set", i) + } + switch name := f.Name(); { + case strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go"): + goFiles[name] = files[i] + case strings.HasSuffix(name, "_test.go"): + testGoFiles = append(testGoFiles, files[i]) + default: + return nil, fmt.Errorf("file files[%d] filename %q does not have a .go extension", i, name) + } + } + + // TODO(dmitshur,gri): A relatively high level call to ast.NewPackage with a simpleImporter + // ast.Importer implementation is made below. It might be possible to short-circuit and simplify. + + // Compute package documentation. + pkg, _ := ast.NewPackage(fset, goFiles, simpleImporter, nil) // Ignore errors that can happen due to unresolved identifiers. + p := New(pkg, importPath, mode) + classifyExamples(p, Examples(testGoFiles...)) + return p, nil +} + +// simpleImporter returns a (dummy) package object named by the last path +// component of the provided package path (as is the convention for packages). +// This is sufficient to resolve package identifiers without doing an actual +// import. It never returns an error. +func simpleImporter(imports map[string]*ast.Object, path string) (*ast.Object, error) { + pkg := imports[path] + if pkg == nil { + // note that strings.LastIndex returns -1 if there is no "/" + pkg = ast.NewObj(ast.Pkg, path[strings.LastIndex(path, "/")+1:]) + pkg.Data = ast.NewScope(nil) // required by ast.NewPackage for dot-import + imports[path] = pkg + } + return pkg, nil +} + +// lookupSym reports whether the package has a given symbol or method. +// +// If recv == "", HasSym reports whether the package has a top-level +// const, func, type, or var named name. +// +// If recv != "", HasSym reports whether the package has a type +// named recv with a method named name. +func (p *Package) lookupSym(recv, name string) bool { + if recv != "" { + return p.syms[recv+"."+name] + } + return p.syms[name] +} + +// lookupPackage returns the import path identified by name +// in the given package. If name uniquely identifies a single import, +// then lookupPackage returns that import. +// If multiple packages are imported as name, importPath returns "", false. +// Otherwise, if name is the name of p itself, importPath returns "", true, +// to signal a reference to p. +// Otherwise, importPath returns "", false. +func (p *Package) lookupPackage(name string) (importPath string, ok bool) { + if path, ok := p.importByName[name]; ok { + if path == "" { + return "", false // multiple imports used the name + } + return path, true // found import + } + if p.Name == name { + return "", true // allow reference to this package + } + return "", false // unknown name +} + +// Parser returns a doc comment parser configured +// for parsing doc comments from package p. +// Each call returns a new parser, so that the caller may +// customize it before use. +func (p *Package) Parser() *comment.Parser { + return &comment.Parser{ + LookupPackage: p.lookupPackage, + LookupSym: p.lookupSym, + } +} + +// Printer returns a doc comment printer configured +// for printing doc comments from package p. +// Each call returns a new printer, so that the caller may +// customize it before use. +func (p *Package) Printer() *comment.Printer { + // No customization today, but having p.Printer() + // gives us flexibility in the future, and it is convenient for callers. + return &comment.Printer{} +} + +// HTML returns formatted HTML for the doc comment text. +// +// To customize details of the HTML, use [Package.Printer] +// to obtain a [comment.Printer], and configure it +// before calling its HTML method. +func (p *Package) HTML(text string) []byte { + return p.Printer().HTML(p.Parser().Parse(text)) +} + +// Markdown returns formatted Markdown for the doc comment text. +// +// To customize details of the Markdown, use [Package.Printer] +// to obtain a [comment.Printer], and configure it +// before calling its Markdown method. +func (p *Package) Markdown(text string) []byte { + return p.Printer().Markdown(p.Parser().Parse(text)) +} + +// Text returns formatted text for the doc comment text, +// wrapped to 80 Unicode code points and using tabs for +// code block indentation. +// +// To customize details of the formatting, use [Package.Printer] +// to obtain a [comment.Printer], and configure it +// before calling its Text method. +func (p *Package) Text(text string) []byte { + return p.Printer().Text(p.Parser().Parse(text)) +} diff --git a/src/go/doc/doc_test.go b/src/go/doc/doc_test.go new file mode 100644 index 0000000..b79087e --- /dev/null +++ b/src/go/doc/doc_test.go @@ -0,0 +1,292 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "text/template" +) + +var update = flag.Bool("update", false, "update golden (.out) files") +var files = flag.String("files", "", "consider only Go test files matching this regular expression") + +const dataDir = "testdata" + +var templateTxt = readTemplate("template.txt") + +func readTemplate(filename string) *template.Template { + t := template.New(filename) + t.Funcs(template.FuncMap{ + "node": nodeFmt, + "synopsis": synopsisFmt, + "indent": indentFmt, + }) + return template.Must(t.ParseFiles(filepath.Join(dataDir, filename))) +} + +func nodeFmt(node any, fset *token.FileSet) string { + var buf bytes.Buffer + printer.Fprint(&buf, fset, node) + return strings.ReplaceAll(strings.TrimSpace(buf.String()), "\n", "\n\t") +} + +func synopsisFmt(s string) string { + const n = 64 + if len(s) > n { + // cut off excess text and go back to a word boundary + s = s[0:n] + if i := strings.LastIndexAny(s, "\t\n "); i >= 0 { + s = s[0:i] + } + s = strings.TrimSpace(s) + " ..." + } + return "// " + strings.ReplaceAll(s, "\n", " ") +} + +func indentFmt(indent, s string) string { + end := "" + if strings.HasSuffix(s, "\n") { + end = "\n" + s = s[:len(s)-1] + } + return indent + strings.ReplaceAll(s, "\n", "\n"+indent) + end +} + +func isGoFile(fi fs.FileInfo) bool { + name := fi.Name() + return !fi.IsDir() && + len(name) > 0 && name[0] != '.' && // ignore .files + filepath.Ext(name) == ".go" +} + +type bundle struct { + *Package + FSet *token.FileSet +} + +func test(t *testing.T, mode Mode) { + // determine file filter + filter := isGoFile + if *files != "" { + rx, err := regexp.Compile(*files) + if err != nil { + t.Fatal(err) + } + filter = func(fi fs.FileInfo) bool { + return isGoFile(fi) && rx.MatchString(fi.Name()) + } + } + + // get packages + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, dataDir, filter, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + + // test packages + for _, pkg := range pkgs { + t.Run(pkg.Name, func(t *testing.T) { + importPath := dataDir + "/" + pkg.Name + var files []*ast.File + for _, f := range pkg.Files { + files = append(files, f) + } + doc, err := NewFromFiles(fset, files, importPath, mode) + if err != nil { + t.Fatal(err) + } + + // golden files always use / in filenames - canonicalize them + for i, filename := range doc.Filenames { + doc.Filenames[i] = filepath.ToSlash(filename) + } + + // print documentation + var buf bytes.Buffer + if err := templateTxt.Execute(&buf, bundle{doc, fset}); err != nil { + t.Fatal(err) + } + got := buf.Bytes() + + // update golden file if necessary + golden := filepath.Join(dataDir, fmt.Sprintf("%s.%d.golden", pkg.Name, mode)) + if *update { + err := os.WriteFile(golden, got, 0644) + if err != nil { + t.Fatal(err) + } + } + + // get golden file + want, err := os.ReadFile(golden) + if err != nil { + t.Fatal(err) + } + + // compare + if !bytes.Equal(got, want) { + t.Errorf("package %s\n\tgot:\n%s\n\twant:\n%s", pkg.Name, got, want) + } + }) + } +} + +func Test(t *testing.T) { + t.Run("default", func(t *testing.T) { test(t, 0) }) + t.Run("AllDecls", func(t *testing.T) { test(t, AllDecls) }) + t.Run("AllMethods", func(t *testing.T) { test(t, AllMethods) }) +} + +func TestFuncs(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "funcs.go", strings.NewReader(funcsTestFile), parser.ParseComments) + if err != nil { + t.Fatal(err) + } + doc, err := NewFromFiles(fset, []*ast.File{file}, "importPath", Mode(0)) + if err != nil { + t.Fatal(err) + } + + for _, f := range doc.Funcs { + f.Decl = nil + } + for _, ty := range doc.Types { + for _, f := range ty.Funcs { + f.Decl = nil + } + for _, m := range ty.Methods { + m.Decl = nil + } + } + + compareFuncs := func(t *testing.T, msg string, got, want *Func) { + // ignore Decl and Examples + got.Decl = nil + got.Examples = nil + if !(got.Doc == want.Doc && + got.Name == want.Name && + got.Recv == want.Recv && + got.Orig == want.Orig && + got.Level == want.Level) { + t.Errorf("%s:\ngot %+v\nwant %+v", msg, got, want) + } + } + + compareSlices(t, "Funcs", doc.Funcs, funcsPackage.Funcs, compareFuncs) + compareSlices(t, "Types", doc.Types, funcsPackage.Types, func(t *testing.T, msg string, got, want *Type) { + if got.Name != want.Name { + t.Errorf("%s.Name: got %q, want %q", msg, got.Name, want.Name) + } else { + compareSlices(t, got.Name+".Funcs", got.Funcs, want.Funcs, compareFuncs) + compareSlices(t, got.Name+".Methods", got.Methods, want.Methods, compareFuncs) + } + }) +} + +func compareSlices[E any](t *testing.T, name string, got, want []E, compareElem func(*testing.T, string, E, E)) { + if len(got) != len(want) { + t.Errorf("%s: got %d, want %d", name, len(got), len(want)) + } + for i := 0; i < len(got) && i < len(want); i++ { + compareElem(t, fmt.Sprintf("%s[%d]", name, i), got[i], want[i]) + } +} + +const funcsTestFile = ` +package funcs + +func F() {} + +type S1 struct { + S2 // embedded, exported + s3 // embedded, unexported +} + +func NewS1() S1 {return S1{} } +func NewS1p() *S1 { return &S1{} } + +func (S1) M1() {} +func (r S1) M2() {} +func(S1) m3() {} // unexported not shown +func (*S1) P1() {} // pointer receiver + +type S2 int +func (S2) M3() {} // shown on S2 + +type s3 int +func (s3) M4() {} // shown on S1 + +type G1[T any] struct { + *s3 +} + +func NewG1[T any]() G1[T] { return G1[T]{} } + +func (G1[T]) MG1() {} +func (*G1[U]) MG2() {} + +type G2[T, U any] struct {} + +func NewG2[T, U any]() G2[T, U] { return G2[T, U]{} } + +func (G2[T, U]) MG3() {} +func (*G2[A, B]) MG4() {} + + +` + +var funcsPackage = &Package{ + Funcs: []*Func{{Name: "F"}}, + Types: []*Type{ + { + Name: "G1", + Funcs: []*Func{{Name: "NewG1"}}, + Methods: []*Func{ + {Name: "M4", Recv: "G1", // TODO: synthesize a param for G1? + Orig: "s3", Level: 1}, + {Name: "MG1", Recv: "G1[T]", Orig: "G1[T]", Level: 0}, + {Name: "MG2", Recv: "*G1[U]", Orig: "*G1[U]", Level: 0}, + }, + }, + { + Name: "G2", + Funcs: []*Func{{Name: "NewG2"}}, + Methods: []*Func{ + {Name: "MG3", Recv: "G2[T, U]", Orig: "G2[T, U]", Level: 0}, + {Name: "MG4", Recv: "*G2[A, B]", Orig: "*G2[A, B]", Level: 0}, + }, + }, + { + Name: "S1", + Funcs: []*Func{{Name: "NewS1"}, {Name: "NewS1p"}}, + Methods: []*Func{ + {Name: "M1", Recv: "S1", Orig: "S1", Level: 0}, + {Name: "M2", Recv: "S1", Orig: "S1", Level: 0}, + {Name: "M4", Recv: "S1", Orig: "s3", Level: 1}, + {Name: "P1", Recv: "*S1", Orig: "*S1", Level: 0}, + }, + }, + { + Name: "S2", + Methods: []*Func{ + {Name: "M3", Recv: "S2", Orig: "S2", Level: 0}, + }, + }, + }, +} diff --git a/src/go/doc/example.go b/src/go/doc/example.go new file mode 100644 index 0000000..65ca885 --- /dev/null +++ b/src/go/doc/example.go @@ -0,0 +1,722 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Extract example functions from file ASTs. + +package doc + +import ( + "go/ast" + "go/token" + "internal/lazyregexp" + "path" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// An Example represents an example function found in a test source file. +type Example struct { + Name string // name of the item being exemplified (including optional suffix) + Suffix string // example suffix, without leading '_' (only populated by NewFromFiles) + Doc string // example function doc string + Code ast.Node + Play *ast.File // a whole program version of the example + Comments []*ast.CommentGroup + Output string // expected output + Unordered bool + EmptyOutput bool // expect empty output + Order int // original source code order +} + +// Examples returns the examples found in testFiles, sorted by Name field. +// The Order fields record the order in which the examples were encountered. +// The Suffix field is not populated when Examples is called directly, it is +// only populated by NewFromFiles for examples it finds in _test.go files. +// +// Playable Examples must be in a package whose name ends in "_test". +// An Example is "playable" (the Play field is non-nil) in either of these +// circumstances: +// - The example function is self-contained: the function references only +// identifiers from other packages (or predeclared identifiers, such as +// "int") and the test file does not include a dot import. +// - The entire test file is the example: the file contains exactly one +// example function, zero test, fuzz test, or benchmark function, and at +// least one top-level function, type, variable, or constant declaration +// other than the example function. +func Examples(testFiles ...*ast.File) []*Example { + var list []*Example + for _, file := range testFiles { + hasTests := false // file contains tests, fuzz test, or benchmarks + numDecl := 0 // number of non-import declarations in the file + var flist []*Example + for _, decl := range file.Decls { + if g, ok := decl.(*ast.GenDecl); ok && g.Tok != token.IMPORT { + numDecl++ + continue + } + f, ok := decl.(*ast.FuncDecl) + if !ok || f.Recv != nil { + continue + } + numDecl++ + name := f.Name.Name + if isTest(name, "Test") || isTest(name, "Benchmark") || isTest(name, "Fuzz") { + hasTests = true + continue + } + if !isTest(name, "Example") { + continue + } + if params := f.Type.Params; len(params.List) != 0 { + continue // function has params; not a valid example + } + if f.Body == nil { // ast.File.Body nil dereference (see issue 28044) + continue + } + var doc string + if f.Doc != nil { + doc = f.Doc.Text() + } + output, unordered, hasOutput := exampleOutput(f.Body, file.Comments) + flist = append(flist, &Example{ + Name: name[len("Example"):], + Doc: doc, + Code: f.Body, + Play: playExample(file, f), + Comments: file.Comments, + Output: output, + Unordered: unordered, + EmptyOutput: output == "" && hasOutput, + Order: len(flist), + }) + } + if !hasTests && numDecl > 1 && len(flist) == 1 { + // If this file only has one example function, some + // other top-level declarations, and no tests or + // benchmarks, use the whole file as the example. + flist[0].Code = file + flist[0].Play = playExampleFile(file) + } + list = append(list, flist...) + } + // sort by name + sort.Slice(list, func(i, j int) bool { + return list[i].Name < list[j].Name + }) + return list +} + +var outputPrefix = lazyregexp.New(`(?i)^[[:space:]]*(unordered )?output:`) + +// Extracts the expected output and whether there was a valid output comment. +func exampleOutput(b *ast.BlockStmt, comments []*ast.CommentGroup) (output string, unordered, ok bool) { + if _, last := lastComment(b, comments); last != nil { + // test that it begins with the correct prefix + text := last.Text() + if loc := outputPrefix.FindStringSubmatchIndex(text); loc != nil { + if loc[2] != -1 { + unordered = true + } + text = text[loc[1]:] + // Strip zero or more spaces followed by \n or a single space. + text = strings.TrimLeft(text, " ") + if len(text) > 0 && text[0] == '\n' { + text = text[1:] + } + return text, unordered, true + } + } + return "", false, false // no suitable comment found +} + +// isTest tells whether name looks like a test, example, fuzz test, or +// benchmark. It is a Test (say) if there is a character after Test that is not +// a lower-case letter. (We don't want Testiness.) +func isTest(name, prefix string) bool { + if !strings.HasPrefix(name, prefix) { + return false + } + if len(name) == len(prefix) { // "Test" is ok + return true + } + rune, _ := utf8.DecodeRuneInString(name[len(prefix):]) + return !unicode.IsLower(rune) +} + +// playExample synthesizes a new *ast.File based on the provided +// file with the provided function body as the body of main. +func playExample(file *ast.File, f *ast.FuncDecl) *ast.File { + body := f.Body + + if !strings.HasSuffix(file.Name.Name, "_test") { + // We don't support examples that are part of the + // greater package (yet). + return nil + } + + // Collect top-level declarations in the file. + topDecls := make(map[*ast.Object]ast.Decl) + typMethods := make(map[string][]ast.Decl) + + for _, decl := range file.Decls { + switch d := decl.(type) { + case *ast.FuncDecl: + if d.Recv == nil { + topDecls[d.Name.Obj] = d + } else { + if len(d.Recv.List) == 1 { + t := d.Recv.List[0].Type + tname, _ := baseTypeName(t) + typMethods[tname] = append(typMethods[tname], d) + } + } + case *ast.GenDecl: + for _, spec := range d.Specs { + switch s := spec.(type) { + case *ast.TypeSpec: + topDecls[s.Name.Obj] = d + case *ast.ValueSpec: + for _, name := range s.Names { + topDecls[name.Obj] = d + } + } + } + } + } + + // Find unresolved identifiers and uses of top-level declarations. + depDecls, unresolved := findDeclsAndUnresolved(body, topDecls, typMethods) + + // Remove predeclared identifiers from unresolved list. + for n := range unresolved { + if predeclaredTypes[n] || predeclaredConstants[n] || predeclaredFuncs[n] { + delete(unresolved, n) + } + } + + // Use unresolved identifiers to determine the imports used by this + // example. The heuristic assumes package names match base import + // paths for imports w/o renames (should be good enough most of the time). + var namedImports []ast.Spec + var blankImports []ast.Spec // _ imports + + // To preserve the blank lines between groups of imports, find the + // start position of each group, and assign that position to all + // imports from that group. + groupStarts := findImportGroupStarts(file.Imports) + groupStart := func(s *ast.ImportSpec) token.Pos { + for i, start := range groupStarts { + if s.Path.ValuePos < start { + return groupStarts[i-1] + } + } + return groupStarts[len(groupStarts)-1] + } + + for _, s := range file.Imports { + p, err := strconv.Unquote(s.Path.Value) + if err != nil { + continue + } + if p == "syscall/js" { + // We don't support examples that import syscall/js, + // because the package syscall/js is not available in the playground. + return nil + } + n := path.Base(p) + if s.Name != nil { + n = s.Name.Name + switch n { + case "_": + blankImports = append(blankImports, s) + continue + case ".": + // We can't resolve dot imports (yet). + return nil + } + } + if unresolved[n] { + // Copy the spec and its path to avoid modifying the original. + spec := *s + path := *s.Path + spec.Path = &path + spec.Path.ValuePos = groupStart(&spec) + namedImports = append(namedImports, &spec) + delete(unresolved, n) + } + } + + // If there are other unresolved identifiers, give up because this + // synthesized file is not going to build. + if len(unresolved) > 0 { + return nil + } + + // Include documentation belonging to blank imports. + var comments []*ast.CommentGroup + for _, s := range blankImports { + if c := s.(*ast.ImportSpec).Doc; c != nil { + comments = append(comments, c) + } + } + + // Include comments that are inside the function body. + for _, c := range file.Comments { + if body.Pos() <= c.Pos() && c.End() <= body.End() { + comments = append(comments, c) + } + } + + // Strip the "Output:" or "Unordered output:" comment and adjust body + // end position. + body, comments = stripOutputComment(body, comments) + + // Include documentation belonging to dependent declarations. + for _, d := range depDecls { + switch d := d.(type) { + case *ast.GenDecl: + if d.Doc != nil { + comments = append(comments, d.Doc) + } + case *ast.FuncDecl: + if d.Doc != nil { + comments = append(comments, d.Doc) + } + } + } + + // Synthesize import declaration. + importDecl := &ast.GenDecl{ + Tok: token.IMPORT, + Lparen: 1, // Need non-zero Lparen and Rparen so that printer + Rparen: 1, // treats this as a factored import. + } + importDecl.Specs = append(namedImports, blankImports...) + + // Synthesize main function. + funcDecl := &ast.FuncDecl{ + Name: ast.NewIdent("main"), + Type: f.Type, + Body: body, + } + + decls := make([]ast.Decl, 0, 2+len(depDecls)) + decls = append(decls, importDecl) + decls = append(decls, depDecls...) + decls = append(decls, funcDecl) + + sort.Slice(decls, func(i, j int) bool { + return decls[i].Pos() < decls[j].Pos() + }) + sort.Slice(comments, func(i, j int) bool { + return comments[i].Pos() < comments[j].Pos() + }) + + // Synthesize file. + return &ast.File{ + Name: ast.NewIdent("main"), + Decls: decls, + Comments: comments, + } +} + +// findDeclsAndUnresolved returns all the top-level declarations mentioned in +// the body, and a set of unresolved symbols (those that appear in the body but +// have no declaration in the program). +// +// topDecls maps objects to the top-level declaration declaring them (not +// necessarily obj.Decl, as obj.Decl will be a Spec for GenDecls, but +// topDecls[obj] will be the GenDecl itself). +func findDeclsAndUnresolved(body ast.Node, topDecls map[*ast.Object]ast.Decl, typMethods map[string][]ast.Decl) ([]ast.Decl, map[string]bool) { + // This function recursively finds every top-level declaration used + // transitively by the body, populating usedDecls and usedObjs. Then it + // trims down the declarations to include only the symbols actually + // referenced by the body. + + unresolved := make(map[string]bool) + var depDecls []ast.Decl + usedDecls := make(map[ast.Decl]bool) // set of top-level decls reachable from the body + usedObjs := make(map[*ast.Object]bool) // set of objects reachable from the body (each declared by a usedDecl) + + var inspectFunc func(ast.Node) bool + inspectFunc = func(n ast.Node) bool { + switch e := n.(type) { + case *ast.Ident: + if e.Obj == nil && e.Name != "_" { + unresolved[e.Name] = true + } else if d := topDecls[e.Obj]; d != nil { + + usedObjs[e.Obj] = true + if !usedDecls[d] { + usedDecls[d] = true + depDecls = append(depDecls, d) + } + } + return true + case *ast.SelectorExpr: + // For selector expressions, only inspect the left hand side. + // (For an expression like fmt.Println, only add "fmt" to the + // set of unresolved names, not "Println".) + ast.Inspect(e.X, inspectFunc) + return false + case *ast.KeyValueExpr: + // For key value expressions, only inspect the value + // as the key should be resolved by the type of the + // composite literal. + ast.Inspect(e.Value, inspectFunc) + return false + } + return true + } + + inspectFieldList := func(fl *ast.FieldList) { + if fl != nil { + for _, f := range fl.List { + ast.Inspect(f.Type, inspectFunc) + } + } + } + + // Find the decls immediately referenced by body. + ast.Inspect(body, inspectFunc) + // Now loop over them, adding to the list when we find a new decl that the + // body depends on. Keep going until we don't find anything new. + for i := 0; i < len(depDecls); i++ { + switch d := depDecls[i].(type) { + case *ast.FuncDecl: + // Inpect type parameters. + inspectFieldList(d.Type.TypeParams) + // Inspect types of parameters and results. See #28492. + inspectFieldList(d.Type.Params) + inspectFieldList(d.Type.Results) + + // Functions might not have a body. See #42706. + if d.Body != nil { + ast.Inspect(d.Body, inspectFunc) + } + case *ast.GenDecl: + for _, spec := range d.Specs { + switch s := spec.(type) { + case *ast.TypeSpec: + inspectFieldList(s.TypeParams) + ast.Inspect(s.Type, inspectFunc) + depDecls = append(depDecls, typMethods[s.Name.Name]...) + case *ast.ValueSpec: + if s.Type != nil { + ast.Inspect(s.Type, inspectFunc) + } + for _, val := range s.Values { + ast.Inspect(val, inspectFunc) + } + } + } + } + } + + // Some decls include multiple specs, such as a variable declaration with + // multiple variables on the same line, or a parenthesized declaration. Trim + // the declarations to include only the specs that are actually mentioned. + // However, if there is a constant group with iota, leave it all: later + // constant declarations in the group may have no value and so cannot stand + // on their own, and removing any constant from the group could change the + // values of subsequent ones. + // See testdata/examples/iota.go for a minimal example. + var ds []ast.Decl + for _, d := range depDecls { + switch d := d.(type) { + case *ast.FuncDecl: + ds = append(ds, d) + case *ast.GenDecl: + containsIota := false // does any spec have iota? + // Collect all Specs that were mentioned in the example. + var specs []ast.Spec + for _, s := range d.Specs { + switch s := s.(type) { + case *ast.TypeSpec: + if usedObjs[s.Name.Obj] { + specs = append(specs, s) + } + case *ast.ValueSpec: + if !containsIota { + containsIota = hasIota(s) + } + // A ValueSpec may have multiple names (e.g. "var a, b int"). + // Keep only the names that were mentioned in the example. + // Exception: the multiple names have a single initializer (which + // would be a function call with multiple return values). In that + // case, keep everything. + if len(s.Names) > 1 && len(s.Values) == 1 { + specs = append(specs, s) + continue + } + ns := *s + ns.Names = nil + ns.Values = nil + for i, n := range s.Names { + if usedObjs[n.Obj] { + ns.Names = append(ns.Names, n) + if s.Values != nil { + ns.Values = append(ns.Values, s.Values[i]) + } + } + } + if len(ns.Names) > 0 { + specs = append(specs, &ns) + } + } + } + if len(specs) > 0 { + // Constant with iota? Keep it all. + if d.Tok == token.CONST && containsIota { + ds = append(ds, d) + } else { + // Synthesize a GenDecl with just the Specs we need. + nd := *d // copy the GenDecl + nd.Specs = specs + if len(specs) == 1 { + // Remove grouping parens if there is only one spec. + nd.Lparen = 0 + } + ds = append(ds, &nd) + } + } + } + } + return ds, unresolved +} + +func hasIota(s ast.Spec) bool { + has := false + ast.Inspect(s, func(n ast.Node) bool { + // Check that this is the special built-in "iota" identifier, not + // a user-defined shadow. + if id, ok := n.(*ast.Ident); ok && id.Name == "iota" && id.Obj == nil { + has = true + return false + } + return true + }) + return has +} + +// findImportGroupStarts finds the start positions of each sequence of import +// specs that are not separated by a blank line. +func findImportGroupStarts(imps []*ast.ImportSpec) []token.Pos { + startImps := findImportGroupStarts1(imps) + groupStarts := make([]token.Pos, len(startImps)) + for i, imp := range startImps { + groupStarts[i] = imp.Pos() + } + return groupStarts +} + +// Helper for findImportGroupStarts to ease testing. +func findImportGroupStarts1(origImps []*ast.ImportSpec) []*ast.ImportSpec { + // Copy to avoid mutation. + imps := make([]*ast.ImportSpec, len(origImps)) + copy(imps, origImps) + // Assume the imports are sorted by position. + sort.Slice(imps, func(i, j int) bool { return imps[i].Pos() < imps[j].Pos() }) + // Assume gofmt has been applied, so there is a blank line between adjacent imps + // if and only if they are more than 2 positions apart (newline, tab). + var groupStarts []*ast.ImportSpec + prevEnd := token.Pos(-2) + for _, imp := range imps { + if imp.Pos()-prevEnd > 2 { + groupStarts = append(groupStarts, imp) + } + prevEnd = imp.End() + // Account for end-of-line comments. + if imp.Comment != nil { + prevEnd = imp.Comment.End() + } + } + return groupStarts +} + +// playExampleFile takes a whole file example and synthesizes a new *ast.File +// such that the example is function main in package main. +func playExampleFile(file *ast.File) *ast.File { + // Strip copyright comment if present. + comments := file.Comments + if len(comments) > 0 && strings.HasPrefix(comments[0].Text(), "Copyright") { + comments = comments[1:] + } + + // Copy declaration slice, rewriting the ExampleX function to main. + var decls []ast.Decl + for _, d := range file.Decls { + if f, ok := d.(*ast.FuncDecl); ok && isTest(f.Name.Name, "Example") { + // Copy the FuncDecl, as it may be used elsewhere. + newF := *f + newF.Name = ast.NewIdent("main") + newF.Body, comments = stripOutputComment(f.Body, comments) + d = &newF + } + decls = append(decls, d) + } + + // Copy the File, as it may be used elsewhere. + f := *file + f.Name = ast.NewIdent("main") + f.Decls = decls + f.Comments = comments + return &f +} + +// stripOutputComment finds and removes the "Output:" or "Unordered output:" +// comment from body and comments, and adjusts the body block's end position. +func stripOutputComment(body *ast.BlockStmt, comments []*ast.CommentGroup) (*ast.BlockStmt, []*ast.CommentGroup) { + // Do nothing if there is no "Output:" or "Unordered output:" comment. + i, last := lastComment(body, comments) + if last == nil || !outputPrefix.MatchString(last.Text()) { + return body, comments + } + + // Copy body and comments, as the originals may be used elsewhere. + newBody := &ast.BlockStmt{ + Lbrace: body.Lbrace, + List: body.List, + Rbrace: last.Pos(), + } + newComments := make([]*ast.CommentGroup, len(comments)-1) + copy(newComments, comments[:i]) + copy(newComments[i:], comments[i+1:]) + return newBody, newComments +} + +// lastComment returns the last comment inside the provided block. +func lastComment(b *ast.BlockStmt, c []*ast.CommentGroup) (i int, last *ast.CommentGroup) { + if b == nil { + return + } + pos, end := b.Pos(), b.End() + for j, cg := range c { + if cg.Pos() < pos { + continue + } + if cg.End() > end { + break + } + i, last = j, cg + } + return +} + +// classifyExamples classifies examples and assigns them to the Examples field +// of the relevant Func, Type, or Package that the example is associated with. +// +// The classification process is ambiguous in some cases: +// +// - ExampleFoo_Bar matches a type named Foo_Bar +// or a method named Foo.Bar. +// - ExampleFoo_bar matches a type named Foo_bar +// or Foo (with a "bar" suffix). +// +// Examples with malformed names are not associated with anything. +func classifyExamples(p *Package, examples []*Example) { + if len(examples) == 0 { + return + } + // Mapping of names for funcs, types, and methods to the example listing. + ids := make(map[string]*[]*Example) + ids[""] = &p.Examples // package-level examples have an empty name + for _, f := range p.Funcs { + if !token.IsExported(f.Name) { + continue + } + ids[f.Name] = &f.Examples + } + for _, t := range p.Types { + if !token.IsExported(t.Name) { + continue + } + ids[t.Name] = &t.Examples + for _, f := range t.Funcs { + if !token.IsExported(f.Name) { + continue + } + ids[f.Name] = &f.Examples + } + for _, m := range t.Methods { + if !token.IsExported(m.Name) { + continue + } + ids[strings.TrimPrefix(nameWithoutInst(m.Recv), "*")+"_"+m.Name] = &m.Examples + } + } + + // Group each example with the associated func, type, or method. + for _, ex := range examples { + // Consider all possible split points for the suffix + // by starting at the end of string (no suffix case), + // then trying all positions that contain a '_' character. + // + // An association is made on the first successful match. + // Examples with malformed names that match nothing are skipped. + for i := len(ex.Name); i >= 0; i = strings.LastIndexByte(ex.Name[:i], '_') { + prefix, suffix, ok := splitExampleName(ex.Name, i) + if !ok { + continue + } + exs, ok := ids[prefix] + if !ok { + continue + } + ex.Suffix = suffix + *exs = append(*exs, ex) + break + } + } + + // Sort list of example according to the user-specified suffix name. + for _, exs := range ids { + sort.Slice((*exs), func(i, j int) bool { + return (*exs)[i].Suffix < (*exs)[j].Suffix + }) + } +} + +// nameWithoutInst returns name if name has no brackets. If name contains +// brackets, then it returns name with all the contents between (and including) +// the outermost left and right bracket removed. +// +// Adapted from debug/gosym/symtab.go:Sym.nameWithoutInst. +func nameWithoutInst(name string) string { + start := strings.Index(name, "[") + if start < 0 { + return name + } + end := strings.LastIndex(name, "]") + if end < 0 { + // Malformed name, should contain closing bracket too. + return name + } + return name[0:start] + name[end+1:] +} + +// splitExampleName attempts to split example name s at index i, +// and reports if that produces a valid split. The suffix may be +// absent. Otherwise, it must start with a lower-case letter and +// be preceded by '_'. +// +// One of i == len(s) or s[i] == '_' must be true. +func splitExampleName(s string, i int) (prefix, suffix string, ok bool) { + if i == len(s) { + return s, "", true + } + if i == len(s)-1 { + return "", "", false + } + prefix, suffix = s[:i], s[i+1:] + return prefix, suffix, isExampleSuffix(suffix) +} + +func isExampleSuffix(s string) bool { + r, size := utf8.DecodeRuneInString(s) + return size > 0 && unicode.IsLower(r) +} diff --git a/src/go/doc/example_internal_test.go b/src/go/doc/example_internal_test.go new file mode 100644 index 0000000..08ddfaf --- /dev/null +++ b/src/go/doc/example_internal_test.go @@ -0,0 +1,121 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "go/parser" + "go/token" + "reflect" + "strconv" + "strings" + "testing" +) + +func TestImportGroupStarts(t *testing.T) { + for _, test := range []struct { + name string + in string + want []string // paths of group-starting imports + }{ + { + name: "one group", + in: `package p +import ( + "a" + "b" + "c" + "d" +) +`, + want: []string{"a"}, + }, + { + name: "several groups", + in: `package p +import ( + "a" + + "b" + "c" + + "d" +) +`, + want: []string{"a", "b", "d"}, + }, + { + name: "extra space", + in: `package p +import ( + "a" + + + "b" + "c" + + + "d" +) +`, + want: []string{"a", "b", "d"}, + }, + { + name: "line comment", + in: `package p +import ( + "a" // comment + "b" // comment + + "c" +)`, + want: []string{"a", "c"}, + }, + { + name: "named import", + in: `package p +import ( + "a" + n "b" + + m "c" + "d" +)`, + want: []string{"a", "c"}, + }, + { + name: "blank import", + in: `package p +import ( + "a" + + _ "b" + + _ "c" + "d" +)`, + want: []string{"a", "b", "c"}, + }, + } { + t.Run(test.name, func(t *testing.T) { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "test.go", strings.NewReader(test.in), parser.ParseComments) + if err != nil { + t.Fatal(err) + } + imps := findImportGroupStarts1(file.Imports) + got := make([]string, len(imps)) + for i, imp := range imps { + got[i], err = strconv.Unquote(imp.Path.Value) + if err != nil { + t.Fatal(err) + } + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("got %v, want %v", got, test.want) + } + }) + } + +} diff --git a/src/go/doc/example_test.go b/src/go/doc/example_test.go new file mode 100644 index 0000000..7919c3a --- /dev/null +++ b/src/go/doc/example_test.go @@ -0,0 +1,336 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc_test + +import ( + "bytes" + "fmt" + "go/ast" + "go/doc" + "go/format" + "go/parser" + "go/token" + "internal/diff" + "internal/txtar" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestExamples(t *testing.T) { + dir := filepath.Join("testdata", "examples") + filenames, err := filepath.Glob(filepath.Join(dir, "*.go")) + if err != nil { + t.Fatal(err) + } + for _, filename := range filenames { + t.Run(strings.TrimSuffix(filepath.Base(filename), ".go"), func(t *testing.T) { + fset := token.NewFileSet() + astFile, err := parser.ParseFile(fset, filename, nil, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + goldenFilename := strings.TrimSuffix(filename, ".go") + ".golden" + archive, err := txtar.ParseFile(goldenFilename) + if err != nil { + t.Fatal(err) + } + golden := map[string]string{} + for _, f := range archive.Files { + golden[f.Name] = strings.TrimSpace(string(f.Data)) + } + + // Collect the results of doc.Examples in a map keyed by example name. + examples := map[string]*doc.Example{} + for _, e := range doc.Examples(astFile) { + examples[e.Name] = e + // Treat missing sections in the golden as empty. + for _, kind := range []string{"Play", "Output"} { + key := e.Name + "." + kind + if _, ok := golden[key]; !ok { + golden[key] = "" + } + } + } + + // Each section in the golden file corresponds to an example we expect + // to see. + for sectionName, want := range golden { + name, kind, found := strings.Cut(sectionName, ".") + if !found { + t.Fatalf("bad section name %q, want EXAMPLE_NAME.KIND", sectionName) + } + ex := examples[name] + if ex == nil { + t.Fatalf("no example named %q", name) + } + + var got string + switch kind { + case "Play": + got = strings.TrimSpace(formatFile(t, fset, ex.Play)) + + case "Output": + got = strings.TrimSpace(ex.Output) + default: + t.Fatalf("bad section kind %q", kind) + } + + if got != want { + t.Errorf("%s mismatch:\n%s", sectionName, + diff.Diff("want", []byte(want), "got", []byte(got))) + } + } + }) + } +} + +func formatFile(t *testing.T, fset *token.FileSet, n *ast.File) string { + t.Helper() + if n == nil { + return "" + } + var buf bytes.Buffer + if err := format.Node(&buf, fset, n); err != nil { + t.Fatal(err) + } + return buf.String() +} + +// This example illustrates how to use NewFromFiles +// to compute package documentation with examples. +func ExampleNewFromFiles() { + // src and test are two source files that make up + // a package whose documentation will be computed. + const src = ` +// This is the package comment. +package p + +import "fmt" + +// This comment is associated with the Greet function. +func Greet(who string) { + fmt.Printf("Hello, %s!\n", who) +} +` + const test = ` +package p_test + +// This comment is associated with the ExampleGreet_world example. +func ExampleGreet_world() { + Greet("world") +} +` + + // Create the AST by parsing src and test. + fset := token.NewFileSet() + files := []*ast.File{ + mustParse(fset, "src.go", src), + mustParse(fset, "src_test.go", test), + } + + // Compute package documentation with examples. + p, err := doc.NewFromFiles(fset, files, "example.com/p") + if err != nil { + panic(err) + } + + fmt.Printf("package %s - %s", p.Name, p.Doc) + fmt.Printf("func %s - %s", p.Funcs[0].Name, p.Funcs[0].Doc) + fmt.Printf(" ā¤· example with suffix %q - %s", p.Funcs[0].Examples[0].Suffix, p.Funcs[0].Examples[0].Doc) + + // Output: + // package p - This is the package comment. + // func Greet - This comment is associated with the Greet function. + // ā¤· example with suffix "world" - This comment is associated with the ExampleGreet_world example. +} + +func TestClassifyExamples(t *testing.T) { + const src = ` +package p + +const Const1 = 0 +var Var1 = 0 + +type ( + Type1 int + Type1_Foo int + Type1_foo int + type2 int + + Embed struct { Type1 } + Uembed struct { type2 } +) + +func Func1() {} +func Func1_Foo() {} +func Func1_foo() {} +func func2() {} + +func (Type1) Func1() {} +func (Type1) Func1_Foo() {} +func (Type1) Func1_foo() {} +func (Type1) func2() {} + +func (type2) Func1() {} + +type ( + Conflict int + Conflict_Conflict int + Conflict_conflict int +) + +func (Conflict) Conflict() {} + +func GFunc[T any]() {} + +type GType[T any] int + +func (GType[T]) M() {} +` + const test = ` +package p_test + +func ExampleConst1() {} // invalid - no support for consts and vars +func ExampleVar1() {} // invalid - no support for consts and vars + +func Example() {} +func Example_() {} // invalid - suffix must start with a lower-case letter +func Example_suffix() {} +func Example_suffix_xX_X_x() {} +func Example_äø–ē•Œ() {} // invalid - suffix must start with a lower-case letter +func Example_123() {} // invalid - suffix must start with a lower-case letter +func Example_BadSuffix() {} // invalid - suffix must start with a lower-case letter + +func ExampleType1() {} +func ExampleType1_() {} // invalid - suffix must start with a lower-case letter +func ExampleType1_suffix() {} +func ExampleType1_BadSuffix() {} // invalid - suffix must start with a lower-case letter +func ExampleType1_Foo() {} +func ExampleType1_Foo_suffix() {} +func ExampleType1_Foo_BadSuffix() {} // invalid - suffix must start with a lower-case letter +func ExampleType1_foo() {} +func ExampleType1_foo_suffix() {} +func ExampleType1_foo_Suffix() {} // matches Type1, instead of Type1_foo +func Exampletype2() {} // invalid - cannot match unexported + +func ExampleFunc1() {} +func ExampleFunc1_() {} // invalid - suffix must start with a lower-case letter +func ExampleFunc1_suffix() {} +func ExampleFunc1_BadSuffix() {} // invalid - suffix must start with a lower-case letter +func ExampleFunc1_Foo() {} +func ExampleFunc1_Foo_suffix() {} +func ExampleFunc1_Foo_BadSuffix() {} // invalid - suffix must start with a lower-case letter +func ExampleFunc1_foo() {} +func ExampleFunc1_foo_suffix() {} +func ExampleFunc1_foo_Suffix() {} // matches Func1, instead of Func1_foo +func Examplefunc1() {} // invalid - cannot match unexported + +func ExampleType1_Func1() {} +func ExampleType1_Func1_() {} // invalid - suffix must start with a lower-case letter +func ExampleType1_Func1_suffix() {} +func ExampleType1_Func1_BadSuffix() {} // invalid - suffix must start with a lower-case letter +func ExampleType1_Func1_Foo() {} +func ExampleType1_Func1_Foo_suffix() {} +func ExampleType1_Func1_Foo_BadSuffix() {} // invalid - suffix must start with a lower-case letter +func ExampleType1_Func1_foo() {} +func ExampleType1_Func1_foo_suffix() {} +func ExampleType1_Func1_foo_Suffix() {} // matches Type1.Func1, instead of Type1.Func1_foo +func ExampleType1_func2() {} // matches Type1, instead of Type1.func2 + +func ExampleEmbed_Func1() {} // invalid - no support for forwarded methods from embedding exported type +func ExampleUembed_Func1() {} // methods from embedding unexported types are OK +func ExampleUembed_Func1_suffix() {} + +func ExampleConflict_Conflict() {} // ambiguous with either Conflict or Conflict_Conflict type +func ExampleConflict_conflict() {} // ambiguous with either Conflict or Conflict_conflict type +func ExampleConflict_Conflict_suffix() {} // ambiguous with either Conflict or Conflict_Conflict type +func ExampleConflict_conflict_suffix() {} // ambiguous with either Conflict or Conflict_conflict type + +func ExampleGFunc() {} +func ExampleGFunc_suffix() {} + +func ExampleGType_M() {} +func ExampleGType_M_suffix() {} +` + + // Parse literal source code as a *doc.Package. + fset := token.NewFileSet() + files := []*ast.File{ + mustParse(fset, "src.go", src), + mustParse(fset, "src_test.go", test), + } + p, err := doc.NewFromFiles(fset, files, "example.com/p") + if err != nil { + t.Fatalf("doc.NewFromFiles: %v", err) + } + + // Collect the association of examples to top-level identifiers. + got := map[string][]string{} + got[""] = exampleNames(p.Examples) + for _, f := range p.Funcs { + got[f.Name] = exampleNames(f.Examples) + } + for _, t := range p.Types { + got[t.Name] = exampleNames(t.Examples) + for _, f := range t.Funcs { + got[f.Name] = exampleNames(f.Examples) + } + for _, m := range t.Methods { + got[t.Name+"."+m.Name] = exampleNames(m.Examples) + } + } + + want := map[string][]string{ + "": {"", "suffix", "suffix_xX_X_x"}, // Package-level examples. + + "Type1": {"", "foo_Suffix", "func2", "suffix"}, + "Type1_Foo": {"", "suffix"}, + "Type1_foo": {"", "suffix"}, + + "Func1": {"", "foo_Suffix", "suffix"}, + "Func1_Foo": {"", "suffix"}, + "Func1_foo": {"", "suffix"}, + + "Type1.Func1": {"", "foo_Suffix", "suffix"}, + "Type1.Func1_Foo": {"", "suffix"}, + "Type1.Func1_foo": {"", "suffix"}, + + "Uembed.Func1": {"", "suffix"}, + + // These are implementation dependent due to the ambiguous parsing. + "Conflict_Conflict": {"", "suffix"}, + "Conflict_conflict": {"", "suffix"}, + + "GFunc": {"", "suffix"}, + "GType.M": {"", "suffix"}, + } + + for id := range got { + if !reflect.DeepEqual(got[id], want[id]) { + t.Errorf("classification mismatch for %q:\ngot %q\nwant %q", id, got[id], want[id]) + } + delete(want, id) + } + if len(want) > 0 { + t.Errorf("did not find:\n%q", want) + } +} + +func exampleNames(exs []*doc.Example) (out []string) { + for _, ex := range exs { + out = append(out, ex.Suffix) + } + return out +} + +func mustParse(fset *token.FileSet, filename, src string) *ast.File { + f, err := parser.ParseFile(fset, filename, src, parser.ParseComments) + if err != nil { + panic(err) + } + return f +} diff --git a/src/go/doc/exports.go b/src/go/doc/exports.go new file mode 100644 index 0000000..655e889 --- /dev/null +++ b/src/go/doc/exports.go @@ -0,0 +1,324 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements export filtering of an AST. + +package doc + +import ( + "go/ast" + "go/token" +) + +// filterIdentList removes unexported names from list in place +// and returns the resulting list. +func filterIdentList(list []*ast.Ident) []*ast.Ident { + j := 0 + for _, x := range list { + if token.IsExported(x.Name) { + list[j] = x + j++ + } + } + return list[0:j] +} + +var underscore = ast.NewIdent("_") + +func filterCompositeLit(lit *ast.CompositeLit, filter Filter, export bool) { + n := len(lit.Elts) + lit.Elts = filterExprList(lit.Elts, filter, export) + if len(lit.Elts) < n { + lit.Incomplete = true + } +} + +func filterExprList(list []ast.Expr, filter Filter, export bool) []ast.Expr { + j := 0 + for _, exp := range list { + switch x := exp.(type) { + case *ast.CompositeLit: + filterCompositeLit(x, filter, export) + case *ast.KeyValueExpr: + if x, ok := x.Key.(*ast.Ident); ok && !filter(x.Name) { + continue + } + if x, ok := x.Value.(*ast.CompositeLit); ok { + filterCompositeLit(x, filter, export) + } + } + list[j] = exp + j++ + } + return list[0:j] +} + +// updateIdentList replaces all unexported identifiers with underscore +// and reports whether at least one exported name exists. +func updateIdentList(list []*ast.Ident) (hasExported bool) { + for i, x := range list { + if token.IsExported(x.Name) { + hasExported = true + } else { + list[i] = underscore + } + } + return hasExported +} + +// hasExportedName reports whether list contains any exported names. +func hasExportedName(list []*ast.Ident) bool { + for _, x := range list { + if x.IsExported() { + return true + } + } + return false +} + +// removeAnonymousField removes anonymous fields named name from an interface. +func removeAnonymousField(name string, ityp *ast.InterfaceType) { + list := ityp.Methods.List // we know that ityp.Methods != nil + j := 0 + for _, field := range list { + keepField := true + if n := len(field.Names); n == 0 { + // anonymous field + if fname, _ := baseTypeName(field.Type); fname == name { + keepField = false + } + } + if keepField { + list[j] = field + j++ + } + } + if j < len(list) { + ityp.Incomplete = true + } + ityp.Methods.List = list[0:j] +} + +// filterFieldList removes unexported fields (field names) from the field list +// in place and reports whether fields were removed. Anonymous fields are +// recorded with the parent type. filterType is called with the types of +// all remaining fields. +func (r *reader) filterFieldList(parent *namedType, fields *ast.FieldList, ityp *ast.InterfaceType) (removedFields bool) { + if fields == nil { + return + } + list := fields.List + j := 0 + for _, field := range list { + keepField := false + if n := len(field.Names); n == 0 { + // anonymous field or embedded type or union element + fname := r.recordAnonymousField(parent, field.Type) + if fname != "" { + if token.IsExported(fname) { + keepField = true + } else if ityp != nil && predeclaredTypes[fname] { + // possibly an embedded predeclared type; keep it for now but + // remember this interface so that it can be fixed if name is also + // defined locally + keepField = true + r.remember(fname, ityp) + } + } else { + // If we're operating on an interface, assume that this is an embedded + // type or union element. + // + // TODO(rfindley): consider traversing into approximation/unions + // elements to see if they are entirely unexported. + keepField = ityp != nil + } + } else { + field.Names = filterIdentList(field.Names) + if len(field.Names) < n { + removedFields = true + } + if len(field.Names) > 0 { + keepField = true + } + } + if keepField { + r.filterType(nil, field.Type) + list[j] = field + j++ + } + } + if j < len(list) { + removedFields = true + } + fields.List = list[0:j] + return +} + +// filterParamList applies filterType to each parameter type in fields. +func (r *reader) filterParamList(fields *ast.FieldList) { + if fields != nil { + for _, f := range fields.List { + r.filterType(nil, f.Type) + } + } +} + +// filterType strips any unexported struct fields or method types from typ +// in place. If fields (or methods) have been removed, the corresponding +// struct or interface type has the Incomplete field set to true. +func (r *reader) filterType(parent *namedType, typ ast.Expr) { + switch t := typ.(type) { + case *ast.Ident: + // nothing to do + case *ast.ParenExpr: + r.filterType(nil, t.X) + case *ast.StarExpr: // possibly an embedded type literal + r.filterType(nil, t.X) + case *ast.UnaryExpr: + if t.Op == token.TILDE { // approximation element + r.filterType(nil, t.X) + } + case *ast.BinaryExpr: + if t.Op == token.OR { // union + r.filterType(nil, t.X) + r.filterType(nil, t.Y) + } + case *ast.ArrayType: + r.filterType(nil, t.Elt) + case *ast.StructType: + if r.filterFieldList(parent, t.Fields, nil) { + t.Incomplete = true + } + case *ast.FuncType: + r.filterParamList(t.TypeParams) + r.filterParamList(t.Params) + r.filterParamList(t.Results) + case *ast.InterfaceType: + if r.filterFieldList(parent, t.Methods, t) { + t.Incomplete = true + } + case *ast.MapType: + r.filterType(nil, t.Key) + r.filterType(nil, t.Value) + case *ast.ChanType: + r.filterType(nil, t.Value) + } +} + +func (r *reader) filterSpec(spec ast.Spec) bool { + switch s := spec.(type) { + case *ast.ImportSpec: + // always keep imports so we can collect them + return true + case *ast.ValueSpec: + s.Values = filterExprList(s.Values, token.IsExported, true) + if len(s.Values) > 0 || s.Type == nil && len(s.Values) == 0 { + // If there are values declared on RHS, just replace the unexported + // identifiers on the LHS with underscore, so that it matches + // the sequence of expression on the RHS. + // + // Similarly, if there are no type and values, then this expression + // must be following an iota expression, where order matters. + if updateIdentList(s.Names) { + r.filterType(nil, s.Type) + return true + } + } else { + s.Names = filterIdentList(s.Names) + if len(s.Names) > 0 { + r.filterType(nil, s.Type) + return true + } + } + case *ast.TypeSpec: + // Don't filter type parameters here, by analogy with function parameters + // which are not filtered for top-level function declarations. + if name := s.Name.Name; token.IsExported(name) { + r.filterType(r.lookupType(s.Name.Name), s.Type) + return true + } else if IsPredeclared(name) { + if r.shadowedPredecl == nil { + r.shadowedPredecl = make(map[string]bool) + } + r.shadowedPredecl[name] = true + } + } + return false +} + +// copyConstType returns a copy of typ with position pos. +// typ must be a valid constant type. +// In practice, only (possibly qualified) identifiers are possible. +func copyConstType(typ ast.Expr, pos token.Pos) ast.Expr { + switch typ := typ.(type) { + case *ast.Ident: + return &ast.Ident{Name: typ.Name, NamePos: pos} + case *ast.SelectorExpr: + if id, ok := typ.X.(*ast.Ident); ok { + // presumably a qualified identifier + return &ast.SelectorExpr{ + Sel: ast.NewIdent(typ.Sel.Name), + X: &ast.Ident{Name: id.Name, NamePos: pos}, + } + } + } + return nil // shouldn't happen, but be conservative and don't panic +} + +func (r *reader) filterSpecList(list []ast.Spec, tok token.Token) []ast.Spec { + if tok == token.CONST { + // Propagate any type information that would get lost otherwise + // when unexported constants are filtered. + var prevType ast.Expr + for _, spec := range list { + spec := spec.(*ast.ValueSpec) + if spec.Type == nil && len(spec.Values) == 0 && prevType != nil { + // provide current spec with an explicit type + spec.Type = copyConstType(prevType, spec.Pos()) + } + if hasExportedName(spec.Names) { + // exported names are preserved so there's no need to propagate the type + prevType = nil + } else { + prevType = spec.Type + } + } + } + + j := 0 + for _, s := range list { + if r.filterSpec(s) { + list[j] = s + j++ + } + } + return list[0:j] +} + +func (r *reader) filterDecl(decl ast.Decl) bool { + switch d := decl.(type) { + case *ast.GenDecl: + d.Specs = r.filterSpecList(d.Specs, d.Tok) + return len(d.Specs) > 0 + case *ast.FuncDecl: + // ok to filter these methods early because any + // conflicting method will be filtered here, too - + // thus, removing these methods early will not lead + // to the false removal of possible conflicts + return token.IsExported(d.Name.Name) + } + return false +} + +// fileExports removes unexported declarations from src in place. +func (r *reader) fileExports(src *ast.File) { + j := 0 + for _, d := range src.Decls { + if r.filterDecl(d) { + src.Decls[j] = d + j++ + } + } + src.Decls = src.Decls[0:j] +} diff --git a/src/go/doc/filter.go b/src/go/doc/filter.go new file mode 100644 index 0000000..f8d3e1f --- /dev/null +++ b/src/go/doc/filter.go @@ -0,0 +1,106 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import "go/ast" + +type Filter func(string) bool + +func matchFields(fields *ast.FieldList, f Filter) bool { + if fields != nil { + for _, field := range fields.List { + for _, name := range field.Names { + if f(name.Name) { + return true + } + } + } + } + return false +} + +func matchDecl(d *ast.GenDecl, f Filter) bool { + for _, d := range d.Specs { + switch v := d.(type) { + case *ast.ValueSpec: + for _, name := range v.Names { + if f(name.Name) { + return true + } + } + case *ast.TypeSpec: + if f(v.Name.Name) { + return true + } + // We don't match ordinary parameters in filterFuncs, so by analogy don't + // match type parameters here. + switch t := v.Type.(type) { + case *ast.StructType: + if matchFields(t.Fields, f) { + return true + } + case *ast.InterfaceType: + if matchFields(t.Methods, f) { + return true + } + } + } + } + return false +} + +func filterValues(a []*Value, f Filter) []*Value { + w := 0 + for _, vd := range a { + if matchDecl(vd.Decl, f) { + a[w] = vd + w++ + } + } + return a[0:w] +} + +func filterFuncs(a []*Func, f Filter) []*Func { + w := 0 + for _, fd := range a { + if f(fd.Name) { + a[w] = fd + w++ + } + } + return a[0:w] +} + +func filterTypes(a []*Type, f Filter) []*Type { + w := 0 + for _, td := range a { + n := 0 // number of matches + if matchDecl(td.Decl, f) { + n = 1 + } else { + // type name doesn't match, but we may have matching consts, vars, factories or methods + td.Consts = filterValues(td.Consts, f) + td.Vars = filterValues(td.Vars, f) + td.Funcs = filterFuncs(td.Funcs, f) + td.Methods = filterFuncs(td.Methods, f) + n += len(td.Consts) + len(td.Vars) + len(td.Funcs) + len(td.Methods) + } + if n > 0 { + a[w] = td + w++ + } + } + return a[0:w] +} + +// Filter eliminates documentation for names that don't pass through the filter f. +// TODO(gri): Recognize "Type.Method" as a name. +func (p *Package) Filter(f Filter) { + p.Consts = filterValues(p.Consts, f) + p.Vars = filterValues(p.Vars, f) + p.Types = filterTypes(p.Types, f) + p.Funcs = filterFuncs(p.Funcs, f) + p.Doc = "" // don't show top-level package doc +} diff --git a/src/go/doc/headscan.go b/src/go/doc/headscan.go new file mode 100644 index 0000000..82f5fed --- /dev/null +++ b/src/go/doc/headscan.go @@ -0,0 +1,109 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ignore + +/* +The headscan command extracts comment headings from package files; +it is used to detect false positives which may require an adjustment +to the comment formatting heuristics in comment.go. + +Usage: headscan [-root root_directory] + +By default, the $GOROOT/src directory is scanned. +*/ +package main + +import ( + "flag" + "fmt" + "go/doc" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" +) + +var ( + root = flag.String("root", filepath.Join(runtime.GOROOT(), "src"), "root of filesystem tree to scan") + verbose = flag.Bool("v", false, "verbose mode") +) + +// ToHTML in comment.go assigns a (possibly blank) ID to each heading +var html_h = regexp.MustCompile(`

`) + +const html_endh = "

\n" + +func isGoFile(fi fs.FileInfo) bool { + return strings.HasSuffix(fi.Name(), ".go") && + !strings.HasSuffix(fi.Name(), "_test.go") +} + +func appendHeadings(list []string, comment string) []string { + var buf strings.Builder + doc.ToHTML(&buf, comment, nil) + for s := buf.String(); s != ""; { + loc := html_h.FindStringIndex(s) + if len(loc) == 0 { + break + } + var inner string + inner, s, _ = strings.Cut(s[loc[1]:], html_endh) + list = append(list, inner) + } + return list +} + +func main() { + flag.Parse() + fset := token.NewFileSet() + nheadings := 0 + err := filepath.WalkDir(*root, func(path string, info fs.DirEntry, err error) error { + if !info.IsDir() { + return nil + } + pkgs, err := parser.ParseDir(fset, path, isGoFile, parser.ParseComments) + if err != nil { + if *verbose { + fmt.Fprintln(os.Stderr, err) + } + return nil + } + for _, pkg := range pkgs { + d := doc.New(pkg, path, doc.Mode(0)) + list := appendHeadings(nil, d.Doc) + for _, d := range d.Consts { + list = appendHeadings(list, d.Doc) + } + for _, d := range d.Types { + list = appendHeadings(list, d.Doc) + } + for _, d := range d.Vars { + list = appendHeadings(list, d.Doc) + } + for _, d := range d.Funcs { + list = appendHeadings(list, d.Doc) + } + if len(list) > 0 { + // directories may contain multiple packages; + // print path and package name + fmt.Printf("%s (package %s)\n", path, pkg.Name) + for _, h := range list { + fmt.Printf("\t%s\n", h) + } + nheadings += len(list) + } + } + return nil + }) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Println(nheadings, "headings found") +} diff --git a/src/go/doc/reader.go b/src/go/doc/reader.go new file mode 100644 index 0000000..8f9fda4 --- /dev/null +++ b/src/go/doc/reader.go @@ -0,0 +1,1029 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "fmt" + "go/ast" + "go/token" + "internal/lazyregexp" + "path" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// ---------------------------------------------------------------------------- +// function/method sets +// +// Internally, we treat functions like methods and collect them in method sets. + +// A methodSet describes a set of methods. Entries where Decl == nil are conflict +// entries (more than one method with the same name at the same embedding level). +type methodSet map[string]*Func + +// recvString returns a string representation of recv of the form "T", "*T", +// "T[A, ...]", "*T[A, ...]" or "BADRECV" (if not a proper receiver type). +func recvString(recv ast.Expr) string { + switch t := recv.(type) { + case *ast.Ident: + return t.Name + case *ast.StarExpr: + return "*" + recvString(t.X) + case *ast.IndexExpr: + // Generic type with one parameter. + return fmt.Sprintf("%s[%s]", recvString(t.X), recvParam(t.Index)) + case *ast.IndexListExpr: + // Generic type with multiple parameters. + if len(t.Indices) > 0 { + var b strings.Builder + b.WriteString(recvString(t.X)) + b.WriteByte('[') + b.WriteString(recvParam(t.Indices[0])) + for _, e := range t.Indices[1:] { + b.WriteString(", ") + b.WriteString(recvParam(e)) + } + b.WriteByte(']') + return b.String() + } + } + return "BADRECV" +} + +func recvParam(p ast.Expr) string { + if id, ok := p.(*ast.Ident); ok { + return id.Name + } + return "BADPARAM" +} + +// set creates the corresponding Func for f and adds it to mset. +// If there are multiple f's with the same name, set keeps the first +// one with documentation; conflicts are ignored. The boolean +// specifies whether to leave the AST untouched. +func (mset methodSet) set(f *ast.FuncDecl, preserveAST bool) { + name := f.Name.Name + if g := mset[name]; g != nil && g.Doc != "" { + // A function with the same name has already been registered; + // since it has documentation, assume f is simply another + // implementation and ignore it. This does not happen if the + // caller is using go/build.ScanDir to determine the list of + // files implementing a package. + return + } + // function doesn't exist or has no documentation; use f + recv := "" + if f.Recv != nil { + var typ ast.Expr + // be careful in case of incorrect ASTs + if list := f.Recv.List; len(list) == 1 { + typ = list[0].Type + } + recv = recvString(typ) + } + mset[name] = &Func{ + Doc: f.Doc.Text(), + Name: name, + Decl: f, + Recv: recv, + Orig: recv, + } + if !preserveAST { + f.Doc = nil // doc consumed - remove from AST + } +} + +// add adds method m to the method set; m is ignored if the method set +// already contains a method with the same name at the same or a higher +// level than m. +func (mset methodSet) add(m *Func) { + old := mset[m.Name] + if old == nil || m.Level < old.Level { + mset[m.Name] = m + return + } + if m.Level == old.Level { + // conflict - mark it using a method with nil Decl + mset[m.Name] = &Func{ + Name: m.Name, + Level: m.Level, + } + } +} + +// ---------------------------------------------------------------------------- +// Named types + +// baseTypeName returns the name of the base type of x (or "") +// and whether the type is imported or not. +func baseTypeName(x ast.Expr) (name string, imported bool) { + switch t := x.(type) { + case *ast.Ident: + return t.Name, false + case *ast.IndexExpr: + return baseTypeName(t.X) + case *ast.IndexListExpr: + return baseTypeName(t.X) + case *ast.SelectorExpr: + if _, ok := t.X.(*ast.Ident); ok { + // only possible for qualified type names; + // assume type is imported + return t.Sel.Name, true + } + case *ast.ParenExpr: + return baseTypeName(t.X) + case *ast.StarExpr: + return baseTypeName(t.X) + } + return "", false +} + +// An embeddedSet describes a set of embedded types. +type embeddedSet map[*namedType]bool + +// A namedType represents a named unqualified (package local, or possibly +// predeclared) type. The namedType for a type name is always found via +// reader.lookupType. +type namedType struct { + doc string // doc comment for type + name string // type name + decl *ast.GenDecl // nil if declaration hasn't been seen yet + + isEmbedded bool // true if this type is embedded + isStruct bool // true if this type is a struct + embedded embeddedSet // true if the embedded type is a pointer + + // associated declarations + values []*Value // consts and vars + funcs methodSet + methods methodSet +} + +// ---------------------------------------------------------------------------- +// AST reader + +// reader accumulates documentation for a single package. +// It modifies the AST: Comments (declaration documentation) +// that have been collected by the reader are set to nil +// in the respective AST nodes so that they are not printed +// twice (once when printing the documentation and once when +// printing the corresponding AST node). +type reader struct { + mode Mode + + // package properties + doc string // package documentation, if any + filenames []string + notes map[string][]*Note + + // imports + imports map[string]int + hasDotImp bool // if set, package contains a dot import + importByName map[string]string + + // declarations + values []*Value // consts and vars + order int // sort order of const and var declarations (when we can't use a name) + types map[string]*namedType + funcs methodSet + + // support for package-local shadowing of predeclared types + shadowedPredecl map[string]bool + fixmap map[string][]*ast.InterfaceType +} + +func (r *reader) isVisible(name string) bool { + return r.mode&AllDecls != 0 || token.IsExported(name) +} + +// lookupType returns the base type with the given name. +// If the base type has not been encountered yet, a new +// type with the given name but no associated declaration +// is added to the type map. +func (r *reader) lookupType(name string) *namedType { + if name == "" || name == "_" { + return nil // no type docs for anonymous types + } + if typ, found := r.types[name]; found { + return typ + } + // type not found - add one without declaration + typ := &namedType{ + name: name, + embedded: make(embeddedSet), + funcs: make(methodSet), + methods: make(methodSet), + } + r.types[name] = typ + return typ +} + +// recordAnonymousField registers fieldType as the type of an +// anonymous field in the parent type. If the field is imported +// (qualified name) or the parent is nil, the field is ignored. +// The function returns the field name. +func (r *reader) recordAnonymousField(parent *namedType, fieldType ast.Expr) (fname string) { + fname, imp := baseTypeName(fieldType) + if parent == nil || imp { + return + } + if ftype := r.lookupType(fname); ftype != nil { + ftype.isEmbedded = true + _, ptr := fieldType.(*ast.StarExpr) + parent.embedded[ftype] = ptr + } + return +} + +func (r *reader) readDoc(comment *ast.CommentGroup) { + // By convention there should be only one package comment + // but collect all of them if there are more than one. + text := comment.Text() + if r.doc == "" { + r.doc = text + return + } + r.doc += "\n" + text +} + +func (r *reader) remember(predecl string, typ *ast.InterfaceType) { + if r.fixmap == nil { + r.fixmap = make(map[string][]*ast.InterfaceType) + } + r.fixmap[predecl] = append(r.fixmap[predecl], typ) +} + +func specNames(specs []ast.Spec) []string { + names := make([]string, 0, len(specs)) // reasonable estimate + for _, s := range specs { + // s guaranteed to be an *ast.ValueSpec by readValue + for _, ident := range s.(*ast.ValueSpec).Names { + names = append(names, ident.Name) + } + } + return names +} + +// readValue processes a const or var declaration. +func (r *reader) readValue(decl *ast.GenDecl) { + // determine if decl should be associated with a type + // Heuristic: For each typed entry, determine the type name, if any. + // If there is exactly one type name that is sufficiently + // frequent, associate the decl with the respective type. + domName := "" + domFreq := 0 + prev := "" + n := 0 + for _, spec := range decl.Specs { + s, ok := spec.(*ast.ValueSpec) + if !ok { + continue // should not happen, but be conservative + } + name := "" + switch { + case s.Type != nil: + // a type is present; determine its name + if n, imp := baseTypeName(s.Type); !imp { + name = n + } + case decl.Tok == token.CONST && len(s.Values) == 0: + // no type or value is present but we have a constant declaration; + // use the previous type name (possibly the empty string) + name = prev + } + if name != "" { + // entry has a named type + if domName != "" && domName != name { + // more than one type name - do not associate + // with any type + domName = "" + break + } + domName = name + domFreq++ + } + prev = name + n++ + } + + // nothing to do w/o a legal declaration + if n == 0 { + return + } + + // determine values list with which to associate the Value for this decl + values := &r.values + const threshold = 0.75 + if domName != "" && r.isVisible(domName) && domFreq >= int(float64(len(decl.Specs))*threshold) { + // typed entries are sufficiently frequent + if typ := r.lookupType(domName); typ != nil { + values = &typ.values // associate with that type + } + } + + *values = append(*values, &Value{ + Doc: decl.Doc.Text(), + Names: specNames(decl.Specs), + Decl: decl, + order: r.order, + }) + if r.mode&PreserveAST == 0 { + decl.Doc = nil // doc consumed - remove from AST + } + // Note: It's important that the order used here is global because the cleanupTypes + // methods may move values associated with types back into the global list. If the + // order is list-specific, sorting is not deterministic because the same order value + // may appear multiple times (was bug, found when fixing #16153). + r.order++ +} + +// fields returns a struct's fields or an interface's methods. +func fields(typ ast.Expr) (list []*ast.Field, isStruct bool) { + var fields *ast.FieldList + switch t := typ.(type) { + case *ast.StructType: + fields = t.Fields + isStruct = true + case *ast.InterfaceType: + fields = t.Methods + } + if fields != nil { + list = fields.List + } + return +} + +// readType processes a type declaration. +func (r *reader) readType(decl *ast.GenDecl, spec *ast.TypeSpec) { + typ := r.lookupType(spec.Name.Name) + if typ == nil { + return // no name or blank name - ignore the type + } + + // A type should be added at most once, so typ.decl + // should be nil - if it is not, simply overwrite it. + typ.decl = decl + + // compute documentation + doc := spec.Doc + if doc == nil { + // no doc associated with the spec, use the declaration doc, if any + doc = decl.Doc + } + if r.mode&PreserveAST == 0 { + spec.Doc = nil // doc consumed - remove from AST + decl.Doc = nil // doc consumed - remove from AST + } + typ.doc = doc.Text() + + // record anonymous fields (they may contribute methods) + // (some fields may have been recorded already when filtering + // exports, but that's ok) + var list []*ast.Field + list, typ.isStruct = fields(spec.Type) + for _, field := range list { + if len(field.Names) == 0 { + r.recordAnonymousField(typ, field.Type) + } + } +} + +// isPredeclared reports whether n denotes a predeclared type. +func (r *reader) isPredeclared(n string) bool { + return predeclaredTypes[n] && r.types[n] == nil +} + +// readFunc processes a func or method declaration. +func (r *reader) readFunc(fun *ast.FuncDecl) { + // strip function body if requested. + if r.mode&PreserveAST == 0 { + fun.Body = nil + } + + // associate methods with the receiver type, if any + if fun.Recv != nil { + // method + if len(fun.Recv.List) == 0 { + // should not happen (incorrect AST); (See issue 17788) + // don't show this method + return + } + recvTypeName, imp := baseTypeName(fun.Recv.List[0].Type) + if imp { + // should not happen (incorrect AST); + // don't show this method + return + } + if typ := r.lookupType(recvTypeName); typ != nil { + typ.methods.set(fun, r.mode&PreserveAST != 0) + } + // otherwise ignore the method + // TODO(gri): There may be exported methods of non-exported types + // that can be called because of exported values (consts, vars, or + // function results) of that type. Could determine if that is the + // case and then show those methods in an appropriate section. + return + } + + // Associate factory functions with the first visible result type, as long as + // others are predeclared types. + if fun.Type.Results.NumFields() >= 1 { + var typ *namedType // type to associate the function with + numResultTypes := 0 + for _, res := range fun.Type.Results.List { + factoryType := res.Type + if t, ok := factoryType.(*ast.ArrayType); ok { + // We consider functions that return slices or arrays of type + // T (or pointers to T) as factory functions of T. + factoryType = t.Elt + } + if n, imp := baseTypeName(factoryType); !imp && r.isVisible(n) && !r.isPredeclared(n) { + if lookupTypeParam(n, fun.Type.TypeParams) != nil { + // Issue #49477: don't associate fun with its type parameter result. + // A type parameter is not a defined type. + continue + } + if t := r.lookupType(n); t != nil { + typ = t + numResultTypes++ + if numResultTypes > 1 { + break + } + } + } + } + // If there is exactly one result type, + // associate the function with that type. + if numResultTypes == 1 { + typ.funcs.set(fun, r.mode&PreserveAST != 0) + return + } + } + + // just an ordinary function + r.funcs.set(fun, r.mode&PreserveAST != 0) +} + +// lookupTypeParam searches for type parameters named name within the tparams +// field list, returning the relevant identifier if found, or nil if not. +func lookupTypeParam(name string, tparams *ast.FieldList) *ast.Ident { + if tparams == nil { + return nil + } + for _, field := range tparams.List { + for _, id := range field.Names { + if id.Name == name { + return id + } + } + } + return nil +} + +var ( + noteMarker = `([A-Z][A-Z]+)\(([^)]+)\):?` // MARKER(uid), MARKER at least 2 chars, uid at least 1 char + noteMarkerRx = lazyregexp.New(`^[ \t]*` + noteMarker) // MARKER(uid) at text start + noteCommentRx = lazyregexp.New(`^/[/*][ \t]*` + noteMarker) // MARKER(uid) at comment start +) + +// clean replaces each sequence of space, \r, or \t characters +// with a single space and removes any trailing and leading spaces. +func clean(s string) string { + var b []byte + p := byte(' ') + for i := 0; i < len(s); i++ { + q := s[i] + if q == '\r' || q == '\t' { + q = ' ' + } + if q != ' ' || p != ' ' { + b = append(b, q) + p = q + } + } + // remove trailing blank, if any + if n := len(b); n > 0 && p == ' ' { + b = b[0 : n-1] + } + return string(b) +} + +// readNote collects a single note from a sequence of comments. +func (r *reader) readNote(list []*ast.Comment) { + text := (&ast.CommentGroup{List: list}).Text() + if m := noteMarkerRx.FindStringSubmatchIndex(text); m != nil { + // The note body starts after the marker. + // We remove any formatting so that we don't + // get spurious line breaks/indentation when + // showing the TODO body. + body := clean(text[m[1]:]) + if body != "" { + marker := text[m[2]:m[3]] + r.notes[marker] = append(r.notes[marker], &Note{ + Pos: list[0].Pos(), + End: list[len(list)-1].End(), + UID: text[m[4]:m[5]], + Body: body, + }) + } + } +} + +// readNotes extracts notes from comments. +// A note must start at the beginning of a comment with "MARKER(uid):" +// and is followed by the note body (e.g., "// BUG(gri): fix this"). +// The note ends at the end of the comment group or at the start of +// another note in the same comment group, whichever comes first. +func (r *reader) readNotes(comments []*ast.CommentGroup) { + for _, group := range comments { + i := -1 // comment index of most recent note start, valid if >= 0 + list := group.List + for j, c := range list { + if noteCommentRx.MatchString(c.Text) { + if i >= 0 { + r.readNote(list[i:j]) + } + i = j + } + } + if i >= 0 { + r.readNote(list[i:]) + } + } +} + +// readFile adds the AST for a source file to the reader. +func (r *reader) readFile(src *ast.File) { + // add package documentation + if src.Doc != nil { + r.readDoc(src.Doc) + if r.mode&PreserveAST == 0 { + src.Doc = nil // doc consumed - remove from AST + } + } + + // add all declarations but for functions which are processed in a separate pass + for _, decl := range src.Decls { + switch d := decl.(type) { + case *ast.GenDecl: + switch d.Tok { + case token.IMPORT: + // imports are handled individually + for _, spec := range d.Specs { + if s, ok := spec.(*ast.ImportSpec); ok { + if import_, err := strconv.Unquote(s.Path.Value); err == nil { + r.imports[import_] = 1 + var name string + if s.Name != nil { + name = s.Name.Name + if name == "." { + r.hasDotImp = true + } + } + if name != "." { + if name == "" { + name = assumedPackageName(import_) + } + old, ok := r.importByName[name] + if !ok { + r.importByName[name] = import_ + } else if old != import_ && old != "" { + r.importByName[name] = "" // ambiguous + } + } + } + } + } + case token.CONST, token.VAR: + // constants and variables are always handled as a group + r.readValue(d) + case token.TYPE: + // types are handled individually + if len(d.Specs) == 1 && !d.Lparen.IsValid() { + // common case: single declaration w/o parentheses + // (if a single declaration is parenthesized, + // create a new fake declaration below, so that + // go/doc type declarations always appear w/o + // parentheses) + if s, ok := d.Specs[0].(*ast.TypeSpec); ok { + r.readType(d, s) + } + break + } + for _, spec := range d.Specs { + if s, ok := spec.(*ast.TypeSpec); ok { + // use an individual (possibly fake) declaration + // for each type; this also ensures that each type + // gets to (re-)use the declaration documentation + // if there's none associated with the spec itself + fake := &ast.GenDecl{ + Doc: d.Doc, + // don't use the existing TokPos because it + // will lead to the wrong selection range for + // the fake declaration if there are more + // than one type in the group (this affects + // src/cmd/godoc/godoc.go's posLink_urlFunc) + TokPos: s.Pos(), + Tok: token.TYPE, + Specs: []ast.Spec{s}, + } + r.readType(fake, s) + } + } + } + } + } + + // collect MARKER(...): annotations + r.readNotes(src.Comments) + if r.mode&PreserveAST == 0 { + src.Comments = nil // consumed unassociated comments - remove from AST + } +} + +func (r *reader) readPackage(pkg *ast.Package, mode Mode) { + // initialize reader + r.filenames = make([]string, len(pkg.Files)) + r.imports = make(map[string]int) + r.mode = mode + r.types = make(map[string]*namedType) + r.funcs = make(methodSet) + r.notes = make(map[string][]*Note) + r.importByName = make(map[string]string) + + // sort package files before reading them so that the + // result does not depend on map iteration order + i := 0 + for filename := range pkg.Files { + r.filenames[i] = filename + i++ + } + sort.Strings(r.filenames) + + // process files in sorted order + for _, filename := range r.filenames { + f := pkg.Files[filename] + if mode&AllDecls == 0 { + r.fileExports(f) + } + r.readFile(f) + } + + for name, path := range r.importByName { + if path == "" { + delete(r.importByName, name) + } + } + + // process functions now that we have better type information + for _, f := range pkg.Files { + for _, decl := range f.Decls { + if d, ok := decl.(*ast.FuncDecl); ok { + r.readFunc(d) + } + } + } +} + +// ---------------------------------------------------------------------------- +// Types + +func customizeRecv(f *Func, recvTypeName string, embeddedIsPtr bool, level int) *Func { + if f == nil || f.Decl == nil || f.Decl.Recv == nil || len(f.Decl.Recv.List) != 1 { + return f // shouldn't happen, but be safe + } + + // copy existing receiver field and set new type + newField := *f.Decl.Recv.List[0] + origPos := newField.Type.Pos() + _, origRecvIsPtr := newField.Type.(*ast.StarExpr) + newIdent := &ast.Ident{NamePos: origPos, Name: recvTypeName} + var typ ast.Expr = newIdent + if !embeddedIsPtr && origRecvIsPtr { + newIdent.NamePos++ // '*' is one character + typ = &ast.StarExpr{Star: origPos, X: newIdent} + } + newField.Type = typ + + // copy existing receiver field list and set new receiver field + newFieldList := *f.Decl.Recv + newFieldList.List = []*ast.Field{&newField} + + // copy existing function declaration and set new receiver field list + newFuncDecl := *f.Decl + newFuncDecl.Recv = &newFieldList + + // copy existing function documentation and set new declaration + newF := *f + newF.Decl = &newFuncDecl + newF.Recv = recvString(typ) + // the Orig field never changes + newF.Level = level + + return &newF +} + +// collectEmbeddedMethods collects the embedded methods of typ in mset. +func (r *reader) collectEmbeddedMethods(mset methodSet, typ *namedType, recvTypeName string, embeddedIsPtr bool, level int, visited embeddedSet) { + visited[typ] = true + for embedded, isPtr := range typ.embedded { + // Once an embedded type is embedded as a pointer type + // all embedded types in those types are treated like + // pointer types for the purpose of the receiver type + // computation; i.e., embeddedIsPtr is sticky for this + // embedding hierarchy. + thisEmbeddedIsPtr := embeddedIsPtr || isPtr + for _, m := range embedded.methods { + // only top-level methods are embedded + if m.Level == 0 { + mset.add(customizeRecv(m, recvTypeName, thisEmbeddedIsPtr, level)) + } + } + if !visited[embedded] { + r.collectEmbeddedMethods(mset, embedded, recvTypeName, thisEmbeddedIsPtr, level+1, visited) + } + } + delete(visited, typ) +} + +// computeMethodSets determines the actual method sets for each type encountered. +func (r *reader) computeMethodSets() { + for _, t := range r.types { + // collect embedded methods for t + if t.isStruct { + // struct + r.collectEmbeddedMethods(t.methods, t, t.name, false, 1, make(embeddedSet)) + } else { + // interface + // TODO(gri) fix this + } + } + + // For any predeclared names that are declared locally, don't treat them as + // exported fields anymore. + for predecl := range r.shadowedPredecl { + for _, ityp := range r.fixmap[predecl] { + removeAnonymousField(predecl, ityp) + } + } +} + +// cleanupTypes removes the association of functions and methods with +// types that have no declaration. Instead, these functions and methods +// are shown at the package level. It also removes types with missing +// declarations or which are not visible. +func (r *reader) cleanupTypes() { + for _, t := range r.types { + visible := r.isVisible(t.name) + predeclared := predeclaredTypes[t.name] + + if t.decl == nil && (predeclared || visible && (t.isEmbedded || r.hasDotImp)) { + // t.name is a predeclared type (and was not redeclared in this package), + // or it was embedded somewhere but its declaration is missing (because + // the AST is incomplete), or we have a dot-import (and all bets are off): + // move any associated values, funcs, and methods back to the top-level so + // that they are not lost. + // 1) move values + r.values = append(r.values, t.values...) + // 2) move factory functions + for name, f := range t.funcs { + // in a correct AST, package-level function names + // are all different - no need to check for conflicts + r.funcs[name] = f + } + // 3) move methods + if !predeclared { + for name, m := range t.methods { + // don't overwrite functions with the same name - drop them + if _, found := r.funcs[name]; !found { + r.funcs[name] = m + } + } + } + } + // remove types w/o declaration or which are not visible + if t.decl == nil || !visible { + delete(r.types, t.name) + } + } +} + +// ---------------------------------------------------------------------------- +// Sorting + +type data struct { + n int + swap func(i, j int) + less func(i, j int) bool +} + +func (d *data) Len() int { return d.n } +func (d *data) Swap(i, j int) { d.swap(i, j) } +func (d *data) Less(i, j int) bool { return d.less(i, j) } + +// sortBy is a helper function for sorting. +func sortBy(less func(i, j int) bool, swap func(i, j int), n int) { + sort.Sort(&data{n, swap, less}) +} + +func sortedKeys(m map[string]int) []string { + list := make([]string, len(m)) + i := 0 + for key := range m { + list[i] = key + i++ + } + sort.Strings(list) + return list +} + +// sortingName returns the name to use when sorting d into place. +func sortingName(d *ast.GenDecl) string { + if len(d.Specs) == 1 { + if s, ok := d.Specs[0].(*ast.ValueSpec); ok { + return s.Names[0].Name + } + } + return "" +} + +func sortedValues(m []*Value, tok token.Token) []*Value { + list := make([]*Value, len(m)) // big enough in any case + i := 0 + for _, val := range m { + if val.Decl.Tok == tok { + list[i] = val + i++ + } + } + list = list[0:i] + + sortBy( + func(i, j int) bool { + if ni, nj := sortingName(list[i].Decl), sortingName(list[j].Decl); ni != nj { + return ni < nj + } + return list[i].order < list[j].order + }, + func(i, j int) { list[i], list[j] = list[j], list[i] }, + len(list), + ) + + return list +} + +func sortedTypes(m map[string]*namedType, allMethods bool) []*Type { + list := make([]*Type, len(m)) + i := 0 + for _, t := range m { + list[i] = &Type{ + Doc: t.doc, + Name: t.name, + Decl: t.decl, + Consts: sortedValues(t.values, token.CONST), + Vars: sortedValues(t.values, token.VAR), + Funcs: sortedFuncs(t.funcs, true), + Methods: sortedFuncs(t.methods, allMethods), + } + i++ + } + + sortBy( + func(i, j int) bool { return list[i].Name < list[j].Name }, + func(i, j int) { list[i], list[j] = list[j], list[i] }, + len(list), + ) + + return list +} + +func removeStar(s string) string { + if len(s) > 0 && s[0] == '*' { + return s[1:] + } + return s +} + +func sortedFuncs(m methodSet, allMethods bool) []*Func { + list := make([]*Func, len(m)) + i := 0 + for _, m := range m { + // determine which methods to include + switch { + case m.Decl == nil: + // exclude conflict entry + case allMethods, m.Level == 0, !token.IsExported(removeStar(m.Orig)): + // forced inclusion, method not embedded, or method + // embedded but original receiver type not exported + list[i] = m + i++ + } + } + list = list[0:i] + sortBy( + func(i, j int) bool { return list[i].Name < list[j].Name }, + func(i, j int) { list[i], list[j] = list[j], list[i] }, + len(list), + ) + return list +} + +// noteBodies returns a list of note body strings given a list of notes. +// This is only used to populate the deprecated Package.Bugs field. +func noteBodies(notes []*Note) []string { + var list []string + for _, n := range notes { + list = append(list, n.Body) + } + return list +} + +// ---------------------------------------------------------------------------- +// Predeclared identifiers + +// IsPredeclared reports whether s is a predeclared identifier. +func IsPredeclared(s string) bool { + return predeclaredTypes[s] || predeclaredFuncs[s] || predeclaredConstants[s] +} + +var predeclaredTypes = map[string]bool{ + "any": true, + "bool": true, + "byte": true, + "comparable": true, + "complex64": true, + "complex128": true, + "error": true, + "float32": true, + "float64": true, + "int": true, + "int8": true, + "int16": true, + "int32": true, + "int64": true, + "rune": true, + "string": true, + "uint": true, + "uint8": true, + "uint16": true, + "uint32": true, + "uint64": true, + "uintptr": true, +} + +var predeclaredFuncs = map[string]bool{ + "append": true, + "cap": true, + "close": true, + "complex": true, + "copy": true, + "delete": true, + "imag": true, + "len": true, + "make": true, + "new": true, + "panic": true, + "print": true, + "println": true, + "real": true, + "recover": true, +} + +var predeclaredConstants = map[string]bool{ + "false": true, + "iota": true, + "nil": true, + "true": true, +} + +// assumedPackageName returns the assumed package name +// for a given import path. This is a copy of +// golang.org/x/tools/internal/imports.ImportPathToAssumedName. +func assumedPackageName(importPath string) string { + notIdentifier := func(ch rune) bool { + return !('a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || + '0' <= ch && ch <= '9' || + ch == '_' || + ch >= utf8.RuneSelf && (unicode.IsLetter(ch) || unicode.IsDigit(ch))) + } + + base := path.Base(importPath) + if strings.HasPrefix(base, "v") { + if _, err := strconv.Atoi(base[1:]); err == nil { + dir := path.Dir(importPath) + if dir != "." { + base = path.Base(dir) + } + } + } + base = strings.TrimPrefix(base, "go-") + if i := strings.IndexFunc(base, notIdentifier); i >= 0 { + base = base[:i] + } + return base +} diff --git a/src/go/doc/synopsis.go b/src/go/doc/synopsis.go new file mode 100644 index 0000000..3c9e7e9 --- /dev/null +++ b/src/go/doc/synopsis.go @@ -0,0 +1,78 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import ( + "go/doc/comment" + "strings" + "unicode" +) + +// firstSentence returns the first sentence in s. +// The sentence ends after the first period followed by space and +// not preceded by exactly one uppercase letter. +func firstSentence(s string) string { + var ppp, pp, p rune + for i, q := range s { + if q == '\n' || q == '\r' || q == '\t' { + q = ' ' + } + if q == ' ' && p == '.' && (!unicode.IsUpper(pp) || unicode.IsUpper(ppp)) { + return s[:i] + } + if p == '怂' || p == 'ļ¼Ž' { + return s[:i] + } + ppp, pp, p = pp, p, q + } + return s +} + +// Synopsis returns a cleaned version of the first sentence in text. +// +// Deprecated: New programs should use [Package.Synopsis] instead, +// which handles links in text properly. +func Synopsis(text string) string { + var p Package + return p.Synopsis(text) +} + +// IllegalPrefixes is a list of lower-case prefixes that identify +// a comment as not being a doc comment. +// This helps to avoid misinterpreting the common mistake +// of a copyright notice immediately before a package statement +// as being a doc comment. +var IllegalPrefixes = []string{ + "copyright", + "all rights", + "author", +} + +// Synopsis returns a cleaned version of the first sentence in text. +// That sentence ends after the first period followed by space and not +// preceded by exactly one uppercase letter, or at the first paragraph break. +// The result string has no \n, \r, or \t characters and uses only single +// spaces between words. If text starts with any of the IllegalPrefixes, +// the result is the empty string. +func (p *Package) Synopsis(text string) string { + text = firstSentence(text) + lower := strings.ToLower(text) + for _, prefix := range IllegalPrefixes { + if strings.HasPrefix(lower, prefix) { + return "" + } + } + pr := p.Printer() + pr.TextWidth = -1 + d := p.Parser().Parse(text) + if len(d.Content) == 0 { + return "" + } + if _, ok := d.Content[0].(*comment.Paragraph); !ok { + return "" + } + d.Content = d.Content[:1] // might be blank lines, code blocks, etc in ā€œfirst sentenceā€ + return strings.TrimSpace(string(pr.Text(d))) +} diff --git a/src/go/doc/synopsis_test.go b/src/go/doc/synopsis_test.go new file mode 100644 index 0000000..158c734 --- /dev/null +++ b/src/go/doc/synopsis_test.go @@ -0,0 +1,52 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package doc + +import "testing" + +var tests = []struct { + txt string + fsl int + syn string +}{ + {"", 0, ""}, + {"foo", 3, "foo"}, + {"foo.", 4, "foo."}, + {"foo.bar", 7, "foo.bar"}, + {" foo. ", 6, "foo."}, + {" foo\t bar.\n", 12, "foo bar."}, + {" foo\t bar.\n", 12, "foo bar."}, + {"a b\n\nc\r\rd\t\t", 12, "a b"}, + {"a b\n\nc\r\rd\t\t . BLA", 15, "a b"}, + {"Package poems by T.S.Eliot. To rhyme...", 27, "Package poems by T.S.Eliot."}, + {"Package poems by T. S. Eliot. To rhyme...", 29, "Package poems by T. S. Eliot."}, + {"foo implements the foo ABI. The foo ABI is...", 27, "foo implements the foo ABI."}, + {"Package\nfoo. ..", 12, "Package foo."}, + {"P . Q.", 3, "P ."}, + {"P. Q. ", 8, "P. Q."}, + {"Package ĪšĪ±Ī»Ī·Ī¼Ī­ĻĪ± ĪŗĻŒĻƒĪ¼Īµ.", 36, "Package ĪšĪ±Ī»Ī·Ī¼Ī­ĻĪ± ĪŗĻŒĻƒĪ¼Īµ."}, + {"Package 恓悓恫恔ćÆ äø–ē•Œ\n", 31, "Package 恓悓恫恔ćÆ äø–ē•Œ"}, + {"Package 恓悓恫恔ćÆ怂äø–ē•Œ", 26, "Package 恓悓恫恔ćÆ怂"}, + {"Package ģ•ˆė…•ļ¼Žäø–ē•Œ", 17, "Package ģ•ˆė…•ļ¼Ž"}, + {"Package foo does bar.", 21, "Package foo does bar."}, + {"Copyright 2012 Google, Inc. Package foo does bar.", 27, ""}, + {"All Rights reserved. Package foo does bar.", 20, ""}, + {"All rights reserved. Package foo does bar.", 20, ""}, + {"Authors: foo@bar.com. Package foo does bar.", 21, ""}, + {"typically invoked as ``go tool asm'',", 37, "typically invoked as ā€œgo tool asmā€,"}, +} + +func TestSynopsis(t *testing.T) { + for _, e := range tests { + fs := firstSentence(e.txt) + if fs != e.txt[:e.fsl] { + t.Errorf("firstSentence(%q) = %q, want %q", e.txt, fs, e.txt[:e.fsl]) + } + syn := Synopsis(e.txt) + if syn != e.syn { + t.Errorf("Synopsis(%q) = %q, want %q", e.txt, syn, e.syn) + } + } +} diff --git a/src/go/doc/testdata/a.0.golden b/src/go/doc/testdata/a.0.golden new file mode 100644 index 0000000..7e680b8 --- /dev/null +++ b/src/go/doc/testdata/a.0.golden @@ -0,0 +1,52 @@ +// comment 0 comment 1 +PACKAGE a + +IMPORTPATH + testdata/a + +FILENAMES + testdata/a0.go + testdata/a1.go + +BUGS .Bugs is now deprecated, please use .Notes instead + bug0 + + bug1 + + +BUGS +BUG(uid) bug0 + +BUG(uid) bug1 + + +NOTES +NOTE(uid) + +NOTE(foo) 1 of 4 - this is the first line of note 1 + - note 1 continues on this 2nd line + - note 1 continues on this 3rd line + +NOTE(foo) 2 of 4 + +NOTE(bar) 3 of 4 + +NOTE(bar) 4 of 4 + - this is the last line of note 4 + +NOTE(bam) This note which contains a (parenthesized) subphrase + must appear in its entirety. + +NOTE(xxx) The ':' after the marker and uid is optional. + + +SECBUGS +SECBUG(uid) sec hole 0 + need to fix asap + + +TODOS +TODO(uid) todo0 + +TODO(uid) todo1 + diff --git a/src/go/doc/testdata/a.1.golden b/src/go/doc/testdata/a.1.golden new file mode 100644 index 0000000..7e680b8 --- /dev/null +++ b/src/go/doc/testdata/a.1.golden @@ -0,0 +1,52 @@ +// comment 0 comment 1 +PACKAGE a + +IMPORTPATH + testdata/a + +FILENAMES + testdata/a0.go + testdata/a1.go + +BUGS .Bugs is now deprecated, please use .Notes instead + bug0 + + bug1 + + +BUGS +BUG(uid) bug0 + +BUG(uid) bug1 + + +NOTES +NOTE(uid) + +NOTE(foo) 1 of 4 - this is the first line of note 1 + - note 1 continues on this 2nd line + - note 1 continues on this 3rd line + +NOTE(foo) 2 of 4 + +NOTE(bar) 3 of 4 + +NOTE(bar) 4 of 4 + - this is the last line of note 4 + +NOTE(bam) This note which contains a (parenthesized) subphrase + must appear in its entirety. + +NOTE(xxx) The ':' after the marker and uid is optional. + + +SECBUGS +SECBUG(uid) sec hole 0 + need to fix asap + + +TODOS +TODO(uid) todo0 + +TODO(uid) todo1 + diff --git a/src/go/doc/testdata/a.2.golden b/src/go/doc/testdata/a.2.golden new file mode 100644 index 0000000..7e680b8 --- /dev/null +++ b/src/go/doc/testdata/a.2.golden @@ -0,0 +1,52 @@ +// comment 0 comment 1 +PACKAGE a + +IMPORTPATH + testdata/a + +FILENAMES + testdata/a0.go + testdata/a1.go + +BUGS .Bugs is now deprecated, please use .Notes instead + bug0 + + bug1 + + +BUGS +BUG(uid) bug0 + +BUG(uid) bug1 + + +NOTES +NOTE(uid) + +NOTE(foo) 1 of 4 - this is the first line of note 1 + - note 1 continues on this 2nd line + - note 1 continues on this 3rd line + +NOTE(foo) 2 of 4 + +NOTE(bar) 3 of 4 + +NOTE(bar) 4 of 4 + - this is the last line of note 4 + +NOTE(bam) This note which contains a (parenthesized) subphrase + must appear in its entirety. + +NOTE(xxx) The ':' after the marker and uid is optional. + + +SECBUGS +SECBUG(uid) sec hole 0 + need to fix asap + + +TODOS +TODO(uid) todo0 + +TODO(uid) todo1 + diff --git a/src/go/doc/testdata/a0.go b/src/go/doc/testdata/a0.go new file mode 100644 index 0000000..2420c8a --- /dev/null +++ b/src/go/doc/testdata/a0.go @@ -0,0 +1,40 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// comment 0 +package a + +//BUG(uid): bug0 + +//TODO(uid): todo0 + +// A note with some spaces after it, should be ignored (watch out for +// emacs modes that remove trailing whitespace). +//NOTE(uid): + +// SECBUG(uid): sec hole 0 +// need to fix asap + +// Multiple notes may be in the same comment group and should be +// recognized individually. Notes may start in the middle of a +// comment group as long as they start at the beginning of an +// individual comment. +// +// NOTE(foo): 1 of 4 - this is the first line of note 1 +// - note 1 continues on this 2nd line +// - note 1 continues on this 3rd line +// NOTE(foo): 2 of 4 +// NOTE(bar): 3 of 4 +/* NOTE(bar): 4 of 4 */ +// - this is the last line of note 4 +// +// + +// NOTE(bam): This note which contains a (parenthesized) subphrase +// must appear in its entirety. + +// NOTE(xxx) The ':' after the marker and uid is optional. + +// NOTE(): NO uid - should not show up. +// NOTE() NO uid - should not show up. diff --git a/src/go/doc/testdata/a1.go b/src/go/doc/testdata/a1.go new file mode 100644 index 0000000..9fad1e0 --- /dev/null +++ b/src/go/doc/testdata/a1.go @@ -0,0 +1,12 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// comment 1 +package a + +//BUG(uid): bug1 + +//TODO(uid): todo1 + +//TODO(): ignored diff --git a/src/go/doc/testdata/b.0.golden b/src/go/doc/testdata/b.0.golden new file mode 100644 index 0000000..c06246a --- /dev/null +++ b/src/go/doc/testdata/b.0.golden @@ -0,0 +1,74 @@ +// +PACKAGE b + +IMPORTPATH + testdata/b + +IMPORTS + a + +FILENAMES + testdata/b.go + +CONSTANTS + // + const ( + C1 notExported = iota + C2 + + C4 + C5 + ) + + // + const C notExported = 0 + + // + const Pi = 3.14 // Pi + + +VARIABLES + // + var ( + U1, U2, U4, U5 notExported + + U7 notExported = 7 + ) + + // + var MaxInt int // MaxInt + + // + var V notExported + + // + var V1, V2, V4, V5 notExported + + +FUNCTIONS + // Associated with comparable type if AllDecls is set. + func ComparableFactory() comparable + + // + func F(x int) int + + // + func F1() notExported + + // Always under the package functions list. + func NotAFactory() int + + // Associated with uint type if AllDecls is set. + func UintFactory() uint + + +TYPES + // + type T struct{} // T + + // + var V T // v + + // + func (x *T) M() + diff --git a/src/go/doc/testdata/b.1.golden b/src/go/doc/testdata/b.1.golden new file mode 100644 index 0000000..2b62c34 --- /dev/null +++ b/src/go/doc/testdata/b.1.golden @@ -0,0 +1,89 @@ +// +PACKAGE b + +IMPORTPATH + testdata/b + +IMPORTS + a + +FILENAMES + testdata/b.go + +CONSTANTS + // + const Pi = 3.14 // Pi + + +VARIABLES + // + var MaxInt int // MaxInt + + +FUNCTIONS + // + func F(x int) int + + // Always under the package functions list. + func NotAFactory() int + + +TYPES + // + type T struct{} // T + + // + var V T // v + + // + func (x *T) M() + + // Should only appear if AllDecls is set. + type comparable struct{} // overrides a predeclared type comparable + + // Associated with comparable type if AllDecls is set. + func ComparableFactory() comparable + + // + type notExported int + + // + const ( + C1 notExported = iota + C2 + c3 + C4 + C5 + ) + + // + const C notExported = 0 + + // + var ( + U1, U2, u3, U4, U5 notExported + u6 notExported + U7 notExported = 7 + ) + + // + var V notExported + + // + var V1, V2, v3, V4, V5 notExported + + // + func F1() notExported + + // + func f2() notExported + + // Should only appear if AllDecls is set. + type uint struct{} // overrides a predeclared type uint + + // Associated with uint type if AllDecls is set. + func UintFactory() uint + + // Associated with uint type if AllDecls is set. + func uintFactory() uint + diff --git a/src/go/doc/testdata/b.2.golden b/src/go/doc/testdata/b.2.golden new file mode 100644 index 0000000..c06246a --- /dev/null +++ b/src/go/doc/testdata/b.2.golden @@ -0,0 +1,74 @@ +// +PACKAGE b + +IMPORTPATH + testdata/b + +IMPORTS + a + +FILENAMES + testdata/b.go + +CONSTANTS + // + const ( + C1 notExported = iota + C2 + + C4 + C5 + ) + + // + const C notExported = 0 + + // + const Pi = 3.14 // Pi + + +VARIABLES + // + var ( + U1, U2, U4, U5 notExported + + U7 notExported = 7 + ) + + // + var MaxInt int // MaxInt + + // + var V notExported + + // + var V1, V2, V4, V5 notExported + + +FUNCTIONS + // Associated with comparable type if AllDecls is set. + func ComparableFactory() comparable + + // + func F(x int) int + + // + func F1() notExported + + // Always under the package functions list. + func NotAFactory() int + + // Associated with uint type if AllDecls is set. + func UintFactory() uint + + +TYPES + // + type T struct{} // T + + // + var V T // v + + // + func (x *T) M() + diff --git a/src/go/doc/testdata/b.go b/src/go/doc/testdata/b.go new file mode 100644 index 0000000..61b512b --- /dev/null +++ b/src/go/doc/testdata/b.go @@ -0,0 +1,64 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package b + +import "a" + +// ---------------------------------------------------------------------------- +// Basic declarations + +const Pi = 3.14 // Pi +var MaxInt int // MaxInt +type T struct{} // T +var V T // v +func F(x int) int {} // F +func (x *T) M() {} // M + +// Corner cases: association with (presumed) predeclared types + +// Always under the package functions list. +func NotAFactory() int {} + +// Associated with uint type if AllDecls is set. +func UintFactory() uint {} + +// Associated with uint type if AllDecls is set. +func uintFactory() uint {} + +// Associated with comparable type if AllDecls is set. +func ComparableFactory() comparable {} + +// Should only appear if AllDecls is set. +type uint struct{} // overrides a predeclared type uint + +// Should only appear if AllDecls is set. +type comparable struct{} // overrides a predeclared type comparable + +// ---------------------------------------------------------------------------- +// Exported declarations associated with non-exported types must always be shown. + +type notExported int + +const C notExported = 0 + +const ( + C1 notExported = iota + C2 + c3 + C4 + C5 +) + +var V notExported +var V1, V2, v3, V4, V5 notExported + +var ( + U1, U2, u3, U4, U5 notExported + u6 notExported + U7 notExported = 7 +) + +func F1() notExported {} +func f2() notExported {} diff --git a/src/go/doc/testdata/benchmark.go b/src/go/doc/testdata/benchmark.go new file mode 100644 index 0000000..dbf6b4f --- /dev/null +++ b/src/go/doc/testdata/benchmark.go @@ -0,0 +1,293 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +import ( + "flag" + "fmt" + "os" + "runtime" + "time" +) + +var matchBenchmarks = flag.String("test.bench", "", "regular expression to select benchmarks to run") +var benchTime = flag.Duration("test.benchtime", 1*time.Second, "approximate run time for each benchmark") + +// An internal type but exported because it is cross-package; part of the implementation +// of go test. +type InternalBenchmark struct { + Name string + F func(b *B) +} + +// B is a type passed to Benchmark functions to manage benchmark +// timing and to specify the number of iterations to run. +type B struct { + common + N int + benchmark InternalBenchmark + bytes int64 + timerOn bool + result BenchmarkResult +} + +// StartTimer starts timing a test. This function is called automatically +// before a benchmark starts, but it can also used to resume timing after +// a call to StopTimer. +func (b *B) StartTimer() { + if !b.timerOn { + b.start = time.Now() + b.timerOn = true + } +} + +// StopTimer stops timing a test. This can be used to pause the timer +// while performing complex initialization that you don't +// want to measure. +func (b *B) StopTimer() { + if b.timerOn { + b.duration += time.Since(b.start) + b.timerOn = false + } +} + +// ResetTimer sets the elapsed benchmark time to zero. +// It does not affect whether the timer is running. +func (b *B) ResetTimer() { + if b.timerOn { + b.start = time.Now() + } + b.duration = 0 +} + +// SetBytes records the number of bytes processed in a single operation. +// If this is called, the benchmark will report ns/op and MB/s. +func (b *B) SetBytes(n int64) { b.bytes = n } + +func (b *B) nsPerOp() int64 { + if b.N <= 0 { + return 0 + } + return b.duration.Nanoseconds() / int64(b.N) +} + +// runN runs a single benchmark for the specified number of iterations. +func (b *B) runN(n int) { + // Try to get a comparable environment for each run + // by clearing garbage from previous runs. + runtime.GC() + b.N = n + b.ResetTimer() + b.StartTimer() + b.benchmark.F(b) + b.StopTimer() +} + +func min(x, y int) int { + if x > y { + return y + } + return x +} + +func max(x, y int) int { + if x < y { + return y + } + return x +} + +// roundDown10 rounds a number down to the nearest power of 10. +func roundDown10(n int) int { + var tens = 0 + // tens = floor(log_10(n)) + for n > 10 { + n = n / 10 + tens++ + } + // result = 10^tens + result := 1 + for i := 0; i < tens; i++ { + result *= 10 + } + return result +} + +// roundUp rounds x up to a number of the form [1eX, 2eX, 5eX]. +func roundUp(n int) int { + base := roundDown10(n) + if n < (2 * base) { + return 2 * base + } + if n < (5 * base) { + return 5 * base + } + return 10 * base +} + +// run times the benchmark function in a separate goroutine. +func (b *B) run() BenchmarkResult { + go b.launch() + <-b.signal + return b.result +} + +// launch launches the benchmark function. It gradually increases the number +// of benchmark iterations until the benchmark runs for a second in order +// to get a reasonable measurement. It prints timing information in this form +// testing.BenchmarkHello 100000 19 ns/op +// launch is run by the fun function as a separate goroutine. +func (b *B) launch() { + // Run the benchmark for a single iteration in case it's expensive. + n := 1 + + // Signal that we're done whether we return normally + // or by FailNow's runtime.Goexit. + defer func() { + b.signal <- b + }() + + b.runN(n) + // Run the benchmark for at least the specified amount of time. + d := *benchTime + for !b.failed && b.duration < d && n < 1e9 { + last := n + // Predict iterations/sec. + if b.nsPerOp() == 0 { + n = 1e9 + } else { + n = int(d.Nanoseconds() / b.nsPerOp()) + } + // Run more iterations than we think we'll need for a second (1.5x). + // Don't grow too fast in case we had timing errors previously. + // Be sure to run at least one more than last time. + n = max(min(n+n/2, 100*last), last+1) + // Round up to something easy to read. + n = roundUp(n) + b.runN(n) + } + b.result = BenchmarkResult{b.N, b.duration, b.bytes} +} + +// The results of a benchmark run. +type BenchmarkResult struct { + N int // The number of iterations. + T time.Duration // The total time taken. + Bytes int64 // Bytes processed in one iteration. +} + +func (r BenchmarkResult) NsPerOp() int64 { + if r.N <= 0 { + return 0 + } + return r.T.Nanoseconds() / int64(r.N) +} + +func (r BenchmarkResult) mbPerSec() float64 { + if r.Bytes <= 0 || r.T <= 0 || r.N <= 0 { + return 0 + } + return (float64(r.Bytes) * float64(r.N) / 1e6) / r.T.Seconds() +} + +func (r BenchmarkResult) String() string { + mbs := r.mbPerSec() + mb := "" + if mbs != 0 { + mb = fmt.Sprintf("\t%7.2f MB/s", mbs) + } + nsop := r.NsPerOp() + ns := fmt.Sprintf("%10d ns/op", nsop) + if r.N > 0 && nsop < 100 { + // The format specifiers here make sure that + // the ones digits line up for all three possible formats. + if nsop < 10 { + ns = fmt.Sprintf("%13.2f ns/op", float64(r.T.Nanoseconds())/float64(r.N)) + } else { + ns = fmt.Sprintf("%12.1f ns/op", float64(r.T.Nanoseconds())/float64(r.N)) + } + } + return fmt.Sprintf("%8d\t%s%s", r.N, ns, mb) +} + +// An internal function but exported because it is cross-package; part of the implementation +// of go test. +func RunBenchmarks(matchString func(pat, str string) (bool, error), benchmarks []InternalBenchmark) { + // If no flag was specified, don't run benchmarks. + if len(*matchBenchmarks) == 0 { + return + } + for _, Benchmark := range benchmarks { + matched, err := matchString(*matchBenchmarks, Benchmark.Name) + if err != nil { + fmt.Fprintf(os.Stderr, "testing: invalid regexp for -test.bench: %s\n", err) + os.Exit(1) + } + if !matched { + continue + } + for _, procs := range cpuList { + runtime.GOMAXPROCS(procs) + b := &B{ + common: common{ + signal: make(chan any), + }, + benchmark: Benchmark, + } + benchName := Benchmark.Name + if procs != 1 { + benchName = fmt.Sprintf("%s-%d", Benchmark.Name, procs) + } + fmt.Printf("%s\t", benchName) + r := b.run() + if b.failed { + // The output could be very long here, but probably isn't. + // We print it all, regardless, because we don't want to trim the reason + // the benchmark failed. + fmt.Printf("--- FAIL: %s\n%s", benchName, b.output) + continue + } + fmt.Printf("%v\n", r) + // Unlike with tests, we ignore the -chatty flag and always print output for + // benchmarks since the output generation time will skew the results. + if len(b.output) > 0 { + b.trimOutput() + fmt.Printf("--- BENCH: %s\n%s", benchName, b.output) + } + if p := runtime.GOMAXPROCS(-1); p != procs { + fmt.Fprintf(os.Stderr, "testing: %s left GOMAXPROCS set to %d\n", benchName, p) + } + } + } +} + +// trimOutput shortens the output from a benchmark, which can be very long. +func (b *B) trimOutput() { + // The output is likely to appear multiple times because the benchmark + // is run multiple times, but at least it will be seen. This is not a big deal + // because benchmarks rarely print, but just in case, we trim it if it's too long. + const maxNewlines = 10 + for nlCount, j := 0, 0; j < len(b.output); j++ { + if b.output[j] == '\n' { + nlCount++ + if nlCount >= maxNewlines { + b.output = append(b.output[:j], "\n\t... [output truncated]\n"...) + break + } + } + } +} + +// Benchmark benchmarks a single function. Useful for creating +// custom benchmarks that do not use go test. +func Benchmark(f func(b *B)) BenchmarkResult { + b := &B{ + common: common{ + signal: make(chan any), + }, + benchmark: InternalBenchmark{"", f}, + } + return b.run() +} diff --git a/src/go/doc/testdata/blank.0.golden b/src/go/doc/testdata/blank.0.golden new file mode 100644 index 0000000..70f2929 --- /dev/null +++ b/src/go/doc/testdata/blank.0.golden @@ -0,0 +1,62 @@ +// Package blank is a go/doc test for the handling of _. See issue ... +PACKAGE blank + +IMPORTPATH + testdata/blank + +IMPORTS + os + +FILENAMES + testdata/blank.go + +CONSTANTS + // T constants counting from unexported constants. + const ( + C1 T + C2 + + C3 + + C4 int + ) + + // Constants with a single type that is not propagated. + const ( + Default = 0644 + Useless = 0312 + WideOpen = 0777 + ) + + // Constants with an imported type that is propagated. + const ( + M1 os.FileMode + M2 + M3 + ) + + // Package constants. + const ( + I1 int + I2 + ) + + +TYPES + // S has a padding field. + type S struct { + H uint32 + + A uint8 + // contains filtered or unexported fields + } + + // + type T int + + // T constants counting from a blank constant. + const ( + T1 T + T2 + ) + diff --git a/src/go/doc/testdata/blank.1.golden b/src/go/doc/testdata/blank.1.golden new file mode 100644 index 0000000..8098cb6 --- /dev/null +++ b/src/go/doc/testdata/blank.1.golden @@ -0,0 +1,83 @@ +// Package blank is a go/doc test for the handling of _. See issue ... +PACKAGE blank + +IMPORTPATH + testdata/blank + +IMPORTS + os + +FILENAMES + testdata/blank.go + +CONSTANTS + // T constants counting from unexported constants. + const ( + tweedledee T = iota + tweedledum + C1 + C2 + alice + C3 + redQueen int = iota + C4 + ) + + // Constants with a single type that is not propagated. + const ( + zero os.FileMode = 0 + Default = 0644 + Useless = 0312 + WideOpen = 0777 + ) + + // Constants with an imported type that is propagated. + const ( + zero os.FileMode = 0 + M1 + M2 + M3 + ) + + // Package constants. + const ( + _ int = iota + I1 + I2 + ) + + // Unexported constants counting from blank iota. See issue 9615. + const ( + _ = iota + one = iota + 1 + ) + + +VARIABLES + // + var _ = T(55) + + +FUNCTIONS + // + func _() + + +TYPES + // S has a padding field. + type S struct { + H uint32 + _ uint8 + A uint8 + } + + // + type T int + + // T constants counting from a blank constant. + const ( + _ T = iota + T1 + T2 + ) + diff --git a/src/go/doc/testdata/blank.2.golden b/src/go/doc/testdata/blank.2.golden new file mode 100644 index 0000000..70f2929 --- /dev/null +++ b/src/go/doc/testdata/blank.2.golden @@ -0,0 +1,62 @@ +// Package blank is a go/doc test for the handling of _. See issue ... +PACKAGE blank + +IMPORTPATH + testdata/blank + +IMPORTS + os + +FILENAMES + testdata/blank.go + +CONSTANTS + // T constants counting from unexported constants. + const ( + C1 T + C2 + + C3 + + C4 int + ) + + // Constants with a single type that is not propagated. + const ( + Default = 0644 + Useless = 0312 + WideOpen = 0777 + ) + + // Constants with an imported type that is propagated. + const ( + M1 os.FileMode + M2 + M3 + ) + + // Package constants. + const ( + I1 int + I2 + ) + + +TYPES + // S has a padding field. + type S struct { + H uint32 + + A uint8 + // contains filtered or unexported fields + } + + // + type T int + + // T constants counting from a blank constant. + const ( + T1 T + T2 + ) + diff --git a/src/go/doc/testdata/blank.go b/src/go/doc/testdata/blank.go new file mode 100644 index 0000000..5ea6186 --- /dev/null +++ b/src/go/doc/testdata/blank.go @@ -0,0 +1,75 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package blank is a go/doc test for the handling of _. +// See issue 5397. +package blank + +import "os" + +type T int + +// T constants counting from a blank constant. +const ( + _ T = iota + T1 + T2 +) + +// T constants counting from unexported constants. +const ( + tweedledee T = iota + tweedledum + C1 + C2 + alice + C3 + redQueen int = iota + C4 +) + +// Constants with a single type that is not propagated. +const ( + zero os.FileMode = 0 + Default = 0644 + Useless = 0312 + WideOpen = 0777 +) + +// Constants with an imported type that is propagated. +const ( + zero os.FileMode = 0 + M1 + M2 + M3 +) + +// Package constants. +const ( + _ int = iota + I1 + I2 +) + +// Unexported constants counting from blank iota. +// See issue 9615. +const ( + _ = iota + one = iota + 1 +) + +// Blanks not in doc output: + +// S has a padding field. +type S struct { + H uint32 + _ uint8 + A uint8 +} + +func _() {} + +type _ T + +var _ = T(55) diff --git a/src/go/doc/testdata/bugpara.0.golden b/src/go/doc/testdata/bugpara.0.golden new file mode 100644 index 0000000..5804859 --- /dev/null +++ b/src/go/doc/testdata/bugpara.0.golden @@ -0,0 +1,20 @@ +// +PACKAGE bugpara + +IMPORTPATH + testdata/bugpara + +FILENAMES + testdata/bugpara.go + +BUGS .Bugs is now deprecated, please use .Notes instead + Sometimes bugs have multiple paragraphs. + + Like this one. + + +BUGS +BUG(rsc) Sometimes bugs have multiple paragraphs. + + Like this one. + diff --git a/src/go/doc/testdata/bugpara.1.golden b/src/go/doc/testdata/bugpara.1.golden new file mode 100644 index 0000000..5804859 --- /dev/null +++ b/src/go/doc/testdata/bugpara.1.golden @@ -0,0 +1,20 @@ +// +PACKAGE bugpara + +IMPORTPATH + testdata/bugpara + +FILENAMES + testdata/bugpara.go + +BUGS .Bugs is now deprecated, please use .Notes instead + Sometimes bugs have multiple paragraphs. + + Like this one. + + +BUGS +BUG(rsc) Sometimes bugs have multiple paragraphs. + + Like this one. + diff --git a/src/go/doc/testdata/bugpara.2.golden b/src/go/doc/testdata/bugpara.2.golden new file mode 100644 index 0000000..5804859 --- /dev/null +++ b/src/go/doc/testdata/bugpara.2.golden @@ -0,0 +1,20 @@ +// +PACKAGE bugpara + +IMPORTPATH + testdata/bugpara + +FILENAMES + testdata/bugpara.go + +BUGS .Bugs is now deprecated, please use .Notes instead + Sometimes bugs have multiple paragraphs. + + Like this one. + + +BUGS +BUG(rsc) Sometimes bugs have multiple paragraphs. + + Like this one. + diff --git a/src/go/doc/testdata/bugpara.go b/src/go/doc/testdata/bugpara.go new file mode 100644 index 0000000..0360a6f --- /dev/null +++ b/src/go/doc/testdata/bugpara.go @@ -0,0 +1,9 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package bugpara + +// BUG(rsc): Sometimes bugs have multiple paragraphs. +// +// Like this one. diff --git a/src/go/doc/testdata/c.0.golden b/src/go/doc/testdata/c.0.golden new file mode 100644 index 0000000..e21959b --- /dev/null +++ b/src/go/doc/testdata/c.0.golden @@ -0,0 +1,48 @@ +// +PACKAGE c + +IMPORTPATH + testdata/c + +IMPORTS + a + +FILENAMES + testdata/c.go + +TYPES + // A (should see this) + type A struct{} + + // B (should see this) + type B struct{} + + // C (should see this) + type C struct{} + + // D (should see this) + type D struct{} + + // E1 (should see this) + type E1 struct{} + + // E (should see this for E2 and E3) + type E2 struct{} + + // E (should see this for E2 and E3) + type E3 struct{} + + // E4 (should see this) + type E4 struct{} + + // + type T1 struct{} + + // + func (t1 *T1) M() + + // T2 must not show methods of local T1 + type T2 struct { + a.T1 // not the same as locally declared T1 + } + diff --git a/src/go/doc/testdata/c.1.golden b/src/go/doc/testdata/c.1.golden new file mode 100644 index 0000000..e21959b --- /dev/null +++ b/src/go/doc/testdata/c.1.golden @@ -0,0 +1,48 @@ +// +PACKAGE c + +IMPORTPATH + testdata/c + +IMPORTS + a + +FILENAMES + testdata/c.go + +TYPES + // A (should see this) + type A struct{} + + // B (should see this) + type B struct{} + + // C (should see this) + type C struct{} + + // D (should see this) + type D struct{} + + // E1 (should see this) + type E1 struct{} + + // E (should see this for E2 and E3) + type E2 struct{} + + // E (should see this for E2 and E3) + type E3 struct{} + + // E4 (should see this) + type E4 struct{} + + // + type T1 struct{} + + // + func (t1 *T1) M() + + // T2 must not show methods of local T1 + type T2 struct { + a.T1 // not the same as locally declared T1 + } + diff --git a/src/go/doc/testdata/c.2.golden b/src/go/doc/testdata/c.2.golden new file mode 100644 index 0000000..e21959b --- /dev/null +++ b/src/go/doc/testdata/c.2.golden @@ -0,0 +1,48 @@ +// +PACKAGE c + +IMPORTPATH + testdata/c + +IMPORTS + a + +FILENAMES + testdata/c.go + +TYPES + // A (should see this) + type A struct{} + + // B (should see this) + type B struct{} + + // C (should see this) + type C struct{} + + // D (should see this) + type D struct{} + + // E1 (should see this) + type E1 struct{} + + // E (should see this for E2 and E3) + type E2 struct{} + + // E (should see this for E2 and E3) + type E3 struct{} + + // E4 (should see this) + type E4 struct{} + + // + type T1 struct{} + + // + func (t1 *T1) M() + + // T2 must not show methods of local T1 + type T2 struct { + a.T1 // not the same as locally declared T1 + } + diff --git a/src/go/doc/testdata/c.go b/src/go/doc/testdata/c.go new file mode 100644 index 0000000..e0f3919 --- /dev/null +++ b/src/go/doc/testdata/c.go @@ -0,0 +1,62 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package c + +import "a" + +// ---------------------------------------------------------------------------- +// Test that empty declarations don't cause problems + +const () + +type () + +var () + +// ---------------------------------------------------------------------------- +// Test that types with documentation on both, the Decl and the Spec node +// are handled correctly. + +// A (should see this) +type A struct{} + +// B (should see this) +type ( + B struct{} +) + +type ( + // C (should see this) + C struct{} +) + +// D (should not see this) +type ( + // D (should see this) + D struct{} +) + +// E (should see this for E2 and E3) +type ( + // E1 (should see this) + E1 struct{} + E2 struct{} + E3 struct{} + // E4 (should see this) + E4 struct{} +) + +// ---------------------------------------------------------------------------- +// Test that local and imported types are different when +// handling anonymous fields. + +type T1 struct{} + +func (t1 *T1) M() {} + +// T2 must not show methods of local T1 +type T2 struct { + a.T1 // not the same as locally declared T1 +} diff --git a/src/go/doc/testdata/d.0.golden b/src/go/doc/testdata/d.0.golden new file mode 100644 index 0000000..c005199 --- /dev/null +++ b/src/go/doc/testdata/d.0.golden @@ -0,0 +1,104 @@ +// +PACKAGE d + +IMPORTPATH + testdata/d + +FILENAMES + testdata/d1.go + testdata/d2.go + +CONSTANTS + // CBx constants should appear before CAx constants. + const ( + CB2 = iota // before CB1 + CB1 // before CB0 + CB0 // at end + ) + + // CAx constants should appear after CBx constants. + const ( + CA2 = iota // before CA1 + CA1 // before CA0 + CA0 // at end + ) + + // C0 should be first. + const C0 = 0 + + // C1 should be second. + const C1 = 1 + + // C2 should be third. + const C2 = 2 + + // + const ( + // Single const declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Cungrouped = 0 + ) + + +VARIABLES + // VBx variables should appear before VAx variables. + var ( + VB2 int // before VB1 + VB1 int // before VB0 + VB0 int // at end + ) + + // VAx variables should appear after VBx variables. + var ( + VA2 int // before VA1 + VA1 int // before VA0 + VA0 int // at end + ) + + // V0 should be first. + var V0 uintptr + + // V1 should be second. + var V1 uint + + // V2 should be third. + var V2 int + + // + var ( + // Single var declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Vungrouped = 0 + ) + + +FUNCTIONS + // F0 should be first. + func F0() + + // F1 should be second. + func F1() + + // F2 should be third. + func F2() + + +TYPES + // T0 should be first. + type T0 struct{} + + // T1 should be second. + type T1 struct{} + + // T2 should be third. + type T2 struct{} + + // TG0 should be first. + type TG0 struct{} + + // TG1 should be second. + type TG1 struct{} + + // TG2 should be third. + type TG2 struct{} + diff --git a/src/go/doc/testdata/d.1.golden b/src/go/doc/testdata/d.1.golden new file mode 100644 index 0000000..c005199 --- /dev/null +++ b/src/go/doc/testdata/d.1.golden @@ -0,0 +1,104 @@ +// +PACKAGE d + +IMPORTPATH + testdata/d + +FILENAMES + testdata/d1.go + testdata/d2.go + +CONSTANTS + // CBx constants should appear before CAx constants. + const ( + CB2 = iota // before CB1 + CB1 // before CB0 + CB0 // at end + ) + + // CAx constants should appear after CBx constants. + const ( + CA2 = iota // before CA1 + CA1 // before CA0 + CA0 // at end + ) + + // C0 should be first. + const C0 = 0 + + // C1 should be second. + const C1 = 1 + + // C2 should be third. + const C2 = 2 + + // + const ( + // Single const declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Cungrouped = 0 + ) + + +VARIABLES + // VBx variables should appear before VAx variables. + var ( + VB2 int // before VB1 + VB1 int // before VB0 + VB0 int // at end + ) + + // VAx variables should appear after VBx variables. + var ( + VA2 int // before VA1 + VA1 int // before VA0 + VA0 int // at end + ) + + // V0 should be first. + var V0 uintptr + + // V1 should be second. + var V1 uint + + // V2 should be third. + var V2 int + + // + var ( + // Single var declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Vungrouped = 0 + ) + + +FUNCTIONS + // F0 should be first. + func F0() + + // F1 should be second. + func F1() + + // F2 should be third. + func F2() + + +TYPES + // T0 should be first. + type T0 struct{} + + // T1 should be second. + type T1 struct{} + + // T2 should be third. + type T2 struct{} + + // TG0 should be first. + type TG0 struct{} + + // TG1 should be second. + type TG1 struct{} + + // TG2 should be third. + type TG2 struct{} + diff --git a/src/go/doc/testdata/d.2.golden b/src/go/doc/testdata/d.2.golden new file mode 100644 index 0000000..c005199 --- /dev/null +++ b/src/go/doc/testdata/d.2.golden @@ -0,0 +1,104 @@ +// +PACKAGE d + +IMPORTPATH + testdata/d + +FILENAMES + testdata/d1.go + testdata/d2.go + +CONSTANTS + // CBx constants should appear before CAx constants. + const ( + CB2 = iota // before CB1 + CB1 // before CB0 + CB0 // at end + ) + + // CAx constants should appear after CBx constants. + const ( + CA2 = iota // before CA1 + CA1 // before CA0 + CA0 // at end + ) + + // C0 should be first. + const C0 = 0 + + // C1 should be second. + const C1 = 1 + + // C2 should be third. + const C2 = 2 + + // + const ( + // Single const declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Cungrouped = 0 + ) + + +VARIABLES + // VBx variables should appear before VAx variables. + var ( + VB2 int // before VB1 + VB1 int // before VB0 + VB0 int // at end + ) + + // VAx variables should appear after VBx variables. + var ( + VA2 int // before VA1 + VA1 int // before VA0 + VA0 int // at end + ) + + // V0 should be first. + var V0 uintptr + + // V1 should be second. + var V1 uint + + // V2 should be third. + var V2 int + + // + var ( + // Single var declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Vungrouped = 0 + ) + + +FUNCTIONS + // F0 should be first. + func F0() + + // F1 should be second. + func F1() + + // F2 should be third. + func F2() + + +TYPES + // T0 should be first. + type T0 struct{} + + // T1 should be second. + type T1 struct{} + + // T2 should be third. + type T2 struct{} + + // TG0 should be first. + type TG0 struct{} + + // TG1 should be second. + type TG1 struct{} + + // TG2 should be third. + type TG2 struct{} + diff --git a/src/go/doc/testdata/d1.go b/src/go/doc/testdata/d1.go new file mode 100644 index 0000000..ebd6941 --- /dev/null +++ b/src/go/doc/testdata/d1.go @@ -0,0 +1,57 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test cases for sort order of declarations. + +package d + +// C2 should be third. +const C2 = 2 + +// V2 should be third. +var V2 int + +// CBx constants should appear before CAx constants. +const ( + CB2 = iota // before CB1 + CB1 // before CB0 + CB0 // at end +) + +// VBx variables should appear before VAx variables. +var ( + VB2 int // before VB1 + VB1 int // before VB0 + VB0 int // at end +) + +const ( + // Single const declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Cungrouped = 0 +) + +var ( + // Single var declarations inside ()'s are considered ungrouped + // and show up in sorted order. + Vungrouped = 0 +) + +// T2 should be third. +type T2 struct{} + +// Grouped types are sorted nevertheless. +type ( + // TG2 should be third. + TG2 struct{} + + // TG1 should be second. + TG1 struct{} + + // TG0 should be first. + TG0 struct{} +) + +// F2 should be third. +func F2() {} diff --git a/src/go/doc/testdata/d2.go b/src/go/doc/testdata/d2.go new file mode 100644 index 0000000..2f56f4f --- /dev/null +++ b/src/go/doc/testdata/d2.go @@ -0,0 +1,45 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test cases for sort order of declarations. + +package d + +// C1 should be second. +const C1 = 1 + +// C0 should be first. +const C0 = 0 + +// V1 should be second. +var V1 uint + +// V0 should be first. +var V0 uintptr + +// CAx constants should appear after CBx constants. +const ( + CA2 = iota // before CA1 + CA1 // before CA0 + CA0 // at end +) + +// VAx variables should appear after VBx variables. +var ( + VA2 int // before VA1 + VA1 int // before VA0 + VA0 int // at end +) + +// T1 should be second. +type T1 struct{} + +// T0 should be first. +type T0 struct{} + +// F1 should be second. +func F1() {} + +// F0 should be first. +func F0() {} diff --git a/src/go/doc/testdata/e.0.golden b/src/go/doc/testdata/e.0.golden new file mode 100644 index 0000000..6987e58 --- /dev/null +++ b/src/go/doc/testdata/e.0.golden @@ -0,0 +1,109 @@ +// The package e is a go/doc test for embedded methods. +PACKAGE e + +IMPORTPATH + testdata/e + +FILENAMES + testdata/e.go + +TYPES + // T1 has no embedded (level 1) M method due to conflict. + type T1 struct { + // contains filtered or unexported fields + } + + // T2 has only M as top-level method. + type T2 struct { + // contains filtered or unexported fields + } + + // T2.M should appear as method of T2. + func (T2) M() + + // T3 has only M as top-level method. + type T3 struct { + // contains filtered or unexported fields + } + + // T3.M should appear as method of T3. + func (T3) M() + + // + type T4 struct{} + + // T4.M should appear as method of T5 only if AllMethods is set. + func (*T4) M() + + // + type T5 struct { + T4 + } + + // + type U1 struct { + *U1 + } + + // U1.M should appear as method of U1. + func (*U1) M() + + // + type U2 struct { + *U3 + } + + // U2.M should appear as method of U2 and as method of U3 only if ... + func (*U2) M() + + // + type U3 struct { + *U2 + } + + // U3.N should appear as method of U3 and as method of U2 only if ... + func (*U3) N() + + // + type U4 struct { + // contains filtered or unexported fields + } + + // U4.M should appear as method of U4. + func (*U4) M() + + // + type V1 struct { + *V2 + *V5 + } + + // + type V2 struct { + *V3 + } + + // + type V3 struct { + *V4 + } + + // + type V4 struct { + *V5 + } + + // V4.M should appear as method of V2 and V3 if AllMethods is set. + func (*V4) M() + + // + type V5 struct { + *V6 + } + + // + type V6 struct{} + + // V6.M should appear as method of V1 and V5 if AllMethods is set. + func (*V6) M() + diff --git a/src/go/doc/testdata/e.1.golden b/src/go/doc/testdata/e.1.golden new file mode 100644 index 0000000..cbe22e0 --- /dev/null +++ b/src/go/doc/testdata/e.1.golden @@ -0,0 +1,144 @@ +// The package e is a go/doc test for embedded methods. +PACKAGE e + +IMPORTPATH + testdata/e + +FILENAMES + testdata/e.go + +TYPES + // T1 has no embedded (level 1) M method due to conflict. + type T1 struct { + t1 + t2 + } + + // T2 has only M as top-level method. + type T2 struct { + t1 + } + + // T2.M should appear as method of T2. + func (T2) M() + + // T3 has only M as top-level method. + type T3 struct { + t1e + t2e + } + + // T3.M should appear as method of T3. + func (T3) M() + + // + type T4 struct{} + + // T4.M should appear as method of T5 only if AllMethods is set. + func (*T4) M() + + // + type T5 struct { + T4 + } + + // + type U1 struct { + *U1 + } + + // U1.M should appear as method of U1. + func (*U1) M() + + // + type U2 struct { + *U3 + } + + // U2.M should appear as method of U2 and as method of U3 only if ... + func (*U2) M() + + // + type U3 struct { + *U2 + } + + // U3.N should appear as method of U3 and as method of U2 only if ... + func (*U3) N() + + // + type U4 struct { + *u5 + } + + // U4.M should appear as method of U4. + func (*U4) M() + + // + type V1 struct { + *V2 + *V5 + } + + // + type V2 struct { + *V3 + } + + // + type V3 struct { + *V4 + } + + // + type V4 struct { + *V5 + } + + // V4.M should appear as method of V2 and V3 if AllMethods is set. + func (*V4) M() + + // + type V5 struct { + *V6 + } + + // + type V6 struct{} + + // V6.M should appear as method of V1 and V5 if AllMethods is set. + func (*V6) M() + + // + type t1 struct{} + + // t1.M should not appear as method in a Tx type. + func (t1) M() + + // + type t1e struct { + t1 + } + + // t1.M should not appear as method in a Tx type. + func (t1e) M() + + // + type t2 struct{} + + // t2.M should not appear as method in a Tx type. + func (t2) M() + + // + type t2e struct { + t2 + } + + // t2.M should not appear as method in a Tx type. + func (t2e) M() + + // + type u5 struct { + *U4 + } + diff --git a/src/go/doc/testdata/e.2.golden b/src/go/doc/testdata/e.2.golden new file mode 100644 index 0000000..e7b05e8 --- /dev/null +++ b/src/go/doc/testdata/e.2.golden @@ -0,0 +1,130 @@ +// The package e is a go/doc test for embedded methods. +PACKAGE e + +IMPORTPATH + testdata/e + +FILENAMES + testdata/e.go + +TYPES + // T1 has no embedded (level 1) M method due to conflict. + type T1 struct { + // contains filtered or unexported fields + } + + // T2 has only M as top-level method. + type T2 struct { + // contains filtered or unexported fields + } + + // T2.M should appear as method of T2. + func (T2) M() + + // T3 has only M as top-level method. + type T3 struct { + // contains filtered or unexported fields + } + + // T3.M should appear as method of T3. + func (T3) M() + + // + type T4 struct{} + + // T4.M should appear as method of T5 only if AllMethods is set. + func (*T4) M() + + // + type T5 struct { + T4 + } + + // T4.M should appear as method of T5 only if AllMethods is set. + func (*T5) M() + + // + type U1 struct { + *U1 + } + + // U1.M should appear as method of U1. + func (*U1) M() + + // + type U2 struct { + *U3 + } + + // U2.M should appear as method of U2 and as method of U3 only if ... + func (*U2) M() + + // U3.N should appear as method of U3 and as method of U2 only if ... + func (U2) N() + + // + type U3 struct { + *U2 + } + + // U2.M should appear as method of U2 and as method of U3 only if ... + func (U3) M() + + // U3.N should appear as method of U3 and as method of U2 only if ... + func (*U3) N() + + // + type U4 struct { + // contains filtered or unexported fields + } + + // U4.M should appear as method of U4. + func (*U4) M() + + // + type V1 struct { + *V2 + *V5 + } + + // V6.M should appear as method of V1 and V5 if AllMethods is set. + func (V1) M() + + // + type V2 struct { + *V3 + } + + // V4.M should appear as method of V2 and V3 if AllMethods is set. + func (V2) M() + + // + type V3 struct { + *V4 + } + + // V4.M should appear as method of V2 and V3 if AllMethods is set. + func (V3) M() + + // + type V4 struct { + *V5 + } + + // V4.M should appear as method of V2 and V3 if AllMethods is set. + func (*V4) M() + + // + type V5 struct { + *V6 + } + + // V6.M should appear as method of V1 and V5 if AllMethods is set. + func (V5) M() + + // + type V6 struct{} + + // V6.M should appear as method of V1 and V5 if AllMethods is set. + func (*V6) M() + diff --git a/src/go/doc/testdata/e.go b/src/go/doc/testdata/e.go new file mode 100644 index 0000000..ec432e3 --- /dev/null +++ b/src/go/doc/testdata/e.go @@ -0,0 +1,147 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The package e is a go/doc test for embedded methods. +package e + +// ---------------------------------------------------------------------------- +// Conflicting methods M must not show up. + +type t1 struct{} + +// t1.M should not appear as method in a Tx type. +func (t1) M() {} + +type t2 struct{} + +// t2.M should not appear as method in a Tx type. +func (t2) M() {} + +// T1 has no embedded (level 1) M method due to conflict. +type T1 struct { + t1 + t2 +} + +// ---------------------------------------------------------------------------- +// Higher-level method M wins over lower-level method M. + +// T2 has only M as top-level method. +type T2 struct { + t1 +} + +// T2.M should appear as method of T2. +func (T2) M() {} + +// ---------------------------------------------------------------------------- +// Higher-level method M wins over lower-level conflicting methods M. + +type t1e struct { + t1 +} + +type t2e struct { + t2 +} + +// T3 has only M as top-level method. +type T3 struct { + t1e + t2e +} + +// T3.M should appear as method of T3. +func (T3) M() {} + +// ---------------------------------------------------------------------------- +// Don't show conflicting methods M embedded via an exported and non-exported +// type. + +// T1 has no embedded (level 1) M method due to conflict. +type T4 struct { + t2 + T2 +} + +// ---------------------------------------------------------------------------- +// Don't show embedded methods of exported anonymous fields unless AllMethods +// is set. + +type T4 struct{} + +// T4.M should appear as method of T5 only if AllMethods is set. +func (*T4) M() {} + +type T5 struct { + T4 +} + +// ---------------------------------------------------------------------------- +// Recursive type declarations must not lead to endless recursion. + +type U1 struct { + *U1 +} + +// U1.M should appear as method of U1. +func (*U1) M() {} + +type U2 struct { + *U3 +} + +// U2.M should appear as method of U2 and as method of U3 only if AllMethods is set. +func (*U2) M() {} + +type U3 struct { + *U2 +} + +// U3.N should appear as method of U3 and as method of U2 only if AllMethods is set. +func (*U3) N() {} + +type U4 struct { + *u5 +} + +// U4.M should appear as method of U4. +func (*U4) M() {} + +type u5 struct { + *U4 +} + +// ---------------------------------------------------------------------------- +// A higher-level embedded type (and its methods) wins over the same type (and +// its methods) embedded at a lower level. + +type V1 struct { + *V2 + *V5 +} + +type V2 struct { + *V3 +} + +type V3 struct { + *V4 +} + +type V4 struct { + *V5 +} + +type V5 struct { + *V6 +} + +type V6 struct{} + +// V4.M should appear as method of V2 and V3 if AllMethods is set. +func (*V4) M() {} + +// V6.M should appear as method of V1 and V5 if AllMethods is set. +func (*V6) M() {} diff --git a/src/go/doc/testdata/error1.0.golden b/src/go/doc/testdata/error1.0.golden new file mode 100644 index 0000000..6c6fe5d --- /dev/null +++ b/src/go/doc/testdata/error1.0.golden @@ -0,0 +1,30 @@ +// +PACKAGE error1 + +IMPORTPATH + testdata/error1 + +FILENAMES + testdata/error1.go + +TYPES + // + type I0 interface { + // When embedded, the predeclared error interface + // must remain visible in interface types. + error + } + + // + type S0 struct { + // contains filtered or unexported fields + } + + // + type T0 struct { + ExportedField interface { + // error should be visible + error + } + } + diff --git a/src/go/doc/testdata/error1.1.golden b/src/go/doc/testdata/error1.1.golden new file mode 100644 index 0000000..a8dc2e7 --- /dev/null +++ b/src/go/doc/testdata/error1.1.golden @@ -0,0 +1,32 @@ +// +PACKAGE error1 + +IMPORTPATH + testdata/error1 + +FILENAMES + testdata/error1.go + +TYPES + // + type I0 interface { + // When embedded, the predeclared error interface + // must remain visible in interface types. + error + } + + // + type S0 struct { + // In struct types, an embedded error must only be visible + // if AllDecls is set. + error + } + + // + type T0 struct { + ExportedField interface { + // error should be visible + error + } + } + diff --git a/src/go/doc/testdata/error1.2.golden b/src/go/doc/testdata/error1.2.golden new file mode 100644 index 0000000..6c6fe5d --- /dev/null +++ b/src/go/doc/testdata/error1.2.golden @@ -0,0 +1,30 @@ +// +PACKAGE error1 + +IMPORTPATH + testdata/error1 + +FILENAMES + testdata/error1.go + +TYPES + // + type I0 interface { + // When embedded, the predeclared error interface + // must remain visible in interface types. + error + } + + // + type S0 struct { + // contains filtered or unexported fields + } + + // + type T0 struct { + ExportedField interface { + // error should be visible + error + } + } + diff --git a/src/go/doc/testdata/error1.go b/src/go/doc/testdata/error1.go new file mode 100644 index 0000000..3c777a7 --- /dev/null +++ b/src/go/doc/testdata/error1.go @@ -0,0 +1,24 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package error1 + +type I0 interface { + // When embedded, the predeclared error interface + // must remain visible in interface types. + error +} + +type T0 struct { + ExportedField interface { + // error should be visible + error + } +} + +type S0 struct { + // In struct types, an embedded error must only be visible + // if AllDecls is set. + error +} diff --git a/src/go/doc/testdata/error2.0.golden b/src/go/doc/testdata/error2.0.golden new file mode 100644 index 0000000..dedfe41 --- /dev/null +++ b/src/go/doc/testdata/error2.0.golden @@ -0,0 +1,27 @@ +// +PACKAGE error2 + +IMPORTPATH + testdata/error2 + +FILENAMES + testdata/error2.go + +TYPES + // + type I0 interface { + // contains filtered or unexported methods + } + + // + type S0 struct { + // contains filtered or unexported fields + } + + // + type T0 struct { + ExportedField interface { + // contains filtered or unexported methods + } + } + diff --git a/src/go/doc/testdata/error2.1.golden b/src/go/doc/testdata/error2.1.golden new file mode 100644 index 0000000..dbcc1b0 --- /dev/null +++ b/src/go/doc/testdata/error2.1.golden @@ -0,0 +1,37 @@ +// +PACKAGE error2 + +IMPORTPATH + testdata/error2 + +FILENAMES + testdata/error2.go + +TYPES + // + type I0 interface { + // When embedded, the locally-declared error interface + // is only visible if all declarations are shown. + error + } + + // + type S0 struct { + // In struct types, an embedded error must only be visible + // if AllDecls is set. + error + } + + // + type T0 struct { + ExportedField interface { + // error should not be visible + error + } + } + + // This error declaration shadows the predeclared error type. + type error interface { + Error() string + } + diff --git a/src/go/doc/testdata/error2.2.golden b/src/go/doc/testdata/error2.2.golden new file mode 100644 index 0000000..dedfe41 --- /dev/null +++ b/src/go/doc/testdata/error2.2.golden @@ -0,0 +1,27 @@ +// +PACKAGE error2 + +IMPORTPATH + testdata/error2 + +FILENAMES + testdata/error2.go + +TYPES + // + type I0 interface { + // contains filtered or unexported methods + } + + // + type S0 struct { + // contains filtered or unexported fields + } + + // + type T0 struct { + ExportedField interface { + // contains filtered or unexported methods + } + } + diff --git a/src/go/doc/testdata/error2.go b/src/go/doc/testdata/error2.go new file mode 100644 index 0000000..6ee96c2 --- /dev/null +++ b/src/go/doc/testdata/error2.go @@ -0,0 +1,29 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package error2 + +type I0 interface { + // When embedded, the locally-declared error interface + // is only visible if all declarations are shown. + error +} + +type T0 struct { + ExportedField interface { + // error should not be visible + error + } +} + +type S0 struct { + // In struct types, an embedded error must only be visible + // if AllDecls is set. + error +} + +// This error declaration shadows the predeclared error type. +type error interface { + Error() string +} diff --git a/src/go/doc/testdata/example.go b/src/go/doc/testdata/example.go new file mode 100644 index 0000000..0b70339 --- /dev/null +++ b/src/go/doc/testdata/example.go @@ -0,0 +1,81 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package testing + +import ( + "bytes" + "fmt" + "io" + "os" + "strings" + "time" +) + +type InternalExample struct { + Name string + F func() + Output string +} + +func RunExamples(examples []InternalExample) (ok bool) { + ok = true + + var eg InternalExample + + stdout, stderr := os.Stdout, os.Stderr + defer func() { + os.Stdout, os.Stderr = stdout, stderr + if e := recover(); e != nil { + fmt.Printf("--- FAIL: %s\npanic: %v\n", eg.Name, e) + os.Exit(1) + } + }() + + for _, eg = range examples { + if *chatty { + fmt.Printf("=== RUN: %s\n", eg.Name) + } + + // capture stdout and stderr + r, w, err := os.Pipe() + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Stdout, os.Stderr = w, w + outC := make(chan string) + go func() { + buf := new(bytes.Buffer) + _, err := io.Copy(buf, r) + if err != nil { + fmt.Fprintf(stderr, "testing: copying pipe: %v\n", err) + os.Exit(1) + } + outC <- buf.String() + }() + + // run example + t0 := time.Now() + eg.F() + dt := time.Since(t0) + + // close pipe, restore stdout/stderr, get output + w.Close() + os.Stdout, os.Stderr = stdout, stderr + out := <-outC + + // report any errors + tstr := fmt.Sprintf("(%.2f seconds)", dt.Seconds()) + if g, e := strings.TrimSpace(out), strings.TrimSpace(eg.Output); g != e { + fmt.Printf("--- FAIL: %s %s\ngot:\n%s\nwant:\n%s\n", + eg.Name, tstr, g, e) + ok = false + } else if *chatty { + fmt.Printf("--- PASS: %s %s\n", eg.Name, tstr) + } + } + + return +} diff --git a/src/go/doc/testdata/examples/README.md b/src/go/doc/testdata/examples/README.md new file mode 100644 index 0000000..a1c18e8 --- /dev/null +++ b/src/go/doc/testdata/examples/README.md @@ -0,0 +1,12 @@ +These files are processed by example_test.go:TestExamples. + +A .golden file is a txtar file with two sections for each example that should be +created by doc.Examples from the corresponding .go file. + +One section, named EXAMPLE_NAME.Output, contains the example's output, +the value of the field Example.Output. + +The other, named EXAMPLE_NAME.Play, contains the formatted code for a playable +version of the example, the value of the field Example.Play. + +If a section is missing, it is treated as being empty. diff --git a/src/go/doc/testdata/examples/empty.go b/src/go/doc/testdata/examples/empty.go new file mode 100644 index 0000000..0b10420 --- /dev/null +++ b/src/go/doc/testdata/examples/empty.go @@ -0,0 +1,8 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func Example() {} +func Example_a() diff --git a/src/go/doc/testdata/examples/empty.golden b/src/go/doc/testdata/examples/empty.golden new file mode 100644 index 0000000..2aafd20 --- /dev/null +++ b/src/go/doc/testdata/examples/empty.golden @@ -0,0 +1,6 @@ +-- .Play -- +package main + +func main() {} +func main() + diff --git a/src/go/doc/testdata/examples/generic_constraints.go b/src/go/doc/testdata/examples/generic_constraints.go new file mode 100644 index 0000000..ea5d2b3 --- /dev/null +++ b/src/go/doc/testdata/examples/generic_constraints.go @@ -0,0 +1,38 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p_test + +import ( + "fmt" + "time" +) + +type C1 interface { + string | int +} + +type C2 interface { + M(time.Time) +} + +type G[T C1] int + +func g[T C2](x T) {} + +type Tm int + +func (Tm) M(time.Time) {} + +type Foo int + +func Example() { + fmt.Println("hello") +} + +func ExampleGeneric() { + var x G[string] + g(Tm(3)) + fmt.Println(x) +} diff --git a/src/go/doc/testdata/examples/generic_constraints.golden b/src/go/doc/testdata/examples/generic_constraints.golden new file mode 100644 index 0000000..6c7b0ed --- /dev/null +++ b/src/go/doc/testdata/examples/generic_constraints.golden @@ -0,0 +1,39 @@ +-- .Play -- +package main + +import ( + "fmt" +) + +func main() { + fmt.Println("hello") +} +-- Generic.Play -- +package main + +import ( + "fmt" + "time" +) + +type C1 interface { + string | int +} + +type C2 interface { + M(time.Time) +} + +type G[T C1] int + +func g[T C2](x T) {} + +type Tm int + +func (Tm) M(time.Time) {} + +func main() { + var x G[string] + g(Tm(3)) + fmt.Println(x) +} diff --git a/src/go/doc/testdata/examples/import_groups.go b/src/go/doc/testdata/examples/import_groups.go new file mode 100644 index 0000000..05f21ca --- /dev/null +++ b/src/go/doc/testdata/examples/import_groups.go @@ -0,0 +1,23 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo_test + +import ( + "fmt" + "time" + + "golang.org/x/time/rate" +) + +func Example() { + fmt.Println("Hello, world!") + // Output: Hello, world! +} + +func ExampleLimiter() { + // Uses fmt, time and rate. + l := rate.NewLimiter(rate.Every(time.Second), 1) + fmt.Println(l) +} diff --git a/src/go/doc/testdata/examples/import_groups.golden b/src/go/doc/testdata/examples/import_groups.golden new file mode 100644 index 0000000..efe2cc1 --- /dev/null +++ b/src/go/doc/testdata/examples/import_groups.golden @@ -0,0 +1,27 @@ +-- .Play -- +package main + +import ( + "fmt" +) + +func main() { + fmt.Println("Hello, world!") +} +-- .Output -- +Hello, world! +-- Limiter.Play -- +package main + +import ( + "fmt" + "time" + + "golang.org/x/time/rate" +) + +func main() { + // Uses fmt, time and rate. + l := rate.NewLimiter(rate.Every(time.Second), 1) + fmt.Println(l) +} diff --git a/src/go/doc/testdata/examples/import_groups_named.go b/src/go/doc/testdata/examples/import_groups_named.go new file mode 100644 index 0000000..377022b --- /dev/null +++ b/src/go/doc/testdata/examples/import_groups_named.go @@ -0,0 +1,23 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo_test + +import ( + "fmt" + tm "time" + + r "golang.org/x/time/rate" +) + +func Example() { + fmt.Println("Hello, world!") + // Output: Hello, world! +} + +func ExampleLimiter() { + // Uses fmt, time and rate. + l := r.NewLimiter(r.Every(tm.Second), 1) + fmt.Println(l) +} diff --git a/src/go/doc/testdata/examples/import_groups_named.golden b/src/go/doc/testdata/examples/import_groups_named.golden new file mode 100644 index 0000000..9baf373 --- /dev/null +++ b/src/go/doc/testdata/examples/import_groups_named.golden @@ -0,0 +1,27 @@ +-- .Play -- +package main + +import ( + "fmt" +) + +func main() { + fmt.Println("Hello, world!") +} +-- .Output -- +Hello, world! +-- Limiter.Play -- +package main + +import ( + "fmt" + tm "time" + + r "golang.org/x/time/rate" +) + +func main() { + // Uses fmt, time and rate. + l := r.NewLimiter(r.Every(tm.Second), 1) + fmt.Println(l) +} diff --git a/src/go/doc/testdata/examples/inspect_signature.go b/src/go/doc/testdata/examples/inspect_signature.go new file mode 100644 index 0000000..c4a36e7 --- /dev/null +++ b/src/go/doc/testdata/examples/inspect_signature.go @@ -0,0 +1,23 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo_test + +import ( + "bytes" + "io" +) + +func getReader() io.Reader { return nil } + +func do(b bytes.Reader) {} + +func Example() { + getReader() + do() + // Output: +} + +func ExampleIgnored() { +} diff --git a/src/go/doc/testdata/examples/inspect_signature.golden b/src/go/doc/testdata/examples/inspect_signature.golden new file mode 100644 index 0000000..c0d9b2e --- /dev/null +++ b/src/go/doc/testdata/examples/inspect_signature.golden @@ -0,0 +1,24 @@ +-- .Play -- +package main + +import ( + "bytes" + "io" +) + +func getReader() io.Reader { return nil } + +func do(b bytes.Reader) {} + +func main() { + getReader() + do() +} +-- Ignored.Play -- +package main + +import () + +func main() { +} + diff --git a/src/go/doc/testdata/examples/iota.go b/src/go/doc/testdata/examples/iota.go new file mode 100644 index 0000000..c878b77 --- /dev/null +++ b/src/go/doc/testdata/examples/iota.go @@ -0,0 +1,34 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo_test + +const ( + a = iota + b +) + +const ( + c = 3 + d = 4 +) + +const ( + e = iota + f +) + +// The example refers to only one of the constants in the iota group, but we +// must keep all of them because of the iota. The second group of constants can +// be trimmed. The third has an iota, but is unused, so it can be eliminated. + +func Example() { + _ = b + _ = d +} + +// Need two examples to hit the playExample function. + +func Example2() { +} diff --git a/src/go/doc/testdata/examples/iota.golden b/src/go/doc/testdata/examples/iota.golden new file mode 100644 index 0000000..7487702 --- /dev/null +++ b/src/go/doc/testdata/examples/iota.golden @@ -0,0 +1,23 @@ +-- .Play -- +package main + +import () + +const ( + a = iota + b +) + +const d = 4 + +func main() { + _ = b + _ = d +} +-- 2.Play -- +package main + +import () + +func main() { +} diff --git a/src/go/doc/testdata/examples/issue43658.go b/src/go/doc/testdata/examples/issue43658.go new file mode 100644 index 0000000..385223a --- /dev/null +++ b/src/go/doc/testdata/examples/issue43658.go @@ -0,0 +1,223 @@ +// Copyright Ā©2016 The Gonum Authors. All rights reserved. +// Copyright 2021 The Go Authors. All rights reserved. +// (above line required for our license-header checker) +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package community_test + +import ( + "fmt" + "log" + "sort" + + "golang.org/x/exp/rand" + + "gonum.org/v1/gonum/graph/community" + "gonum.org/v1/gonum/graph/internal/ordered" + "gonum.org/v1/gonum/graph/simple" +) + +func ExampleProfile_simple() { + // Profile calls Modularize which implements the Louvain modularization algorithm. + // Since this is a randomized algorithm we use a defined random source to ensure + // consistency between test runs. In practice, results will not differ greatly + // between runs with different PRNG seeds. + src := rand.NewSource(1) + + // Create dumbell graph: + // + // 0 4 + // |\ /| + // | 2 - 3 | + // |/ \| + // 1 5 + // + g := simple.NewUndirectedGraph() + for u, e := range smallDumbell { + for v := range e { + g.SetEdge(simple.Edge{F: simple.Node(u), T: simple.Node(v)}) + } + } + + // Get the profile of internal node weight for resolutions + // between 0.1 and 10 using logarithmic bisection. + p, err := community.Profile( + community.ModularScore(g, community.Weight, 10, src), + true, 1e-3, 0.1, 10, + ) + if err != nil { + log.Fatal(err) + } + + // Print out each step with communities ordered. + for _, d := range p { + comm := d.Communities() + for _, c := range comm { + sort.Sort(ordered.ByID(c)) + } + sort.Sort(ordered.BySliceIDs(comm)) + fmt.Printf("Low:%.2v High:%.2v Score:%v Communities:%v Q=%.3v\n", + d.Low, d.High, d.Score, comm, community.Q(g, comm, d.Low)) + } + + // Output: + // Low:0.1 High:0.29 Score:14 Communities:[[0 1 2 3 4 5]] Q=0.9 + // Low:0.29 High:2.3 Score:12 Communities:[[0 1 2] [3 4 5]] Q=0.714 + // Low:2.3 High:3.5 Score:4 Communities:[[0 1] [2] [3] [4 5]] Q=-0.31 + // Low:3.5 High:10 Score:0 Communities:[[0] [1] [2] [3] [4] [5]] Q=-0.607 +} + +// intset is an integer set. +type intset map[int]struct{} + +func linksTo(i ...int) intset { + if len(i) == 0 { + return nil + } + s := make(intset) + for _, v := range i { + s[v] = struct{}{} + } + return s +} + +var ( + smallDumbell = []intset{ + 0: linksTo(1, 2), + 1: linksTo(2), + 2: linksTo(3), + 3: linksTo(4, 5), + 4: linksTo(5), + 5: nil, + } + + // http://www.slate.com/blogs/the_world_/2014/07/17/the_middle_east_friendship_chart.html + middleEast = struct{ friends, complicated, enemies []intset }{ + // green cells + friends: []intset{ + 0: nil, + 1: linksTo(5, 7, 9, 12), + 2: linksTo(11), + 3: linksTo(4, 5, 10), + 4: linksTo(3, 5, 10), + 5: linksTo(1, 3, 4, 8, 10, 12), + 6: nil, + 7: linksTo(1, 12), + 8: linksTo(5, 9, 11), + 9: linksTo(1, 8, 12), + 10: linksTo(3, 4, 5), + 11: linksTo(2, 8), + 12: linksTo(1, 5, 7, 9), + }, + + // yellow cells + complicated: []intset{ + 0: linksTo(2, 4), + 1: linksTo(4, 8), + 2: linksTo(0, 3, 4, 5, 8, 9), + 3: linksTo(2, 8, 11), + 4: linksTo(0, 1, 2, 8), + 5: linksTo(2), + 6: nil, + 7: linksTo(9, 11), + 8: linksTo(1, 2, 3, 4, 10, 12), + 9: linksTo(2, 7, 11), + 10: linksTo(8), + 11: linksTo(3, 7, 9, 12), + 12: linksTo(8, 11), + }, + + // red cells + enemies: []intset{ + 0: linksTo(1, 3, 5, 6, 7, 8, 9, 10, 11, 12), + 1: linksTo(0, 2, 3, 6, 10, 11), + 2: linksTo(1, 6, 7, 10, 12), + 3: linksTo(0, 1, 6, 7, 9, 12), + 4: linksTo(6, 7, 9, 11, 12), + 5: linksTo(0, 6, 7, 9, 11), + 6: linksTo(0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12), + 7: linksTo(0, 2, 3, 4, 5, 6, 8, 10), + 8: linksTo(0, 6, 7), + 9: linksTo(0, 3, 4, 5, 6, 10), + 10: linksTo(0, 1, 2, 6, 7, 9, 11, 12), + 11: linksTo(0, 1, 4, 5, 6, 10), + 12: linksTo(0, 2, 3, 4, 6, 10), + }, + } +) + +var friends, enemies *simple.WeightedUndirectedGraph + +func init() { + friends = simple.NewWeightedUndirectedGraph(0, 0) + for u, e := range middleEast.friends { + // Ensure unconnected nodes are included. + if friends.Node(int64(u)) == nil { + friends.AddNode(simple.Node(u)) + } + for v := range e { + friends.SetWeightedEdge(simple.WeightedEdge{F: simple.Node(u), T: simple.Node(v), W: 1}) + } + } + enemies = simple.NewWeightedUndirectedGraph(0, 0) + for u, e := range middleEast.enemies { + // Ensure unconnected nodes are included. + if enemies.Node(int64(u)) == nil { + enemies.AddNode(simple.Node(u)) + } + for v := range e { + enemies.SetWeightedEdge(simple.WeightedEdge{F: simple.Node(u), T: simple.Node(v), W: -1}) + } + } +} + +func ExampleProfile_multiplex() { + // Profile calls ModularizeMultiplex which implements the Louvain modularization + // algorithm. Since this is a randomized algorithm we use a defined random source + // to ensure consistency between test runs. In practice, results will not differ + // greatly between runs with different PRNG seeds. + src := rand.NewSource(1) + + // The undirected graphs, friends and enemies, are the political relationships + // in the Middle East as described in the Slate article: + // http://www.slate.com/blogs/the_world_/2014/07/17/the_middle_east_friendship_chart.html + g, err := community.NewUndirectedLayers(friends, enemies) + if err != nil { + log.Fatal(err) + } + weights := []float64{1, -1} + + // Get the profile of internal node weight for resolutions + // between 0.1 and 10 using logarithmic bisection. + p, err := community.Profile( + community.ModularMultiplexScore(g, weights, true, community.WeightMultiplex, 10, src), + true, 1e-3, 0.1, 10, + ) + if err != nil { + log.Fatal(err) + } + + // Print out each step with communities ordered. + for _, d := range p { + comm := d.Communities() + for _, c := range comm { + sort.Sort(ordered.ByID(c)) + } + sort.Sort(ordered.BySliceIDs(comm)) + fmt.Printf("Low:%.2v High:%.2v Score:%v Communities:%v Q=%.3v\n", + d.Low, d.High, d.Score, comm, community.QMultiplex(g, comm, weights, []float64{d.Low})) + } + + // Output: + // Low:0.1 High:0.72 Score:26 Communities:[[0] [1 7 9 12] [2 8 11] [3 4 5 10] [6]] Q=[24.7 1.97] + // Low:0.72 High:1.1 Score:24 Communities:[[0 6] [1 7 9 12] [2 8 11] [3 4 5 10]] Q=[16.9 14.1] + // Low:1.1 High:1.2 Score:18 Communities:[[0 2 6 11] [1 7 9 12] [3 4 5 8 10]] Q=[9.16 25.1] + // Low:1.2 High:1.6 Score:10 Communities:[[0 3 4 5 6 10] [1 7 9 12] [2 8 11]] Q=[10.5 26.7] + // Low:1.6 High:1.6 Score:8 Communities:[[0 1 6 7 9 12] [2 8 11] [3 4 5 10]] Q=[5.56 39.8] + // Low:1.6 High:1.8 Score:2 Communities:[[0 2 3 4 5 6 10] [1 7 8 9 11 12]] Q=[-1.82 48.6] + // Low:1.8 High:2.3 Score:-6 Communities:[[0 2 3 4 5 6 8 10 11] [1 7 9 12]] Q=[-5 57.5] + // Low:2.3 High:2.4 Score:-10 Communities:[[0 1 2 6 7 8 9 11 12] [3 4 5 10]] Q=[-11.2 79] + // Low:2.4 High:4.3 Score:-52 Communities:[[0 1 2 3 4 5 6 7 8 9 10 11 12]] Q=[-46.1 117] + // Low:4.3 High:10 Score:-54 Communities:[[0 1 2 3 4 6 7 8 9 10 11 12] [5]] Q=[-82 254] +} diff --git a/src/go/doc/testdata/examples/issue43658.golden b/src/go/doc/testdata/examples/issue43658.golden new file mode 100644 index 0000000..5200d14 --- /dev/null +++ b/src/go/doc/testdata/examples/issue43658.golden @@ -0,0 +1,156 @@ +-- Profile_simple.Play -- +package main + +import ( + "fmt" + "log" + "sort" + + "golang.org/x/exp/rand" + + "gonum.org/v1/gonum/graph/community" + "gonum.org/v1/gonum/graph/internal/ordered" + "gonum.org/v1/gonum/graph/simple" +) + +func main() { + // Profile calls Modularize which implements the Louvain modularization algorithm. + // Since this is a randomized algorithm we use a defined random source to ensure + // consistency between test runs. In practice, results will not differ greatly + // between runs with different PRNG seeds. + src := rand.NewSource(1) + + // Create dumbell graph: + // + // 0 4 + // |\ /| + // | 2 - 3 | + // |/ \| + // 1 5 + // + g := simple.NewUndirectedGraph() + for u, e := range smallDumbell { + for v := range e { + g.SetEdge(simple.Edge{F: simple.Node(u), T: simple.Node(v)}) + } + } + + // Get the profile of internal node weight for resolutions + // between 0.1 and 10 using logarithmic bisection. + p, err := community.Profile( + community.ModularScore(g, community.Weight, 10, src), + true, 1e-3, 0.1, 10, + ) + if err != nil { + log.Fatal(err) + } + + // Print out each step with communities ordered. + for _, d := range p { + comm := d.Communities() + for _, c := range comm { + sort.Sort(ordered.ByID(c)) + } + sort.Sort(ordered.BySliceIDs(comm)) + fmt.Printf("Low:%.2v High:%.2v Score:%v Communities:%v Q=%.3v\n", + d.Low, d.High, d.Score, comm, community.Q(g, comm, d.Low)) + } + +} + +// intset is an integer set. +type intset map[int]struct{} + +func linksTo(i ...int) intset { + if len(i) == 0 { + return nil + } + s := make(intset) + for _, v := range i { + s[v] = struct{}{} + } + return s +} + +var smallDumbell = []intset{ + 0: linksTo(1, 2), + 1: linksTo(2), + 2: linksTo(3), + 3: linksTo(4, 5), + 4: linksTo(5), + 5: nil, +} + +-- Profile_simple.Output -- +Low:0.1 High:0.29 Score:14 Communities:[[0 1 2 3 4 5]] Q=0.9 +Low:0.29 High:2.3 Score:12 Communities:[[0 1 2] [3 4 5]] Q=0.714 +Low:2.3 High:3.5 Score:4 Communities:[[0 1] [2] [3] [4 5]] Q=-0.31 +Low:3.5 High:10 Score:0 Communities:[[0] [1] [2] [3] [4] [5]] Q=-0.607 + +-- Profile_multiplex.Play -- + +package main + +import ( + "fmt" + "log" + "sort" + + "golang.org/x/exp/rand" + + "gonum.org/v1/gonum/graph/community" + "gonum.org/v1/gonum/graph/internal/ordered" + "gonum.org/v1/gonum/graph/simple" +) + +var friends, enemies *simple.WeightedUndirectedGraph + +func main() { + // Profile calls ModularizeMultiplex which implements the Louvain modularization + // algorithm. Since this is a randomized algorithm we use a defined random source + // to ensure consistency between test runs. In practice, results will not differ + // greatly between runs with different PRNG seeds. + src := rand.NewSource(1) + + // The undirected graphs, friends and enemies, are the political relationships + // in the Middle East as described in the Slate article: + // http://www.slate.com/blogs/the_world_/2014/07/17/the_middle_east_friendship_chart.html + g, err := community.NewUndirectedLayers(friends, enemies) + if err != nil { + log.Fatal(err) + } + weights := []float64{1, -1} + + // Get the profile of internal node weight for resolutions + // between 0.1 and 10 using logarithmic bisection. + p, err := community.Profile( + community.ModularMultiplexScore(g, weights, true, community.WeightMultiplex, 10, src), + true, 1e-3, 0.1, 10, + ) + if err != nil { + log.Fatal(err) + } + + // Print out each step with communities ordered. + for _, d := range p { + comm := d.Communities() + for _, c := range comm { + sort.Sort(ordered.ByID(c)) + } + sort.Sort(ordered.BySliceIDs(comm)) + fmt.Printf("Low:%.2v High:%.2v Score:%v Communities:%v Q=%.3v\n", + d.Low, d.High, d.Score, comm, community.QMultiplex(g, comm, weights, []float64{d.Low})) + } + +} +-- Profile_multiplex.Output -- +Low:0.1 High:0.72 Score:26 Communities:[[0] [1 7 9 12] [2 8 11] [3 4 5 10] [6]] Q=[24.7 1.97] +Low:0.72 High:1.1 Score:24 Communities:[[0 6] [1 7 9 12] [2 8 11] [3 4 5 10]] Q=[16.9 14.1] +Low:1.1 High:1.2 Score:18 Communities:[[0 2 6 11] [1 7 9 12] [3 4 5 8 10]] Q=[9.16 25.1] +Low:1.2 High:1.6 Score:10 Communities:[[0 3 4 5 6 10] [1 7 9 12] [2 8 11]] Q=[10.5 26.7] +Low:1.6 High:1.6 Score:8 Communities:[[0 1 6 7 9 12] [2 8 11] [3 4 5 10]] Q=[5.56 39.8] +Low:1.6 High:1.8 Score:2 Communities:[[0 2 3 4 5 6 10] [1 7 8 9 11 12]] Q=[-1.82 48.6] +Low:1.8 High:2.3 Score:-6 Communities:[[0 2 3 4 5 6 8 10 11] [1 7 9 12]] Q=[-5 57.5] +Low:2.3 High:2.4 Score:-10 Communities:[[0 1 2 6 7 8 9 11 12] [3 4 5 10]] Q=[-11.2 79] +Low:2.4 High:4.3 Score:-52 Communities:[[0 1 2 3 4 5 6 7 8 9 10 11 12]] Q=[-46.1 117] +Low:4.3 High:10 Score:-54 Communities:[[0 1 2 3 4 6 7 8 9 10 11 12] [5]] Q=[-82 254] diff --git a/src/go/doc/testdata/examples/multiple.go b/src/go/doc/testdata/examples/multiple.go new file mode 100644 index 0000000..2728264 --- /dev/null +++ b/src/go/doc/testdata/examples/multiple.go @@ -0,0 +1,98 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo_test + +import ( + "flag" + "fmt" + "log" + "os/exec" + "sort" +) + +func ExampleHello() { + fmt.Println("Hello, world!") + // Output: Hello, world! +} + +func ExampleImport() { + out, err := exec.Command("date").Output() + if err != nil { + log.Fatal(err) + } + fmt.Printf("The date is %s\n", out) +} + +func ExampleKeyValue() { + v := struct { + a string + b int + }{ + a: "A", + b: 1, + } + fmt.Print(v) + // Output: a: "A", b: 1 +} + +func ExampleKeyValueImport() { + f := flag.Flag{ + Name: "play", + } + fmt.Print(f) + // Output: Name: "play" +} + +var keyValueTopDecl = struct { + a string + b int +}{ + a: "B", + b: 2, +} + +func ExampleKeyValueTopDecl() { + fmt.Print(keyValueTopDecl) + // Output: a: "B", b: 2 +} + +// Person represents a person by name and age. +type Person struct { + Name string + Age int +} + +// String returns a string representation of the Person. +func (p Person) String() string { + return fmt.Sprintf("%s: %d", p.Name, p.Age) +} + +// ByAge implements sort.Interface for []Person based on +// the Age field. +type ByAge []Person + +// Len returns the number of elements in ByAge. +func (a ByAge) Len() int { return len(a) } + +// Swap swaps the elements in ByAge. +func (a ByAge) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByAge) Less(i, j int) bool { return a[i].Age < a[j].Age } + +// people is the array of Person +var people = []Person{ + {"Bob", 31}, + {"John", 42}, + {"Michael", 17}, + {"Jenny", 26}, +} + +func ExampleSort() { + fmt.Println(people) + sort.Sort(ByAge(people)) + fmt.Println(people) + // Output: + // [Bob: 31 John: 42 Michael: 17 Jenny: 26] + // [Michael: 17 Jenny: 26 Bob: 31 John: 42] +} diff --git a/src/go/doc/testdata/examples/multiple.golden b/src/go/doc/testdata/examples/multiple.golden new file mode 100644 index 0000000..d2d791e --- /dev/null +++ b/src/go/doc/testdata/examples/multiple.golden @@ -0,0 +1,129 @@ +-- Hello.Play -- +package main + +import ( + "fmt" +) + +func main() { + fmt.Println("Hello, world!") +} +-- Hello.Output -- +Hello, world! +-- Import.Play -- +package main + +import ( + "fmt" + "log" + "os/exec" +) + +func main() { + out, err := exec.Command("date").Output() + if err != nil { + log.Fatal(err) + } + fmt.Printf("The date is %s\n", out) +} +-- KeyValue.Play -- +package main + +import ( + "fmt" +) + +func main() { + v := struct { + a string + b int + }{ + a: "A", + b: 1, + } + fmt.Print(v) +} +-- KeyValue.Output -- +a: "A", b: 1 +-- KeyValueImport.Play -- +package main + +import ( + "flag" + "fmt" +) + +func main() { + f := flag.Flag{ + Name: "play", + } + fmt.Print(f) +} +-- KeyValueImport.Output -- +Name: "play" +-- KeyValueTopDecl.Play -- +package main + +import ( + "fmt" +) + +var keyValueTopDecl = struct { + a string + b int +}{ + a: "B", + b: 2, +} + +func main() { + fmt.Print(keyValueTopDecl) +} +-- KeyValueTopDecl.Output -- +a: "B", b: 2 +-- Sort.Play -- +package main + +import ( + "fmt" + "sort" +) + +// Person represents a person by name and age. +type Person struct { + Name string + Age int +} + +// String returns a string representation of the Person. +func (p Person) String() string { + return fmt.Sprintf("%s: %d", p.Name, p.Age) +} + +// ByAge implements sort.Interface for []Person based on +// the Age field. +type ByAge []Person + +// Len returns the number of elements in ByAge. +func (a ByAge) Len() int { return len(a) } + +// Swap swaps the elements in ByAge. +func (a ByAge) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByAge) Less(i, j int) bool { return a[i].Age < a[j].Age } + +// people is the array of Person +var people = []Person{ + {"Bob", 31}, + {"John", 42}, + {"Michael", 17}, + {"Jenny", 26}, +} + +func main() { + fmt.Println(people) + sort.Sort(ByAge(people)) + fmt.Println(people) +} +-- Sort.Output -- +[Bob: 31 John: 42 Michael: 17 Jenny: 26] +[Michael: 17 Jenny: 26 Bob: 31 John: 42] diff --git a/src/go/doc/testdata/examples/values.go b/src/go/doc/testdata/examples/values.go new file mode 100644 index 0000000..64b0de4 --- /dev/null +++ b/src/go/doc/testdata/examples/values.go @@ -0,0 +1,22 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo_test + +// Variable declaration with fewer values than names. + +func f() (int, int) { + return 1, 2 +} + +var a, b = f() + +// Need two examples to hit playExample. + +func ExampleA() { + _ = a +} + +func ExampleB() { +} diff --git a/src/go/doc/testdata/examples/values.golden b/src/go/doc/testdata/examples/values.golden new file mode 100644 index 0000000..00c1991 --- /dev/null +++ b/src/go/doc/testdata/examples/values.golden @@ -0,0 +1,21 @@ +-- A.Play -- +package main + +import () + +func f() (int, int) { + return 1, 2 +} + +var a, b = f() + +func main() { + _ = a +} +-- B.Play -- +package main + +import () + +func main() { +} diff --git a/src/go/doc/testdata/examples/whole_file.go b/src/go/doc/testdata/examples/whole_file.go new file mode 100644 index 0000000..61954ce --- /dev/null +++ b/src/go/doc/testdata/examples/whole_file.go @@ -0,0 +1,23 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo_test + +import "fmt" + +type X int + +func (X) Foo() { +} + +func (X) TestBlah() { +} + +func (X) BenchmarkFoo() { +} + +func Example() { + fmt.Println("Hello, world!") + // Output: Hello, world! +} diff --git a/src/go/doc/testdata/examples/whole_file.golden b/src/go/doc/testdata/examples/whole_file.golden new file mode 100644 index 0000000..74a2291 --- /dev/null +++ b/src/go/doc/testdata/examples/whole_file.golden @@ -0,0 +1,21 @@ +-- .Play -- +package main + +import "fmt" + +type X int + +func (X) Foo() { +} + +func (X) TestBlah() { +} + +func (X) BenchmarkFoo() { +} + +func main() { + fmt.Println("Hello, world!") +} +-- .Output -- +Hello, world! diff --git a/src/go/doc/testdata/examples/whole_function.go b/src/go/doc/testdata/examples/whole_function.go new file mode 100644 index 0000000..1754ee3 --- /dev/null +++ b/src/go/doc/testdata/examples/whole_function.go @@ -0,0 +1,13 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo_test + +func Foo(x int) { +} + +func Example() { + fmt.Println("Hello, world!") + // Output: Hello, world! +} diff --git a/src/go/doc/testdata/examples/whole_function.golden b/src/go/doc/testdata/examples/whole_function.golden new file mode 100644 index 0000000..7d5b5cb --- /dev/null +++ b/src/go/doc/testdata/examples/whole_function.golden @@ -0,0 +1,11 @@ +-- .Play -- +package main + +func Foo(x int) { +} + +func main() { + fmt.Println("Hello, world!") +} +-- .Output -- +Hello, world! diff --git a/src/go/doc/testdata/examples/whole_function_external.go b/src/go/doc/testdata/examples/whole_function_external.go new file mode 100644 index 0000000..0e0e2f5 --- /dev/null +++ b/src/go/doc/testdata/examples/whole_function_external.go @@ -0,0 +1,12 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package foo_test + +func foo(int) + +func Example() { + foo(42) + // Output: +} diff --git a/src/go/doc/testdata/examples/whole_function_external.golden b/src/go/doc/testdata/examples/whole_function_external.golden new file mode 100644 index 0000000..ec8f114 --- /dev/null +++ b/src/go/doc/testdata/examples/whole_function_external.golden @@ -0,0 +1,8 @@ +-- .Play -- +package main + +func foo(int) + +func main() { + foo(42) +} diff --git a/src/go/doc/testdata/f.0.golden b/src/go/doc/testdata/f.0.golden new file mode 100644 index 0000000..8175901 --- /dev/null +++ b/src/go/doc/testdata/f.0.golden @@ -0,0 +1,13 @@ +// The package f is a go/doc test for functions and factory ... +PACKAGE f + +IMPORTPATH + testdata/f + +FILENAMES + testdata/f.go + +FUNCTIONS + // Exported must always be visible. Was issue 2824. + func Exported() private + diff --git a/src/go/doc/testdata/f.1.golden b/src/go/doc/testdata/f.1.golden new file mode 100644 index 0000000..ba68e88 --- /dev/null +++ b/src/go/doc/testdata/f.1.golden @@ -0,0 +1,16 @@ +// The package f is a go/doc test for functions and factory ... +PACKAGE f + +IMPORTPATH + testdata/f + +FILENAMES + testdata/f.go + +TYPES + // + type private struct{} + + // Exported must always be visible. Was issue 2824. + func Exported() private + diff --git a/src/go/doc/testdata/f.2.golden b/src/go/doc/testdata/f.2.golden new file mode 100644 index 0000000..8175901 --- /dev/null +++ b/src/go/doc/testdata/f.2.golden @@ -0,0 +1,13 @@ +// The package f is a go/doc test for functions and factory ... +PACKAGE f + +IMPORTPATH + testdata/f + +FILENAMES + testdata/f.go + +FUNCTIONS + // Exported must always be visible. Was issue 2824. + func Exported() private + diff --git a/src/go/doc/testdata/f.go b/src/go/doc/testdata/f.go new file mode 100644 index 0000000..7e9add9 --- /dev/null +++ b/src/go/doc/testdata/f.go @@ -0,0 +1,14 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The package f is a go/doc test for functions and factory methods. +package f + +// ---------------------------------------------------------------------------- +// Factory functions for non-exported types must not get lost. + +type private struct{} + +// Exported must always be visible. Was issue 2824. +func Exported() private {} diff --git a/src/go/doc/testdata/g.0.golden b/src/go/doc/testdata/g.0.golden new file mode 100644 index 0000000..487cf06 --- /dev/null +++ b/src/go/doc/testdata/g.0.golden @@ -0,0 +1,32 @@ +// The package g is a go/doc test for mixed exported/unexported ... +PACKAGE g + +IMPORTPATH + testdata/g + +FILENAMES + testdata/g.go + +CONSTANTS + // + const ( + A, _ = iota, iota + _, D + E, _ + G, H + ) + + +VARIABLES + // + var ( + _, C2, _ = 1, 2, 3 + C4, _, C6 = 4, 5, 6 + _, C8, _ = 7, 8, 9 + ) + + // + var ( + _, X = f() + ) + diff --git a/src/go/doc/testdata/g.1.golden b/src/go/doc/testdata/g.1.golden new file mode 100644 index 0000000..438441a --- /dev/null +++ b/src/go/doc/testdata/g.1.golden @@ -0,0 +1,34 @@ +// The package g is a go/doc test for mixed exported/unexported ... +PACKAGE g + +IMPORTPATH + testdata/g + +FILENAMES + testdata/g.go + +CONSTANTS + // + const ( + A, b = iota, iota + c, D + E, f + G, H + ) + + +VARIABLES + // + var ( + c1, C2, c3 = 1, 2, 3 + C4, c5, C6 = 4, 5, 6 + c7, C8, c9 = 7, 8, 9 + xx, yy, zz = 0, 0, 0 // all unexported and hidden + ) + + // + var ( + x, X = f() + y, z = f() + ) + diff --git a/src/go/doc/testdata/g.2.golden b/src/go/doc/testdata/g.2.golden new file mode 100644 index 0000000..487cf06 --- /dev/null +++ b/src/go/doc/testdata/g.2.golden @@ -0,0 +1,32 @@ +// The package g is a go/doc test for mixed exported/unexported ... +PACKAGE g + +IMPORTPATH + testdata/g + +FILENAMES + testdata/g.go + +CONSTANTS + // + const ( + A, _ = iota, iota + _, D + E, _ + G, H + ) + + +VARIABLES + // + var ( + _, C2, _ = 1, 2, 3 + C4, _, C6 = 4, 5, 6 + _, C8, _ = 7, 8, 9 + ) + + // + var ( + _, X = f() + ) + diff --git a/src/go/doc/testdata/g.go b/src/go/doc/testdata/g.go new file mode 100644 index 0000000..ceeb417 --- /dev/null +++ b/src/go/doc/testdata/g.go @@ -0,0 +1,25 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The package g is a go/doc test for mixed exported/unexported values. +package g + +const ( + A, b = iota, iota + c, D + E, f + G, H +) + +var ( + c1, C2, c3 = 1, 2, 3 + C4, c5, C6 = 4, 5, 6 + c7, C8, c9 = 7, 8, 9 + xx, yy, zz = 0, 0, 0 // all unexported and hidden +) + +var ( + x, X = f() + y, z = f() +) diff --git a/src/go/doc/testdata/generics.0.golden b/src/go/doc/testdata/generics.0.golden new file mode 100644 index 0000000..91c874c --- /dev/null +++ b/src/go/doc/testdata/generics.0.golden @@ -0,0 +1,76 @@ +// Package generics contains the new syntax supporting generic ... +PACKAGE generics + +IMPORTPATH + testdata/generics + +FILENAMES + testdata/generics.go + +FUNCTIONS + // AnotherFunc has an implicit constraint interface. Neither type ... + func AnotherFunc[T ~struct{ f int }](_ struct{ f int }) + + // Func has an instantiated constraint. + func Func[T Constraint[string, Type[int]]]() + + // Single is not a factory function. + func Single[T any]() *T + + // Slice is not a factory function. + func Slice[T any]() []T + + +TYPES + // AFuncType demonstrates filtering of parameters and type ... + type AFuncType[T ~struct{ f int }] func(_ struct { + // contains filtered or unexported fields + }) + + // Constraint is a constraint interface with two type parameters. + type Constraint[P, Q interface{ string | ~int | Type[int] }] interface { + ~int | ~byte | Type[string] + M() P + } + + // NewEmbeddings demonstrates how we filter the new embedded ... + type NewEmbeddings interface { + string // should not be filtered + + struct { + // contains filtered or unexported fields + } + ~struct { + // contains filtered or unexported fields + } + *struct { + // contains filtered or unexported fields + } + struct { + // contains filtered or unexported fields + } | ~struct { + // contains filtered or unexported fields + } + // contains filtered or unexported methods + } + + // Parameterized types should be shown. + type Type[P any] struct { + Field P + } + + // Variables with an instantiated type should be shown. + var X Type[int] + + // Constructors for parameterized types should be shown. + func Constructor[lowerCase any]() Type[lowerCase] + + // MethodA uses a different name for its receiver type parameter. + func (t Type[A]) MethodA(p A) + + // MethodB has a blank receiver type parameter. + func (t Type[_]) MethodB() + + // MethodC has a lower-case receiver type parameter. + func (t Type[c]) MethodC() + diff --git a/src/go/doc/testdata/generics.1.golden b/src/go/doc/testdata/generics.1.golden new file mode 100644 index 0000000..923a4ce --- /dev/null +++ b/src/go/doc/testdata/generics.1.golden @@ -0,0 +1,66 @@ +// Package generics contains the new syntax supporting generic ... +PACKAGE generics + +IMPORTPATH + testdata/generics + +FILENAMES + testdata/generics.go + +FUNCTIONS + // AnotherFunc has an implicit constraint interface. Neither type ... + func AnotherFunc[T ~struct{ f int }](_ struct{ f int }) + + // Func has an instantiated constraint. + func Func[T Constraint[string, Type[int]]]() + + // Single is not a factory function. + func Single[T any]() *T + + // Slice is not a factory function. + func Slice[T any]() []T + + +TYPES + // AFuncType demonstrates filtering of parameters and type ... + type AFuncType[T ~struct{ f int }] func(_ struct{ f int }) + + // Constraint is a constraint interface with two type parameters. + type Constraint[P, Q interface{ string | ~int | Type[int] }] interface { + ~int | ~byte | Type[string] + M() P + } + + // NewEmbeddings demonstrates how we filter the new embedded ... + type NewEmbeddings interface { + string // should not be filtered + int16 + struct{ f int } + ~struct{ f int } + *struct{ f int } + struct{ f int } | ~struct{ f int } + } + + // Parameterized types should be shown. + type Type[P any] struct { + Field P + } + + // Variables with an instantiated type should be shown. + var X Type[int] + + // Constructors for parameterized types should be shown. + func Constructor[lowerCase any]() Type[lowerCase] + + // MethodA uses a different name for its receiver type parameter. + func (t Type[A]) MethodA(p A) + + // MethodB has a blank receiver type parameter. + func (t Type[_]) MethodB() + + // MethodC has a lower-case receiver type parameter. + func (t Type[c]) MethodC() + + // int16 shadows the predeclared type int16. + type int16 int + diff --git a/src/go/doc/testdata/generics.2.golden b/src/go/doc/testdata/generics.2.golden new file mode 100644 index 0000000..91c874c --- /dev/null +++ b/src/go/doc/testdata/generics.2.golden @@ -0,0 +1,76 @@ +// Package generics contains the new syntax supporting generic ... +PACKAGE generics + +IMPORTPATH + testdata/generics + +FILENAMES + testdata/generics.go + +FUNCTIONS + // AnotherFunc has an implicit constraint interface. Neither type ... + func AnotherFunc[T ~struct{ f int }](_ struct{ f int }) + + // Func has an instantiated constraint. + func Func[T Constraint[string, Type[int]]]() + + // Single is not a factory function. + func Single[T any]() *T + + // Slice is not a factory function. + func Slice[T any]() []T + + +TYPES + // AFuncType demonstrates filtering of parameters and type ... + type AFuncType[T ~struct{ f int }] func(_ struct { + // contains filtered or unexported fields + }) + + // Constraint is a constraint interface with two type parameters. + type Constraint[P, Q interface{ string | ~int | Type[int] }] interface { + ~int | ~byte | Type[string] + M() P + } + + // NewEmbeddings demonstrates how we filter the new embedded ... + type NewEmbeddings interface { + string // should not be filtered + + struct { + // contains filtered or unexported fields + } + ~struct { + // contains filtered or unexported fields + } + *struct { + // contains filtered or unexported fields + } + struct { + // contains filtered or unexported fields + } | ~struct { + // contains filtered or unexported fields + } + // contains filtered or unexported methods + } + + // Parameterized types should be shown. + type Type[P any] struct { + Field P + } + + // Variables with an instantiated type should be shown. + var X Type[int] + + // Constructors for parameterized types should be shown. + func Constructor[lowerCase any]() Type[lowerCase] + + // MethodA uses a different name for its receiver type parameter. + func (t Type[A]) MethodA(p A) + + // MethodB has a blank receiver type parameter. + func (t Type[_]) MethodB() + + // MethodC has a lower-case receiver type parameter. + func (t Type[c]) MethodC() + diff --git a/src/go/doc/testdata/generics.go b/src/go/doc/testdata/generics.go new file mode 100644 index 0000000..ba7187e --- /dev/null +++ b/src/go/doc/testdata/generics.go @@ -0,0 +1,74 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package generics contains the new syntax supporting generic programming in +// Go. +package generics + +// Variables with an instantiated type should be shown. +var X Type[int] + +// Parameterized types should be shown. +type Type[P any] struct { + Field P +} + +// Constructors for parameterized types should be shown. +func Constructor[lowerCase any]() Type[lowerCase] { + return Type[lowerCase]{} +} + +// MethodA uses a different name for its receiver type parameter. +func (t Type[A]) MethodA(p A) {} + +// MethodB has a blank receiver type parameter. +func (t Type[_]) MethodB() {} + +// MethodC has a lower-case receiver type parameter. +func (t Type[c]) MethodC() {} + +// Constraint is a constraint interface with two type parameters. +type Constraint[P, Q interface{ string | ~int | Type[int] }] interface { + ~int | ~byte | Type[string] + M() P +} + +// int16 shadows the predeclared type int16. +type int16 int + +// NewEmbeddings demonstrates how we filter the new embedded elements. +type NewEmbeddings interface { + string // should not be filtered + int16 + struct{ f int } + ~struct{ f int } + *struct{ f int } + struct{ f int } | ~struct{ f int } +} + +// Func has an instantiated constraint. +func Func[T Constraint[string, Type[int]]]() {} + +// AnotherFunc has an implicit constraint interface. +// +// Neither type parameters nor regular parameters should be filtered. +func AnotherFunc[T ~struct{ f int }](_ struct{ f int }) {} + +// AFuncType demonstrates filtering of parameters and type parameters. Here we +// don't filter type parameters (to be consistent with function declarations), +// but DO filter the RHS. +type AFuncType[T ~struct{ f int }] func(_ struct{ f int }) + +// See issue #49477: type parameters should not be interpreted as named types +// for the purpose of determining whether a function is a factory function. + +// Slice is not a factory function. +func Slice[T any]() []T { + return nil +} + +// Single is not a factory function. +func Single[T any]() *T { + return nil +} diff --git a/src/go/doc/testdata/issue12839.0.golden b/src/go/doc/testdata/issue12839.0.golden new file mode 100644 index 0000000..6b59774 --- /dev/null +++ b/src/go/doc/testdata/issue12839.0.golden @@ -0,0 +1,51 @@ +// Package issue12839 is a go/doc test to test association of a ... +PACKAGE issue12839 + +IMPORTPATH + testdata/issue12839 + +IMPORTS + p + +FILENAMES + testdata/issue12839.go + +FUNCTIONS + // F1 should not be associated with T1 + func F1() (*T1, *T2) + + // F10 should not be associated with T1. + func F10() (T1, T2, error) + + // F4 should not be associated with a type (same as F1) + func F4() (a T1, b T2) + + // F9 should not be associated with T1. + func F9() (int, T1, T2) + + +TYPES + // + type T1 struct{} + + // F2 should be associated with T1 + func F2() (a, b, c T1) + + // F3 should be associated with T1 because b.T3 is from a ... + func F3() (a T1, b p.T3) + + // F5 should be associated with T1. + func F5() (T1, error) + + // F6 should be associated with T1. + func F6() (*T1, error) + + // F7 should be associated with T1. + func F7() (T1, string) + + // F8 should be associated with T1. + func F8() (int, T1, string) + + // + type T2 struct{} + diff --git a/src/go/doc/testdata/issue12839.1.golden b/src/go/doc/testdata/issue12839.1.golden new file mode 100644 index 0000000..4b9b9f6 --- /dev/null +++ b/src/go/doc/testdata/issue12839.1.golden @@ -0,0 +1,54 @@ +// Package issue12839 is a go/doc test to test association of a ... +PACKAGE issue12839 + +IMPORTPATH + testdata/issue12839 + +IMPORTS + p + +FILENAMES + testdata/issue12839.go + +FUNCTIONS + // F1 should not be associated with T1 + func F1() (*T1, *T2) + + // F10 should not be associated with T1. + func F10() (T1, T2, error) + + // F4 should not be associated with a type (same as F1) + func F4() (a T1, b T2) + + // F9 should not be associated with T1. + func F9() (int, T1, T2) + + +TYPES + // + type T1 struct{} + + // F2 should be associated with T1 + func F2() (a, b, c T1) + + // F3 should be associated with T1 because b.T3 is from a ... + func F3() (a T1, b p.T3) + + // F5 should be associated with T1. + func F5() (T1, error) + + // F6 should be associated with T1. + func F6() (*T1, error) + + // F7 should be associated with T1. + func F7() (T1, string) + + // F8 should be associated with T1. + func F8() (int, T1, string) + + // + func (t T1) hello() string + + // + type T2 struct{} + diff --git a/src/go/doc/testdata/issue12839.2.golden b/src/go/doc/testdata/issue12839.2.golden new file mode 100644 index 0000000..6b59774 --- /dev/null +++ b/src/go/doc/testdata/issue12839.2.golden @@ -0,0 +1,51 @@ +// Package issue12839 is a go/doc test to test association of a ... +PACKAGE issue12839 + +IMPORTPATH + testdata/issue12839 + +IMPORTS + p + +FILENAMES + testdata/issue12839.go + +FUNCTIONS + // F1 should not be associated with T1 + func F1() (*T1, *T2) + + // F10 should not be associated with T1. + func F10() (T1, T2, error) + + // F4 should not be associated with a type (same as F1) + func F4() (a T1, b T2) + + // F9 should not be associated with T1. + func F9() (int, T1, T2) + + +TYPES + // + type T1 struct{} + + // F2 should be associated with T1 + func F2() (a, b, c T1) + + // F3 should be associated with T1 because b.T3 is from a ... + func F3() (a T1, b p.T3) + + // F5 should be associated with T1. + func F5() (T1, error) + + // F6 should be associated with T1. + func F6() (*T1, error) + + // F7 should be associated with T1. + func F7() (T1, string) + + // F8 should be associated with T1. + func F8() (int, T1, string) + + // + type T2 struct{} + diff --git a/src/go/doc/testdata/issue12839.go b/src/go/doc/testdata/issue12839.go new file mode 100644 index 0000000..51c7ac1 --- /dev/null +++ b/src/go/doc/testdata/issue12839.go @@ -0,0 +1,69 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package issue12839 is a go/doc test to test association of a function +// that returns multiple types. +// See golang.org/issue/12839. +// (See also golang.org/issue/27928.) +package issue12839 + +import "p" + +type T1 struct{} + +type T2 struct{} + +func (t T1) hello() string { + return "hello" +} + +// F1 should not be associated with T1 +func F1() (*T1, *T2) { + return &T1{}, &T2{} +} + +// F2 should be associated with T1 +func F2() (a, b, c T1) { + return T1{}, T1{}, T1{} +} + +// F3 should be associated with T1 because b.T3 is from a different package +func F3() (a T1, b p.T3) { + return T1{}, p.T3{} +} + +// F4 should not be associated with a type (same as F1) +func F4() (a T1, b T2) { + return T1{}, T2{} +} + +// F5 should be associated with T1. +func F5() (T1, error) { + return T1{}, nil +} + +// F6 should be associated with T1. +func F6() (*T1, error) { + return &T1{}, nil +} + +// F7 should be associated with T1. +func F7() (T1, string) { + return T1{}, nil +} + +// F8 should be associated with T1. +func F8() (int, T1, string) { + return 0, T1{}, nil +} + +// F9 should not be associated with T1. +func F9() (int, T1, T2) { + return 0, T1{}, T2{} +} + +// F10 should not be associated with T1. +func F10() (T1, T2, error) { + return T1{}, T2{}, nil +} diff --git a/src/go/doc/testdata/issue13742.0.golden b/src/go/doc/testdata/issue13742.0.golden new file mode 100644 index 0000000..8dee9aa --- /dev/null +++ b/src/go/doc/testdata/issue13742.0.golden @@ -0,0 +1,25 @@ +// +PACKAGE issue13742 + +IMPORTPATH + testdata/issue13742 + +IMPORTS + go/ast + +FILENAMES + testdata/issue13742.go + +FUNCTIONS + // Both F0 and G0 should appear as functions. + func F0(Node) + + // Both F1 and G1 should appear as functions. + func F1(ast.Node) + + // + func G0() Node + + // + func G1() ast.Node + diff --git a/src/go/doc/testdata/issue13742.1.golden b/src/go/doc/testdata/issue13742.1.golden new file mode 100644 index 0000000..8dee9aa --- /dev/null +++ b/src/go/doc/testdata/issue13742.1.golden @@ -0,0 +1,25 @@ +// +PACKAGE issue13742 + +IMPORTPATH + testdata/issue13742 + +IMPORTS + go/ast + +FILENAMES + testdata/issue13742.go + +FUNCTIONS + // Both F0 and G0 should appear as functions. + func F0(Node) + + // Both F1 and G1 should appear as functions. + func F1(ast.Node) + + // + func G0() Node + + // + func G1() ast.Node + diff --git a/src/go/doc/testdata/issue13742.2.golden b/src/go/doc/testdata/issue13742.2.golden new file mode 100644 index 0000000..8dee9aa --- /dev/null +++ b/src/go/doc/testdata/issue13742.2.golden @@ -0,0 +1,25 @@ +// +PACKAGE issue13742 + +IMPORTPATH + testdata/issue13742 + +IMPORTS + go/ast + +FILENAMES + testdata/issue13742.go + +FUNCTIONS + // Both F0 and G0 should appear as functions. + func F0(Node) + + // Both F1 and G1 should appear as functions. + func F1(ast.Node) + + // + func G0() Node + + // + func G1() ast.Node + diff --git a/src/go/doc/testdata/issue13742.go b/src/go/doc/testdata/issue13742.go new file mode 100644 index 0000000..dbc1941 --- /dev/null +++ b/src/go/doc/testdata/issue13742.go @@ -0,0 +1,18 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue13742 + +import ( + "go/ast" + . "go/ast" +) + +// Both F0 and G0 should appear as functions. +func F0(Node) {} +func G0() Node { return nil } + +// Both F1 and G1 should appear as functions. +func F1(ast.Node) {} +func G1() ast.Node { return nil } diff --git a/src/go/doc/testdata/issue16153.0.golden b/src/go/doc/testdata/issue16153.0.golden new file mode 100644 index 0000000..189260b --- /dev/null +++ b/src/go/doc/testdata/issue16153.0.golden @@ -0,0 +1,32 @@ +// +PACKAGE issue16153 + +IMPORTPATH + testdata/issue16153 + +FILENAMES + testdata/issue16153.go + +CONSTANTS + // + const ( + X3 int64 = iota + Y3 = 1 + ) + + // + const ( + X4 int64 = iota + Y4 + ) + + // original test case + const ( + Y1 = 256 + ) + + // variations + const ( + Y2 uint8 + ) + diff --git a/src/go/doc/testdata/issue16153.1.golden b/src/go/doc/testdata/issue16153.1.golden new file mode 100644 index 0000000..803df3e --- /dev/null +++ b/src/go/doc/testdata/issue16153.1.golden @@ -0,0 +1,34 @@ +// +PACKAGE issue16153 + +IMPORTPATH + testdata/issue16153 + +FILENAMES + testdata/issue16153.go + +CONSTANTS + // original test case + const ( + x1 uint8 = 255 + Y1 = 256 + ) + + // variations + const ( + x2 uint8 = 255 + Y2 + ) + + // + const ( + X3 int64 = iota + Y3 = 1 + ) + + // + const ( + X4 int64 = iota + Y4 + ) + diff --git a/src/go/doc/testdata/issue16153.2.golden b/src/go/doc/testdata/issue16153.2.golden new file mode 100644 index 0000000..189260b --- /dev/null +++ b/src/go/doc/testdata/issue16153.2.golden @@ -0,0 +1,32 @@ +// +PACKAGE issue16153 + +IMPORTPATH + testdata/issue16153 + +FILENAMES + testdata/issue16153.go + +CONSTANTS + // + const ( + X3 int64 = iota + Y3 = 1 + ) + + // + const ( + X4 int64 = iota + Y4 + ) + + // original test case + const ( + Y1 = 256 + ) + + // variations + const ( + Y2 uint8 + ) + diff --git a/src/go/doc/testdata/issue16153.go b/src/go/doc/testdata/issue16153.go new file mode 100644 index 0000000..528be42 --- /dev/null +++ b/src/go/doc/testdata/issue16153.go @@ -0,0 +1,27 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue16153 + +// original test case +const ( + x1 uint8 = 255 + Y1 = 256 +) + +// variations +const ( + x2 uint8 = 255 + Y2 +) + +const ( + X3 int64 = iota + Y3 = 1 +) + +const ( + X4 int64 = iota + Y4 +) diff --git a/src/go/doc/testdata/issue17788.0.golden b/src/go/doc/testdata/issue17788.0.golden new file mode 100644 index 0000000..42c00da --- /dev/null +++ b/src/go/doc/testdata/issue17788.0.golden @@ -0,0 +1,8 @@ +// +PACKAGE issue17788 + +IMPORTPATH + testdata/issue17788 + +FILENAMES + testdata/issue17788.go diff --git a/src/go/doc/testdata/issue17788.1.golden b/src/go/doc/testdata/issue17788.1.golden new file mode 100644 index 0000000..42c00da --- /dev/null +++ b/src/go/doc/testdata/issue17788.1.golden @@ -0,0 +1,8 @@ +// +PACKAGE issue17788 + +IMPORTPATH + testdata/issue17788 + +FILENAMES + testdata/issue17788.go diff --git a/src/go/doc/testdata/issue17788.2.golden b/src/go/doc/testdata/issue17788.2.golden new file mode 100644 index 0000000..42c00da --- /dev/null +++ b/src/go/doc/testdata/issue17788.2.golden @@ -0,0 +1,8 @@ +// +PACKAGE issue17788 + +IMPORTPATH + testdata/issue17788 + +FILENAMES + testdata/issue17788.go diff --git a/src/go/doc/testdata/issue17788.go b/src/go/doc/testdata/issue17788.go new file mode 100644 index 0000000..883ad5f --- /dev/null +++ b/src/go/doc/testdata/issue17788.go @@ -0,0 +1,8 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue17788 + +func ( /* receiver type */ ) f0() { +} diff --git a/src/go/doc/testdata/issue22856.0.golden b/src/go/doc/testdata/issue22856.0.golden new file mode 100644 index 0000000..a88f43f --- /dev/null +++ b/src/go/doc/testdata/issue22856.0.golden @@ -0,0 +1,45 @@ +// +PACKAGE issue22856 + +IMPORTPATH + testdata/issue22856 + +FILENAMES + testdata/issue22856.go + +FUNCTIONS + // NewPointerSliceOfSlice is not a factory function because slices ... + func NewPointerSliceOfSlice() [][]*T + + // NewSlice3 is not a factory function because 3 nested slices of ... + func NewSlice3() [][][]T + + // NewSliceOfSlice is not a factory function because slices of a ... + func NewSliceOfSlice() [][]T + + +TYPES + // + type T struct{} + + // + func New() T + + // + func NewArray() [1]T + + // + func NewPointer() *T + + // + func NewPointerArray() [1]*T + + // + func NewPointerOfPointer() **T + + // + func NewPointerSlice() []*T + + // + func NewSlice() []T + diff --git a/src/go/doc/testdata/issue22856.1.golden b/src/go/doc/testdata/issue22856.1.golden new file mode 100644 index 0000000..a88f43f --- /dev/null +++ b/src/go/doc/testdata/issue22856.1.golden @@ -0,0 +1,45 @@ +// +PACKAGE issue22856 + +IMPORTPATH + testdata/issue22856 + +FILENAMES + testdata/issue22856.go + +FUNCTIONS + // NewPointerSliceOfSlice is not a factory function because slices ... + func NewPointerSliceOfSlice() [][]*T + + // NewSlice3 is not a factory function because 3 nested slices of ... + func NewSlice3() [][][]T + + // NewSliceOfSlice is not a factory function because slices of a ... + func NewSliceOfSlice() [][]T + + +TYPES + // + type T struct{} + + // + func New() T + + // + func NewArray() [1]T + + // + func NewPointer() *T + + // + func NewPointerArray() [1]*T + + // + func NewPointerOfPointer() **T + + // + func NewPointerSlice() []*T + + // + func NewSlice() []T + diff --git a/src/go/doc/testdata/issue22856.2.golden b/src/go/doc/testdata/issue22856.2.golden new file mode 100644 index 0000000..a88f43f --- /dev/null +++ b/src/go/doc/testdata/issue22856.2.golden @@ -0,0 +1,45 @@ +// +PACKAGE issue22856 + +IMPORTPATH + testdata/issue22856 + +FILENAMES + testdata/issue22856.go + +FUNCTIONS + // NewPointerSliceOfSlice is not a factory function because slices ... + func NewPointerSliceOfSlice() [][]*T + + // NewSlice3 is not a factory function because 3 nested slices of ... + func NewSlice3() [][][]T + + // NewSliceOfSlice is not a factory function because slices of a ... + func NewSliceOfSlice() [][]T + + +TYPES + // + type T struct{} + + // + func New() T + + // + func NewArray() [1]T + + // + func NewPointer() *T + + // + func NewPointerArray() [1]*T + + // + func NewPointerOfPointer() **T + + // + func NewPointerSlice() []*T + + // + func NewSlice() []T + diff --git a/src/go/doc/testdata/issue22856.go b/src/go/doc/testdata/issue22856.go new file mode 100644 index 0000000..f456998 --- /dev/null +++ b/src/go/doc/testdata/issue22856.go @@ -0,0 +1,27 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue22856 + +type T struct{} + +func New() T { return T{} } +func NewPointer() *T { return &T{} } +func NewPointerSlice() []*T { return []*T{&T{}} } +func NewSlice() []T { return []T{T{}} } +func NewPointerOfPointer() **T { x := &T{}; return &x } +func NewArray() [1]T { return [1]T{T{}} } +func NewPointerArray() [1]*T { return [1]*T{&T{}} } + +// NewSliceOfSlice is not a factory function because slices of a slice of +// type *T are not factory functions of type T. +func NewSliceOfSlice() [][]T { return []T{[]T{}} } + +// NewPointerSliceOfSlice is not a factory function because slices of a +// slice of type *T are not factory functions of type T. +func NewPointerSliceOfSlice() [][]*T { return []*T{[]*T{}} } + +// NewSlice3 is not a factory function because 3 nested slices of type T +// are not factory functions of type T. +func NewSlice3() [][][]T { return []T{[]T{[]T{}}} } diff --git a/src/go/doc/testdata/pkgdoc/doc.go b/src/go/doc/testdata/pkgdoc/doc.go new file mode 100644 index 0000000..3f822c7 --- /dev/null +++ b/src/go/doc/testdata/pkgdoc/doc.go @@ -0,0 +1,24 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pkgdoc + +import ( + crand "crypto/rand" + "math/rand" +) + +type T int + +type U int + +func (T) M() {} + +var _ = rand.Int +var _ = crand.Reader + +type G[T any] struct{ x T } + +func (g G[T]) M1() {} +func (g *G[T]) M2() {} diff --git a/src/go/doc/testdata/predeclared.0.golden b/src/go/doc/testdata/predeclared.0.golden new file mode 100644 index 0000000..9f37b06 --- /dev/null +++ b/src/go/doc/testdata/predeclared.0.golden @@ -0,0 +1,8 @@ +// Package predeclared is a go/doc test for handling of exported ... +PACKAGE predeclared + +IMPORTPATH + testdata/predeclared + +FILENAMES + testdata/predeclared.go diff --git a/src/go/doc/testdata/predeclared.1.golden b/src/go/doc/testdata/predeclared.1.golden new file mode 100644 index 0000000..2ff8ee6 --- /dev/null +++ b/src/go/doc/testdata/predeclared.1.golden @@ -0,0 +1,22 @@ +// Package predeclared is a go/doc test for handling of exported ... +PACKAGE predeclared + +IMPORTPATH + testdata/predeclared + +FILENAMES + testdata/predeclared.go + +TYPES + // + type bool int + + // Must not be visible. + func (b bool) String() string + + // + type error struct{} + + // Must not be visible. + func (e error) Error() string + diff --git a/src/go/doc/testdata/predeclared.2.golden b/src/go/doc/testdata/predeclared.2.golden new file mode 100644 index 0000000..9f37b06 --- /dev/null +++ b/src/go/doc/testdata/predeclared.2.golden @@ -0,0 +1,8 @@ +// Package predeclared is a go/doc test for handling of exported ... +PACKAGE predeclared + +IMPORTPATH + testdata/predeclared + +FILENAMES + testdata/predeclared.go diff --git a/src/go/doc/testdata/predeclared.go b/src/go/doc/testdata/predeclared.go new file mode 100644 index 0000000..c6dd806 --- /dev/null +++ b/src/go/doc/testdata/predeclared.go @@ -0,0 +1,22 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package predeclared is a go/doc test for handling of +// exported methods on locally-defined predeclared types. +// See issue 9860. +package predeclared + +type error struct{} + +// Must not be visible. +func (e error) Error() string { + return "" +} + +type bool int + +// Must not be visible. +func (b bool) String() string { + return "" +} diff --git a/src/go/doc/testdata/template.txt b/src/go/doc/testdata/template.txt new file mode 100644 index 0000000..1b07382 --- /dev/null +++ b/src/go/doc/testdata/template.txt @@ -0,0 +1,68 @@ +{{synopsis .Doc}} +PACKAGE {{.Name}} + +IMPORTPATH + {{.ImportPath}} + +{{with .Imports}}IMPORTS +{{range .}} {{.}} +{{end}} +{{end}}{{/* + +*/}}FILENAMES +{{range .Filenames}} {{.}} +{{end}}{{/* + +*/}}{{with .Consts}} +CONSTANTS +{{range .}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{end}}{{/* + +*/}}{{with .Vars}} +VARIABLES +{{range .}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{end}}{{/* + +*/}}{{with .Funcs}} +FUNCTIONS +{{range .}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{end}}{{/* + +*/}}{{with .Types}} +TYPES +{{range .}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{range .Consts}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{/* + +*/}}{{range .Vars}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{/* + +*/}}{{range .Funcs}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{/* + +*/}}{{range .Methods}} {{synopsis .Doc}} + {{node .Decl $.FSet}} + +{{end}}{{end}}{{end}}{{/* + +*/}}{{with .Bugs}} +BUGS .Bugs is now deprecated, please use .Notes instead +{{range .}}{{indent "\t" .}} +{{end}}{{end}}{{with .Notes}}{{range $marker, $content := .}} +{{$marker}}S +{{range $content}}{{$marker}}({{.UID}}){{indent "\t" .Body}} +{{end}}{{end}}{{end}} \ No newline at end of file diff --git a/src/go/doc/testdata/testing.0.golden b/src/go/doc/testdata/testing.0.golden new file mode 100644 index 0000000..61dac8b --- /dev/null +++ b/src/go/doc/testdata/testing.0.golden @@ -0,0 +1,156 @@ +// Package testing provides support for automated testing of Go ... +PACKAGE testing + +IMPORTPATH + testdata/testing + +IMPORTS + bytes + flag + fmt + io + os + runtime + runtime/pprof + strconv + strings + time + +FILENAMES + testdata/benchmark.go + testdata/example.go + testdata/testing.go + +FUNCTIONS + // An internal function but exported because it is cross-package; ... + func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) + + // An internal function but exported because it is cross-package; ... + func RunBenchmarks(matchString func(pat, str string) (bool, error), benchmarks []InternalBenchmark) + + // + func RunExamples(examples []InternalExample) (ok bool) + + // + func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) + + // Short reports whether the -test.short flag is set. + func Short() bool + + +TYPES + // B is a type passed to Benchmark functions to manage benchmark ... + type B struct { + N int + // contains filtered or unexported fields + } + + // Error is equivalent to Log() followed by Fail(). + func (c *B) Error(args ...any) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *B) Errorf(format string, args ...any) + + // Fail marks the function as having failed but continues ... + func (c *B) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *B) FailNow() + + // Failed reports whether the function has failed. + func (c *B) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *B) Fatal(args ...any) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *B) Fatalf(format string, args ...any) + + // Log formats its arguments using default formatting, analogous ... + func (c *B) Log(args ...any) + + // Logf formats its arguments according to the format, analogous ... + func (c *B) Logf(format string, args ...any) + + // ResetTimer sets the elapsed benchmark time to zero. It does not ... + func (b *B) ResetTimer() + + // SetBytes records the number of bytes processed in a single ... + func (b *B) SetBytes(n int64) + + // StartTimer starts timing a test. This function is called ... + func (b *B) StartTimer() + + // StopTimer stops timing a test. This can be used to pause the ... + func (b *B) StopTimer() + + // The results of a benchmark run. + type BenchmarkResult struct { + N int // The number of iterations. + T time.Duration // The total time taken. + Bytes int64 // Bytes processed in one iteration. + } + + // Benchmark benchmarks a single function. Useful for creating ... + func Benchmark(f func(b *B)) BenchmarkResult + + // + func (r BenchmarkResult) NsPerOp() int64 + + // + func (r BenchmarkResult) String() string + + // An internal type but exported because it is cross-package; part ... + type InternalBenchmark struct { + Name string + F func(b *B) + } + + // + type InternalExample struct { + Name string + F func() + Output string + } + + // An internal type but exported because it is cross-package; part ... + type InternalTest struct { + Name string + F func(*T) + } + + // T is a type passed to Test functions to manage test state and ... + type T struct { + // contains filtered or unexported fields + } + + // Error is equivalent to Log() followed by Fail(). + func (c *T) Error(args ...any) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *T) Errorf(format string, args ...any) + + // Fail marks the function as having failed but continues ... + func (c *T) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *T) FailNow() + + // Failed reports whether the function has failed. + func (c *T) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *T) Fatal(args ...any) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *T) Fatalf(format string, args ...any) + + // Log formats its arguments using default formatting, analogous ... + func (c *T) Log(args ...any) + + // Logf formats its arguments according to the format, analogous ... + func (c *T) Logf(format string, args ...any) + + // Parallel signals that this test is to be run in parallel with ... + func (t *T) Parallel() + diff --git a/src/go/doc/testdata/testing.1.golden b/src/go/doc/testdata/testing.1.golden new file mode 100644 index 0000000..1655af1 --- /dev/null +++ b/src/go/doc/testdata/testing.1.golden @@ -0,0 +1,298 @@ +// Package testing provides support for automated testing of Go ... +PACKAGE testing + +IMPORTPATH + testdata/testing + +IMPORTS + bytes + flag + fmt + io + os + runtime + runtime/pprof + strconv + strings + time + +FILENAMES + testdata/benchmark.go + testdata/example.go + testdata/testing.go + +VARIABLES + // + var ( + // The short flag requests that tests run more quickly, but its functionality + // is provided by test writers themselves. The testing package is just its + // home. The all.bash installation script sets it to make installation more + // efficient, but by default the flag is off so a plain "go test" will do a + // full test of the package. + short = flag.Bool("test.short", false, "run smaller test suite to save time") + + // Report as tests are run; default is silent for success. + chatty = flag.Bool("test.v", false, "verbose: print additional output") + match = flag.String("test.run", "", "regular expression to select tests to run") + memProfile = flag.String("test.memprofile", "", "write a memory profile to the named file after execution") + memProfileRate = flag.Int("test.memprofilerate", 0, "if >=0, sets runtime.MemProfileRate") + cpuProfile = flag.String("test.cpuprofile", "", "write a cpu profile to the named file during execution") + timeout = flag.Duration("test.timeout", 0, "if positive, sets an aggregate time limit for all tests") + cpuListStr = flag.String("test.cpu", "", "comma-separated list of number of CPUs to use for each test") + parallel = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "maximum test parallelism") + + cpuList []int + ) + + // + var benchTime = flag.Duration("test.benchtime", 1*time.Second, "approximate run time for each benchmark") + + // + var matchBenchmarks = flag.String("test.bench", "", "regular expression to select benchmarks to run") + + // + var timer *time.Timer + + +FUNCTIONS + // An internal function but exported because it is cross-package; ... + func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) + + // An internal function but exported because it is cross-package; ... + func RunBenchmarks(matchString func(pat, str string) (bool, error), benchmarks []InternalBenchmark) + + // + func RunExamples(examples []InternalExample) (ok bool) + + // + func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) + + // Short reports whether the -test.short flag is set. + func Short() bool + + // after runs after all testing. + func after() + + // alarm is called if the timeout expires. + func alarm() + + // before runs before all testing. + func before() + + // decorate inserts the final newline if needed and indentation ... + func decorate(s string, addFileLine bool) string + + // + func max(x, y int) int + + // + func min(x, y int) int + + // + func parseCpuList() + + // roundDown10 rounds a number down to the nearest power of 10. + func roundDown10(n int) int + + // roundUp rounds x up to a number of the form [1eX, 2eX, 5eX]. + func roundUp(n int) int + + // startAlarm starts an alarm if requested. + func startAlarm() + + // stopAlarm turns off the alarm. + func stopAlarm() + + // + func tRunner(t *T, test *InternalTest) + + +TYPES + // B is a type passed to Benchmark functions to manage benchmark ... + type B struct { + common + N int + benchmark InternalBenchmark + bytes int64 + timerOn bool + result BenchmarkResult + } + + // Error is equivalent to Log() followed by Fail(). + func (c *B) Error(args ...any) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *B) Errorf(format string, args ...any) + + // Fail marks the function as having failed but continues ... + func (c *B) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *B) FailNow() + + // Failed reports whether the function has failed. + func (c *B) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *B) Fatal(args ...any) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *B) Fatalf(format string, args ...any) + + // Log formats its arguments using default formatting, analogous ... + func (c *B) Log(args ...any) + + // Logf formats its arguments according to the format, analogous ... + func (c *B) Logf(format string, args ...any) + + // ResetTimer sets the elapsed benchmark time to zero. It does not ... + func (b *B) ResetTimer() + + // SetBytes records the number of bytes processed in a single ... + func (b *B) SetBytes(n int64) + + // StartTimer starts timing a test. This function is called ... + func (b *B) StartTimer() + + // StopTimer stops timing a test. This can be used to pause the ... + func (b *B) StopTimer() + + // launch launches the benchmark function. It gradually increases ... + func (b *B) launch() + + // log generates the output. It's always at the same stack depth. + func (c *B) log(s string) + + // + func (b *B) nsPerOp() int64 + + // run times the benchmark function in a separate goroutine. + func (b *B) run() BenchmarkResult + + // runN runs a single benchmark for the specified number of ... + func (b *B) runN(n int) + + // trimOutput shortens the output from a benchmark, which can be ... + func (b *B) trimOutput() + + // The results of a benchmark run. + type BenchmarkResult struct { + N int // The number of iterations. + T time.Duration // The total time taken. + Bytes int64 // Bytes processed in one iteration. + } + + // Benchmark benchmarks a single function. Useful for creating ... + func Benchmark(f func(b *B)) BenchmarkResult + + // + func (r BenchmarkResult) NsPerOp() int64 + + // + func (r BenchmarkResult) String() string + + // + func (r BenchmarkResult) mbPerSec() float64 + + // An internal type but exported because it is cross-package; part ... + type InternalBenchmark struct { + Name string + F func(b *B) + } + + // + type InternalExample struct { + Name string + F func() + Output string + } + + // An internal type but exported because it is cross-package; part ... + type InternalTest struct { + Name string + F func(*T) + } + + // T is a type passed to Test functions to manage test state and ... + type T struct { + common + name string // Name of test. + startParallel chan bool // Parallel tests will wait on this. + } + + // Error is equivalent to Log() followed by Fail(). + func (c *T) Error(args ...any) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *T) Errorf(format string, args ...any) + + // Fail marks the function as having failed but continues ... + func (c *T) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *T) FailNow() + + // Failed reports whether the function has failed. + func (c *T) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *T) Fatal(args ...any) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *T) Fatalf(format string, args ...any) + + // Log formats its arguments using default formatting, analogous ... + func (c *T) Log(args ...any) + + // Logf formats its arguments according to the format, analogous ... + func (c *T) Logf(format string, args ...any) + + // Parallel signals that this test is to be run in parallel with ... + func (t *T) Parallel() + + // log generates the output. It's always at the same stack depth. + func (c *T) log(s string) + + // + func (t *T) report() + + // common holds the elements common between T and B and captures ... + type common struct { + output []byte // Output generated by test or benchmark. + failed bool // Test or benchmark has failed. + start time.Time // Time test or benchmark started + duration time.Duration + self any // To be sent on signal channel when done. + signal chan any // Output for serial tests. + } + + // Error is equivalent to Log() followed by Fail(). + func (c *common) Error(args ...any) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *common) Errorf(format string, args ...any) + + // Fail marks the function as having failed but continues ... + func (c *common) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *common) FailNow() + + // Failed reports whether the function has failed. + func (c *common) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *common) Fatal(args ...any) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *common) Fatalf(format string, args ...any) + + // Log formats its arguments using default formatting, analogous ... + func (c *common) Log(args ...any) + + // Logf formats its arguments according to the format, analogous ... + func (c *common) Logf(format string, args ...any) + + // log generates the output. It's always at the same stack depth. + func (c *common) log(s string) + diff --git a/src/go/doc/testdata/testing.2.golden b/src/go/doc/testdata/testing.2.golden new file mode 100644 index 0000000..61dac8b --- /dev/null +++ b/src/go/doc/testdata/testing.2.golden @@ -0,0 +1,156 @@ +// Package testing provides support for automated testing of Go ... +PACKAGE testing + +IMPORTPATH + testdata/testing + +IMPORTS + bytes + flag + fmt + io + os + runtime + runtime/pprof + strconv + strings + time + +FILENAMES + testdata/benchmark.go + testdata/example.go + testdata/testing.go + +FUNCTIONS + // An internal function but exported because it is cross-package; ... + func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) + + // An internal function but exported because it is cross-package; ... + func RunBenchmarks(matchString func(pat, str string) (bool, error), benchmarks []InternalBenchmark) + + // + func RunExamples(examples []InternalExample) (ok bool) + + // + func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) + + // Short reports whether the -test.short flag is set. + func Short() bool + + +TYPES + // B is a type passed to Benchmark functions to manage benchmark ... + type B struct { + N int + // contains filtered or unexported fields + } + + // Error is equivalent to Log() followed by Fail(). + func (c *B) Error(args ...any) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *B) Errorf(format string, args ...any) + + // Fail marks the function as having failed but continues ... + func (c *B) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *B) FailNow() + + // Failed reports whether the function has failed. + func (c *B) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *B) Fatal(args ...any) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *B) Fatalf(format string, args ...any) + + // Log formats its arguments using default formatting, analogous ... + func (c *B) Log(args ...any) + + // Logf formats its arguments according to the format, analogous ... + func (c *B) Logf(format string, args ...any) + + // ResetTimer sets the elapsed benchmark time to zero. It does not ... + func (b *B) ResetTimer() + + // SetBytes records the number of bytes processed in a single ... + func (b *B) SetBytes(n int64) + + // StartTimer starts timing a test. This function is called ... + func (b *B) StartTimer() + + // StopTimer stops timing a test. This can be used to pause the ... + func (b *B) StopTimer() + + // The results of a benchmark run. + type BenchmarkResult struct { + N int // The number of iterations. + T time.Duration // The total time taken. + Bytes int64 // Bytes processed in one iteration. + } + + // Benchmark benchmarks a single function. Useful for creating ... + func Benchmark(f func(b *B)) BenchmarkResult + + // + func (r BenchmarkResult) NsPerOp() int64 + + // + func (r BenchmarkResult) String() string + + // An internal type but exported because it is cross-package; part ... + type InternalBenchmark struct { + Name string + F func(b *B) + } + + // + type InternalExample struct { + Name string + F func() + Output string + } + + // An internal type but exported because it is cross-package; part ... + type InternalTest struct { + Name string + F func(*T) + } + + // T is a type passed to Test functions to manage test state and ... + type T struct { + // contains filtered or unexported fields + } + + // Error is equivalent to Log() followed by Fail(). + func (c *T) Error(args ...any) + + // Errorf is equivalent to Logf() followed by Fail(). + func (c *T) Errorf(format string, args ...any) + + // Fail marks the function as having failed but continues ... + func (c *T) Fail() + + // FailNow marks the function as having failed and stops its ... + func (c *T) FailNow() + + // Failed reports whether the function has failed. + func (c *T) Failed() bool + + // Fatal is equivalent to Log() followed by FailNow(). + func (c *T) Fatal(args ...any) + + // Fatalf is equivalent to Logf() followed by FailNow(). + func (c *T) Fatalf(format string, args ...any) + + // Log formats its arguments using default formatting, analogous ... + func (c *T) Log(args ...any) + + // Logf formats its arguments according to the format, analogous ... + func (c *T) Logf(format string, args ...any) + + // Parallel signals that this test is to be run in parallel with ... + func (t *T) Parallel() + diff --git a/src/go/doc/testdata/testing.go b/src/go/doc/testdata/testing.go new file mode 100644 index 0000000..d3076c9 --- /dev/null +++ b/src/go/doc/testdata/testing.go @@ -0,0 +1,404 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package testing provides support for automated testing of Go packages. +// It is intended to be used in concert with the ā€œgo testā€ utility, which automates +// execution of any function of the form +// func TestXxx(*testing.T) +// where Xxx can be any alphanumeric string (but the first letter must not be in +// [a-z]) and serves to identify the test routine. +// These TestXxx routines should be declared within the package they are testing. +// +// Functions of the form +// func BenchmarkXxx(*testing.B) +// are considered benchmarks, and are executed by go test when the -test.bench +// flag is provided. +// +// A sample benchmark function looks like this: +// func BenchmarkHello(b *testing.B) { +// for i := 0; i < b.N; i++ { +// fmt.Sprintf("hello") +// } +// } +// The benchmark package will vary b.N until the benchmark function lasts +// long enough to be timed reliably. The output +// testing.BenchmarkHello 10000000 282 ns/op +// means that the loop ran 10000000 times at a speed of 282 ns per loop. +// +// If a benchmark needs some expensive setup before running, the timer +// may be stopped: +// func BenchmarkBigLen(b *testing.B) { +// b.StopTimer() +// big := NewBig() +// b.StartTimer() +// for i := 0; i < b.N; i++ { +// big.Len() +// } +// } +package testing + +import ( + "flag" + "fmt" + "os" + "runtime" + "runtime/pprof" + "strconv" + "strings" + "time" +) + +var ( + // The short flag requests that tests run more quickly, but its functionality + // is provided by test writers themselves. The testing package is just its + // home. The all.bash installation script sets it to make installation more + // efficient, but by default the flag is off so a plain "go test" will do a + // full test of the package. + short = flag.Bool("test.short", false, "run smaller test suite to save time") + + // Report as tests are run; default is silent for success. + chatty = flag.Bool("test.v", false, "verbose: print additional output") + match = flag.String("test.run", "", "regular expression to select tests to run") + memProfile = flag.String("test.memprofile", "", "write a memory profile to the named file after execution") + memProfileRate = flag.Int("test.memprofilerate", 0, "if >=0, sets runtime.MemProfileRate") + cpuProfile = flag.String("test.cpuprofile", "", "write a cpu profile to the named file during execution") + timeout = flag.Duration("test.timeout", 0, "if positive, sets an aggregate time limit for all tests") + cpuListStr = flag.String("test.cpu", "", "comma-separated list of number of CPUs to use for each test") + parallel = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "maximum test parallelism") + + cpuList []int +) + +// common holds the elements common between T and B and +// captures common methods such as Errorf. +type common struct { + output []byte // Output generated by test or benchmark. + failed bool // Test or benchmark has failed. + start time.Time // Time test or benchmark started + duration time.Duration + self any // To be sent on signal channel when done. + signal chan any // Output for serial tests. +} + +// Short reports whether the -test.short flag is set. +func Short() bool { + return *short +} + +// decorate inserts the final newline if needed and indentation tabs for formatting. +// If addFileLine is true, it also prefixes the string with the file and line of the call site. +func decorate(s string, addFileLine bool) string { + if addFileLine { + _, file, line, ok := runtime.Caller(3) // decorate + log + public function. + if ok { + // Truncate file name at last file name separator. + if index := strings.LastIndex(file, "/"); index >= 0 { + file = file[index+1:] + } else if index = strings.LastIndex(file, "\\"); index >= 0 { + file = file[index+1:] + } + } else { + file = "???" + line = 1 + } + s = fmt.Sprintf("%s:%d: %s", file, line, s) + } + s = "\t" + s // Every line is indented at least one tab. + n := len(s) + if n > 0 && s[n-1] != '\n' { + s += "\n" + n++ + } + for i := 0; i < n-1; i++ { // -1 to avoid final newline + if s[i] == '\n' { + // Second and subsequent lines are indented an extra tab. + return s[0:i+1] + "\t" + decorate(s[i+1:n], false) + } + } + return s +} + +// T is a type passed to Test functions to manage test state and support formatted test logs. +// Logs are accumulated during execution and dumped to standard error when done. +type T struct { + common + name string // Name of test. + startParallel chan bool // Parallel tests will wait on this. +} + +// Fail marks the function as having failed but continues execution. +func (c *common) Fail() { c.failed = true } + +// Failed reports whether the function has failed. +func (c *common) Failed() bool { return c.failed } + +// FailNow marks the function as having failed and stops its execution. +// Execution will continue at the next Test. +func (c *common) FailNow() { + c.Fail() + + // Calling runtime.Goexit will exit the goroutine, which + // will run the deferred functions in this goroutine, + // which will eventually run the deferred lines in tRunner, + // which will signal to the test loop that this test is done. + // + // A previous version of this code said: + // + // c.duration = ... + // c.signal <- c.self + // runtime.Goexit() + // + // This previous version duplicated code (those lines are in + // tRunner no matter what), but worse the goroutine teardown + // implicit in runtime.Goexit was not guaranteed to complete + // before the test exited. If a test deferred an important cleanup + // function (like removing temporary files), there was no guarantee + // it would run on a test failure. Because we send on c.signal during + // a top-of-stack deferred function now, we know that the send + // only happens after any other stacked defers have completed. + runtime.Goexit() +} + +// log generates the output. It's always at the same stack depth. +func (c *common) log(s string) { + c.output = append(c.output, decorate(s, true)...) +} + +// Log formats its arguments using default formatting, analogous to Println(), +// and records the text in the error log. +func (c *common) Log(args ...any) { c.log(fmt.Sprintln(args...)) } + +// Logf formats its arguments according to the format, analogous to Printf(), +// and records the text in the error log. +func (c *common) Logf(format string, args ...any) { c.log(fmt.Sprintf(format, args...)) } + +// Error is equivalent to Log() followed by Fail(). +func (c *common) Error(args ...any) { + c.log(fmt.Sprintln(args...)) + c.Fail() +} + +// Errorf is equivalent to Logf() followed by Fail(). +func (c *common) Errorf(format string, args ...any) { + c.log(fmt.Sprintf(format, args...)) + c.Fail() +} + +// Fatal is equivalent to Log() followed by FailNow(). +func (c *common) Fatal(args ...any) { + c.log(fmt.Sprintln(args...)) + c.FailNow() +} + +// Fatalf is equivalent to Logf() followed by FailNow(). +func (c *common) Fatalf(format string, args ...any) { + c.log(fmt.Sprintf(format, args...)) + c.FailNow() +} + +// Parallel signals that this test is to be run in parallel with (and only with) +// other parallel tests in this CPU group. +func (t *T) Parallel() { + t.signal <- (*T)(nil) // Release main testing loop + <-t.startParallel // Wait for serial tests to finish +} + +// An internal type but exported because it is cross-package; part of the implementation +// of go test. +type InternalTest struct { + Name string + F func(*T) +} + +func tRunner(t *T, test *InternalTest) { + t.start = time.Now() + + // When this goroutine is done, either because test.F(t) + // returned normally or because a test failure triggered + // a call to runtime.Goexit, record the duration and send + // a signal saying that the test is done. + defer func() { + t.duration = time.Since(t.start) + t.signal <- t + }() + + test.F(t) +} + +// An internal function but exported because it is cross-package; part of the implementation +// of go test. +func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { + flag.Parse() + parseCpuList() + + before() + startAlarm() + testOk := RunTests(matchString, tests) + exampleOk := RunExamples(examples) + if !testOk || !exampleOk { + fmt.Println("FAIL") + os.Exit(1) + } + fmt.Println("PASS") + stopAlarm() + RunBenchmarks(matchString, benchmarks) + after() +} + +func (t *T) report() { + tstr := fmt.Sprintf("(%.2f seconds)", t.duration.Seconds()) + format := "--- %s: %s %s\n%s" + if t.failed { + fmt.Printf(format, "FAIL", t.name, tstr, t.output) + } else if *chatty { + fmt.Printf(format, "PASS", t.name, tstr, t.output) + } +} + +func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) { + ok = true + if len(tests) == 0 { + fmt.Fprintln(os.Stderr, "testing: warning: no tests to run") + return + } + for _, procs := range cpuList { + runtime.GOMAXPROCS(procs) + // We build a new channel tree for each run of the loop. + // collector merges in one channel all the upstream signals from parallel tests. + // If all tests pump to the same channel, a bug can occur where a test + // kicks off a goroutine that Fails, yet the test still delivers a completion signal, + // which skews the counting. + var collector = make(chan any) + + numParallel := 0 + startParallel := make(chan bool) + + for i := 0; i < len(tests); i++ { + matched, err := matchString(*match, tests[i].Name) + if err != nil { + fmt.Fprintf(os.Stderr, "testing: invalid regexp for -test.run: %s\n", err) + os.Exit(1) + } + if !matched { + continue + } + testName := tests[i].Name + if procs != 1 { + testName = fmt.Sprintf("%s-%d", tests[i].Name, procs) + } + t := &T{ + common: common{ + signal: make(chan any), + }, + name: testName, + startParallel: startParallel, + } + t.self = t + if *chatty { + fmt.Printf("=== RUN %s\n", t.name) + } + go tRunner(t, &tests[i]) + out := (<-t.signal).(*T) + if out == nil { // Parallel run. + go func() { + collector <- <-t.signal + }() + numParallel++ + continue + } + t.report() + ok = ok && !out.failed + } + + running := 0 + for numParallel+running > 0 { + if running < *parallel && numParallel > 0 { + startParallel <- true + running++ + numParallel-- + continue + } + t := (<-collector).(*T) + t.report() + ok = ok && !t.failed + running-- + } + } + return +} + +// before runs before all testing. +func before() { + if *memProfileRate > 0 { + runtime.MemProfileRate = *memProfileRate + } + if *cpuProfile != "" { + f, err := os.Create(*cpuProfile) + if err != nil { + fmt.Fprintf(os.Stderr, "testing: %s", err) + return + } + if err := pprof.StartCPUProfile(f); err != nil { + fmt.Fprintf(os.Stderr, "testing: can't start cpu profile: %s", err) + f.Close() + return + } + // Could save f so after can call f.Close; not worth the effort. + } + +} + +// after runs after all testing. +func after() { + if *cpuProfile != "" { + pprof.StopCPUProfile() // flushes profile to disk + } + if *memProfile != "" { + f, err := os.Create(*memProfile) + if err != nil { + fmt.Fprintf(os.Stderr, "testing: %s", err) + return + } + if err = pprof.WriteHeapProfile(f); err != nil { + fmt.Fprintf(os.Stderr, "testing: can't write %s: %s", *memProfile, err) + } + f.Close() + } +} + +var timer *time.Timer + +// startAlarm starts an alarm if requested. +func startAlarm() { + if *timeout > 0 { + timer = time.AfterFunc(*timeout, alarm) + } +} + +// stopAlarm turns off the alarm. +func stopAlarm() { + if *timeout > 0 { + timer.Stop() + } +} + +// alarm is called if the timeout expires. +func alarm() { + panic("test timed out") +} + +func parseCpuList() { + if len(*cpuListStr) == 0 { + cpuList = append(cpuList, runtime.GOMAXPROCS(-1)) + } else { + for _, val := range strings.Split(*cpuListStr, ",") { + cpu, err := strconv.Atoi(val) + if err != nil || cpu <= 0 { + fmt.Fprintf(os.Stderr, "testing: invalid value %q for -test.cpu", val) + os.Exit(1) + } + cpuList = append(cpuList, cpu) + } + } +} diff --git a/src/go/format/benchmark_test.go b/src/go/format/benchmark_test.go new file mode 100644 index 0000000..f42aac4 --- /dev/null +++ b/src/go/format/benchmark_test.go @@ -0,0 +1,91 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file provides a simple framework to add benchmarks +// based on generated input (source) files. + +package format_test + +import ( + "bytes" + "flag" + "fmt" + "go/format" + "os" + "testing" +) + +var debug = flag.Bool("debug", false, "write .src files containing formatting input; for debugging") + +// array1 generates an array literal with n elements of the form: +// +// var _ = [...]byte{ +// +// // 0 +// 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, +// 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, +// 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, +// 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, +// 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, +// // 40 +// 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, +// 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, +// ... +func array1(buf *bytes.Buffer, n int) { + buf.WriteString("var _ = [...]byte{\n") + for i := 0; i < n; { + if i%10 == 0 { + fmt.Fprintf(buf, "\t// %d\n", i) + } + buf.WriteByte('\t') + for j := 0; j < 8; j++ { + fmt.Fprintf(buf, "0x%02x, ", byte(i)) + i++ + } + buf.WriteString("\n") + } + buf.WriteString("}\n") +} + +var tests = []struct { + name string + gen func(*bytes.Buffer, int) + n int +}{ + {"array1", array1, 10000}, + // add new test cases here as needed +} + +func BenchmarkFormat(b *testing.B) { + var src bytes.Buffer + for _, t := range tests { + src.Reset() + src.WriteString("package p\n") + t.gen(&src, t.n) + data := src.Bytes() + + if *debug { + filename := t.name + ".src" + err := os.WriteFile(filename, data, 0660) + if err != nil { + b.Fatalf("couldn't write %s: %v", filename, err) + } + } + + b.Run(fmt.Sprintf("%s-%d", t.name, t.n), func(b *testing.B) { + b.SetBytes(int64(len(data))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + var err error + sink, err = format.Source(data) + if err != nil { + b.Fatal(err) + } + } + }) + } +} + +var sink []byte diff --git a/src/go/format/example_test.go b/src/go/format/example_test.go new file mode 100644 index 0000000..5b6789a --- /dev/null +++ b/src/go/format/example_test.go @@ -0,0 +1,39 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package format_test + +import ( + "bytes" + "fmt" + "go/format" + "go/parser" + "go/token" + "log" +) + +func ExampleNode() { + const expr = "(6+2*3)/4" + + // parser.ParseExpr parses the argument and returns the + // corresponding ast.Node. + node, err := parser.ParseExpr(expr) + if err != nil { + log.Fatal(err) + } + + // Create a FileSet for node. Since the node does not come + // from a real source file, fset will be empty. + fset := token.NewFileSet() + + var buf bytes.Buffer + err = format.Node(&buf, fset, node) + if err != nil { + log.Fatal(err) + } + + fmt.Println(buf.String()) + + // Output: (6 + 2*3) / 4 +} diff --git a/src/go/format/format.go b/src/go/format/format.go new file mode 100644 index 0000000..3837cb4 --- /dev/null +++ b/src/go/format/format.go @@ -0,0 +1,133 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package format implements standard formatting of Go source. +// +// Note that formatting of Go source code changes over time, so tools relying on +// consistent formatting should execute a specific version of the gofmt binary +// instead of using this package. That way, the formatting will be stable, and +// the tools won't need to be recompiled each time gofmt changes. +// +// For example, pre-submit checks that use this package directly would behave +// differently depending on what Go version each developer uses, causing the +// check to be inherently fragile. +package format + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "io" +) + +// Keep these in sync with cmd/gofmt/gofmt.go. +const ( + tabWidth = 8 + printerMode = printer.UseSpaces | printer.TabIndent | printerNormalizeNumbers + + // printerNormalizeNumbers means to canonicalize number literal prefixes + // and exponents while printing. See https://golang.org/doc/go1.13#gofmt. + // + // This value is defined in go/printer specifically for go/format and cmd/gofmt. + printerNormalizeNumbers = 1 << 30 +) + +var config = printer.Config{Mode: printerMode, Tabwidth: tabWidth} + +const parserMode = parser.ParseComments | parser.SkipObjectResolution + +// Node formats node in canonical gofmt style and writes the result to dst. +// +// The node type must be *ast.File, *printer.CommentedNode, []ast.Decl, +// []ast.Stmt, or assignment-compatible to ast.Expr, ast.Decl, ast.Spec, +// or ast.Stmt. Node does not modify node. Imports are not sorted for +// nodes representing partial source files (for instance, if the node is +// not an *ast.File or a *printer.CommentedNode not wrapping an *ast.File). +// +// The function may return early (before the entire result is written) +// and return a formatting error, for instance due to an incorrect AST. +func Node(dst io.Writer, fset *token.FileSet, node any) error { + // Determine if we have a complete source file (file != nil). + var file *ast.File + var cnode *printer.CommentedNode + switch n := node.(type) { + case *ast.File: + file = n + case *printer.CommentedNode: + if f, ok := n.Node.(*ast.File); ok { + file = f + cnode = n + } + } + + // Sort imports if necessary. + if file != nil && hasUnsortedImports(file) { + // Make a copy of the AST because ast.SortImports is destructive. + // TODO(gri) Do this more efficiently. + var buf bytes.Buffer + err := config.Fprint(&buf, fset, file) + if err != nil { + return err + } + file, err = parser.ParseFile(fset, "", buf.Bytes(), parserMode) + if err != nil { + // We should never get here. If we do, provide good diagnostic. + return fmt.Errorf("format.Node internal error (%s)", err) + } + ast.SortImports(fset, file) + + // Use new file with sorted imports. + node = file + if cnode != nil { + node = &printer.CommentedNode{Node: file, Comments: cnode.Comments} + } + } + + return config.Fprint(dst, fset, node) +} + +// Source formats src in canonical gofmt style and returns the result +// or an (I/O or syntax) error. src is expected to be a syntactically +// correct Go source file, or a list of Go declarations or statements. +// +// If src is a partial source file, the leading and trailing space of src +// is applied to the result (such that it has the same leading and trailing +// space as src), and the result is indented by the same amount as the first +// line of src containing code. Imports are not sorted for partial source files. +func Source(src []byte) ([]byte, error) { + fset := token.NewFileSet() + file, sourceAdj, indentAdj, err := parse(fset, "", src, true) + if err != nil { + return nil, err + } + + if sourceAdj == nil { + // Complete source file. + // TODO(gri) consider doing this always. + ast.SortImports(fset, file) + } + + return format(fset, file, sourceAdj, indentAdj, src, config) +} + +func hasUnsortedImports(file *ast.File) bool { + for _, d := range file.Decls { + d, ok := d.(*ast.GenDecl) + if !ok || d.Tok != token.IMPORT { + // Not an import declaration, so we're done. + // Imports are always first. + return false + } + if d.Lparen.IsValid() { + // For now assume all grouped imports are unsorted. + // TODO(gri) Should check if they are sorted already. + return true + } + // Ungrouped imports are sorted by default. + } + return false +} diff --git a/src/go/format/format_test.go b/src/go/format/format_test.go new file mode 100644 index 0000000..6cc0278 --- /dev/null +++ b/src/go/format/format_test.go @@ -0,0 +1,187 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package format + +import ( + "bytes" + "go/ast" + "go/parser" + "go/token" + "os" + "strings" + "testing" +) + +const testfile = "format_test.go" + +func diff(t *testing.T, dst, src []byte) { + line := 1 + offs := 0 // line offset + for i := 0; i < len(dst) && i < len(src); i++ { + d := dst[i] + s := src[i] + if d != s { + t.Errorf("dst:%d: %s\n", line, dst[offs:i+1]) + t.Errorf("src:%d: %s\n", line, src[offs:i+1]) + return + } + if s == '\n' { + line++ + offs = i + 1 + } + } + if len(dst) != len(src) { + t.Errorf("len(dst) = %d, len(src) = %d\nsrc = %q", len(dst), len(src), src) + } +} + +func TestNode(t *testing.T) { + src, err := os.ReadFile(testfile) + if err != nil { + t.Fatal(err) + } + + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, testfile, src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + + if err = Node(&buf, fset, file); err != nil { + t.Fatal("Node failed:", err) + } + + diff(t, buf.Bytes(), src) +} + +// Node is documented to not modify the AST. +// Test that it is so even when numbers are normalized. +func TestNodeNoModify(t *testing.T) { + const ( + src = "package p\n\nconst _ = 0000000123i\n" + golden = "package p\n\nconst _ = 123i\n" + ) + + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + + // Capture original address and value of a BasicLit node + // which will undergo formatting changes during printing. + wantLit := file.Decls[0].(*ast.GenDecl).Specs[0].(*ast.ValueSpec).Values[0].(*ast.BasicLit) + wantVal := wantLit.Value + + var buf bytes.Buffer + if err = Node(&buf, fset, file); err != nil { + t.Fatal("Node failed:", err) + } + diff(t, buf.Bytes(), []byte(golden)) + + // Check if anything changed after Node returned. + gotLit := file.Decls[0].(*ast.GenDecl).Specs[0].(*ast.ValueSpec).Values[0].(*ast.BasicLit) + gotVal := gotLit.Value + + if gotLit != wantLit { + t.Errorf("got *ast.BasicLit address %p, want %p", gotLit, wantLit) + } + if gotVal != wantVal { + t.Errorf("got *ast.BasicLit value %q, want %q", gotVal, wantVal) + } +} + +func TestSource(t *testing.T) { + src, err := os.ReadFile(testfile) + if err != nil { + t.Fatal(err) + } + + res, err := Source(src) + if err != nil { + t.Fatal("Source failed:", err) + } + + diff(t, res, src) +} + +// Test cases that are expected to fail are marked by the prefix "ERROR". +// The formatted result must look the same as the input for successful tests. +var tests = []string{ + // declaration lists + `import "go/format"`, + "var x int", + "var x int\n\ntype T struct{}", + + // statement lists + "x := 0", + "f(a, b, c)\nvar x int = f(1, 2, 3)", + + // indentation, leading and trailing space + "\tx := 0\n\tgo f()", + "\tx := 0\n\tgo f()\n\n\n", + "\n\t\t\n\n\tx := 0\n\tgo f()\n\n\n", + "\n\t\t\n\n\t\t\tx := 0\n\t\t\tgo f()\n\n\n", + "\n\t\t\n\n\t\t\tx := 0\n\t\t\tconst s = `\nfoo\n`\n\n\n", // no indentation added inside raw strings + "\n\t\t\n\n\t\t\tx := 0\n\t\t\tconst s = `\n\t\tfoo\n`\n\n\n", // no indentation removed inside raw strings + + // comments + "/* Comment */", + "\t/* Comment */ ", + "\n/* Comment */ ", + "i := 5 /* Comment */", // issue #5551 + "\ta()\n//line :1", // issue #11276 + "\t//xxx\n\ta()\n//line :2", // issue #11276 + "\ta() //line :1\n\tb()\n", // issue #11276 + "x := 0\n//line :1\n//line :2", // issue #11276 + + // whitespace + "", // issue #11275 + " ", // issue #11275 + "\t", // issue #11275 + "\t\t", // issue #11275 + "\n", // issue #11275 + "\n\n", // issue #11275 + "\t\n", // issue #11275 + + // erroneous programs + "ERROR1 + 2 +", + "ERRORx := 0", + + // build comments + "// copyright\n\n//go:build x\n\npackage p\n", + "// copyright\n\n//go:build x\n// +build x\n\npackage p\n", +} + +func String(s string) (string, error) { + res, err := Source([]byte(s)) + if err != nil { + return "", err + } + return string(res), nil +} + +func TestPartial(t *testing.T) { + for _, src := range tests { + if strings.HasPrefix(src, "ERROR") { + // test expected to fail + src = src[5:] // remove ERROR prefix + res, err := String(src) + if err == nil && res == src { + t.Errorf("formatting succeeded but was expected to fail:\n%q", src) + } + } else { + // test expected to succeed + res, err := String(src) + if err != nil { + t.Errorf("formatting failed (%s):\n%q", err, src) + } else if res != src { + t.Errorf("formatting incorrect:\nsource: %q\nresult: %q", src, res) + } + } + } +} diff --git a/src/go/format/internal.go b/src/go/format/internal.go new file mode 100644 index 0000000..2f3b0e4 --- /dev/null +++ b/src/go/format/internal.go @@ -0,0 +1,176 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TODO(gri): This file and the file src/cmd/gofmt/internal.go are +// the same (but for this comment and the package name). Do not modify +// one without the other. Determine if we can factor out functionality +// in a public API. See also #11844 for context. + +package format + +import ( + "bytes" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "strings" +) + +// parse parses src, which was read from the named file, +// as a Go source file, declaration, or statement list. +func parse(fset *token.FileSet, filename string, src []byte, fragmentOk bool) ( + file *ast.File, + sourceAdj func(src []byte, indent int) []byte, + indentAdj int, + err error, +) { + // Try as whole source file. + file, err = parser.ParseFile(fset, filename, src, parserMode) + // If there's no error, return. If the error is that the source file didn't begin with a + // package line and source fragments are ok, fall through to + // try as a source fragment. Stop and return on any other error. + if err == nil || !fragmentOk || !strings.Contains(err.Error(), "expected 'package'") { + return + } + + // If this is a declaration list, make it a source file + // by inserting a package clause. + // Insert using a ';', not a newline, so that the line numbers + // in psrc match the ones in src. + psrc := append([]byte("package p;"), src...) + file, err = parser.ParseFile(fset, filename, psrc, parserMode) + if err == nil { + sourceAdj = func(src []byte, indent int) []byte { + // Remove the package clause. + // Gofmt has turned the ';' into a '\n'. + src = src[indent+len("package p\n"):] + return bytes.TrimSpace(src) + } + return + } + // If the error is that the source file didn't begin with a + // declaration, fall through to try as a statement list. + // Stop and return on any other error. + if !strings.Contains(err.Error(), "expected declaration") { + return + } + + // If this is a statement list, make it a source file + // by inserting a package clause and turning the list + // into a function body. This handles expressions too. + // Insert using a ';', not a newline, so that the line numbers + // in fsrc match the ones in src. Add an extra '\n' before the '}' + // to make sure comments are flushed before the '}'. + fsrc := append(append([]byte("package p; func _() {"), src...), '\n', '\n', '}') + file, err = parser.ParseFile(fset, filename, fsrc, parserMode) + if err == nil { + sourceAdj = func(src []byte, indent int) []byte { + // Cap adjusted indent to zero. + if indent < 0 { + indent = 0 + } + // Remove the wrapping. + // Gofmt has turned the "; " into a "\n\n". + // There will be two non-blank lines with indent, hence 2*indent. + src = src[2*indent+len("package p\n\nfunc _() {"):] + // Remove only the "}\n" suffix: remaining whitespaces will be trimmed anyway + src = src[:len(src)-len("}\n")] + return bytes.TrimSpace(src) + } + // Gofmt has also indented the function body one level. + // Adjust that with indentAdj. + indentAdj = -1 + } + + // Succeeded, or out of options. + return +} + +// format formats the given package file originally obtained from src +// and adjusts the result based on the original source via sourceAdj +// and indentAdj. +func format( + fset *token.FileSet, + file *ast.File, + sourceAdj func(src []byte, indent int) []byte, + indentAdj int, + src []byte, + cfg printer.Config, +) ([]byte, error) { + if sourceAdj == nil { + // Complete source file. + var buf bytes.Buffer + err := cfg.Fprint(&buf, fset, file) + if err != nil { + return nil, err + } + return buf.Bytes(), nil + } + + // Partial source file. + // Determine and prepend leading space. + i, j := 0, 0 + for j < len(src) && isSpace(src[j]) { + if src[j] == '\n' { + i = j + 1 // byte offset of last line in leading space + } + j++ + } + var res []byte + res = append(res, src[:i]...) + + // Determine and prepend indentation of first code line. + // Spaces are ignored unless there are no tabs, + // in which case spaces count as one tab. + indent := 0 + hasSpace := false + for _, b := range src[i:j] { + switch b { + case ' ': + hasSpace = true + case '\t': + indent++ + } + } + if indent == 0 && hasSpace { + indent = 1 + } + for i := 0; i < indent; i++ { + res = append(res, '\t') + } + + // Format the source. + // Write it without any leading and trailing space. + cfg.Indent = indent + indentAdj + var buf bytes.Buffer + err := cfg.Fprint(&buf, fset, file) + if err != nil { + return nil, err + } + out := sourceAdj(buf.Bytes(), cfg.Indent) + + // If the adjusted output is empty, the source + // was empty but (possibly) for white space. + // The result is the incoming source. + if len(out) == 0 { + return src, nil + } + + // Otherwise, append output to leading space. + res = append(res, out...) + + // Determine and append trailing space. + i = len(src) + for i > 0 && isSpace(src[i-1]) { + i-- + } + return append(res, src[i:]...), nil +} + +// isSpace reports whether the byte is a space character. +// isSpace defines a space as being among the following bytes: ' ', '\t', '\n' and '\r'. +func isSpace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' +} diff --git a/src/go/importer/importer.go b/src/go/importer/importer.go new file mode 100644 index 0000000..23118d3 --- /dev/null +++ b/src/go/importer/importer.go @@ -0,0 +1,122 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package importer provides access to export data importers. +package importer + +import ( + "go/build" + "go/internal/gccgoimporter" + "go/internal/gcimporter" + "go/internal/srcimporter" + "go/token" + "go/types" + "io" + "runtime" +) + +// A Lookup function returns a reader to access package data for +// a given import path, or an error if no matching package is found. +type Lookup func(path string) (io.ReadCloser, error) + +// ForCompiler returns an Importer for importing from installed packages +// for the compilers "gc" and "gccgo", or for importing directly +// from the source if the compiler argument is "source". In this +// latter case, importing may fail under circumstances where the +// exported API is not entirely defined in pure Go source code +// (if the package API depends on cgo-defined entities, the type +// checker won't have access to those). +// +// The lookup function is called each time the resulting importer needs +// to resolve an import path. In this mode the importer can only be +// invoked with canonical import paths (not relative or absolute ones); +// it is assumed that the translation to canonical import paths is being +// done by the client of the importer. +// +// A lookup function must be provided for correct module-aware operation. +// Deprecated: If lookup is nil, for backwards-compatibility, the importer +// will attempt to resolve imports in the $GOPATH workspace. +func ForCompiler(fset *token.FileSet, compiler string, lookup Lookup) types.Importer { + switch compiler { + case "gc": + return &gcimports{ + fset: fset, + packages: make(map[string]*types.Package), + lookup: lookup, + } + + case "gccgo": + var inst gccgoimporter.GccgoInstallation + if err := inst.InitFromDriver("gccgo"); err != nil { + return nil + } + return &gccgoimports{ + packages: make(map[string]*types.Package), + importer: inst.GetImporter(nil, nil), + lookup: lookup, + } + + case "source": + if lookup != nil { + panic("source importer for custom import path lookup not supported (issue #13847).") + } + + return srcimporter.New(&build.Default, fset, make(map[string]*types.Package)) + } + + // compiler not supported + return nil +} + +// For calls ForCompiler with a new FileSet. +// +// Deprecated: Use ForCompiler, which populates a FileSet +// with the positions of objects created by the importer. +func For(compiler string, lookup Lookup) types.Importer { + return ForCompiler(token.NewFileSet(), compiler, lookup) +} + +// Default returns an Importer for the compiler that built the running binary. +// If available, the result implements types.ImporterFrom. +func Default() types.Importer { + return For(runtime.Compiler, nil) +} + +// gc importer + +type gcimports struct { + fset *token.FileSet + packages map[string]*types.Package + lookup Lookup +} + +func (m *gcimports) Import(path string) (*types.Package, error) { + return m.ImportFrom(path, "" /* no vendoring */, 0) +} + +func (m *gcimports) ImportFrom(path, srcDir string, mode types.ImportMode) (*types.Package, error) { + if mode != 0 { + panic("mode must be 0") + } + return gcimporter.Import(m.fset, m.packages, path, srcDir, m.lookup) +} + +// gccgo importer + +type gccgoimports struct { + packages map[string]*types.Package + importer gccgoimporter.Importer + lookup Lookup +} + +func (m *gccgoimports) Import(path string) (*types.Package, error) { + return m.ImportFrom(path, "" /* no vendoring */, 0) +} + +func (m *gccgoimports) ImportFrom(path, srcDir string, mode types.ImportMode) (*types.Package, error) { + if mode != 0 { + panic("mode must be 0") + } + return m.importer(m.packages, path, srcDir, m.lookup) +} diff --git a/src/go/importer/importer_test.go b/src/go/importer/importer_test.go new file mode 100644 index 0000000..142efd3 --- /dev/null +++ b/src/go/importer/importer_test.go @@ -0,0 +1,96 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package importer + +import ( + "go/build" + "go/token" + "internal/buildcfg" + "internal/testenv" + "io" + "os" + "strings" + "testing" +) + +func TestMain(m *testing.M) { + build.Default.GOROOT = testenv.GOROOT(nil) + os.Exit(m.Run()) +} + +func TestForCompiler(t *testing.T) { + testenv.MustHaveGoBuild(t) + + const thePackage = "math/big" + out, err := testenv.Command(t, testenv.GoToolPath(t), "list", "-export", "-f={{context.Compiler}}:{{.Export}}", thePackage).CombinedOutput() + if err != nil { + t.Fatalf("go list %s: %v\n%s", thePackage, err, out) + } + export := strings.TrimSpace(string(out)) + compiler, target, _ := strings.Cut(export, ":") + + if compiler == "gccgo" { + t.Skip("golang.org/issue/22500") + } + + fset := token.NewFileSet() + + t.Run("LookupDefault", func(t *testing.T) { + imp := ForCompiler(fset, compiler, nil) + pkg, err := imp.Import(thePackage) + if err != nil { + t.Fatal(err) + } + if pkg.Path() != thePackage { + t.Fatalf("Path() = %q, want %q", pkg.Path(), thePackage) + } + + // Check that the fileset positions are accurate. + // https://github.com/golang/go#28995 + mathBigInt := pkg.Scope().Lookup("Int") + posn := fset.Position(mathBigInt.Pos()) // "$GOROOT/src/math/big/int.go:25:1" + filename := strings.Replace(posn.Filename, "$GOROOT", testenv.GOROOT(t), 1) + data, err := os.ReadFile(filename) + if err != nil { + t.Fatalf("can't read file containing declaration of math/big.Int: %v", err) + } + lines := strings.Split(string(data), "\n") + if posn.Line > len(lines) || !strings.HasPrefix(lines[posn.Line-1], "type Int") { + t.Fatalf("Object %v position %s does not contain its declaration", + mathBigInt, posn) + } + }) + + t.Run("LookupCustom", func(t *testing.T) { + // TODO(mdempsky): Decide whether to remove this test, or to fix + // support for it in unified IR. It's not clear that we actually + // need to support importing "math/big" as "math/bigger", for + // example. cmd/link no longer supports that. + if buildcfg.Experiment.Unified { + t.Skip("not supported by GOEXPERIMENT=unified; see go.dev/cl/406319") + } + + lookup := func(path string) (io.ReadCloser, error) { + if path != "math/bigger" { + t.Fatalf("lookup called with unexpected path %q", path) + } + f, err := os.Open(target) + if err != nil { + t.Fatal(err) + } + return f, nil + } + imp := ForCompiler(fset, compiler, lookup) + pkg, err := imp.Import("math/bigger") + if err != nil { + t.Fatal(err) + } + // Even though we open math/big.a, the import request was for math/bigger + // and that should be recorded in pkg.Path(), at least for the gc toolchain. + if pkg.Path() != "math/bigger" { + t.Fatalf("Path() = %q, want %q", pkg.Path(), "math/bigger") + } + }) +} diff --git a/src/go/internal/gccgoimporter/ar.go b/src/go/internal/gccgoimporter/ar.go new file mode 100644 index 0000000..9df7934 --- /dev/null +++ b/src/go/internal/gccgoimporter/ar.go @@ -0,0 +1,171 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gccgoimporter + +import ( + "bytes" + "debug/elf" + "errors" + "fmt" + "internal/xcoff" + "io" + "strconv" + "strings" +) + +// Magic strings for different archive file formats. +const ( + armag = "!\n" + armagt = "!\n" + armagb = "\n" +) + +// Offsets and sizes for fields in a standard archive header. +const ( + arNameOff = 0 + arNameSize = 16 + arDateOff = arNameOff + arNameSize + arDateSize = 12 + arUIDOff = arDateOff + arDateSize + arUIDSize = 6 + arGIDOff = arUIDOff + arUIDSize + arGIDSize = 6 + arModeOff = arGIDOff + arGIDSize + arModeSize = 8 + arSizeOff = arModeOff + arModeSize + arSizeSize = 10 + arFmagOff = arSizeOff + arSizeSize + arFmagSize = 2 + + arHdrSize = arFmagOff + arFmagSize +) + +// The contents of the fmag field of a standard archive header. +const arfmag = "`\n" + +// arExportData takes an archive file and returns a ReadSeeker for the +// export data in that file. This assumes that there is only one +// object in the archive containing export data, which is not quite +// what gccgo does; gccgo concatenates together all the export data +// for all the objects in the file. In practice that case does not arise. +func arExportData(archive io.ReadSeeker) (io.ReadSeeker, error) { + if _, err := archive.Seek(0, io.SeekStart); err != nil { + return nil, err + } + + var buf [len(armag)]byte + if _, err := archive.Read(buf[:]); err != nil { + return nil, err + } + + switch string(buf[:]) { + case armag: + return standardArExportData(archive) + case armagt: + return nil, errors.New("unsupported thin archive") + case armagb: + return aixBigArExportData(archive) + default: + return nil, fmt.Errorf("unrecognized archive file format %q", buf[:]) + } +} + +// standardArExportData returns export data from a standard archive. +func standardArExportData(archive io.ReadSeeker) (io.ReadSeeker, error) { + off := int64(len(armag)) + for { + var hdrBuf [arHdrSize]byte + if _, err := archive.Read(hdrBuf[:]); err != nil { + return nil, err + } + off += arHdrSize + + if !bytes.Equal(hdrBuf[arFmagOff:arFmagOff+arFmagSize], []byte(arfmag)) { + return nil, fmt.Errorf("archive header format header (%q)", hdrBuf[:]) + } + + size, err := strconv.ParseInt(strings.TrimSpace(string(hdrBuf[arSizeOff:arSizeOff+arSizeSize])), 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing size in archive header (%q): %v", hdrBuf[:], err) + } + + fn := hdrBuf[arNameOff : arNameOff+arNameSize] + if fn[0] == '/' && (fn[1] == ' ' || fn[1] == '/' || string(fn[:8]) == "/SYM64/ ") { + // Archive symbol table or extended name table, + // which we don't care about. + } else { + archiveAt := readerAtFromSeeker(archive) + ret, err := elfFromAr(io.NewSectionReader(archiveAt, off, size)) + if ret != nil || err != nil { + return ret, err + } + } + + if size&1 != 0 { + size++ + } + off += size + if _, err := archive.Seek(off, io.SeekStart); err != nil { + return nil, err + } + } +} + +// elfFromAr tries to get export data from an archive member as an ELF file. +// If there is no export data, this returns nil, nil. +func elfFromAr(member *io.SectionReader) (io.ReadSeeker, error) { + ef, err := elf.NewFile(member) + if err != nil { + return nil, err + } + sec := ef.Section(".go_export") + if sec == nil { + return nil, nil + } + return sec.Open(), nil +} + +// aixBigArExportData returns export data from an AIX big archive. +func aixBigArExportData(archive io.ReadSeeker) (io.ReadSeeker, error) { + archiveAt := readerAtFromSeeker(archive) + arch, err := xcoff.NewArchive(archiveAt) + if err != nil { + return nil, err + } + + for _, mem := range arch.Members { + f, err := arch.GetFile(mem.Name) + if err != nil { + return nil, err + } + sdat := f.CSect(".go_export") + if sdat != nil { + return bytes.NewReader(sdat), nil + } + } + + return nil, fmt.Errorf(".go_export not found in this archive") +} + +// readerAtFromSeeker turns an io.ReadSeeker into an io.ReaderAt. +// This is only safe because there won't be any concurrent seeks +// while this code is executing. +func readerAtFromSeeker(rs io.ReadSeeker) io.ReaderAt { + if ret, ok := rs.(io.ReaderAt); ok { + return ret + } + return seekerReadAt{rs} +} + +type seekerReadAt struct { + seeker io.ReadSeeker +} + +func (sra seekerReadAt) ReadAt(p []byte, off int64) (int, error) { + if _, err := sra.seeker.Seek(off, io.SeekStart); err != nil { + return 0, err + } + return sra.seeker.Read(p) +} diff --git a/src/go/internal/gccgoimporter/gccgoinstallation.go b/src/go/internal/gccgoimporter/gccgoinstallation.go new file mode 100644 index 0000000..8fc7ce3 --- /dev/null +++ b/src/go/internal/gccgoimporter/gccgoinstallation.go @@ -0,0 +1,97 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gccgoimporter + +import ( + "bufio" + "go/types" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// Information about a specific installation of gccgo. +type GccgoInstallation struct { + // Version of gcc (e.g. 4.8.0). + GccVersion string + + // Target triple (e.g. x86_64-unknown-linux-gnu). + TargetTriple string + + // Built-in library paths used by this installation. + LibPaths []string +} + +// Ask the driver at the given path for information for this GccgoInstallation. +// The given arguments are passed directly to the call of the driver. +func (inst *GccgoInstallation) InitFromDriver(gccgoPath string, args ...string) (err error) { + argv := append([]string{"-###", "-S", "-x", "go", "-"}, args...) + cmd := exec.Command(gccgoPath, argv...) + stderr, err := cmd.StderrPipe() + if err != nil { + return + } + + err = cmd.Start() + if err != nil { + return + } + + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + line := scanner.Text() + switch { + case strings.HasPrefix(line, "Target: "): + inst.TargetTriple = line[8:] + + case line[0] == ' ': + args := strings.Fields(line) + for _, arg := range args[1:] { + if strings.HasPrefix(arg, "-L") { + inst.LibPaths = append(inst.LibPaths, arg[2:]) + } + } + } + } + + argv = append([]string{"-dumpversion"}, args...) + stdout, err := exec.Command(gccgoPath, argv...).Output() + if err != nil { + return + } + inst.GccVersion = strings.TrimSpace(string(stdout)) + + return +} + +// Return the list of export search paths for this GccgoInstallation. +func (inst *GccgoInstallation) SearchPaths() (paths []string) { + for _, lpath := range inst.LibPaths { + spath := filepath.Join(lpath, "go", inst.GccVersion) + fi, err := os.Stat(spath) + if err != nil || !fi.IsDir() { + continue + } + paths = append(paths, spath) + + spath = filepath.Join(spath, inst.TargetTriple) + fi, err = os.Stat(spath) + if err != nil || !fi.IsDir() { + continue + } + paths = append(paths, spath) + } + + paths = append(paths, inst.LibPaths...) + + return +} + +// Return an importer that searches incpaths followed by the gcc installation's +// built-in search paths and the current directory. +func (inst *GccgoInstallation) GetImporter(incpaths []string, initmap map[*types.Package]InitData) Importer { + return GetImporter(append(append(incpaths, inst.SearchPaths()...), "."), initmap) +} diff --git a/src/go/internal/gccgoimporter/gccgoinstallation_test.go b/src/go/internal/gccgoimporter/gccgoinstallation_test.go new file mode 100644 index 0000000..df0188a --- /dev/null +++ b/src/go/internal/gccgoimporter/gccgoinstallation_test.go @@ -0,0 +1,192 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gccgoimporter + +import ( + "go/types" + "testing" +) + +// importablePackages is a list of packages that we verify that we can +// import. This should be all standard library packages in all relevant +// versions of gccgo. Note that since gccgo follows a different release +// cycle, and since different systems have different versions installed, +// we can't use the last-two-versions rule of the gc toolchain. +var importablePackages = [...]string{ + "archive/tar", + "archive/zip", + "bufio", + "bytes", + "compress/bzip2", + "compress/flate", + "compress/gzip", + "compress/lzw", + "compress/zlib", + "container/heap", + "container/list", + "container/ring", + "crypto/aes", + "crypto/cipher", + "crypto/des", + "crypto/dsa", + "crypto/ecdsa", + "crypto/elliptic", + "crypto", + "crypto/hmac", + "crypto/md5", + "crypto/rand", + "crypto/rc4", + "crypto/rsa", + "crypto/sha1", + "crypto/sha256", + "crypto/sha512", + "crypto/subtle", + "crypto/tls", + "crypto/x509", + "crypto/x509/pkix", + "database/sql/driver", + "database/sql", + "debug/dwarf", + "debug/elf", + "debug/gosym", + "debug/macho", + "debug/pe", + "encoding/ascii85", + "encoding/asn1", + "encoding/base32", + "encoding/base64", + "encoding/binary", + "encoding/csv", + "encoding/gob", + // "encoding", // Added in GCC 4.9. + "encoding/hex", + "encoding/json", + "encoding/pem", + "encoding/xml", + "errors", + "expvar", + "flag", + "fmt", + "go/ast", + "go/build", + "go/doc", + // "go/format", // Added in GCC 4.8. + "go/parser", + "go/printer", + "go/scanner", + "go/token", + "hash/adler32", + "hash/crc32", + "hash/crc64", + "hash/fnv", + "hash", + "html", + "html/template", + "image/color", + // "image/color/palette", // Added in GCC 4.9. + "image/draw", + "image/gif", + "image", + "image/jpeg", + "image/png", + "index/suffixarray", + "io", + "io/ioutil", + "log", + "log/syslog", + "math/big", + "math/cmplx", + "math", + "math/rand", + "mime", + "mime/multipart", + "net", + "net/http/cgi", + // "net/http/cookiejar", // Added in GCC 4.8. + "net/http/fcgi", + "net/http", + "net/http/httptest", + "net/http/httputil", + "net/http/pprof", + "net/mail", + "net/rpc", + "net/rpc/jsonrpc", + "net/smtp", + "net/textproto", + "net/url", + "os/exec", + "os", + "os/signal", + "os/user", + "path/filepath", + "path", + "reflect", + "regexp", + "regexp/syntax", + "runtime/debug", + "runtime", + "runtime/pprof", + "sort", + "strconv", + "strings", + "sync/atomic", + "sync", + "syscall", + "testing", + "testing/iotest", + "testing/quick", + "text/scanner", + "text/tabwriter", + "text/template", + "text/template/parse", + "time", + "unicode", + "unicode/utf16", + "unicode/utf8", +} + +func TestInstallationImporter(t *testing.T) { + // This test relies on gccgo being around. + gpath := gccgoPath() + if gpath == "" { + t.Skip("This test needs gccgo") + } + + var inst GccgoInstallation + err := inst.InitFromDriver(gpath) + if err != nil { + t.Fatal(err) + } + imp := inst.GetImporter(nil, nil) + + // Ensure we don't regress the number of packages we can parse. First import + // all packages into the same map and then each individually. + pkgMap := make(map[string]*types.Package) + for _, pkg := range importablePackages { + _, err = imp(pkgMap, pkg, ".", nil) + if err != nil { + t.Error(err) + } + } + + for _, pkg := range importablePackages { + _, err = imp(make(map[string]*types.Package), pkg, ".", nil) + if err != nil { + t.Error(err) + } + } + + // Test for certain specific entities in the imported data. + for _, test := range [...]importerTest{ + {pkgpath: "io", name: "Reader", want: "type Reader interface{Read(p []byte) (n int, err error)}"}, + {pkgpath: "io", name: "ReadWriter", want: "type ReadWriter interface{Reader; Writer}"}, + {pkgpath: "math", name: "Pi", want: "const Pi untyped float"}, + {pkgpath: "math", name: "Sin", want: "func Sin(x float64) float64"}, + {pkgpath: "sort", name: "Search", want: "func Search(n int, f func(int) bool) int"}, + {pkgpath: "unsafe", name: "Pointer", want: "type Pointer"}, + } { + runImporterTest(t, imp, nil, &test) + } +} diff --git a/src/go/internal/gccgoimporter/importer.go b/src/go/internal/gccgoimporter/importer.go new file mode 100644 index 0000000..94f2def --- /dev/null +++ b/src/go/internal/gccgoimporter/importer.go @@ -0,0 +1,261 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gccgoimporter implements Import for gccgo-generated object files. +package gccgoimporter // import "go/internal/gccgoimporter" + +import ( + "bytes" + "debug/elf" + "fmt" + "go/types" + "internal/xcoff" + "io" + "os" + "path/filepath" + "strings" +) + +// A PackageInit describes an imported package that needs initialization. +type PackageInit struct { + Name string // short package name + InitFunc string // name of init function + Priority int // priority of init function, see InitData.Priority +} + +// The gccgo-specific init data for a package. +type InitData struct { + // Initialization priority of this package relative to other packages. + // This is based on the maximum depth of the package's dependency graph; + // it is guaranteed to be greater than that of its dependencies. + Priority int + + // The list of packages which this package depends on to be initialized, + // including itself if needed. This is the subset of the transitive closure of + // the package's dependencies that need initialization. + Inits []PackageInit +} + +// Locate the file from which to read export data. +// This is intended to replicate the logic in gofrontend. +func findExportFile(searchpaths []string, pkgpath string) (string, error) { + for _, spath := range searchpaths { + pkgfullpath := filepath.Join(spath, pkgpath) + pkgdir, name := filepath.Split(pkgfullpath) + + for _, filepath := range [...]string{ + pkgfullpath, + pkgfullpath + ".gox", + pkgdir + "lib" + name + ".so", + pkgdir + "lib" + name + ".a", + pkgfullpath + ".o", + } { + fi, err := os.Stat(filepath) + if err == nil && !fi.IsDir() { + return filepath, nil + } + } + } + + return "", fmt.Errorf("%s: could not find export data (tried %s)", pkgpath, strings.Join(searchpaths, ":")) +} + +const ( + gccgov1Magic = "v1;\n" + gccgov2Magic = "v2;\n" + gccgov3Magic = "v3;\n" + goimporterMagic = "\n$$ " + archiveMagic = "!" + // Take name from Name method (like on os.File) if present. + if n, ok := rc.(interface{ Name() string }); ok { + fpath = n.Name() + } + } else { + fpath, err = findExportFile(searchpaths, pkgpath) + if err != nil { + return nil, err + } + + r, closer, err := openExportFile(fpath) + if err != nil { + return nil, err + } + if closer != nil { + defer closer.Close() + } + reader = r + } + + var magics string + magics, err = readMagic(reader) + if err != nil { + return + } + + if magics == archiveMagic || magics == aixbigafMagic { + reader, err = arExportData(reader) + if err != nil { + return + } + magics, err = readMagic(reader) + if err != nil { + return + } + } + + switch magics { + case gccgov1Magic, gccgov2Magic, gccgov3Magic: + var p parser + p.init(fpath, reader, imports) + pkg = p.parsePackage() + if initmap != nil { + initmap[pkg] = p.initdata + } + + // Excluded for now: Standard gccgo doesn't support this import format currently. + // case goimporterMagic: + // var data []byte + // data, err = io.ReadAll(reader) + // if err != nil { + // return + // } + // var n int + // n, pkg, err = importer.ImportData(imports, data) + // if err != nil { + // return + // } + + // if initmap != nil { + // suffixreader := bytes.NewReader(data[n:]) + // var p parser + // p.init(fpath, suffixreader, nil) + // p.parseInitData() + // initmap[pkg] = p.initdata + // } + + default: + err = fmt.Errorf("unrecognized magic string: %q", magics) + } + + return + } +} + +// readMagic reads the four bytes at the start of a ReadSeeker and +// returns them as a string. +func readMagic(reader io.ReadSeeker) (string, error) { + var magic [4]byte + if _, err := reader.Read(magic[:]); err != nil { + return "", err + } + if _, err := reader.Seek(0, io.SeekStart); err != nil { + return "", err + } + return string(magic[:]), nil +} diff --git a/src/go/internal/gccgoimporter/importer_test.go b/src/go/internal/gccgoimporter/importer_test.go new file mode 100644 index 0000000..76b4500 --- /dev/null +++ b/src/go/internal/gccgoimporter/importer_test.go @@ -0,0 +1,199 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gccgoimporter + +import ( + "go/types" + "internal/testenv" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "testing" +) + +type importerTest struct { + pkgpath, name, want, wantval string + wantinits []string + gccgoVersion int // minimum gccgo version (0 => any) +} + +func runImporterTest(t *testing.T, imp Importer, initmap map[*types.Package]InitData, test *importerTest) { + pkg, err := imp(make(map[string]*types.Package), test.pkgpath, ".", nil) + if err != nil { + t.Error(err) + return + } + + if test.name != "" { + obj := pkg.Scope().Lookup(test.name) + if obj == nil { + t.Errorf("%s: object not found", test.name) + return + } + + got := types.ObjectString(obj, types.RelativeTo(pkg)) + if got != test.want { + t.Errorf("%s: got %q; want %q", test.name, got, test.want) + } + + if test.wantval != "" { + gotval := obj.(*types.Const).Val().String() + if gotval != test.wantval { + t.Errorf("%s: got val %q; want val %q", test.name, gotval, test.wantval) + } + } + } + + if len(test.wantinits) > 0 { + initdata := initmap[pkg] + found := false + // Check that the package's own init function has the package's priority + for _, pkginit := range initdata.Inits { + if pkginit.InitFunc == test.wantinits[0] { + found = true + break + } + } + + if !found { + t.Errorf("%s: could not find expected function %q", test.pkgpath, test.wantinits[0]) + } + + // FIXME: the original version of this test was written against + // the v1 export data scheme for capturing init functions, so it + // verified the priority values. We moved away from the priority + // scheme some time ago; it is not clear how much work it would be + // to validate the new init export data. + } +} + +// When adding tests to this list, be sure to set the 'gccgoVersion' +// field if the testcases uses a "recent" Go addition (ex: aliases). +var importerTests = [...]importerTest{ + {pkgpath: "pointer", name: "Int8Ptr", want: "type Int8Ptr *int8"}, + {pkgpath: "complexnums", name: "NN", want: "const NN untyped complex", wantval: "(-1 + -1i)"}, + {pkgpath: "complexnums", name: "NP", want: "const NP untyped complex", wantval: "(-1 + 1i)"}, + {pkgpath: "complexnums", name: "PN", want: "const PN untyped complex", wantval: "(1 + -1i)"}, + {pkgpath: "complexnums", name: "PP", want: "const PP untyped complex", wantval: "(1 + 1i)"}, + {pkgpath: "conversions", name: "Bits", want: "const Bits Units", wantval: `"bits"`}, + {pkgpath: "time", name: "Duration", want: "type Duration int64"}, + {pkgpath: "time", name: "Nanosecond", want: "const Nanosecond Duration", wantval: "1"}, + {pkgpath: "unicode", name: "IsUpper", want: "func IsUpper(r rune) bool"}, + {pkgpath: "unicode", name: "MaxRune", want: "const MaxRune untyped rune", wantval: "1114111"}, + {pkgpath: "imports", wantinits: []string{"imports..import", "fmt..import"}}, + {pkgpath: "importsar", name: "Hello", want: "var Hello string"}, + {pkgpath: "aliases", name: "A14", gccgoVersion: 7, want: "type A14 = func(int, T0) chan T2"}, + {pkgpath: "aliases", name: "C0", gccgoVersion: 7, want: "type C0 struct{f1 C1; f2 C1}"}, + {pkgpath: "escapeinfo", name: "NewT", want: "func NewT(data []byte) *T"}, + {pkgpath: "issue27856", name: "M", gccgoVersion: 7, want: "type M struct{E F}"}, + {pkgpath: "v1reflect", name: "Type", want: "type Type interface{Align() int; AssignableTo(u Type) bool; Bits() int; ChanDir() ChanDir; Elem() Type; Field(i int) StructField; FieldAlign() int; FieldByIndex(index []int) StructField; FieldByName(name string) (StructField, bool); FieldByNameFunc(match func(string) bool) (StructField, bool); Implements(u Type) bool; In(i int) Type; IsVariadic() bool; Key() Type; Kind() Kind; Len() int; Method(int) Method; MethodByName(string) (Method, bool); Name() string; NumField() int; NumIn() int; NumMethod() int; NumOut() int; Out(i int) Type; PkgPath() string; Size() uintptr; String() string; common() *commonType; rawString() string; runtimeType() *runtimeType; uncommon() *uncommonType}"}, + {pkgpath: "nointerface", name: "I", want: "type I int"}, + {pkgpath: "issue29198", name: "FooServer", gccgoVersion: 7, want: "type FooServer struct{FooServer *FooServer; user string; ctx context.Context}"}, + {pkgpath: "issue30628", name: "Apple", want: "type Apple struct{hey sync.RWMutex; x int; RQ [517]struct{Count uintptr; NumBytes uintptr; Last uintptr}}"}, + {pkgpath: "issue31540", name: "S", gccgoVersion: 7, want: "type S struct{b int; map[Y]Z}"}, // should want "type S struct{b int; A2}" (issue #44410) + {pkgpath: "issue34182", name: "T1", want: "type T1 struct{f *T2}"}, + {pkgpath: "notinheap", name: "S", want: "type S struct{}"}, +} + +func TestGoxImporter(t *testing.T) { + testenv.MustHaveExec(t) + initmap := make(map[*types.Package]InitData) + imp := GetImporter([]string{"testdata"}, initmap) + + for _, test := range importerTests { + runImporterTest(t, imp, initmap, &test) + } +} + +// gccgoPath returns a path to gccgo if it is present (either in +// path or specified via GCCGO environment variable), or an +// empty string if no gccgo is available. +func gccgoPath() string { + gccgoname := os.Getenv("GCCGO") + if gccgoname == "" { + gccgoname = "gccgo" + } + if gpath, gerr := exec.LookPath(gccgoname); gerr == nil { + return gpath + } + return "" +} + +func TestObjImporter(t *testing.T) { + // This test relies on gccgo being around. + gpath := gccgoPath() + if gpath == "" { + t.Skip("This test needs gccgo") + } + + verout, err := testenv.Command(t, gpath, "--version").CombinedOutput() + if err != nil { + t.Logf("%s", verout) + t.Fatal(err) + } + vers := regexp.MustCompile(`(\d+)\.(\d+)`).FindSubmatch(verout) + if len(vers) == 0 { + t.Fatalf("could not find version number in %s", verout) + } + major, err := strconv.Atoi(string(vers[1])) + if err != nil { + t.Fatal(err) + } + minor, err := strconv.Atoi(string(vers[2])) + if err != nil { + t.Fatal(err) + } + t.Logf("gccgo version %d.%d", major, minor) + + tmpdir := t.TempDir() + initmap := make(map[*types.Package]InitData) + imp := GetImporter([]string{tmpdir}, initmap) + + artmpdir := t.TempDir() + arinitmap := make(map[*types.Package]InitData) + arimp := GetImporter([]string{artmpdir}, arinitmap) + + for _, test := range importerTests { + if major < test.gccgoVersion { + // Support for type aliases was added in GCC 7. + t.Logf("skipping %q: not supported before gccgo version %d", test.pkgpath, test.gccgoVersion) + continue + } + + gofile := filepath.Join("testdata", test.pkgpath+".go") + if _, err := os.Stat(gofile); os.IsNotExist(err) { + continue + } + ofile := filepath.Join(tmpdir, test.pkgpath+".o") + afile := filepath.Join(artmpdir, "lib"+test.pkgpath+".a") + + cmd := testenv.Command(t, gpath, "-fgo-pkgpath="+test.pkgpath, "-c", "-o", ofile, gofile) + out, err := cmd.CombinedOutput() + if err != nil { + t.Logf("%s", out) + t.Fatalf("gccgo %s failed: %s", gofile, err) + } + + runImporterTest(t, imp, initmap, &test) + + cmd = testenv.Command(t, "ar", "cr", afile, ofile) + out, err = cmd.CombinedOutput() + if err != nil { + t.Logf("%s", out) + t.Fatalf("ar cr %s %s failed: %s", afile, ofile, err) + } + + runImporterTest(t, arimp, arinitmap, &test) + + if err = os.Remove(ofile); err != nil { + t.Fatal(err) + } + if err = os.Remove(afile); err != nil { + t.Fatal(err) + } + } +} diff --git a/src/go/internal/gccgoimporter/parser.go b/src/go/internal/gccgoimporter/parser.go new file mode 100644 index 0000000..de9df0b --- /dev/null +++ b/src/go/internal/gccgoimporter/parser.go @@ -0,0 +1,1283 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gccgoimporter + +import ( + "errors" + "fmt" + "go/constant" + "go/token" + "go/types" + "io" + "strconv" + "strings" + "text/scanner" + "unicode/utf8" +) + +type parser struct { + scanner *scanner.Scanner + version string // format version + tok rune // current token + lit string // literal string; only valid for Ident, Int, String tokens + pkgpath string // package path of imported package + pkgname string // name of imported package + pkg *types.Package // reference to imported package + imports map[string]*types.Package // package path -> package object + typeList []types.Type // type number -> type + typeData []string // unparsed type data (v3 and later) + fixups []fixupRecord // fixups to apply at end of parsing + initdata InitData // package init priority data + aliases map[int]string // maps saved type number to alias name +} + +// When reading export data it's possible to encounter a defined type +// N1 with an underlying defined type N2 while we are still reading in +// that defined type N2; see issues #29006 and #29198 for instances +// of this. Example: +// +// type N1 N2 +// type N2 struct { +// ... +// p *N1 +// } +// +// To handle such cases, the parser generates a fixup record (below) and +// delays setting of N1's underlying type until parsing is complete, at +// which point fixups are applied. + +type fixupRecord struct { + toUpdate *types.Named // type to modify when fixup is processed + target types.Type // type that was incomplete when fixup was created +} + +func (p *parser) init(filename string, src io.Reader, imports map[string]*types.Package) { + p.scanner = new(scanner.Scanner) + p.initScanner(filename, src) + p.imports = imports + p.aliases = make(map[int]string) + p.typeList = make([]types.Type, 1 /* type numbers start at 1 */, 16) +} + +func (p *parser) initScanner(filename string, src io.Reader) { + p.scanner.Init(src) + p.scanner.Error = func(_ *scanner.Scanner, msg string) { p.error(msg) } + p.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanFloats | scanner.ScanStrings + p.scanner.Whitespace = 1<<'\t' | 1<<' ' + p.scanner.Filename = filename // for good error messages + p.next() +} + +type importError struct { + pos scanner.Position + err error +} + +func (e importError) Error() string { + return fmt.Sprintf("import error %s (byte offset = %d): %s", e.pos, e.pos.Offset, e.err) +} + +func (p *parser) error(err any) { + if s, ok := err.(string); ok { + err = errors.New(s) + } + // panic with a runtime.Error if err is not an error + panic(importError{p.scanner.Pos(), err.(error)}) +} + +func (p *parser) errorf(format string, args ...any) { + p.error(fmt.Errorf(format, args...)) +} + +func (p *parser) expect(tok rune) string { + lit := p.lit + if p.tok != tok { + p.errorf("expected %s, got %s (%s)", scanner.TokenString(tok), scanner.TokenString(p.tok), lit) + } + p.next() + return lit +} + +func (p *parser) expectEOL() { + if p.version == "v1" || p.version == "v2" { + p.expect(';') + } + p.expect('\n') +} + +func (p *parser) expectKeyword(keyword string) { + lit := p.expect(scanner.Ident) + if lit != keyword { + p.errorf("expected keyword %s, got %q", keyword, lit) + } +} + +func (p *parser) parseString() string { + str, err := strconv.Unquote(p.expect(scanner.String)) + if err != nil { + p.error(err) + } + return str +} + +// unquotedString = { unquotedStringChar } . +// unquotedStringChar = . +func (p *parser) parseUnquotedString() string { + if p.tok == scanner.EOF { + p.error("unexpected EOF") + } + var b strings.Builder + b.WriteString(p.scanner.TokenText()) + // This loop needs to examine each character before deciding whether to consume it. If we see a semicolon, + // we need to let it be consumed by p.next(). + for ch := p.scanner.Peek(); ch != '\n' && ch != ';' && ch != scanner.EOF && p.scanner.Whitespace&(1<" . (optional and ignored) + p.next() + p.expectKeyword("esc") + p.expect(':') + p.expect(scanner.Int) + p.expect('>') + } + if p.tok == '.' { + p.next() + p.expect('.') + p.expect('.') + isVariadic = true + } + typ := p.parseType(pkg) + if isVariadic { + typ = types.NewSlice(typ) + } + param = types.NewParam(token.NoPos, pkg, name, typ) + return +} + +// Var = Name Type . +func (p *parser) parseVar(pkg *types.Package) *types.Var { + name := p.parseName() + v := types.NewVar(token.NoPos, pkg, name, p.parseType(pkg)) + if name[0] == '.' || name[0] == '<' { + // This is an unexported variable, + // or a variable defined in a different package. + // We only want to record exported variables. + return nil + } + return v +} + +// Conversion = "convert" "(" Type "," ConstValue ")" . +func (p *parser) parseConversion(pkg *types.Package) (val constant.Value, typ types.Type) { + p.expectKeyword("convert") + p.expect('(') + typ = p.parseType(pkg) + p.expect(',') + val, _ = p.parseConstValue(pkg) + p.expect(')') + return +} + +// ConstValue = string | "false" | "true" | ["-"] (int ["'"] | FloatOrComplex) | Conversion . +// FloatOrComplex = float ["i" | ("+"|"-") float "i"] . +func (p *parser) parseConstValue(pkg *types.Package) (val constant.Value, typ types.Type) { + // v3 changed to $false, $true, $convert, to avoid confusion + // with variable names in inline function bodies. + if p.tok == '$' { + p.next() + if p.tok != scanner.Ident { + p.errorf("expected identifier after '$', got %s (%q)", scanner.TokenString(p.tok), p.lit) + } + } + + switch p.tok { + case scanner.String: + str := p.parseString() + val = constant.MakeString(str) + typ = types.Typ[types.UntypedString] + return + + case scanner.Ident: + b := false + switch p.lit { + case "false": + case "true": + b = true + + case "convert": + return p.parseConversion(pkg) + + default: + p.errorf("expected const value, got %s (%q)", scanner.TokenString(p.tok), p.lit) + } + + p.next() + val = constant.MakeBool(b) + typ = types.Typ[types.UntypedBool] + return + } + + sign := "" + if p.tok == '-' { + p.next() + sign = "-" + } + + switch p.tok { + case scanner.Int: + val = constant.MakeFromLiteral(sign+p.lit, token.INT, 0) + if val == nil { + p.error("could not parse integer literal") + } + + p.next() + if p.tok == '\'' { + p.next() + typ = types.Typ[types.UntypedRune] + } else { + typ = types.Typ[types.UntypedInt] + } + + case scanner.Float: + re := sign + p.lit + p.next() + + var im string + switch p.tok { + case '+': + p.next() + im = p.expect(scanner.Float) + + case '-': + p.next() + im = "-" + p.expect(scanner.Float) + + case scanner.Ident: + // re is in fact the imaginary component. Expect "i" below. + im = re + re = "0" + + default: + val = constant.MakeFromLiteral(re, token.FLOAT, 0) + if val == nil { + p.error("could not parse float literal") + } + typ = types.Typ[types.UntypedFloat] + return + } + + p.expectKeyword("i") + reval := constant.MakeFromLiteral(re, token.FLOAT, 0) + if reval == nil { + p.error("could not parse real component of complex literal") + } + imval := constant.MakeFromLiteral(im+"i", token.IMAG, 0) + if imval == nil { + p.error("could not parse imag component of complex literal") + } + val = constant.BinaryOp(reval, token.ADD, imval) + typ = types.Typ[types.UntypedComplex] + + default: + p.errorf("expected const value, got %s (%q)", scanner.TokenString(p.tok), p.lit) + } + + return +} + +// Const = Name [Type] "=" ConstValue . +func (p *parser) parseConst(pkg *types.Package) *types.Const { + name := p.parseName() + var typ types.Type + if p.tok == '<' { + typ = p.parseType(pkg) + } + p.expect('=') + val, vtyp := p.parseConstValue(pkg) + if typ == nil { + typ = vtyp + } + return types.NewConst(token.NoPos, pkg, name, typ, val) +} + +// reserved is a singleton type used to fill type map slots that have +// been reserved (i.e., for which a type number has been parsed) but +// which don't have their actual type yet. When the type map is updated, +// the actual type must replace a reserved entry (or we have an internal +// error). Used for self-verification only - not required for correctness. +var reserved = new(struct{ types.Type }) + +// reserve reserves the type map entry n for future use. +func (p *parser) reserve(n int) { + // Notes: + // - for pre-V3 export data, the type numbers we see are + // guaranteed to be in increasing order, so we append a + // reserved entry onto the list. + // - for V3+ export data, type numbers can appear in + // any order, however the 'types' section tells us the + // total number of types, hence typeList is pre-allocated. + if len(p.typeData) == 0 { + if n != len(p.typeList) { + p.errorf("invalid type number %d (out of sync)", n) + } + p.typeList = append(p.typeList, reserved) + } else { + if p.typeList[n] != nil { + p.errorf("previously visited type number %d", n) + } + p.typeList[n] = reserved + } +} + +// update sets the type map entries for the entries in nlist to t. +// An entry in nlist can be a type number in p.typeList, +// used to resolve named types, or it can be a *types.Pointer, +// used to resolve pointers to named types in case they are referenced +// by embedded fields. +func (p *parser) update(t types.Type, nlist []any) { + if t == reserved { + p.errorf("internal error: update(%v) invoked on reserved", nlist) + } + if t == nil { + p.errorf("internal error: update(%v) invoked on nil", nlist) + } + for _, n := range nlist { + switch n := n.(type) { + case int: + if p.typeList[n] == t { + continue + } + if p.typeList[n] != reserved { + p.errorf("internal error: update(%v): %d not reserved", nlist, n) + } + p.typeList[n] = t + case *types.Pointer: + if *n != (types.Pointer{}) { + elem := n.Elem() + if elem == t { + continue + } + p.errorf("internal error: update: pointer already set to %v, expected %v", elem, t) + } + *n = *types.NewPointer(t) + default: + p.errorf("internal error: %T on nlist", n) + } + } +} + +// NamedType = TypeName [ "=" ] Type { Method } . +// TypeName = ExportedName . +// Method = "func" "(" Param ")" Name ParamList ResultList [InlineBody] ";" . +func (p *parser) parseNamedType(nlist []any) types.Type { + pkg, name := p.parseExportedName() + scope := pkg.Scope() + obj := scope.Lookup(name) + if obj != nil && obj.Type() == nil { + p.errorf("%v has nil type", obj) + } + + if p.tok == scanner.Ident && p.lit == "notinheap" { + p.next() + // The go/types package has no way of recording that + // this type is marked notinheap. Presumably no user + // of this package actually cares. + } + + // type alias + if p.tok == '=' { + p.next() + p.aliases[nlist[len(nlist)-1].(int)] = name + if obj != nil { + // use the previously imported (canonical) type + t := obj.Type() + p.update(t, nlist) + p.parseType(pkg) // discard + return t + } + t := p.parseType(pkg, nlist...) + obj = types.NewTypeName(token.NoPos, pkg, name, t) + scope.Insert(obj) + return t + } + + // defined type + if obj == nil { + // A named type may be referred to before the underlying type + // is known - set it up. + tname := types.NewTypeName(token.NoPos, pkg, name, nil) + types.NewNamed(tname, nil, nil) + scope.Insert(tname) + obj = tname + } + + // use the previously imported (canonical), or newly created type + t := obj.Type() + p.update(t, nlist) + + nt, ok := t.(*types.Named) + if !ok { + // This can happen for unsafe.Pointer, which is a TypeName holding a Basic type. + pt := p.parseType(pkg) + if pt != t { + p.error("unexpected underlying type for non-named TypeName") + } + return t + } + + underlying := p.parseType(pkg) + if nt.Underlying() == nil { + if underlying.Underlying() == nil { + fix := fixupRecord{toUpdate: nt, target: underlying} + p.fixups = append(p.fixups, fix) + } else { + nt.SetUnderlying(underlying.Underlying()) + } + } + + if p.tok == '\n' { + p.next() + // collect associated methods + for p.tok == scanner.Ident { + p.expectKeyword("func") + if p.tok == '/' { + // Skip a /*nointerface*/ or /*asm ID */ comment. + p.expect('/') + p.expect('*') + if p.expect(scanner.Ident) == "asm" { + p.parseUnquotedString() + } + p.expect('*') + p.expect('/') + } + p.expect('(') + receiver, _ := p.parseParam(pkg) + p.expect(')') + name := p.parseName() + params, isVariadic := p.parseParamList(pkg) + results := p.parseResultList(pkg) + p.skipInlineBody() + p.expectEOL() + + sig := types.NewSignatureType(receiver, nil, nil, params, results, isVariadic) + nt.AddMethod(types.NewFunc(token.NoPos, pkg, name, sig)) + } + } + + return nt +} + +func (p *parser) parseInt64() int64 { + lit := p.expect(scanner.Int) + n, err := strconv.ParseInt(lit, 10, 64) + if err != nil { + p.error(err) + } + return n +} + +func (p *parser) parseInt() int { + lit := p.expect(scanner.Int) + n, err := strconv.ParseInt(lit, 10, 0 /* int */) + if err != nil { + p.error(err) + } + return int(n) +} + +// ArrayOrSliceType = "[" [ int ] "]" Type . +func (p *parser) parseArrayOrSliceType(pkg *types.Package, nlist []any) types.Type { + p.expect('[') + if p.tok == ']' { + p.next() + + t := new(types.Slice) + p.update(t, nlist) + + *t = *types.NewSlice(p.parseType(pkg)) + return t + } + + t := new(types.Array) + p.update(t, nlist) + + len := p.parseInt64() + p.expect(']') + + *t = *types.NewArray(p.parseType(pkg), len) + return t +} + +// MapType = "map" "[" Type "]" Type . +func (p *parser) parseMapType(pkg *types.Package, nlist []any) types.Type { + p.expectKeyword("map") + + t := new(types.Map) + p.update(t, nlist) + + p.expect('[') + key := p.parseType(pkg) + p.expect(']') + elem := p.parseType(pkg) + + *t = *types.NewMap(key, elem) + return t +} + +// ChanType = "chan" ["<-" | "-<"] Type . +func (p *parser) parseChanType(pkg *types.Package, nlist []any) types.Type { + p.expectKeyword("chan") + + t := new(types.Chan) + p.update(t, nlist) + + dir := types.SendRecv + switch p.tok { + case '-': + p.next() + p.expect('<') + dir = types.SendOnly + + case '<': + // don't consume '<' if it belongs to Type + if p.scanner.Peek() == '-' { + p.next() + p.expect('-') + dir = types.RecvOnly + } + } + + *t = *types.NewChan(dir, p.parseType(pkg)) + return t +} + +// StructType = "struct" "{" { Field } "}" . +func (p *parser) parseStructType(pkg *types.Package, nlist []any) types.Type { + p.expectKeyword("struct") + + t := new(types.Struct) + p.update(t, nlist) + + var fields []*types.Var + var tags []string + + p.expect('{') + for p.tok != '}' && p.tok != scanner.EOF { + field, tag := p.parseField(pkg) + p.expect(';') + fields = append(fields, field) + tags = append(tags, tag) + } + p.expect('}') + + *t = *types.NewStruct(fields, tags) + return t +} + +// ParamList = "(" [ { Parameter "," } Parameter ] ")" . +func (p *parser) parseParamList(pkg *types.Package) (*types.Tuple, bool) { + var list []*types.Var + isVariadic := false + + p.expect('(') + for p.tok != ')' && p.tok != scanner.EOF { + if len(list) > 0 { + p.expect(',') + } + par, variadic := p.parseParam(pkg) + list = append(list, par) + if variadic { + if isVariadic { + p.error("... not on final argument") + } + isVariadic = true + } + } + p.expect(')') + + return types.NewTuple(list...), isVariadic +} + +// ResultList = Type | ParamList . +func (p *parser) parseResultList(pkg *types.Package) *types.Tuple { + switch p.tok { + case '<': + p.next() + if p.tok == scanner.Ident && p.lit == "inl" { + return nil + } + taa, _ := p.parseTypeAfterAngle(pkg) + return types.NewTuple(types.NewParam(token.NoPos, pkg, "", taa)) + + case '(': + params, _ := p.parseParamList(pkg) + return params + + default: + return nil + } +} + +// FunctionType = ParamList ResultList . +func (p *parser) parseFunctionType(pkg *types.Package, nlist []any) *types.Signature { + t := new(types.Signature) + p.update(t, nlist) + + params, isVariadic := p.parseParamList(pkg) + results := p.parseResultList(pkg) + + *t = *types.NewSignatureType(nil, nil, nil, params, results, isVariadic) + return t +} + +// Func = Name FunctionType [InlineBody] . +func (p *parser) parseFunc(pkg *types.Package) *types.Func { + if p.tok == '/' { + // Skip an /*asm ID */ comment. + p.expect('/') + p.expect('*') + if p.expect(scanner.Ident) == "asm" { + p.parseUnquotedString() + } + p.expect('*') + p.expect('/') + } + + name := p.parseName() + f := types.NewFunc(token.NoPos, pkg, name, p.parseFunctionType(pkg, nil)) + p.skipInlineBody() + + if name[0] == '.' || name[0] == '<' || strings.ContainsRune(name, '$') { + // This is an unexported function, + // or a function defined in a different package, + // or a type$equal or type$hash function. + // We only want to record exported functions. + return nil + } + + return f +} + +// InterfaceType = "interface" "{" { ("?" Type | Func) ";" } "}" . +func (p *parser) parseInterfaceType(pkg *types.Package, nlist []any) types.Type { + p.expectKeyword("interface") + + t := new(types.Interface) + p.update(t, nlist) + + var methods []*types.Func + var embeddeds []types.Type + + p.expect('{') + for p.tok != '}' && p.tok != scanner.EOF { + if p.tok == '?' { + p.next() + embeddeds = append(embeddeds, p.parseType(pkg)) + } else { + method := p.parseFunc(pkg) + if method != nil { + methods = append(methods, method) + } + } + p.expect(';') + } + p.expect('}') + + *t = *types.NewInterfaceType(methods, embeddeds) + return t +} + +// PointerType = "*" ("any" | Type) . +func (p *parser) parsePointerType(pkg *types.Package, nlist []any) types.Type { + p.expect('*') + if p.tok == scanner.Ident { + p.expectKeyword("any") + t := types.Typ[types.UnsafePointer] + p.update(t, nlist) + return t + } + + t := new(types.Pointer) + p.update(t, nlist) + + *t = *types.NewPointer(p.parseType(pkg, t)) + + return t +} + +// TypeSpec = NamedType | MapType | ChanType | StructType | InterfaceType | PointerType | ArrayOrSliceType | FunctionType . +func (p *parser) parseTypeSpec(pkg *types.Package, nlist []any) types.Type { + switch p.tok { + case scanner.String: + return p.parseNamedType(nlist) + + case scanner.Ident: + switch p.lit { + case "map": + return p.parseMapType(pkg, nlist) + + case "chan": + return p.parseChanType(pkg, nlist) + + case "struct": + return p.parseStructType(pkg, nlist) + + case "interface": + return p.parseInterfaceType(pkg, nlist) + } + + case '*': + return p.parsePointerType(pkg, nlist) + + case '[': + return p.parseArrayOrSliceType(pkg, nlist) + + case '(': + return p.parseFunctionType(pkg, nlist) + } + + p.errorf("expected type name or literal, got %s", scanner.TokenString(p.tok)) + return nil +} + +const ( + // From gofrontend/go/export.h + // Note that these values are negative in the gofrontend and have been made positive + // in the gccgoimporter. + gccgoBuiltinINT8 = 1 + gccgoBuiltinINT16 = 2 + gccgoBuiltinINT32 = 3 + gccgoBuiltinINT64 = 4 + gccgoBuiltinUINT8 = 5 + gccgoBuiltinUINT16 = 6 + gccgoBuiltinUINT32 = 7 + gccgoBuiltinUINT64 = 8 + gccgoBuiltinFLOAT32 = 9 + gccgoBuiltinFLOAT64 = 10 + gccgoBuiltinINT = 11 + gccgoBuiltinUINT = 12 + gccgoBuiltinUINTPTR = 13 + gccgoBuiltinBOOL = 15 + gccgoBuiltinSTRING = 16 + gccgoBuiltinCOMPLEX64 = 17 + gccgoBuiltinCOMPLEX128 = 18 + gccgoBuiltinERROR = 19 + gccgoBuiltinBYTE = 20 + gccgoBuiltinRUNE = 21 +) + +func lookupBuiltinType(typ int) types.Type { + return [...]types.Type{ + gccgoBuiltinINT8: types.Typ[types.Int8], + gccgoBuiltinINT16: types.Typ[types.Int16], + gccgoBuiltinINT32: types.Typ[types.Int32], + gccgoBuiltinINT64: types.Typ[types.Int64], + gccgoBuiltinUINT8: types.Typ[types.Uint8], + gccgoBuiltinUINT16: types.Typ[types.Uint16], + gccgoBuiltinUINT32: types.Typ[types.Uint32], + gccgoBuiltinUINT64: types.Typ[types.Uint64], + gccgoBuiltinFLOAT32: types.Typ[types.Float32], + gccgoBuiltinFLOAT64: types.Typ[types.Float64], + gccgoBuiltinINT: types.Typ[types.Int], + gccgoBuiltinUINT: types.Typ[types.Uint], + gccgoBuiltinUINTPTR: types.Typ[types.Uintptr], + gccgoBuiltinBOOL: types.Typ[types.Bool], + gccgoBuiltinSTRING: types.Typ[types.String], + gccgoBuiltinCOMPLEX64: types.Typ[types.Complex64], + gccgoBuiltinCOMPLEX128: types.Typ[types.Complex128], + gccgoBuiltinERROR: types.Universe.Lookup("error").Type(), + gccgoBuiltinBYTE: types.Universe.Lookup("byte").Type(), + gccgoBuiltinRUNE: types.Universe.Lookup("rune").Type(), + }[typ] +} + +// Type = "<" "type" ( "-" int | int [ TypeSpec ] ) ">" . +// +// parseType updates the type map to t for all type numbers n. +func (p *parser) parseType(pkg *types.Package, n ...any) types.Type { + p.expect('<') + t, _ := p.parseTypeAfterAngle(pkg, n...) + return t +} + +// (*parser).Type after reading the "<". +func (p *parser) parseTypeAfterAngle(pkg *types.Package, n ...any) (t types.Type, n1 int) { + p.expectKeyword("type") + + n1 = 0 + switch p.tok { + case scanner.Int: + n1 = p.parseInt() + if p.tok == '>' { + if len(p.typeData) > 0 && p.typeList[n1] == nil { + p.parseSavedType(pkg, n1, n) + } + t = p.typeList[n1] + if len(p.typeData) == 0 && t == reserved { + p.errorf("invalid type cycle, type %d not yet defined (nlist=%v)", n1, n) + } + p.update(t, n) + } else { + p.reserve(n1) + t = p.parseTypeSpec(pkg, append(n, n1)) + } + + case '-': + p.next() + n1 := p.parseInt() + t = lookupBuiltinType(n1) + p.update(t, n) + + default: + p.errorf("expected type number, got %s (%q)", scanner.TokenString(p.tok), p.lit) + return nil, 0 + } + + if t == nil || t == reserved { + p.errorf("internal error: bad return from parseType(%v)", n) + } + + p.expect('>') + return +} + +// parseTypeExtended is identical to parseType, but if the type in +// question is a saved type, returns the index as well as the type +// pointer (index returned is zero if we parsed a builtin). +func (p *parser) parseTypeExtended(pkg *types.Package, n ...any) (t types.Type, n1 int) { + p.expect('<') + t, n1 = p.parseTypeAfterAngle(pkg, n...) + return +} + +// InlineBody = "" .{NN} +// Reports whether a body was skipped. +func (p *parser) skipInlineBody() { + // We may or may not have seen the '<' already, depending on + // whether the function had a result type or not. + if p.tok == '<' { + p.next() + p.expectKeyword("inl") + } else if p.tok != scanner.Ident || p.lit != "inl" { + return + } else { + p.next() + } + + p.expect(':') + want := p.parseInt() + p.expect('>') + + defer func(w uint64) { + p.scanner.Whitespace = w + }(p.scanner.Whitespace) + p.scanner.Whitespace = 0 + + got := 0 + for got < want { + r := p.scanner.Next() + if r == scanner.EOF { + p.error("unexpected EOF") + } + got += utf8.RuneLen(r) + } +} + +// Types = "types" maxp1 exportedp1 (offset length)* . +func (p *parser) parseTypes(pkg *types.Package) { + maxp1 := p.parseInt() + exportedp1 := p.parseInt() + p.typeList = make([]types.Type, maxp1, maxp1) + + type typeOffset struct { + offset int + length int + } + var typeOffsets []typeOffset + + total := 0 + for i := 1; i < maxp1; i++ { + len := p.parseInt() + typeOffsets = append(typeOffsets, typeOffset{total, len}) + total += len + } + + defer func(w uint64) { + p.scanner.Whitespace = w + }(p.scanner.Whitespace) + p.scanner.Whitespace = 0 + + // We should now have p.tok pointing to the final newline. + // The next runes from the scanner should be the type data. + + var sb strings.Builder + for sb.Len() < total { + r := p.scanner.Next() + if r == scanner.EOF { + p.error("unexpected EOF") + } + sb.WriteRune(r) + } + allTypeData := sb.String() + + p.typeData = []string{""} // type 0, unused + for _, to := range typeOffsets { + p.typeData = append(p.typeData, allTypeData[to.offset:to.offset+to.length]) + } + + for i := 1; i < int(exportedp1); i++ { + p.parseSavedType(pkg, i, nil) + } +} + +// parseSavedType parses one saved type definition. +func (p *parser) parseSavedType(pkg *types.Package, i int, nlist []any) { + defer func(s *scanner.Scanner, tok rune, lit string) { + p.scanner = s + p.tok = tok + p.lit = lit + }(p.scanner, p.tok, p.lit) + + p.scanner = new(scanner.Scanner) + p.initScanner(p.scanner.Filename, strings.NewReader(p.typeData[i])) + p.expectKeyword("type") + id := p.parseInt() + if id != i { + p.errorf("type ID mismatch: got %d, want %d", id, i) + } + if p.typeList[i] == reserved { + p.errorf("internal error: %d already reserved in parseSavedType", i) + } + if p.typeList[i] == nil { + p.reserve(i) + p.parseTypeSpec(pkg, append(nlist, i)) + } + if p.typeList[i] == nil || p.typeList[i] == reserved { + p.errorf("internal error: parseSavedType(%d,%v) reserved/nil", i, nlist) + } +} + +// PackageInit = unquotedString unquotedString int . +func (p *parser) parsePackageInit() PackageInit { + name := p.parseUnquotedString() + initfunc := p.parseUnquotedString() + priority := -1 + if p.version == "v1" { + priority = p.parseInt() + } + return PackageInit{Name: name, InitFunc: initfunc, Priority: priority} +} + +// Create the package if we have parsed both the package path and package name. +func (p *parser) maybeCreatePackage() { + if p.pkgname != "" && p.pkgpath != "" { + p.pkg = p.getPkg(p.pkgpath, p.pkgname) + } +} + +// InitDataDirective = ( "v1" | "v2" | "v3" ) ";" | +// +// "priority" int ";" | +// "init" { PackageInit } ";" | +// "checksum" unquotedString ";" . +func (p *parser) parseInitDataDirective() { + if p.tok != scanner.Ident { + // unexpected token kind; panic + p.expect(scanner.Ident) + } + + switch p.lit { + case "v1", "v2", "v3": + p.version = p.lit + p.next() + p.expect(';') + p.expect('\n') + + case "priority": + p.next() + p.initdata.Priority = p.parseInt() + p.expectEOL() + + case "init": + p.next() + for p.tok != '\n' && p.tok != ';' && p.tok != scanner.EOF { + p.initdata.Inits = append(p.initdata.Inits, p.parsePackageInit()) + } + p.expectEOL() + + case "init_graph": + p.next() + // The graph data is thrown away for now. + for p.tok != '\n' && p.tok != ';' && p.tok != scanner.EOF { + p.parseInt64() + p.parseInt64() + } + p.expectEOL() + + case "checksum": + // Don't let the scanner try to parse the checksum as a number. + defer func(mode uint) { + p.scanner.Mode = mode + }(p.scanner.Mode) + p.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats + p.next() + p.parseUnquotedString() + p.expectEOL() + + default: + p.errorf("unexpected identifier: %q", p.lit) + } +} + +// Directive = InitDataDirective | +// +// "package" unquotedString [ unquotedString ] [ unquotedString ] ";" | +// "pkgpath" unquotedString ";" | +// "prefix" unquotedString ";" | +// "import" unquotedString unquotedString string ";" | +// "indirectimport" unquotedString unquotedstring ";" | +// "func" Func ";" | +// "type" Type ";" | +// "var" Var ";" | +// "const" Const ";" . +func (p *parser) parseDirective() { + if p.tok != scanner.Ident { + // unexpected token kind; panic + p.expect(scanner.Ident) + } + + switch p.lit { + case "v1", "v2", "v3", "priority", "init", "init_graph", "checksum": + p.parseInitDataDirective() + + case "package": + p.next() + p.pkgname = p.parseUnquotedString() + p.maybeCreatePackage() + if p.version != "v1" && p.tok != '\n' && p.tok != ';' { + p.parseUnquotedString() + p.parseUnquotedString() + } + p.expectEOL() + + case "pkgpath": + p.next() + p.pkgpath = p.parseUnquotedString() + p.maybeCreatePackage() + p.expectEOL() + + case "prefix": + p.next() + p.pkgpath = p.parseUnquotedString() + p.expectEOL() + + case "import": + p.next() + pkgname := p.parseUnquotedString() + pkgpath := p.parseUnquotedString() + p.getPkg(pkgpath, pkgname) + p.parseString() + p.expectEOL() + + case "indirectimport": + p.next() + pkgname := p.parseUnquotedString() + pkgpath := p.parseUnquotedString() + p.getPkg(pkgpath, pkgname) + p.expectEOL() + + case "types": + p.next() + p.parseTypes(p.pkg) + p.expectEOL() + + case "func": + p.next() + fun := p.parseFunc(p.pkg) + if fun != nil { + p.pkg.Scope().Insert(fun) + } + p.expectEOL() + + case "type": + p.next() + p.parseType(p.pkg) + p.expectEOL() + + case "var": + p.next() + v := p.parseVar(p.pkg) + if v != nil { + p.pkg.Scope().Insert(v) + } + p.expectEOL() + + case "const": + p.next() + c := p.parseConst(p.pkg) + p.pkg.Scope().Insert(c) + p.expectEOL() + + default: + p.errorf("unexpected identifier: %q", p.lit) + } +} + +// Package = { Directive } . +func (p *parser) parsePackage() *types.Package { + for p.tok != scanner.EOF { + p.parseDirective() + } + for _, f := range p.fixups { + if f.target.Underlying() == nil { + p.errorf("internal error: fixup can't be applied, loop required") + } + f.toUpdate.SetUnderlying(f.target.Underlying()) + } + p.fixups = nil + for _, typ := range p.typeList { + if it, ok := typ.(*types.Interface); ok { + it.Complete() + } + } + p.pkg.MarkComplete() + return p.pkg +} diff --git a/src/go/internal/gccgoimporter/parser_test.go b/src/go/internal/gccgoimporter/parser_test.go new file mode 100644 index 0000000..00128b4 --- /dev/null +++ b/src/go/internal/gccgoimporter/parser_test.go @@ -0,0 +1,78 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gccgoimporter + +import ( + "bytes" + "go/types" + "strings" + "testing" + "text/scanner" +) + +var typeParserTests = []struct { + id, typ, want, underlying, methods string +}{ + {id: "foo", typ: "", want: "int8"}, + {id: "foo", typ: ">", want: "*error"}, + {id: "foo", typ: "", want: "unsafe.Pointer"}, + {id: "foo", typ: ">>", want: "foo.Bar", underlying: "*foo.Bar"}, + {id: "foo", typ: "\nfunc (? ) M ();\n>", want: "bar.Foo", underlying: "int8", methods: "func (bar.Foo).M()"}, + {id: "foo", typ: ">", want: "bar.foo", underlying: "int8"}, + {id: "foo", typ: ">", want: "[]int8"}, + {id: "foo", typ: ">", want: "[42]int8"}, + {id: "foo", typ: "] >", want: "map[int8]int16"}, + {id: "foo", typ: ">", want: "chan int8"}, + {id: "foo", typ: ">", want: "<-chan int8"}, + {id: "foo", typ: ">", want: "chan<- int8"}, + {id: "foo", typ: "; I16 \"i16\"; }>", want: "struct{I8 int8; I16 int16 \"i16\"}"}, + {id: "foo", typ: ", b ) ; Bar (? , ? ...) (? , ? ); Baz (); }>", want: "interface{Bar(int16, ...int8) (int16, int8); Baz(); Foo(a int8, b int16) int8}"}, + {id: "foo", typ: ") >", want: "func(int8) int16"}, +} + +func TestTypeParser(t *testing.T) { + for _, test := range typeParserTests { + var p parser + p.init("test.gox", strings.NewReader(test.typ), make(map[string]*types.Package)) + p.version = "v2" + p.pkgname = test.id + p.pkgpath = test.id + p.maybeCreatePackage() + typ := p.parseType(p.pkg) + + if p.tok != scanner.EOF { + t.Errorf("expected full parse, stopped at %q", p.lit) + } + + // interfaces must be explicitly completed + if ityp, _ := typ.(*types.Interface); ityp != nil { + ityp.Complete() + } + + got := typ.String() + if got != test.want { + t.Errorf("got type %q, expected %q", got, test.want) + } + + if test.underlying != "" { + underlying := typ.Underlying().String() + if underlying != test.underlying { + t.Errorf("got underlying type %q, expected %q", underlying, test.underlying) + } + } + + if test.methods != "" { + nt := typ.(*types.Named) + var buf bytes.Buffer + for i := 0; i != nt.NumMethods(); i++ { + buf.WriteString(nt.Method(i).String()) + } + methods := buf.String() + if methods != test.methods { + t.Errorf("got methods %q, expected %q", methods, test.methods) + } + } + } +} diff --git a/src/go/internal/gccgoimporter/testdata/aliases.go b/src/go/internal/gccgoimporter/testdata/aliases.go new file mode 100644 index 0000000..cfb59b3 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/aliases.go @@ -0,0 +1,65 @@ +package aliases + +type ( + T0 [10]int + T1 []byte + T2 struct { + x int + } + T3 interface { + m() T2 + } + T4 func(int, T0) chan T2 +) + +// basic aliases +type ( + Ai = int + A0 = T0 + A1 = T1 + A2 = T2 + A3 = T3 + A4 = T4 + + A10 = [10]int + A11 = []byte + A12 = struct { + x int + } + A13 = interface { + m() A2 + } + A14 = func(int, A0) chan A2 +) + +// alias receiver types +func (T0) m1() {} +func (A0) m2() {} + +// alias receiver types (long type declaration chains) +type ( + V0 = V1 + V1 = (V2) + V2 = (V3) + V3 = T0 +) + +func (V1) n() {} + +// cycles +type C0 struct { + f1 C1 + f2 C2 +} + +type ( + C1 *C0 + C2 = C1 +) + +type ( + C5 struct { + f *C6 + } + C6 = C5 +) diff --git a/src/go/internal/gccgoimporter/testdata/aliases.gox b/src/go/internal/gccgoimporter/testdata/aliases.gox new file mode 100644 index 0000000..2428c06 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/aliases.gox @@ -0,0 +1,33 @@ +v2; +package aliases; +prefix go; +package aliases go.aliases go.aliases; +type > + func (? ) .go.aliases.m1 (); + func (? ) .go.aliases.m2 (); + func (? >>>) .go.aliases.n (); +>>; +type >>>; +type >>; +type >>; +type ; }>>; +type ; }>>>; }>>; +type , ? ) >>>; +type ; +type ; }>>>; +type , ? ) >>>>; +type >; +type >>; .go.aliases.f2 >; }>>; +type ; +type ; +type >>; }>>; +type ; +type ; +type ; +type ; +type ; +type ; +type >; +type ; +type ; +type ; diff --git a/src/go/internal/gccgoimporter/testdata/complexnums.go b/src/go/internal/gccgoimporter/testdata/complexnums.go new file mode 100644 index 0000000..a51b6b0 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/complexnums.go @@ -0,0 +1,6 @@ +package complexnums + +const NN = -1 - 1i +const NP = -1 + 1i +const PN = 1 - 1i +const PP = 1 + 1i diff --git a/src/go/internal/gccgoimporter/testdata/complexnums.gox b/src/go/internal/gccgoimporter/testdata/complexnums.gox new file mode 100644 index 0000000..b66524f --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/complexnums.gox @@ -0,0 +1,8 @@ +v1; +package complexnums; +pkgpath complexnums; +priority 1; +const NN = -0.1E1-0.1E1i ; +const NP = -0.1E1+0.1E1i ; +const PN = 0.1E1-0.1E1i ; +const PP = 0.1E1+0.1E1i ; diff --git a/src/go/internal/gccgoimporter/testdata/conversions.go b/src/go/internal/gccgoimporter/testdata/conversions.go new file mode 100644 index 0000000..653927a --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/conversions.go @@ -0,0 +1,5 @@ +package conversions + +type Units string + +const Bits = Units("bits") diff --git a/src/go/internal/gccgoimporter/testdata/conversions.gox b/src/go/internal/gccgoimporter/testdata/conversions.gox new file mode 100644 index 0000000..7de6cda --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/conversions.gox @@ -0,0 +1,6 @@ +v2; +package conversions; +prefix go; +package conversions go.conversions go.conversions; +const Bits > = convert(, "bits"); +type ; diff --git a/src/go/internal/gccgoimporter/testdata/escapeinfo.go b/src/go/internal/gccgoimporter/testdata/escapeinfo.go new file mode 100644 index 0000000..103ad95 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/escapeinfo.go @@ -0,0 +1,13 @@ +// Test case for escape info in export data. To compile and extract .gox file: +// gccgo -fgo-optimize-allocs -c escapeinfo.go +// objcopy -j .go_export escapeinfo.o escapeinfo.gox + +package escapeinfo + +type T struct{ data []byte } + +func NewT(data []byte) *T { + return &T{data} +} + +func (*T) Read(p []byte) {} diff --git a/src/go/internal/gccgoimporter/testdata/escapeinfo.gox b/src/go/internal/gccgoimporter/testdata/escapeinfo.gox new file mode 100644 index 0000000..94ce039 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/escapeinfo.gox @@ -0,0 +1,9 @@ +v2; +package escapeinfo; +prefix go; +package escapeinfo go.escapeinfo go.escapeinfo; +func NewT (data >) >; }> + func (? >) Read (p >); +>>; +type ; +checksum 3500838130783C0059CD0C81527F60E9738E3ACE; diff --git a/src/go/internal/gccgoimporter/testdata/imports.go b/src/go/internal/gccgoimporter/testdata/imports.go new file mode 100644 index 0000000..7907316 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/imports.go @@ -0,0 +1,5 @@ +package imports + +import "fmt" + +var Hello = fmt.Sprintf("Hello, world") diff --git a/src/go/internal/gccgoimporter/testdata/imports.gox b/src/go/internal/gccgoimporter/testdata/imports.gox new file mode 100644 index 0000000..958a4f5 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/imports.gox @@ -0,0 +1,7 @@ +v1; +package imports; +pkgpath imports; +priority 7; +import fmt fmt "fmt"; +init imports imports..import 7 math math..import 1 runtime runtime..import 1 strconv strconv..import 2 io io..import 3 reflect reflect..import 3 syscall syscall..import 3 time time..import 4 os os..import 5 fmt fmt..import 6; +var Hello ; diff --git a/src/go/internal/gccgoimporter/testdata/issue27856.go b/src/go/internal/gccgoimporter/testdata/issue27856.go new file mode 100644 index 0000000..bf361e1 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/issue27856.go @@ -0,0 +1,9 @@ +package lib + +type M struct { + E E +} +type F struct { + _ *M +} +type E = F diff --git a/src/go/internal/gccgoimporter/testdata/issue27856.gox b/src/go/internal/gccgoimporter/testdata/issue27856.gox new file mode 100644 index 0000000..6665e64 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/issue27856.gox @@ -0,0 +1,9 @@ +v2; +package main; +pkgpath main; +import runtime runtime "runtime"; +init runtime runtime..import sys runtime_internal_sys..import; +init_graph 0 1; +type ; }>>>; }>>>; +type ; +type ; diff --git a/src/go/internal/gccgoimporter/testdata/issue29198.go b/src/go/internal/gccgoimporter/testdata/issue29198.go new file mode 100644 index 0000000..75c2162 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/issue29198.go @@ -0,0 +1,37 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package server + +import ( + "context" + "errors" +) + +type A struct { + x int +} + +func (a *A) AMethod(y int) *Server { + return nil +} + +// FooServer is a server that provides Foo services +type FooServer Server + +func (f *FooServer) WriteEvents(ctx context.Context, x int) error { + return errors.New("hey!") +} + +type Server struct { + FooServer *FooServer + user string + ctx context.Context +} + +func New(sctx context.Context, u string) (*Server, error) { + s := &Server{user: u, ctx: sctx} + s.FooServer = (*FooServer)(s) + return s, nil +} diff --git a/src/go/internal/gccgoimporter/testdata/issue29198.gox b/src/go/internal/gccgoimporter/testdata/issue29198.gox new file mode 100644 index 0000000..905c866 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/issue29198.gox @@ -0,0 +1,86 @@ +v2; +package server; +pkgpath issue29198; +import context context "context"; +import errors errors "errors"; +init context context..import fmt fmt..import poll internal_poll..import testlog internal_testlog..import io io..import os os..import reflect reflect..import runtime runtime..import sys runtime_internal_sys..import strconv strconv..import sync sync..import syscall syscall..import time time..import unicode unicode..import; +init_graph 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9 0 10 0 11 0 12 0 13 1 2 1 3 1 4 1 5 1 6 1 7 1 8 1 9 1 10 1 11 1 12 1 13 2 4 2 7 2 8 2 10 2 11 2 12 4 7 4 8 4 10 5 2 5 3 5 4 5 7 5 8 5 10 5 11 5 12 6 7 6 8 6 9 6 10 6 13 7 8 9 7 9 8 10 7 10 8 11 7 11 8 11 10 12 7 12 8 12 10 12 11; +type ; }> + func (a >) AMethod (y ) + func (f >) WriteEvents (ctx ; .time.ext ; .time.loc ; .time.zone ; .time.offset ; .time.isDST ; }>>>; .time.tx ; .time.index ; .time.isstd ; .time.isutc ; }>>>; .time.cacheStart ; .time.cacheEnd ; .time.cacheZone >; }> + func (l >) String () ; + func (l ) .time.lookupFirstZone () ; + func (l ) .time.get () ; + func (l ) .time.lookup (sec ) (name , offset , isDST , start , end ); + func (l ) .time.lookupName (name , unix ) (offset , ok ); + func (l ) .time.firstZoneUsed () ; +>>; }> + func (t ) In (loc ) ; + func (t ) .time.date (full ) (year , month + func (m ) String () ; +>, day , yday ); + func (t ) Sub (u ) + func (d ) Truncate (m ) ; + func (d ) String () ; + func (d ) Round (m ) ; + func (d ) Seconds () ; + func (d ) Nanoseconds () ; + func (d ) Minutes () ; + func (d ) Hours () ; +>; + func (t ) Add (d ) ; + func (t ) UTC () ; + func (t ) AddDate (years , months , days ) ; + func (t ) MarshalBinary () (? >, ? ); + func (t ) Nanosecond () ; + func (t ) Round (d ) ; + func (t ) Minute () ; + func (t ) Clock () (hour , min , sec ); + func (t ) ISOWeek () (year , week ); + func (t ) Day () ; + func (t >) .time.mono () ; + func (t ) UnixNano () ; + func (t ) .time.sec () ; + func (t ) Second () ; + func (t ) Before (u ) ; + func (t ) UnmarshalBinary (data >) ; + func (t ) Month () ; + func (t ) YearDay () ; + func (t ) Location () ; + func (t ) Zone () (name , offset ); + func (t ) Local () ; + func (t ) .time.setLoc (loc ); + func (t ) Truncate (d ) ; + func (t ) MarshalJSON () (? >, ? ); + func (t ) AppendFormat (b >, layout ) >; + func (t ) GobDecode (data >) ; + func (t ) UnmarshalJSON (data >) ; + func (t ) MarshalText () (? >, ? ); + func (t ) GobEncode () (? >, ? ); + func (t ) .time.stripMono (); + func (t ) After (u ) ; + func (t ) Hour () ; + func (t ) UnmarshalText (data >) ; + func (t ) Equal (u ) ; + func (t ) .time.setMono (m ); + func (t ) Year () ; + func (t ) IsZero () ; + func (t ) .time.addSec (d ); + func (t ) Weekday () + func (d ) String () ; +>; + func (t ) String () ; + func (t ) .time.nsec () ; + func (t ) Format (layout ) ; + func (t ) .time.unixSec () ; + func (t ) Unix () ; + func (t ) .time.abs () ; + func (t ) .time.locabs () (name , offset , abs ); + func (t ) Date () (year , month , day ); +>, ok ); Done () >; Err () ; Value (key ) ; }>>, x ) ; +>>; .issue29198.user ; .issue29198.ctx ; }>>>; +>; +type ; +func New (sctx , u ) (? >, ? ); +type ; +checksum 86C8D76B2582F55A8BD2CA9E00060358EC1CE214; diff --git a/src/go/internal/gccgoimporter/testdata/issue30628.go b/src/go/internal/gccgoimporter/testdata/issue30628.go new file mode 100644 index 0000000..8fd7c13 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/issue30628.go @@ -0,0 +1,18 @@ +package issue30628 + +import ( + "os" + "sync" +) + +const numR = int32(os.O_TRUNC + 5) + +type Apple struct { + hey sync.RWMutex + x int + RQ [numR]struct { + Count uintptr + NumBytes uintptr + Last uintptr + } +} diff --git a/src/go/internal/gccgoimporter/testdata/issue30628.gox b/src/go/internal/gccgoimporter/testdata/issue30628.gox new file mode 100644 index 0000000..0ff6259 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/issue30628.gox @@ -0,0 +1,28 @@ +v3; +package issue30628 +pkgpath issue30628 +import os os "os" +import sync sync "sync" +init cpu internal..z2fcpu..import poll internal..z2fpoll..import testlog internal..z2ftestlog..import io io..import os os..import runtime runtime..import sys runtime..z2finternal..z2fsys..import sync sync..import syscall syscall..import time time..import +init_graph 1 0 1 3 1 5 1 6 1 7 1 8 1 9 3 0 3 5 3 6 3 7 4 0 4 1 4 2 4 3 4 5 4 6 4 7 4 8 4 9 5 0 5 6 7 0 7 5 7 6 8 0 8 5 8 6 8 7 9 0 9 5 9 6 9 7 9 8 +types 13 2 24 84 208 17 30 41 147 86 17 64 25 75 +type 1 "Apple" +type 2 struct { .issue30628.hey ; .issue30628.x ; RQ ; } +type 3 "sync.RWMutex" + func (rw ) Lock () + func (rw ) RLocker () ($ret8 ) + func (rw ) RUnlock () + func (rw ) Unlock () + func (rw ) RLock () +type 4 * +type 5 "sync.Locker" +type 6 interface { Lock (); Unlock (); } +type 7 struct { .sync.w ; .sync.writerSem ; .sync.readerSem ; .sync.readerCount ; .sync.readerWait ; } +type 8 "sync.Mutex" + func (m ) Unlock () + func (m ) Lock () +type 9 * +type 10 struct { .sync.state ; .sync.sema ; } +type 11 [517 ] +type 12 struct { Count ; NumBytes ; Last ; } +checksum 199DCF6D3EE2FCF39F715B4E42B5F87F5B15D3AF diff --git a/src/go/internal/gccgoimporter/testdata/issue31540.go b/src/go/internal/gccgoimporter/testdata/issue31540.go new file mode 100644 index 0000000..2c6799e --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/issue31540.go @@ -0,0 +1,26 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue31540 + +type Y struct { + q int +} + +type Z map[int]int + +type X = map[Y]Z + +type A1 = X + +type A2 = A1 + +type S struct { + b int + A2 +} + +func Hallo() S { + return S{} +} diff --git a/src/go/internal/gccgoimporter/testdata/issue31540.gox b/src/go/internal/gccgoimporter/testdata/issue31540.gox new file mode 100644 index 0000000..abdc696 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/issue31540.gox @@ -0,0 +1,16 @@ +v3; +package issue31540 +pkgpath issue31540 +types 11 7 23 23 20 22 20 21 57 31 45 36 +type 1 "A1" = +type 2 "A2" = +type 3 "S" +type 4 "X" = +type 5 "Y" +type 6 "Z" +type 7 struct { .go.mapalias.b ; ? ; } +type 8 map [] +type 9 struct { .go.mapalias.q ; } +type 10 map [] +func Hallo () +checksum C3FAF2524A90BC11225EE65D059BF27DFB69134B diff --git a/src/go/internal/gccgoimporter/testdata/issue34182.go b/src/go/internal/gccgoimporter/testdata/issue34182.go new file mode 100644 index 0000000..2a5c333 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/issue34182.go @@ -0,0 +1,17 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue34182 + +type T1 struct { + f *T2 +} + +type T2 struct { + f T3 +} + +type T3 struct { + *T2 +} diff --git a/src/go/internal/gccgoimporter/testdata/issue34182.gox b/src/go/internal/gccgoimporter/testdata/issue34182.gox new file mode 100644 index 0000000..671a7d6 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/issue34182.gox @@ -0,0 +1,13 @@ +v3; +package issue34182 +pkgpath issue34182 +init issue34182 ~go.issue34182 +types 8 4 21 21 21 17 30 45 45 +type 1 "T1" +type 2 "T2" +type 3 "T3" +type 4 * +type 5 struct { ? ; } +type 6 struct { .go.issue34182.f ; } +type 7 struct { .go.issue34182.f ; } +checksum FF02C49BAF44B06C087ED4E573F7CC880C79C208 diff --git a/src/go/internal/gccgoimporter/testdata/libimportsar.a b/src/go/internal/gccgoimporter/testdata/libimportsar.a new file mode 100644 index 0000000..6f30758 Binary files /dev/null and b/src/go/internal/gccgoimporter/testdata/libimportsar.a differ diff --git a/src/go/internal/gccgoimporter/testdata/nointerface.go b/src/go/internal/gccgoimporter/testdata/nointerface.go new file mode 100644 index 0000000..6a545f2 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/nointerface.go @@ -0,0 +1,12 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package nointerface + +type I int + +//go:nointerface +func (p *I) Get() int { return int(*p) } + +func (p *I) Set(v int) { *p = I(v) } diff --git a/src/go/internal/gccgoimporter/testdata/nointerface.gox b/src/go/internal/gccgoimporter/testdata/nointerface.gox new file mode 100644 index 0000000..7b73d17 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/nointerface.gox @@ -0,0 +1,8 @@ +v3; +package nointerface +pkgpath nointerface +types 3 2 133 17 +type 1 "I" + func /*nointerface*/ (p ) Get () + func (p ) Set (v ) +type 2 * diff --git a/src/go/internal/gccgoimporter/testdata/notinheap.go b/src/go/internal/gccgoimporter/testdata/notinheap.go new file mode 100644 index 0000000..b1ac967 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/notinheap.go @@ -0,0 +1,4 @@ +package notinheap + +//go:notinheap +type S struct{} diff --git a/src/go/internal/gccgoimporter/testdata/notinheap.gox b/src/go/internal/gccgoimporter/testdata/notinheap.gox new file mode 100644 index 0000000..cc438e7 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/notinheap.gox @@ -0,0 +1,7 @@ +v3; +package notinheap +pkgpath notinheap +init notinheap ~notinheap +types 3 2 30 18 +type 1 "S" notinheap +type 2 struct { } diff --git a/src/go/internal/gccgoimporter/testdata/pointer.go b/src/go/internal/gccgoimporter/testdata/pointer.go new file mode 100644 index 0000000..4ebc671 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/pointer.go @@ -0,0 +1,3 @@ +package pointer + +type Int8Ptr *int8 diff --git a/src/go/internal/gccgoimporter/testdata/pointer.gox b/src/go/internal/gccgoimporter/testdata/pointer.gox new file mode 100644 index 0000000..d96ebbd --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/pointer.gox @@ -0,0 +1,4 @@ +v1; +package pointer; +pkgpath pointer; +type >>; diff --git a/src/go/internal/gccgoimporter/testdata/time.gox b/src/go/internal/gccgoimporter/testdata/time.gox new file mode 100644 index 0000000..a6822ea --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/time.gox @@ -0,0 +1,142 @@ +v2; +package time; +pkgpath time; +import errors errors "errors"; +import sync sync "sync"; +import syscall syscall "syscall"; +init time time..import runtime runtime..import sync sync..import syscall syscall..import; +init_graph 0 1 0 2 0 3 2 1 3 1 3 2; +const ANSIC = "Mon Jan _2 15:04:05 2006"; +func After (d + func (d ) String () ; + func (d ) Nanoseconds () ; + func (d ) Seconds () ; + func (d ) Minutes () ; + func (d ) Hours () ; +>) ; .time.nsec ; .time.loc ; .time.zone ; .time.offset ; .time.isDST ; }>>>; .time.tx ; .time.index ; .time.isstd ; .time.isutc ; }>>>; .time.cacheStart ; .time.cacheEnd ; .time.cacheZone >; }> + func (l >) .time.get () ; + func (l ) String () ; + func (l ) .time.lookup (sec ) (name , offset , isDST , start , end ); + func (l ) .time.lookupFirstZone () ; + func (l ) .time.firstZoneUsed () ; + func (l ) .time.lookupName (name , unix ) (offset , isDST , ok ); +>>; }> + func (t ) String () ; + func (t ) Format (layout ) ; + func (t ) AppendFormat (b >, layout ) >; + func (t ) After (u ) ; + func (t ) Before (u ) ; + func (t ) Equal (u ) ; + func (t ) IsZero () ; + func (t ) .time.abs () ; + func (t ) .time.locabs () (name , offset , abs ); + func (t ) Date () (year , month + func (m ) String () ; +>, day ); + func (t ) Year () ; + func (t ) Month () ; + func (t ) Day () ; + func (t ) Weekday () + func (d ) String () ; +>; + func (t ) ISOWeek () (year , week ); + func (t ) Clock () (hour , min , sec ); + func (t ) Hour () ; + func (t ) Minute () ; + func (t ) Second () ; + func (t ) Nanosecond () ; + func (t ) YearDay () ; + func (t ) Add (d ) ; + func (t ) Sub (u ) ; + func (t ) AddDate (years , months , days ) ; + func (t ) .time.date (full ) (year , month , day , yday ); + func (t ) UTC () ; + func (t ) Local () ; + func (t ) In (loc >) ; + func (t ) Location () >; + func (t ) Zone () (name , offset ); + func (t ) Unix () ; + func (t ) UnixNano () ; + func (t ) MarshalBinary () (? >, ? ); + func (t >) UnmarshalBinary (data >) ; + func (t ) GobEncode () (? >, ? ); + func (t ) GobDecode (data >) ; + func (t ) MarshalJSON () (? >, ? ); + func (t ) UnmarshalJSON (data >) ; + func (t ) MarshalText () (? >, ? ); + func (t ) UnmarshalText (data >) ; + func (t ) Truncate (d ) ; + func (t ) Round (d ) ; +>>; +func AfterFunc (d , f ) >; .time.r ; .time.when ; .time.period ; .time.f , ? )>; .time.arg ; .time.seq ; }>>; }> + func (t >) Stop () ; + func (t ) Reset (d ) ; +>>; +const April = 4 ; +const August = 8 ; +func Date (year , month , day , hour , min , sec , nsec , loc >) ; +const December = 12 ; +type ; +const February = 2 ; +func FixedZone (name , offset ) ; +const Friday = 5 ; +const Hour = 3600000000000 ; +const January = 1 ; +const July = 7 ; +const June = 6 ; +const Kitchen = "3:04PM"; +func LoadLocation (name ) (? , ? ); +var Local ; +type ; +var LocationSource ) (? >, ? )>; +const March = 3 ; +const May = 5 ; +const Microsecond = 1000 ; +const Millisecond = 1000000 ; +const Minute = 60000000000 ; +const Monday = 1 ; +type ; +const Nanosecond = 1 ; +func NewTicker (d ) >; .time.r ; }> + func (t >) Stop (); +>>; +func NewTimer (d ) ; +const November = 11 ; +func Now () ; +const October = 10 ; +func Parse (layout , value ) (? , ? ); +func ParseDuration (s ) (? , ? ); +type ; Value ; LayoutElem ; ValueElem ; Message ; }> + func (e >) Error () ; +>; +func ParseInLocation (layout , value , loc >) (? , ? ); +const RFC1123 = "Mon, 02 Jan 2006 15:04:05 MST"; +const RFC1123Z = "Mon, 02 Jan 2006 15:04:05 -0700"; +const RFC3339 = "2006-01-02T15:04:05Z07:00"; +const RFC3339Nano = "2006-01-02T15:04:05.999999999Z07:00"; +const RFC822 = "02 Jan 06 15:04 MST"; +const RFC822Z = "02 Jan 06 15:04 -0700"; +const RFC850 = "Monday, 02-Jan-06 15:04:05 MST"; +const RubyDate = "Mon Jan 02 15:04:05 -0700 2006"; +const Saturday = 6 ; +const Second = 1000000000 ; +const September = 9 ; +func Since (t ) ; +func Sleep (d ); +const Stamp = "Jan _2 15:04:05"; +const StampMicro = "Jan _2 15:04:05.000000"; +const StampMilli = "Jan _2 15:04:05.000"; +const StampNano = "Jan _2 15:04:05.000000000"; +const Sunday = 0 ; +const Thursday = 4 ; +func Tick (d ) >; +type ; +type ; +type ; +const Tuesday = 2 ; +var UTC ; +func Unix (sec , nsec ) ; +const UnixDate = "Mon Jan _2 15:04:05 MST 2006"; +const Wednesday = 3 ; +type ; +checksum C04E7F45B20C621A25DC74E9B13EA4FF6732E226; diff --git a/src/go/internal/gccgoimporter/testdata/unicode.gox b/src/go/internal/gccgoimporter/testdata/unicode.gox new file mode 100644 index 0000000..ae1a6f7 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/unicode.gox @@ -0,0 +1,273 @@ +v2; +package unicode; +pkgpath unicode; +init unicode unicode..import; +var ASCII_Hex_Digit ; Hi ; Stride ; }>>>; R32 ; Hi ; Stride ; }>>>; LatinOffset ; }>>>; +var Adlam ; +var Ahom ; +var Anatolian_Hieroglyphs ; +var Arabic ; +var Armenian ; +var Avestan ; +var AzeriCase ; Hi ; Delta >>; }>>> + func (special ) ToUpper (r ) ; + func (special ) ToTitle (r ) ; + func (special ) ToLower (r ) ; +>; +var Balinese ; +var Bamum ; +var Bassa_Vah ; +var Batak ; +var Bengali ; +var Bhaiksuki ; +var Bidi_Control ; +var Bopomofo ; +var Brahmi ; +var Braille ; +var Buginese ; +var Buhid ; +var C ; +var Canadian_Aboriginal ; +var Carian ; +type ; +var CaseRanges >; +var Categories ] >; +var Caucasian_Albanian ; +var Cc ; +var Cf ; +var Chakma ; +var Cham ; +var Cherokee ; +var Co ; +var Common ; +var Coptic ; +var Cs ; +var Cuneiform ; +var Cypriot ; +var Cyrillic ; +var Dash ; +var Deprecated ; +var Deseret ; +var Devanagari ; +var Diacritic ; +var Digit ; +var Duployan ; +var Egyptian_Hieroglyphs ; +var Elbasan ; +var Ethiopic ; +var Extender ; +var FoldCategory ] >; +var FoldScript ] >; +var Georgian ; +var Glagolitic ; +var Gothic ; +var Grantha ; +var GraphicRanges >>; +var Greek ; +var Gujarati ; +var Gurmukhi ; +var Han ; +var Hangul ; +var Hanunoo ; +var Hatran ; +var Hebrew ; +var Hex_Digit ; +var Hiragana ; +var Hyphen ; +var IDS_Binary_Operator ; +var IDS_Trinary_Operator ; +var Ideographic ; +var Imperial_Aramaic ; +func In (r , ranges ...>) ; +var Inherited ; +var Inscriptional_Pahlavi ; +var Inscriptional_Parthian ; +func Is (rangeTab , r ) ; +func IsControl (r ) ; +func IsDigit (r ) ; +func IsGraphic (r ) ; +func IsLetter (r ) ; +func IsLower (r ) ; +func IsMark (r ) ; +func IsNumber (r ) ; +func IsOneOf (ranges >>, r ) ; +func IsPrint (r ) ; +func IsPunct (r ) ; +func IsSpace (r ) ; +func IsSymbol (r ) ; +func IsTitle (r ) ; +func IsUpper (r ) ; +var Javanese ; +var Join_Control ; +var Kaithi ; +var Kannada ; +var Katakana ; +var Kayah_Li ; +var Kharoshthi ; +var Khmer ; +var Khojki ; +var Khudawadi ; +var L ; +var Lao ; +var Latin ; +var Lepcha ; +var Letter ; +var Limbu ; +var Linear_A ; +var Linear_B ; +var Lisu ; +var Ll ; +var Lm ; +var Lo ; +var Logical_Order_Exception ; +var Lower ; +const LowerCase = 1 ; +var Lt ; +var Lu ; +var Lycian ; +var Lydian ; +var M ; +var Mahajani ; +var Malayalam ; +var Mandaic ; +var Manichaean ; +var Marchen ; +var Mark ; +const MaxASCII = 127' ; +const MaxCase = 3 ; +const MaxLatin1 = 255' ; +const MaxRune = 1114111' ; +var Mc ; +var Me ; +var Meetei_Mayek ; +var Mende_Kikakui ; +var Meroitic_Cursive ; +var Meroitic_Hieroglyphs ; +var Miao ; +var Mn ; +var Modi ; +var Mongolian ; +var Mro ; +var Multani ; +var Myanmar ; +var N ; +var Nabataean ; +var Nd ; +var New_Tai_Lue ; +var Newa ; +var Nko ; +var Nl ; +var No ; +var Noncharacter_Code_Point ; +var Number ; +var Ogham ; +var Ol_Chiki ; +var Old_Hungarian ; +var Old_Italic ; +var Old_North_Arabian ; +var Old_Permic ; +var Old_Persian ; +var Old_South_Arabian ; +var Old_Turkic ; +var Oriya ; +var Osage ; +var Osmanya ; +var Other ; +var Other_Alphabetic ; +var Other_Default_Ignorable_Code_Point ; +var Other_Grapheme_Extend ; +var Other_ID_Continue ; +var Other_ID_Start ; +var Other_Lowercase ; +var Other_Math ; +var Other_Uppercase ; +var P ; +var Pahawh_Hmong ; +var Palmyrene ; +var Pattern_Syntax ; +var Pattern_White_Space ; +var Pau_Cin_Hau ; +var Pc ; +var Pd ; +var Pe ; +var Pf ; +var Phags_Pa ; +var Phoenician ; +var Pi ; +var Po ; +var Prepended_Concatenation_Mark ; +var PrintRanges >>; +var Properties ] >; +var Ps ; +var Psalter_Pahlavi ; +var Punct ; +var Quotation_Mark ; +var Radical ; +type ; +type ; +type ; +var Rejang ; +const ReplacementChar = 65533' ; +var Runic ; +var S ; +var STerm ; +var Samaritan ; +var Saurashtra ; +var Sc ; +var Scripts ] >; +var Sentence_Terminal ; +var Sharada ; +var Shavian ; +var Siddham ; +var SignWriting ; +func SimpleFold (r ) ; +var Sinhala ; +var Sk ; +var Sm ; +var So ; +var Soft_Dotted ; +var Sora_Sompeng ; +var Space ; +type ; +var Sundanese ; +var Syloti_Nagri ; +var Symbol ; +var Syriac ; +var Tagalog ; +var Tagbanwa ; +var Tai_Le ; +var Tai_Tham ; +var Tai_Viet ; +var Takri ; +var Tamil ; +var Tangut ; +var Telugu ; +var Terminal_Punctuation ; +var Thaana ; +var Thai ; +var Tibetan ; +var Tifinagh ; +var Tirhuta ; +var Title ; +const TitleCase = 2 ; +func To (_case , r ) ; +func ToLower (r ) ; +func ToTitle (r ) ; +func ToUpper (r ) ; +var TurkishCase ; +var Ugaritic ; +var Unified_Ideograph ; +var Upper ; +const UpperCase = 0 ; +const UpperLower = 1114112' ; +var Vai ; +var Variation_Selector ; +const Version = "9.0.0"; +var Warang_Citi ; +var White_Space ; +var Yi ; +var Z ; +var Zl ; +var Zp ; +var Zs ; +checksum 7643975C0BE2732C7557F1B2A70796673C11DF4A; diff --git a/src/go/internal/gccgoimporter/testdata/v1reflect.gox b/src/go/internal/gccgoimporter/testdata/v1reflect.gox new file mode 100644 index 0000000..d693fe6 --- /dev/null +++ b/src/go/internal/gccgoimporter/testdata/v1reflect.gox @@ -0,0 +1,184 @@ +v1; +package reflect; +pkgpath reflect; +priority 3; +import math math "math"; +import runtime runtime "runtime"; +import strconv strconv "strconv"; +import sync sync "sync"; +import unsafe unsafe "unsafe"; +init reflect reflect..import 3 math math..import 1 runtime runtime..import 1 strconv strconv..import 2; +func Append (s ; .reflect.align ; .reflect.fieldAlign ; .reflect._ ; .reflect.size ; .reflect.hash ; .reflect.hashfn >, ? )>; .reflect.equalfn , ? , ? )>; .reflect.string >; ? >; .reflect.pkgPath >; .reflect.methods >; .reflect.pkgPath >; .reflect.mtyp >>; .reflect.typ ; .reflect.tfn ; }>>>; }> + func (t >) .reflect.uncommon () ; + func (t ) PkgPath () ; + func (t ) Name () ; + func (t ) Method (i ) (m ; PkgPath ; Type ; FieldAlign () ; Method (? ) ; MethodByName (? ) (? , ? ); NumMethod () ; Name () ; PkgPath () ; Size () ; String () ; .reflect.rawString () ; Kind () + func (k ) String () ; +>; Implements (u ) ; AssignableTo (u ) ; Bits () ; ChanDir () + func (d ) String () ; +>; IsVariadic () ; Elem () ; Field (i ) ; PkgPath ; Type ; Tag + func (tag ) Get (key ) ; +>; Offset ; Index >; Anonymous ; }>>; FieldByIndex (index >) ; FieldByName (name ) (? , ? ); FieldByNameFunc (match ) >) (? , ? ); In (i ) ; Key () ; Len () ; NumField () ; NumIn () ; NumOut () ; Out (i ) ; .reflect.runtimeType () >; .reflect.common () >; .reflect.uncommon () >; }>>; Func ; Index ; }>>); + func (t ) NumMethod () ; + func (t ) MethodByName (name ) (m , ok ); +>>; .reflect.ptrToThis ; }> + func (t >) .reflect.toType () ; + func (t ) .reflect.rawString () ; + func (t ) String () ; + func (t ) Size () ; + func (t ) Bits () ; + func (t ) Align () ; + func (t ) FieldAlign () ; + func (t ) Kind () ; + func (t ) .reflect.common () ; + func (t ) NumMethod () ; + func (t ) Method (i ) (m ); + func (t ) MethodByName (name ) (m , ok ); + func (t ) PkgPath () ; + func (t ) Name () ; + func (t ) ChanDir () ; + func (t ) IsVariadic () ; + func (t ) Elem () ; + func (t ) Field (i ) ; + func (t ) FieldByIndex (index >) ; + func (t ) FieldByName (name ) (? , ? ); + func (t ) FieldByNameFunc (match ) >) (? , ? ); + func (t ) In (i ) ; + func (t ) Key () ; + func (t ) Len () ; + func (t ) NumField () ; + func (t ) NumIn () ; + func (t ) NumOut () ; + func (t ) Out (i ) ; + func (t ) .reflect.runtimeType () ; + func (ct ) .reflect.ptrTo () ; + func (t ) Implements (u ) ; + func (t ) AssignableTo (u ) ; +>>; .reflect.val ; ? + func (f ) .reflect.kind () ; + func (f ) .reflect.mustBe (expected ); + func (f ) .reflect.mustBeExported (); + func (f ) .reflect.mustBeAssignable (); +>; }> + func (v ) .reflect.iword () >; + func (v ) Addr () ; + func (v ) Bool () ; + func (v ) Bytes () >; + func (v ) CanAddr () ; + func (v ) CanSet () ; + func (v ) Call (in >) >; + func (v ) CallSlice (in >) >; + func (v ) .reflect.call (method , in >) >; + func (v ) Cap () ; + func (v ) Close (); + func (v ) Complex () ; + func (v ) Elem () ; + func (v ) Field (i ) ; + func (v ) FieldByIndex (index >) ; + func (v ) FieldByName (name ) ; + func (v ) FieldByNameFunc (match ) >) ; + func (v ) Float () ; + func (v ) Index (i ) ; + func (v ) Int () ; + func (v ) CanInterface () ; + func (v ) Interface () (i ); + func (v ) InterfaceData () >; + func (v ) IsNil () ; + func (v ) IsValid () ; + func (v ) Kind () ; + func (v ) Len () ; + func (v ) MapIndex (key ) ; + func (v ) MapKeys () >; + func (v ) Method (i ) ; + func (v ) NumMethod () ; + func (v ) MethodByName (name ) ; + func (v ) NumField () ; + func (v ) OverflowComplex (x ) ; + func (v ) OverflowFloat (x ) ; + func (v ) OverflowInt (x ) ; + func (v ) OverflowUint (x ) ; + func (v ) Pointer () ; + func (v ) Recv () (x , ok ); + func (v ) .reflect.recv (nb ) (val , ok ); + func (v ) Send (x ); + func (v ) .reflect.send (x , nb ) (selected ); + func (v ) Set (x ); + func (v ) SetBool (x ); + func (v ) SetBytes (x >); + func (v ) SetComplex (x ); + func (v ) SetFloat (x ); + func (v ) SetInt (x ); + func (v ) SetLen (n ); + func (v ) SetMapIndex (key , val ); + func (v ) SetUint (x ); + func (v ) SetPointer (x ); + func (v ) SetString (x ); + func (v ) Slice (beg , end ) ; + func (v ) String () ; + func (v ) TryRecv () (x , ok ); + func (v ) TrySend (x ) ; + func (v ) Type () ; + func (v ) Uint () ; + func (v ) UnsafeAddr () ; + func (v ) .reflect.assignTo (context , dst , target >) ; +>, x ...) ; +func AppendSlice (s , t ) ; +const Array = 17 ; +const Bool = 1 ; +const BothDir = 3 ; +const Chan = 18 ; +type ; +const Complex128 = 16 ; +const Complex64 = 15 ; +func Copy (dst , src ) ; +func DeepEqual (a1 , a2 ) ; +const Float32 = 13 ; +const Float64 = 14 ; +const Func = 19 ; +func Indirect (v ) ; +const Int = 2 ; +const Int16 = 4 ; +const Int32 = 5 ; +const Int64 = 6 ; +const Int8 = 3 ; +const Interface = 20 ; +const Invalid = 0 ; +type ; +func MakeChan (typ , buffer ) ; +func MakeMap (typ ) ; +func MakeSlice (typ , len , cap ) ; +const Map = 21 ; +type ; +func Method$equal (key1 , key2 , key_size ) ; +func Method$hash (key , key_size ) ; +func New (typ ) ; +func NewAt (typ , p ) ; +const Ptr = 22 ; +func PtrTo (t ) ; +const RecvDir = 1 ; +const SendDir = 2 ; +const Slice = 23 ; +type ; Len ; Cap ; }>>; +const String = 24 ; +type ; Len ; }>>; +const Struct = 25 ; +type ; +type ; +type ; +func TypeOf (i ) ; +const Uint = 7 ; +const Uint16 = 9 ; +const Uint32 = 10 ; +const Uint64 = 11 ; +const Uint8 = 8 ; +const Uintptr = 12 ; +const UnsafePointer = 26 ; +type ; +type ; Kind ; }> + func (e >) Error () ; +>; +func ValueError$equal (key1 , key2 , key_size ) ; +func ValueError$hash (key , key_size ) ; +func ValueOf (i ) ; +func Zero (typ ) ; +checksum DD7720796E91D9D24ED12B75BDEA2A714D47B095; diff --git a/src/go/internal/gcimporter/exportdata.go b/src/go/internal/gcimporter/exportdata.go new file mode 100644 index 0000000..4aa22d7 --- /dev/null +++ b/src/go/internal/gcimporter/exportdata.go @@ -0,0 +1,92 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements FindExportData. + +package gcimporter + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" +) + +func readGopackHeader(r *bufio.Reader) (name string, size int, err error) { + // See $GOROOT/include/ar.h. + hdr := make([]byte, 16+12+6+6+8+10+2) + _, err = io.ReadFull(r, hdr) + if err != nil { + return + } + // leave for debugging + if false { + fmt.Printf("header: %s", hdr) + } + s := strings.TrimSpace(string(hdr[16+12+6+6+8:][:10])) + size, err = strconv.Atoi(s) + if err != nil || hdr[len(hdr)-2] != '`' || hdr[len(hdr)-1] != '\n' { + err = fmt.Errorf("invalid archive header") + return + } + name = strings.TrimSpace(string(hdr[:16])) + return +} + +// FindExportData positions the reader r at the beginning of the +// export data section of an underlying GC-created object/archive +// file by reading from it. The reader must be positioned at the +// start of the file before calling this function. The hdr result +// is the string before the export data, either "$$" or "$$B". +func FindExportData(r *bufio.Reader) (hdr string, size int, err error) { + // Read first line to make sure this is an object file. + line, err := r.ReadSlice('\n') + if err != nil { + err = fmt.Errorf("can't find export data (%v)", err) + return + } + + if string(line) == "!\n" { + // Archive file. Scan to __.PKGDEF. + var name string + if name, size, err = readGopackHeader(r); err != nil { + return + } + + // First entry should be __.PKGDEF. + if name != "__.PKGDEF" { + err = fmt.Errorf("go archive is missing __.PKGDEF") + return + } + + // Read first line of __.PKGDEF data, so that line + // is once again the first line of the input. + if line, err = r.ReadSlice('\n'); err != nil { + err = fmt.Errorf("can't find export data (%v)", err) + return + } + } + + // Now at __.PKGDEF in archive or still at beginning of file. + // Either way, line should begin with "go object ". + if !strings.HasPrefix(string(line), "go object ") { + err = fmt.Errorf("not a Go object file") + return + } + size -= len(line) + + // Skip over object header to export data. + // Begins after first line starting with $$. + for line[0] != '$' { + if line, err = r.ReadSlice('\n'); err != nil { + err = fmt.Errorf("can't find export data (%v)", err) + return + } + size -= len(line) + } + hdr = string(line) + + return +} diff --git a/src/go/internal/gcimporter/gcimporter.go b/src/go/internal/gcimporter/gcimporter.go new file mode 100644 index 0000000..284389a --- /dev/null +++ b/src/go/internal/gcimporter/gcimporter.go @@ -0,0 +1,245 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package gcimporter implements Import for gc-generated object files. +package gcimporter // import "go/internal/gcimporter" + +import ( + "bufio" + "bytes" + "fmt" + "go/build" + "go/token" + "go/types" + "internal/pkgbits" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" +) + +// debugging/development support +const debug = false + +var exportMap sync.Map // package dir ā†’ func() (string, bool) + +// lookupGorootExport returns the location of the export data +// (normally found in the build cache, but located in GOROOT/pkg +// in prior Go releases) for the package located in pkgDir. +// +// (We use the package's directory instead of its import path +// mainly to simplify handling of the packages in src/vendor +// and cmd/vendor.) +func lookupGorootExport(pkgDir string) (string, bool) { + f, ok := exportMap.Load(pkgDir) + if !ok { + var ( + listOnce sync.Once + exportPath string + ) + f, _ = exportMap.LoadOrStore(pkgDir, func() (string, bool) { + listOnce.Do(func() { + cmd := exec.Command(filepath.Join(build.Default.GOROOT, "bin", "go"), "list", "-export", "-f", "{{.Export}}", pkgDir) + cmd.Dir = build.Default.GOROOT + cmd.Env = append(cmd.Environ(), "GOROOT="+build.Default.GOROOT) + var output []byte + output, err := cmd.Output() + if err != nil { + return + } + + exports := strings.Split(string(bytes.TrimSpace(output)), "\n") + if len(exports) != 1 { + return + } + + exportPath = exports[0] + }) + + return exportPath, exportPath != "" + }) + } + + return f.(func() (string, bool))() +} + +var pkgExts = [...]string{".a", ".o"} // a file from the build cache will have no extension + +// FindPkg returns the filename and unique package id for an import +// path based on package information provided by build.Import (using +// the build.Default build.Context). A relative srcDir is interpreted +// relative to the current working directory. +// If no file was found, an empty filename is returned. +func FindPkg(path, srcDir string) (filename, id string) { + if path == "" { + return + } + + var noext string + switch { + default: + // "x" -> "$GOPATH/pkg/$GOOS_$GOARCH/x.ext", "x" + // Don't require the source files to be present. + if abs, err := filepath.Abs(srcDir); err == nil { // see issue 14282 + srcDir = abs + } + bp, _ := build.Import(path, srcDir, build.FindOnly|build.AllowBinary) + if bp.PkgObj == "" { + var ok bool + if bp.Goroot && bp.Dir != "" { + filename, ok = lookupGorootExport(bp.Dir) + } + if !ok { + id = path // make sure we have an id to print in error message + return + } + } else { + noext = strings.TrimSuffix(bp.PkgObj, ".a") + } + id = bp.ImportPath + + case build.IsLocalImport(path): + // "./x" -> "/this/directory/x.ext", "/this/directory/x" + noext = filepath.Join(srcDir, path) + id = noext + + case filepath.IsAbs(path): + // for completeness only - go/build.Import + // does not support absolute imports + // "/x" -> "/x.ext", "/x" + noext = path + id = path + } + + if false { // for debugging + if path != id { + fmt.Printf("%s -> %s\n", path, id) + } + } + + if filename != "" { + if f, err := os.Stat(filename); err == nil && !f.IsDir() { + return + } + } + // try extensions + for _, ext := range pkgExts { + filename = noext + ext + if f, err := os.Stat(filename); err == nil && !f.IsDir() { + return + } + } + + filename = "" // not found + return +} + +// Import imports a gc-generated package given its import path and srcDir, adds +// the corresponding package object to the packages map, and returns the object. +// The packages map must contain all packages already imported. +func Import(fset *token.FileSet, packages map[string]*types.Package, path, srcDir string, lookup func(path string) (io.ReadCloser, error)) (pkg *types.Package, err error) { + var rc io.ReadCloser + var id string + if lookup != nil { + // With custom lookup specified, assume that caller has + // converted path to a canonical import path for use in the map. + if path == "unsafe" { + return types.Unsafe, nil + } + id = path + + // No need to re-import if the package was imported completely before. + if pkg = packages[id]; pkg != nil && pkg.Complete() { + return + } + f, err := lookup(path) + if err != nil { + return nil, err + } + rc = f + } else { + var filename string + filename, id = FindPkg(path, srcDir) + if filename == "" { + if path == "unsafe" { + return types.Unsafe, nil + } + return nil, fmt.Errorf("can't find import: %q", id) + } + + // no need to re-import if the package was imported completely before + if pkg = packages[id]; pkg != nil && pkg.Complete() { + return + } + + // open file + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + // add file name to error + err = fmt.Errorf("%s: %v", filename, err) + } + }() + rc = f + } + defer rc.Close() + + buf := bufio.NewReader(rc) + hdr, size, err := FindExportData(buf) + if err != nil { + return + } + + switch hdr { + case "$$\n": + err = fmt.Errorf("import %q: old textual export format no longer supported (recompile library)", path) + + case "$$B\n": + var exportFormat byte + if exportFormat, err = buf.ReadByte(); err != nil { + return + } + + // The unified export format starts with a 'u'; the indexed export + // format starts with an 'i'; and the older binary export format + // starts with a 'c', 'd', or 'v' (from "version"). Select + // appropriate importer. + switch exportFormat { + case 'u': + var data []byte + var r io.Reader = buf + if size >= 0 { + r = io.LimitReader(r, int64(size)) + } + if data, err = io.ReadAll(r); err != nil { + return + } + s := string(data) + s = s[:strings.LastIndex(s, "\n$$\n")] + + input := pkgbits.NewPkgDecoder(id, s) + pkg = readUnifiedPackage(fset, nil, packages, input) + case 'i': + pkg, err = iImportData(fset, packages, buf, id) + default: + err = fmt.Errorf("import %q: old binary export format no longer supported (recompile library)", path) + } + + default: + err = fmt.Errorf("import %q: unknown export data header: %q", path, hdr) + } + + return +} + +type byPath []*types.Package + +func (a byPath) Len() int { return len(a) } +func (a byPath) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byPath) Less(i, j int) bool { return a[i].Path() < a[j].Path() } diff --git a/src/go/internal/gcimporter/gcimporter_test.go b/src/go/internal/gcimporter/gcimporter_test.go new file mode 100644 index 0000000..f2202ab --- /dev/null +++ b/src/go/internal/gcimporter/gcimporter_test.go @@ -0,0 +1,832 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gcimporter_test + +import ( + "bytes" + "fmt" + "internal/goexperiment" + "internal/goroot" + "internal/testenv" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "go/ast" + "go/build" + "go/importer" + "go/parser" + "go/token" + "go/types" + + . "go/internal/gcimporter" +) + +func TestMain(m *testing.M) { + build.Default.GOROOT = testenv.GOROOT(nil) + os.Exit(m.Run()) +} + +// compile runs the compiler on filename, with dirname as the working directory, +// and writes the output file to outdirname. +// compile gives the resulting package a packagepath of testdata/. +func compile(t *testing.T, dirname, filename, outdirname string, packagefiles map[string]string) string { + // filename must end with ".go" + basename, ok := strings.CutSuffix(filepath.Base(filename), ".go") + if !ok { + t.Fatalf("filename doesn't end in .go: %s", filename) + } + objname := basename + ".o" + outname := filepath.Join(outdirname, objname) + + importcfgfile := os.DevNull + if len(packagefiles) > 0 { + importcfgfile = filepath.Join(outdirname, basename) + ".importcfg" + importcfg := new(bytes.Buffer) + fmt.Fprintf(importcfg, "# import config") + for k, v := range packagefiles { + fmt.Fprintf(importcfg, "\npackagefile %s=%s\n", k, v) + } + if err := os.WriteFile(importcfgfile, importcfg.Bytes(), 0655); err != nil { + t.Fatal(err) + } + } + + pkgpath := path.Join("testdata", basename) + cmd := testenv.Command(t, testenv.GoToolPath(t), "tool", "compile", "-p", pkgpath, "-D", "testdata", "-importcfg", importcfgfile, "-o", outname, filename) + cmd.Dir = dirname + out, err := cmd.CombinedOutput() + if err != nil { + t.Logf("%s", out) + t.Fatalf("go tool compile %s failed: %s", filename, err) + } + return outname +} + +func testPath(t *testing.T, path, srcDir string) *types.Package { + t0 := time.Now() + fset := token.NewFileSet() + pkg, err := Import(fset, make(map[string]*types.Package), path, srcDir, nil) + if err != nil { + t.Errorf("testPath(%s): %s", path, err) + return nil + } + t.Logf("testPath(%s): %v", path, time.Since(t0)) + return pkg +} + +var pkgExts = [...]string{".a", ".o"} // keep in sync with gcimporter.go + +func mktmpdir(t *testing.T) string { + tmpdir, err := os.MkdirTemp("", "gcimporter_test") + if err != nil { + t.Fatal("mktmpdir:", err) + } + if err := os.Mkdir(filepath.Join(tmpdir, "testdata"), 0700); err != nil { + os.RemoveAll(tmpdir) + t.Fatal("mktmpdir:", err) + } + return tmpdir +} + +func TestImportTestdata(t *testing.T) { + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + testenv.MustHaveGoBuild(t) + + testfiles := map[string][]string{ + "exports.go": {"go/ast", "go/token"}, + "generics.go": nil, + } + if goexperiment.Unified { + // TODO(mdempsky): Fix test below to flatten the transitive + // Package.Imports graph. Unified IR is more precise about + // recreating the package import graph. + testfiles["exports.go"] = []string{"go/ast"} + } + + for testfile, wantImports := range testfiles { + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + + packageFiles := map[string]string{} + for _, pkg := range wantImports { + export, _ := FindPkg(pkg, "testdata") + if export == "" { + t.Fatalf("no export data found for %s", pkg) + } + packageFiles[pkg] = export + } + + compile(t, "testdata", testfile, filepath.Join(tmpdir, "testdata"), packageFiles) + path := "./testdata/" + strings.TrimSuffix(testfile, ".go") + + if pkg := testPath(t, path, tmpdir); pkg != nil { + // The package's Imports list must include all packages + // explicitly imported by testfile, plus all packages + // referenced indirectly via exported objects in testfile. + got := fmt.Sprint(pkg.Imports()) + for _, want := range wantImports { + if !strings.Contains(got, want) { + t.Errorf(`Package("exports").Imports() = %s, does not contain %s`, got, want) + } + } + } + } +} + +func TestImportTypeparamTests(t *testing.T) { + if testing.Short() { + t.Skipf("in short mode, skipping test that requires export data for all of std") + } + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + testenv.MustHaveGoBuild(t) + + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + + // Check go files in test/typeparam, except those that fail for a known + // reason. + rootDir := filepath.Join(testenv.GOROOT(t), "test", "typeparam") + list, err := os.ReadDir(rootDir) + if err != nil { + t.Fatal(err) + } + + var skip map[string]string + if !goexperiment.Unified { + // The Go 1.18 frontend still fails several cases. + skip = map[string]string{ + "equal.go": "inconsistent embedded sorting", // TODO(rfindley): investigate this. + "nested.go": "fails to compile", // TODO(rfindley): investigate this. + "issue47631.go": "can not handle local type declarations", + "issue55101.go": "fails to compile", + } + } + + for _, entry := range list { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") { + // For now, only consider standalone go files. + continue + } + + t.Run(entry.Name(), func(t *testing.T) { + if reason, ok := skip[entry.Name()]; ok { + t.Skip(reason) + } + + filename := filepath.Join(rootDir, entry.Name()) + src, err := os.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + if !bytes.HasPrefix(src, []byte("// run")) && !bytes.HasPrefix(src, []byte("// compile")) { + // We're bypassing the logic of run.go here, so be conservative about + // the files we consider in an attempt to make this test more robust to + // changes in test/typeparams. + t.Skipf("not detected as a run test") + } + + // Compile and import, and compare the resulting package with the package + // that was type-checked directly. + pkgFiles, err := goroot.PkgfileMap() + if err != nil { + t.Fatal(err) + } + compile(t, rootDir, entry.Name(), filepath.Join(tmpdir, "testdata"), pkgFiles) + pkgName := strings.TrimSuffix(entry.Name(), ".go") + imported := importPkg(t, "./testdata/"+pkgName, tmpdir) + checked := checkFile(t, filename, src) + + seen := make(map[string]bool) + for _, name := range imported.Scope().Names() { + if !token.IsExported(name) { + continue // ignore synthetic names like .inittask and .dict.* + } + seen[name] = true + + importedObj := imported.Scope().Lookup(name) + got := types.ObjectString(importedObj, types.RelativeTo(imported)) + got = sanitizeObjectString(got) + + checkedObj := checked.Scope().Lookup(name) + if checkedObj == nil { + t.Fatalf("imported object %q was not type-checked", name) + } + want := types.ObjectString(checkedObj, types.RelativeTo(checked)) + want = sanitizeObjectString(want) + + if got != want { + t.Errorf("imported %q as %q, want %q", name, got, want) + } + } + + for _, name := range checked.Scope().Names() { + if !token.IsExported(name) || seen[name] { + continue + } + t.Errorf("did not import object %q", name) + } + }) + } +} + +// sanitizeObjectString removes type parameter debugging markers from an object +// string, to normalize it for comparison. +// TODO(rfindley): this should not be necessary. +func sanitizeObjectString(s string) string { + var runes []rune + for _, r := range s { + if 'ā‚€' <= r && r < 'ā‚€'+10 { + continue // trim type parameter subscripts + } + runes = append(runes, r) + } + return string(runes) +} + +func checkFile(t *testing.T, filename string, src []byte) *types.Package { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, filename, src, 0) + if err != nil { + t.Fatal(err) + } + config := types.Config{ + Importer: importer.Default(), + } + pkg, err := config.Check("", fset, []*ast.File{f}, nil) + if err != nil { + t.Fatal(err) + } + return pkg +} + +func TestVersionHandling(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + const dir = "./testdata/versions" + list, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + corruptdir := filepath.Join(tmpdir, "testdata", "versions") + if err := os.Mkdir(corruptdir, 0700); err != nil { + t.Fatal(err) + } + + fset := token.NewFileSet() + + for _, f := range list { + name := f.Name() + if !strings.HasSuffix(name, ".a") { + continue // not a package file + } + if strings.Contains(name, "corrupted") { + continue // don't process a leftover corrupted file + } + pkgpath := "./" + name[:len(name)-2] + + if testing.Verbose() { + t.Logf("importing %s", name) + } + + // test that export data can be imported + _, err := Import(fset, make(map[string]*types.Package), pkgpath, dir, nil) + if err != nil { + // ok to fail if it fails with a no longer supported error for select files + if strings.Contains(err.Error(), "no longer supported") { + switch name { + case "test_go1.7_0.a", "test_go1.7_1.a", + "test_go1.8_4.a", "test_go1.8_5.a", + "test_go1.11_6b.a", "test_go1.11_999b.a": + continue + } + // fall through + } + // ok to fail if it fails with a newer version error for select files + if strings.Contains(err.Error(), "newer version") { + switch name { + case "test_go1.11_999i.a": + continue + } + // fall through + } + t.Errorf("import %q failed: %v", pkgpath, err) + continue + } + + // create file with corrupted export data + // 1) read file + data, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + // 2) find export data + i := bytes.Index(data, []byte("\n$$B\n")) + 5 + j := bytes.Index(data[i:], []byte("\n$$\n")) + i + if i < 0 || j < 0 || i > j { + t.Fatalf("export data section not found (i = %d, j = %d)", i, j) + } + // 3) corrupt the data (increment every 7th byte) + for k := j - 13; k >= i; k -= 7 { + data[k]++ + } + // 4) write the file + pkgpath += "_corrupted" + filename := filepath.Join(corruptdir, pkgpath) + ".a" + os.WriteFile(filename, data, 0666) + + // test that importing the corrupted file results in an error + _, err = Import(fset, make(map[string]*types.Package), pkgpath, corruptdir, nil) + if err == nil { + t.Errorf("import corrupted %q succeeded", pkgpath) + } else if msg := err.Error(); !strings.Contains(msg, "version skew") { + t.Errorf("import %q error incorrect (%s)", pkgpath, msg) + } + } +} + +func TestImportStdLib(t *testing.T) { + if testing.Short() { + t.Skip("the imports can be expensive, and this test is especially slow when the build cache is empty") + } + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // Get list of packages in stdlib. Filter out test-only packages with {{if .GoFiles}} check. + var stderr bytes.Buffer + cmd := exec.Command("go", "list", "-f", "{{if .GoFiles}}{{.ImportPath}}{{end}}", "std") + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + t.Fatalf("failed to run go list to determine stdlib packages: %v\nstderr:\n%v", err, stderr.String()) + } + pkgs := strings.Fields(string(out)) + + var nimports int + for _, pkg := range pkgs { + t.Run(pkg, func(t *testing.T) { + if testPath(t, pkg, filepath.Join(testenv.GOROOT(t), "src", path.Dir(pkg))) != nil { + nimports++ + } + }) + } + const minPkgs = 225 // 'GOOS=plan9 go1.18 list std | wc -l' reports 228; most other platforms have more. + if len(pkgs) < minPkgs { + t.Fatalf("too few packages (%d) were imported", nimports) + } + + t.Logf("tested %d imports", nimports) +} + +var importedObjectTests = []struct { + name string + want string +}{ + // non-interfaces + {"crypto.Hash", "type Hash uint"}, + {"go/ast.ObjKind", "type ObjKind int"}, + {"go/types.Qualifier", "type Qualifier func(*Package) string"}, + {"go/types.Comparable", "func Comparable(T Type) bool"}, + {"math.Pi", "const Pi untyped float"}, + {"math.Sin", "func Sin(x float64) float64"}, + {"go/ast.NotNilFilter", "func NotNilFilter(_ string, v reflect.Value) bool"}, + {"go/internal/gcimporter.FindPkg", "func FindPkg(path string, srcDir string) (filename string, id string)"}, + + // interfaces + {"context.Context", "type Context interface{Deadline() (deadline time.Time, ok bool); Done() <-chan struct{}; Err() error; Value(key any) any}"}, + {"crypto.Decrypter", "type Decrypter interface{Decrypt(rand io.Reader, msg []byte, opts DecrypterOpts) (plaintext []byte, err error); Public() PublicKey}"}, + {"encoding.BinaryMarshaler", "type BinaryMarshaler interface{MarshalBinary() (data []byte, err error)}"}, + {"io.Reader", "type Reader interface{Read(p []byte) (n int, err error)}"}, + {"io.ReadWriter", "type ReadWriter interface{Reader; Writer}"}, + {"go/ast.Node", "type Node interface{End() go/token.Pos; Pos() go/token.Pos}"}, + {"go/types.Type", "type Type interface{String() string; Underlying() Type}"}, +} + +func TestImportedTypes(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + fset := token.NewFileSet() + for _, test := range importedObjectTests { + s := strings.Split(test.name, ".") + if len(s) != 2 { + t.Fatal("inconsistent test data") + } + importPath := s[0] + objName := s[1] + + pkg, err := Import(fset, make(map[string]*types.Package), importPath, ".", nil) + if err != nil { + t.Error(err) + continue + } + + obj := pkg.Scope().Lookup(objName) + if obj == nil { + t.Errorf("%s: object not found", test.name) + continue + } + + got := types.ObjectString(obj, types.RelativeTo(pkg)) + if got != test.want { + t.Errorf("%s: got %q; want %q", test.name, got, test.want) + } + + if named, _ := obj.Type().(*types.Named); named != nil { + verifyInterfaceMethodRecvs(t, named, 0) + } + } +} + +// verifyInterfaceMethodRecvs verifies that method receiver types +// are named if the methods belong to a named interface type. +func verifyInterfaceMethodRecvs(t *testing.T, named *types.Named, level int) { + // avoid endless recursion in case of an embedding bug that lead to a cycle + if level > 10 { + t.Errorf("%s: embeds itself", named) + return + } + + iface, _ := named.Underlying().(*types.Interface) + if iface == nil { + return // not an interface + } + + // check explicitly declared methods + for i := 0; i < iface.NumExplicitMethods(); i++ { + m := iface.ExplicitMethod(i) + recv := m.Type().(*types.Signature).Recv() + if recv == nil { + t.Errorf("%s: missing receiver type", m) + continue + } + if recv.Type() != named { + t.Errorf("%s: got recv type %s; want %s", m, recv.Type(), named) + } + } + + // check embedded interfaces (if they are named, too) + for i := 0; i < iface.NumEmbeddeds(); i++ { + // embedding of interfaces cannot have cycles; recursion will terminate + if etype, _ := iface.EmbeddedType(i).(*types.Named); etype != nil { + verifyInterfaceMethodRecvs(t, etype, level+1) + } + } +} + +func TestIssue5815(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + pkg := importPkg(t, "strings", ".") + + scope := pkg.Scope() + for _, name := range scope.Names() { + obj := scope.Lookup(name) + if obj.Pkg() == nil { + t.Errorf("no pkg for %s", obj) + } + if tname, _ := obj.(*types.TypeName); tname != nil { + named := tname.Type().(*types.Named) + for i := 0; i < named.NumMethods(); i++ { + m := named.Method(i) + if m.Pkg() == nil { + t.Errorf("no pkg for %s", m) + } + } + } + } +} + +// Smoke test to ensure that imported methods get the correct package. +func TestCorrectMethodPackage(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + imports := make(map[string]*types.Package) + fset := token.NewFileSet() + _, err := Import(fset, imports, "net/http", ".", nil) + if err != nil { + t.Fatal(err) + } + + mutex := imports["sync"].Scope().Lookup("Mutex").(*types.TypeName).Type() + mset := types.NewMethodSet(types.NewPointer(mutex)) // methods of *sync.Mutex + sel := mset.Lookup(nil, "Lock") + lock := sel.Obj().(*types.Func) + if got, want := lock.Pkg().Path(), "sync"; got != want { + t.Errorf("got package path %q; want %q", got, want) + } +} + +func TestIssue13566(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // On windows, we have to set the -D option for the compiler to avoid having a drive + // letter and an illegal ':' in the import path - just skip it (see also issue #3483). + if runtime.GOOS == "windows" { + t.Skip("avoid dealing with relative paths/drive letters on windows") + } + + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + testoutdir := filepath.Join(tmpdir, "testdata") + + // b.go needs to be compiled from the output directory so that the compiler can + // find the compiled package a. We pass the full path to compile() so that we + // don't have to copy the file to that directory. + bpath, err := filepath.Abs(filepath.Join("testdata", "b.go")) + if err != nil { + t.Fatal(err) + } + + jsonExport, _ := FindPkg("encoding/json", "testdata") + if jsonExport == "" { + t.Fatalf("no export data found for encoding/json") + } + + compile(t, "testdata", "a.go", testoutdir, map[string]string{"encoding/json": jsonExport}) + compile(t, testoutdir, bpath, testoutdir, map[string]string{"testdata/a": filepath.Join(testoutdir, "a.o")}) + + // import must succeed (test for issue at hand) + pkg := importPkg(t, "./testdata/b", tmpdir) + + // make sure all indirectly imported packages have names + for _, imp := range pkg.Imports() { + if imp.Name() == "" { + t.Errorf("no name for %s package", imp.Path()) + } + } +} + +func TestTypeNamingOrder(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // On windows, we have to set the -D option for the compiler to avoid having a drive + // letter and an illegal ':' in the import path - just skip it (see also issue #3483). + if runtime.GOOS == "windows" { + t.Skip("avoid dealing with relative paths/drive letters on windows") + } + + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + testoutdir := filepath.Join(tmpdir, "testdata") + + compile(t, "testdata", "g.go", testoutdir, nil) + + // import must succeed (test for issue at hand) + _ = importPkg(t, "./testdata/g", tmpdir) +} + +func TestIssue13898(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // import go/internal/gcimporter which imports go/types partially + fset := token.NewFileSet() + imports := make(map[string]*types.Package) + _, err := Import(fset, imports, "go/internal/gcimporter", ".", nil) + if err != nil { + t.Fatal(err) + } + + // look for go/types package + var goTypesPkg *types.Package + for path, pkg := range imports { + if path == "go/types" { + goTypesPkg = pkg + break + } + } + if goTypesPkg == nil { + t.Fatal("go/types not found") + } + + // look for go/types.Object type + obj := lookupObj(t, goTypesPkg.Scope(), "Object") + typ, ok := obj.Type().(*types.Named) + if !ok { + t.Fatalf("go/types.Object type is %v; wanted named type", typ) + } + + // lookup go/types.Object.Pkg method + m, index, indirect := types.LookupFieldOrMethod(typ, false, nil, "Pkg") + if m == nil { + t.Fatalf("go/types.Object.Pkg not found (index = %v, indirect = %v)", index, indirect) + } + + // the method must belong to go/types + if m.Pkg().Path() != "go/types" { + t.Fatalf("found %v; want go/types", m.Pkg()) + } +} + +func TestIssue15517(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // On windows, we have to set the -D option for the compiler to avoid having a drive + // letter and an illegal ':' in the import path - just skip it (see also issue #3483). + if runtime.GOOS == "windows" { + t.Skip("avoid dealing with relative paths/drive letters on windows") + } + + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + + compile(t, "testdata", "p.go", filepath.Join(tmpdir, "testdata"), nil) + + // Multiple imports of p must succeed without redeclaration errors. + // We use an import path that's not cleaned up so that the eventual + // file path for the package is different from the package path; this + // will expose the error if it is present. + // + // (Issue: Both the textual and the binary importer used the file path + // of the package to be imported as key into the shared packages map. + // However, the binary importer then used the package path to identify + // the imported package to mark it as complete; effectively marking the + // wrong package as complete. By using an "unclean" package path, the + // file and package path are different, exposing the problem if present. + // The same issue occurs with vendoring.) + imports := make(map[string]*types.Package) + fset := token.NewFileSet() + for i := 0; i < 3; i++ { + if _, err := Import(fset, imports, "./././testdata/p", tmpdir, nil); err != nil { + t.Fatal(err) + } + } +} + +func TestIssue15920(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // On windows, we have to set the -D option for the compiler to avoid having a drive + // letter and an illegal ':' in the import path - just skip it (see also issue #3483). + if runtime.GOOS == "windows" { + t.Skip("avoid dealing with relative paths/drive letters on windows") + } + + compileAndImportPkg(t, "issue15920") +} + +func TestIssue20046(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // On windows, we have to set the -D option for the compiler to avoid having a drive + // letter and an illegal ':' in the import path - just skip it (see also issue #3483). + if runtime.GOOS == "windows" { + t.Skip("avoid dealing with relative paths/drive letters on windows") + } + + // "./issue20046".V.M must exist + pkg := compileAndImportPkg(t, "issue20046") + obj := lookupObj(t, pkg.Scope(), "V") + if m, index, indirect := types.LookupFieldOrMethod(obj.Type(), false, nil, "M"); m == nil { + t.Fatalf("V.M not found (index = %v, indirect = %v)", index, indirect) + } +} +func TestIssue25301(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // On windows, we have to set the -D option for the compiler to avoid having a drive + // letter and an illegal ':' in the import path - just skip it (see also issue #3483). + if runtime.GOOS == "windows" { + t.Skip("avoid dealing with relative paths/drive letters on windows") + } + + compileAndImportPkg(t, "issue25301") +} + +func TestIssue25596(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // On windows, we have to set the -D option for the compiler to avoid having a drive + // letter and an illegal ':' in the import path - just skip it (see also issue #3483). + if runtime.GOOS == "windows" { + t.Skip("avoid dealing with relative paths/drive letters on windows") + } + + compileAndImportPkg(t, "issue25596") +} + +func TestIssue57015(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // This package only handles gc export data. + if runtime.Compiler != "gc" { + t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler) + } + + // On windows, we have to set the -D option for the compiler to avoid having a drive + // letter and an illegal ':' in the import path - just skip it (see also issue #3483). + if runtime.GOOS == "windows" { + t.Skip("avoid dealing with relative paths/drive letters on windows") + } + + compileAndImportPkg(t, "issue57015") +} + +func importPkg(t *testing.T, path, srcDir string) *types.Package { + fset := token.NewFileSet() + pkg, err := Import(fset, make(map[string]*types.Package), path, srcDir, nil) + if err != nil { + t.Helper() + t.Fatal(err) + } + return pkg +} + +func compileAndImportPkg(t *testing.T, name string) *types.Package { + t.Helper() + tmpdir := mktmpdir(t) + defer os.RemoveAll(tmpdir) + compile(t, "testdata", name+".go", filepath.Join(tmpdir, "testdata"), nil) + return importPkg(t, "./testdata/"+name, tmpdir) +} + +func lookupObj(t *testing.T, scope *types.Scope, name string) types.Object { + if obj := scope.Lookup(name); obj != nil { + return obj + } + t.Helper() + t.Fatalf("%s not found", name) + return nil +} diff --git a/src/go/internal/gcimporter/iimport.go b/src/go/internal/gcimporter/iimport.go new file mode 100644 index 0000000..9e3c945 --- /dev/null +++ b/src/go/internal/gcimporter/iimport.go @@ -0,0 +1,817 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Indexed package import. +// See cmd/compile/internal/gc/iexport.go for the export data format. + +package gcimporter + +import ( + "bufio" + "bytes" + "encoding/binary" + "fmt" + "go/constant" + "go/token" + "go/types" + "internal/saferio" + "io" + "math" + "math/big" + "sort" + "strings" +) + +type intReader struct { + *bufio.Reader + path string +} + +func (r *intReader) int64() int64 { + i, err := binary.ReadVarint(r.Reader) + if err != nil { + errorf("import %q: read varint error: %v", r.path, err) + } + return i +} + +func (r *intReader) uint64() uint64 { + i, err := binary.ReadUvarint(r.Reader) + if err != nil { + errorf("import %q: read varint error: %v", r.path, err) + } + return i +} + +// Keep this in sync with constants in iexport.go. +const ( + iexportVersionGo1_11 = 0 + iexportVersionPosCol = 1 + iexportVersionGenerics = 2 + iexportVersionGo1_18 = 2 + + iexportVersionCurrent = 2 +) + +type ident struct { + pkg *types.Package + name string +} + +const predeclReserved = 32 + +type itag uint64 + +const ( + // Types + definedType itag = iota + pointerType + sliceType + arrayType + chanType + mapType + signatureType + structType + interfaceType + typeParamType + instanceType + unionType +) + +// iImportData imports a package from the serialized package data +// and returns the number of bytes consumed and a reference to the package. +// If the export data version is not recognized or the format is otherwise +// compromised, an error is returned. +func iImportData(fset *token.FileSet, imports map[string]*types.Package, dataReader *bufio.Reader, path string) (pkg *types.Package, err error) { + const currentVersion = iexportVersionCurrent + version := int64(-1) + defer func() { + if e := recover(); e != nil { + if version > currentVersion { + err = fmt.Errorf("cannot import %q (%v), export data is newer version - update tool", path, e) + } else { + err = fmt.Errorf("cannot import %q (%v), possibly version skew - reinstall package", path, e) + } + } + }() + + r := &intReader{dataReader, path} + + version = int64(r.uint64()) + switch version { + case iexportVersionGo1_18, iexportVersionPosCol, iexportVersionGo1_11: + default: + errorf("unknown iexport format version %d", version) + } + + sLen := r.uint64() + dLen := r.uint64() + + if sLen > math.MaxUint64-dLen { + errorf("lengths out of range (%d, %d)", sLen, dLen) + } + + data, err := saferio.ReadData(r, sLen+dLen) + if err != nil { + errorf("cannot read %d bytes of stringData and declData: %s", sLen+dLen, err) + } + stringData := data[:sLen] + declData := data[sLen:] + + p := iimporter{ + exportVersion: version, + ipath: path, + version: int(version), + + stringData: stringData, + stringCache: make(map[uint64]string), + pkgCache: make(map[uint64]*types.Package), + + declData: declData, + pkgIndex: make(map[*types.Package]map[string]uint64), + typCache: make(map[uint64]types.Type), + // Separate map for typeparams, keyed by their package and unique + // name (name with subscript). + tparamIndex: make(map[ident]*types.TypeParam), + + fake: fakeFileSet{ + fset: fset, + files: make(map[string]*fileInfo), + }, + } + defer p.fake.setLines() // set lines for files in fset + + for i, pt := range predeclared { + p.typCache[uint64(i)] = pt + } + + pkgList := make([]*types.Package, r.uint64()) + for i := range pkgList { + pkgPathOff := r.uint64() + pkgPath := p.stringAt(pkgPathOff) + pkgName := p.stringAt(r.uint64()) + _ = r.uint64() // package height; unused by go/types + + if pkgPath == "" { + pkgPath = path + } + pkg := imports[pkgPath] + if pkg == nil { + pkg = types.NewPackage(pkgPath, pkgName) + imports[pkgPath] = pkg + } else if pkg.Name() != pkgName { + errorf("conflicting names %s and %s for package %q", pkg.Name(), pkgName, path) + } + + p.pkgCache[pkgPathOff] = pkg + + nameIndex := make(map[string]uint64) + for nSyms := r.uint64(); nSyms > 0; nSyms-- { + name := p.stringAt(r.uint64()) + nameIndex[name] = r.uint64() + } + + p.pkgIndex[pkg] = nameIndex + pkgList[i] = pkg + } + + localpkg := pkgList[0] + + names := make([]string, 0, len(p.pkgIndex[localpkg])) + for name := range p.pkgIndex[localpkg] { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + p.doDecl(localpkg, name) + } + + // SetConstraint can't be called if the constraint type is not yet complete. + // When type params are created in the 'P' case of (*importReader).obj(), + // the associated constraint type may not be complete due to recursion. + // Therefore, we defer calling SetConstraint there, and call it here instead + // after all types are complete. + for _, d := range p.later { + d.t.SetConstraint(d.constraint) + } + + for _, typ := range p.interfaceList { + typ.Complete() + } + + // record all referenced packages as imports + list := append(([]*types.Package)(nil), pkgList[1:]...) + sort.Sort(byPath(list)) + localpkg.SetImports(list) + + // package was imported completely and without errors + localpkg.MarkComplete() + return localpkg, nil +} + +type setConstraintArgs struct { + t *types.TypeParam + constraint types.Type +} + +type iimporter struct { + exportVersion int64 + ipath string + version int + + stringData []byte + stringCache map[uint64]string + pkgCache map[uint64]*types.Package + + declData []byte + pkgIndex map[*types.Package]map[string]uint64 + typCache map[uint64]types.Type + tparamIndex map[ident]*types.TypeParam + + fake fakeFileSet + interfaceList []*types.Interface + + // Arguments for calls to SetConstraint that are deferred due to recursive types + later []setConstraintArgs +} + +func (p *iimporter) doDecl(pkg *types.Package, name string) { + // See if we've already imported this declaration. + if obj := pkg.Scope().Lookup(name); obj != nil { + return + } + + off, ok := p.pkgIndex[pkg][name] + if !ok { + errorf("%v.%v not in index", pkg, name) + } + + r := &importReader{p: p, currPkg: pkg} + r.declReader.Reset(p.declData[off:]) + + r.obj(name) +} + +func (p *iimporter) stringAt(off uint64) string { + if s, ok := p.stringCache[off]; ok { + return s + } + + slen, n := binary.Uvarint(p.stringData[off:]) + if n <= 0 { + errorf("varint failed") + } + spos := off + uint64(n) + s := string(p.stringData[spos : spos+slen]) + p.stringCache[off] = s + return s +} + +func (p *iimporter) pkgAt(off uint64) *types.Package { + if pkg, ok := p.pkgCache[off]; ok { + return pkg + } + path := p.stringAt(off) + errorf("missing package %q in %q", path, p.ipath) + return nil +} + +func (p *iimporter) typAt(off uint64, base *types.Named) types.Type { + if t, ok := p.typCache[off]; ok && canReuse(base, t) { + return t + } + + if off < predeclReserved { + errorf("predeclared type missing from cache: %v", off) + } + + r := &importReader{p: p} + r.declReader.Reset(p.declData[off-predeclReserved:]) + t := r.doType(base) + + if canReuse(base, t) { + p.typCache[off] = t + } + return t +} + +// canReuse reports whether the type rhs on the RHS of the declaration for def +// may be re-used. +// +// Specifically, if def is non-nil and rhs is an interface type with methods, it +// may not be re-used because we have a convention of setting the receiver type +// for interface methods to def. +func canReuse(def *types.Named, rhs types.Type) bool { + if def == nil { + return true + } + iface, _ := rhs.(*types.Interface) + if iface == nil { + return true + } + // Don't use iface.Empty() here as iface may not be complete. + return iface.NumEmbeddeds() == 0 && iface.NumExplicitMethods() == 0 +} + +type importReader struct { + p *iimporter + declReader bytes.Reader + currPkg *types.Package + prevFile string + prevLine int64 + prevColumn int64 +} + +func (r *importReader) obj(name string) { + tag := r.byte() + pos := r.pos() + + switch tag { + case 'A': + typ := r.typ() + + r.declare(types.NewTypeName(pos, r.currPkg, name, typ)) + + case 'C': + typ, val := r.value() + + r.declare(types.NewConst(pos, r.currPkg, name, typ, val)) + + case 'F', 'G': + var tparams []*types.TypeParam + if tag == 'G' { + tparams = r.tparamList() + } + sig := r.signature(nil, nil, tparams) + r.declare(types.NewFunc(pos, r.currPkg, name, sig)) + + case 'T', 'U': + // Types can be recursive. We need to setup a stub + // declaration before recurring. + obj := types.NewTypeName(pos, r.currPkg, name, nil) + named := types.NewNamed(obj, nil, nil) + // Declare obj before calling r.tparamList, so the new type name is recognized + // if used in the constraint of one of its own typeparams (see #48280). + r.declare(obj) + if tag == 'U' { + tparams := r.tparamList() + named.SetTypeParams(tparams) + } + + underlying := r.p.typAt(r.uint64(), named).Underlying() + named.SetUnderlying(underlying) + + if !isInterface(underlying) { + for n := r.uint64(); n > 0; n-- { + mpos := r.pos() + mname := r.ident() + recv := r.param() + + // If the receiver has any targs, set those as the + // rparams of the method (since those are the + // typeparams being used in the method sig/body). + targs := baseType(recv.Type()).TypeArgs() + var rparams []*types.TypeParam + if targs.Len() > 0 { + rparams = make([]*types.TypeParam, targs.Len()) + for i := range rparams { + rparams[i], _ = targs.At(i).(*types.TypeParam) + } + } + msig := r.signature(recv, rparams, nil) + + named.AddMethod(types.NewFunc(mpos, r.currPkg, mname, msig)) + } + } + + case 'P': + // We need to "declare" a typeparam in order to have a name that + // can be referenced recursively (if needed) in the type param's + // bound. + if r.p.exportVersion < iexportVersionGenerics { + errorf("unexpected type param type") + } + // Remove the "path" from the type param name that makes it unique, + // and revert any unique name used for blank typeparams. + name0 := tparamName(name) + tn := types.NewTypeName(pos, r.currPkg, name0, nil) + t := types.NewTypeParam(tn, nil) + // To handle recursive references to the typeparam within its + // bound, save the partial type in tparamIndex before reading the bounds. + id := ident{r.currPkg, name} + r.p.tparamIndex[id] = t + + var implicit bool + if r.p.exportVersion >= iexportVersionGo1_18 { + implicit = r.bool() + } + constraint := r.typ() + if implicit { + iface, _ := constraint.(*types.Interface) + if iface == nil { + errorf("non-interface constraint marked implicit") + } + iface.MarkImplicit() + } + // The constraint type may not be complete, if we + // are in the middle of a type recursion involving type + // constraints. So, we defer SetConstraint until we have + // completely set up all types in ImportData. + r.p.later = append(r.p.later, setConstraintArgs{t: t, constraint: constraint}) + + case 'V': + typ := r.typ() + + r.declare(types.NewVar(pos, r.currPkg, name, typ)) + + default: + errorf("unexpected tag: %v", tag) + } +} + +func (r *importReader) declare(obj types.Object) { + obj.Pkg().Scope().Insert(obj) +} + +func (r *importReader) value() (typ types.Type, val constant.Value) { + typ = r.typ() + if r.p.exportVersion >= iexportVersionGo1_18 { + // TODO: add support for using the kind + _ = constant.Kind(r.int64()) + } + + switch b := typ.Underlying().(*types.Basic); b.Info() & types.IsConstType { + case types.IsBoolean: + val = constant.MakeBool(r.bool()) + + case types.IsString: + val = constant.MakeString(r.string()) + + case types.IsInteger: + var x big.Int + r.mpint(&x, b) + val = constant.Make(&x) + + case types.IsFloat: + val = r.mpfloat(b) + + case types.IsComplex: + re := r.mpfloat(b) + im := r.mpfloat(b) + val = constant.BinaryOp(re, token.ADD, constant.MakeImag(im)) + + default: + errorf("unexpected type %v", typ) // panics + panic("unreachable") + } + + return +} + +func intSize(b *types.Basic) (signed bool, maxBytes uint) { + if (b.Info() & types.IsUntyped) != 0 { + return true, 64 + } + + switch b.Kind() { + case types.Float32, types.Complex64: + return true, 3 + case types.Float64, types.Complex128: + return true, 7 + } + + signed = (b.Info() & types.IsUnsigned) == 0 + switch b.Kind() { + case types.Int8, types.Uint8: + maxBytes = 1 + case types.Int16, types.Uint16: + maxBytes = 2 + case types.Int32, types.Uint32: + maxBytes = 4 + default: + maxBytes = 8 + } + + return +} + +func (r *importReader) mpint(x *big.Int, typ *types.Basic) { + signed, maxBytes := intSize(typ) + + maxSmall := 256 - maxBytes + if signed { + maxSmall = 256 - 2*maxBytes + } + if maxBytes == 1 { + maxSmall = 256 + } + + n, _ := r.declReader.ReadByte() + if uint(n) < maxSmall { + v := int64(n) + if signed { + v >>= 1 + if n&1 != 0 { + v = ^v + } + } + x.SetInt64(v) + return + } + + v := -n + if signed { + v = -(n &^ 1) >> 1 + } + if v < 1 || uint(v) > maxBytes { + errorf("weird decoding: %v, %v => %v", n, signed, v) + } + b := make([]byte, v) + io.ReadFull(&r.declReader, b) + x.SetBytes(b) + if signed && n&1 != 0 { + x.Neg(x) + } +} + +func (r *importReader) mpfloat(typ *types.Basic) constant.Value { + var mant big.Int + r.mpint(&mant, typ) + var f big.Float + f.SetInt(&mant) + if f.Sign() != 0 { + f.SetMantExp(&f, int(r.int64())) + } + return constant.Make(&f) +} + +func (r *importReader) ident() string { + return r.string() +} + +func (r *importReader) qualifiedIdent() (*types.Package, string) { + name := r.string() + pkg := r.pkg() + return pkg, name +} + +func (r *importReader) pos() token.Pos { + if r.p.version >= 1 { + r.posv1() + } else { + r.posv0() + } + + if r.prevFile == "" && r.prevLine == 0 && r.prevColumn == 0 { + return token.NoPos + } + return r.p.fake.pos(r.prevFile, int(r.prevLine), int(r.prevColumn)) +} + +func (r *importReader) posv0() { + delta := r.int64() + if delta != deltaNewFile { + r.prevLine += delta + } else if l := r.int64(); l == -1 { + r.prevLine += deltaNewFile + } else { + r.prevFile = r.string() + r.prevLine = l + } +} + +func (r *importReader) posv1() { + delta := r.int64() + r.prevColumn += delta >> 1 + if delta&1 != 0 { + delta = r.int64() + r.prevLine += delta >> 1 + if delta&1 != 0 { + r.prevFile = r.string() + } + } +} + +func (r *importReader) typ() types.Type { + return r.p.typAt(r.uint64(), nil) +} + +func isInterface(t types.Type) bool { + _, ok := t.(*types.Interface) + return ok +} + +func (r *importReader) pkg() *types.Package { return r.p.pkgAt(r.uint64()) } +func (r *importReader) string() string { return r.p.stringAt(r.uint64()) } + +func (r *importReader) doType(base *types.Named) types.Type { + switch k := r.kind(); k { + default: + errorf("unexpected kind tag in %q: %v", r.p.ipath, k) + return nil + + case definedType: + pkg, name := r.qualifiedIdent() + r.p.doDecl(pkg, name) + return pkg.Scope().Lookup(name).(*types.TypeName).Type() + case pointerType: + return types.NewPointer(r.typ()) + case sliceType: + return types.NewSlice(r.typ()) + case arrayType: + n := r.uint64() + return types.NewArray(r.typ(), int64(n)) + case chanType: + dir := chanDir(int(r.uint64())) + return types.NewChan(dir, r.typ()) + case mapType: + return types.NewMap(r.typ(), r.typ()) + case signatureType: + r.currPkg = r.pkg() + return r.signature(nil, nil, nil) + + case structType: + r.currPkg = r.pkg() + + fields := make([]*types.Var, r.uint64()) + tags := make([]string, len(fields)) + for i := range fields { + fpos := r.pos() + fname := r.ident() + ftyp := r.typ() + emb := r.bool() + tag := r.string() + + fields[i] = types.NewField(fpos, r.currPkg, fname, ftyp, emb) + tags[i] = tag + } + return types.NewStruct(fields, tags) + + case interfaceType: + r.currPkg = r.pkg() + + embeddeds := make([]types.Type, r.uint64()) + for i := range embeddeds { + _ = r.pos() + embeddeds[i] = r.typ() + } + + methods := make([]*types.Func, r.uint64()) + for i := range methods { + mpos := r.pos() + mname := r.ident() + + // TODO(mdempsky): Matches bimport.go, but I + // don't agree with this. + var recv *types.Var + if base != nil { + recv = types.NewVar(token.NoPos, r.currPkg, "", base) + } + + msig := r.signature(recv, nil, nil) + methods[i] = types.NewFunc(mpos, r.currPkg, mname, msig) + } + + typ := types.NewInterfaceType(methods, embeddeds) + r.p.interfaceList = append(r.p.interfaceList, typ) + return typ + + case typeParamType: + if r.p.exportVersion < iexportVersionGenerics { + errorf("unexpected type param type") + } + pkg, name := r.qualifiedIdent() + id := ident{pkg, name} + if t, ok := r.p.tparamIndex[id]; ok { + // We're already in the process of importing this typeparam. + return t + } + // Otherwise, import the definition of the typeparam now. + r.p.doDecl(pkg, name) + return r.p.tparamIndex[id] + + case instanceType: + if r.p.exportVersion < iexportVersionGenerics { + errorf("unexpected instantiation type") + } + // pos does not matter for instances: they are positioned on the original + // type. + _ = r.pos() + len := r.uint64() + targs := make([]types.Type, len) + for i := range targs { + targs[i] = r.typ() + } + baseType := r.typ() + // The imported instantiated type doesn't include any methods, so + // we must always use the methods of the base (orig) type. + // TODO provide a non-nil *Context + t, _ := types.Instantiate(nil, baseType, targs, false) + return t + + case unionType: + if r.p.exportVersion < iexportVersionGenerics { + errorf("unexpected instantiation type") + } + terms := make([]*types.Term, r.uint64()) + for i := range terms { + terms[i] = types.NewTerm(r.bool(), r.typ()) + } + return types.NewUnion(terms) + } +} + +func (r *importReader) kind() itag { + return itag(r.uint64()) +} + +func (r *importReader) signature(recv *types.Var, rparams, tparams []*types.TypeParam) *types.Signature { + params := r.paramList() + results := r.paramList() + variadic := params.Len() > 0 && r.bool() + return types.NewSignatureType(recv, rparams, tparams, params, results, variadic) +} + +func (r *importReader) tparamList() []*types.TypeParam { + n := r.uint64() + if n == 0 { + return nil + } + xs := make([]*types.TypeParam, n) + for i := range xs { + xs[i], _ = r.typ().(*types.TypeParam) + } + return xs +} + +func (r *importReader) paramList() *types.Tuple { + xs := make([]*types.Var, r.uint64()) + for i := range xs { + xs[i] = r.param() + } + return types.NewTuple(xs...) +} + +func (r *importReader) param() *types.Var { + pos := r.pos() + name := r.ident() + typ := r.typ() + return types.NewParam(pos, r.currPkg, name, typ) +} + +func (r *importReader) bool() bool { + return r.uint64() != 0 +} + +func (r *importReader) int64() int64 { + n, err := binary.ReadVarint(&r.declReader) + if err != nil { + errorf("readVarint: %v", err) + } + return n +} + +func (r *importReader) uint64() uint64 { + n, err := binary.ReadUvarint(&r.declReader) + if err != nil { + errorf("readUvarint: %v", err) + } + return n +} + +func (r *importReader) byte() byte { + x, err := r.declReader.ReadByte() + if err != nil { + errorf("declReader.ReadByte: %v", err) + } + return x +} + +func baseType(typ types.Type) *types.Named { + // pointer receivers are never types.Named types + if p, _ := typ.(*types.Pointer); p != nil { + typ = p.Elem() + } + // receiver base types are always (possibly generic) types.Named types + n, _ := typ.(*types.Named) + return n +} + +const blankMarker = "$" + +// tparamName returns the real name of a type parameter, after stripping its +// qualifying prefix and reverting blank-name encoding. See tparamExportName +// for details. +func tparamName(exportName string) string { + // Remove the "path" from the type param name that makes it unique. + ix := strings.LastIndex(exportName, ".") + if ix < 0 { + errorf("malformed type parameter export name %s: missing prefix", exportName) + } + name := exportName[ix+1:] + if strings.HasPrefix(name, blankMarker) { + return "_" + } + return name +} diff --git a/src/go/internal/gcimporter/support.go b/src/go/internal/gcimporter/support.go new file mode 100644 index 0000000..7ed8c9a --- /dev/null +++ b/src/go/internal/gcimporter/support.go @@ -0,0 +1,183 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements support functionality for iimport.go. + +package gcimporter + +import ( + "fmt" + "go/token" + "go/types" + "internal/pkgbits" + "sync" +) + +func assert(b bool) { + if !b { + panic("assertion failed") + } +} + +func errorf(format string, args ...any) { + panic(fmt.Sprintf(format, args...)) +} + +// deltaNewFile is a magic line delta offset indicating a new file. +// We use -64 because it is rare; see issue 20080 and CL 41619. +// -64 is the smallest int that fits in a single byte as a varint. +const deltaNewFile = -64 + +// Synthesize a token.Pos +type fakeFileSet struct { + fset *token.FileSet + files map[string]*fileInfo +} + +type fileInfo struct { + file *token.File + lastline int +} + +const maxlines = 64 * 1024 + +func (s *fakeFileSet) pos(file string, line, column int) token.Pos { + // TODO(mdempsky): Make use of column. + + // Since we don't know the set of needed file positions, we reserve + // maxlines positions per file. We delay calling token.File.SetLines until + // all positions have been calculated (by way of fakeFileSet.setLines), so + // that we can avoid setting unnecessary lines. See also golang/go#46586. + f := s.files[file] + if f == nil { + f = &fileInfo{file: s.fset.AddFile(file, -1, maxlines)} + s.files[file] = f + } + + if line > maxlines { + line = 1 + } + if line > f.lastline { + f.lastline = line + } + + // Return a fake position assuming that f.file consists only of newlines. + return token.Pos(f.file.Base() + line - 1) +} + +func (s *fakeFileSet) setLines() { + fakeLinesOnce.Do(func() { + fakeLines = make([]int, maxlines) + for i := range fakeLines { + fakeLines[i] = i + } + }) + for _, f := range s.files { + f.file.SetLines(fakeLines[:f.lastline]) + } +} + +var ( + fakeLines []int + fakeLinesOnce sync.Once +) + +func chanDir(d int) types.ChanDir { + // tag values must match the constants in cmd/compile/internal/gc/go.go + switch d { + case 1 /* Crecv */ : + return types.RecvOnly + case 2 /* Csend */ : + return types.SendOnly + case 3 /* Cboth */ : + return types.SendRecv + default: + errorf("unexpected channel dir %d", d) + return 0 + } +} + +var predeclared = []types.Type{ + // basic types + types.Typ[types.Bool], + types.Typ[types.Int], + types.Typ[types.Int8], + types.Typ[types.Int16], + types.Typ[types.Int32], + types.Typ[types.Int64], + types.Typ[types.Uint], + types.Typ[types.Uint8], + types.Typ[types.Uint16], + types.Typ[types.Uint32], + types.Typ[types.Uint64], + types.Typ[types.Uintptr], + types.Typ[types.Float32], + types.Typ[types.Float64], + types.Typ[types.Complex64], + types.Typ[types.Complex128], + types.Typ[types.String], + + // basic type aliases + types.Universe.Lookup("byte").Type(), + types.Universe.Lookup("rune").Type(), + + // error + types.Universe.Lookup("error").Type(), + + // untyped types + types.Typ[types.UntypedBool], + types.Typ[types.UntypedInt], + types.Typ[types.UntypedRune], + types.Typ[types.UntypedFloat], + types.Typ[types.UntypedComplex], + types.Typ[types.UntypedString], + types.Typ[types.UntypedNil], + + // package unsafe + types.Typ[types.UnsafePointer], + + // invalid type + types.Typ[types.Invalid], // only appears in packages with errors + + // used internally by gc; never used by this package or in .a files + // not to be confused with the universe any + anyType{}, + + // comparable + types.Universe.Lookup("comparable").Type(), + + // any + types.Universe.Lookup("any").Type(), +} + +type anyType struct{} + +func (t anyType) Underlying() types.Type { return t } +func (t anyType) String() string { return "any" } + +// See cmd/compile/internal/noder.derivedInfo. +type derivedInfo struct { + idx pkgbits.Index + needed bool +} + +// See cmd/compile/internal/noder.typeInfo. +type typeInfo struct { + idx pkgbits.Index + derived bool +} + +// See cmd/compile/internal/types.SplitVargenSuffix. +func splitVargenSuffix(name string) (base, suffix string) { + i := len(name) + for i > 0 && name[i-1] >= '0' && name[i-1] <= '9' { + i-- + } + const dot = "Ā·" + if i >= len(dot) && name[i-len(dot):i] == dot { + i -= len(dot) + return name[:i], name[i:] + } + return name, "" +} diff --git a/src/go/internal/gcimporter/testdata/a.go b/src/go/internal/gcimporter/testdata/a.go new file mode 100644 index 0000000..56e4292 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/a.go @@ -0,0 +1,14 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Input for TestIssue13566 + +package a + +import "encoding/json" + +type A struct { + a *A + json json.RawMessage +} diff --git a/src/go/internal/gcimporter/testdata/b.go b/src/go/internal/gcimporter/testdata/b.go new file mode 100644 index 0000000..4196678 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/b.go @@ -0,0 +1,11 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Input for TestIssue13566 + +package b + +import "./a" + +type A a.A diff --git a/src/go/internal/gcimporter/testdata/exports.go b/src/go/internal/gcimporter/testdata/exports.go new file mode 100644 index 0000000..3d5a8c9 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/exports.go @@ -0,0 +1,91 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is used to generate an object file which +// serves as test file for gcimporter_test.go. + +package exports + +import "go/ast" + +// Issue 3682: Correctly read dotted identifiers from export data. +const init1 = 0 + +func init() {} + +const ( + C0 int = 0 + C1 = 3.14159265 + C2 = 2.718281828i + C3 = -123.456e-789 + C4 = +123.456e+789 + C5 = 1234i + C6 = "foo\n" + C7 = `bar\n` + C8 = 42 + C9 int = 42 + C10 float64 = 42 +) + +type ( + T1 int + T2 [10]int + T3 []int + T4 *int + T5 chan int + T6a chan<- int + T6b chan (<-chan int) + T6c chan<- (chan int) + T7 <-chan *ast.File + T8 struct{} + T9 struct { + a int + b, c float32 + d []string `go:"tag"` + } + T10 struct { + T8 + T9 + _ *T10 + } + T11 map[int]string + T12 any + T13 interface { + m1() + m2(int) float32 + } + T14 interface { + T12 + T13 + m3(x ...struct{}) []T9 + } + T15 func() + T16 func(int) + T17 func(x int) + T18 func() float32 + T19 func() (x float32) + T20 func(...any) + T21 struct{ next *T21 } + T22 struct{ link *T23 } + T23 struct{ link *T22 } + T24 *T24 + T25 *T26 + T26 *T27 + T27 *T25 + T28 func(T28) T28 +) + +var ( + V0 int + V1 = -991.0 + V2 float32 = 1.2 +) + +func F1() {} +func F2(x int) {} +func F3() int { return 0 } +func F4() float32 { return 0 } +func F5(a, b, c int, u, v, w struct{ x, y T1 }, more ...any) (p, q, r chan<- T10) + +func (p *T1) M1() diff --git a/src/go/internal/gcimporter/testdata/g.go b/src/go/internal/gcimporter/testdata/g.go new file mode 100644 index 0000000..301c142 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/g.go @@ -0,0 +1,23 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Input for TestTypeNamingOrder + +// ensures that the order in which "type A B" declarations are +// processed is correct; this was a problem for unified IR imports. + +package g + +type Client struct { + common service + A *AService + B *BService +} + +type service struct { + client *Client +} + +type AService service +type BService service diff --git a/src/go/internal/gcimporter/testdata/generics.go b/src/go/internal/gcimporter/testdata/generics.go new file mode 100644 index 0000000..00bf040 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/generics.go @@ -0,0 +1,29 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is used to generate an object file which +// serves as test file for gcimporter_test.go. + +package generics + +type Any any + +var x any + +type T[A, B any] struct { + Left A + Right B +} + +var X T[int, string] = T[int, string]{1, "hi"} + +func ToInt[P interface{ ~int }](p P) int { return int(p) } + +var IntID = ToInt[int] + +type G[C comparable] int + +func ImplicitFunc[T ~int]() {} + +type ImplicitType[T ~int] int diff --git a/src/go/internal/gcimporter/testdata/issue15920.go b/src/go/internal/gcimporter/testdata/issue15920.go new file mode 100644 index 0000000..c70f7d8 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/issue15920.go @@ -0,0 +1,11 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +// The underlying type of Error is the underlying type of error. +// Make sure we can import this again without problems. +type Error error + +func F() Error { return nil } diff --git a/src/go/internal/gcimporter/testdata/issue20046.go b/src/go/internal/gcimporter/testdata/issue20046.go new file mode 100644 index 0000000..c63ee82 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/issue20046.go @@ -0,0 +1,9 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +var V interface { + M() +} diff --git a/src/go/internal/gcimporter/testdata/issue25301.go b/src/go/internal/gcimporter/testdata/issue25301.go new file mode 100644 index 0000000..e3dc98b --- /dev/null +++ b/src/go/internal/gcimporter/testdata/issue25301.go @@ -0,0 +1,17 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue25301 + +type ( + A = interface { + M() + } + T interface { + A + } + S struct{} +) + +func (S) M() { println("m") } diff --git a/src/go/internal/gcimporter/testdata/issue25596.go b/src/go/internal/gcimporter/testdata/issue25596.go new file mode 100644 index 0000000..8923373 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/issue25596.go @@ -0,0 +1,13 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue25596 + +type E interface { + M() T +} + +type T interface { + E +} diff --git a/src/go/internal/gcimporter/testdata/issue57015.go b/src/go/internal/gcimporter/testdata/issue57015.go new file mode 100644 index 0000000..b6be811 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/issue57015.go @@ -0,0 +1,16 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue57015 + +type E error + +type X[T any] struct {} + +func F() X[interface { + E +}] { + panic(0) +} + diff --git a/src/go/internal/gcimporter/testdata/p.go b/src/go/internal/gcimporter/testdata/p.go new file mode 100644 index 0000000..9e2e705 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/p.go @@ -0,0 +1,13 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Input for TestIssue15517 + +package p + +const C = 0 + +var V int + +func F() {} diff --git a/src/go/internal/gcimporter/testdata/versions/test.go b/src/go/internal/gcimporter/testdata/versions/test.go new file mode 100644 index 0000000..227fc09 --- /dev/null +++ b/src/go/internal/gcimporter/testdata/versions/test.go @@ -0,0 +1,28 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// To create a test case for a new export format version, +// build this package with the latest compiler and store +// the resulting .a file appropriately named in the versions +// directory. The VersionHandling test will pick it up. +// +// In the testdata/versions: +// +// go build -o test_go1.$X_$Y.a test.go +// +// with $X = Go version and $Y = export format version +// (add 'b' or 'i' to distinguish between binary and +// indexed format starting with 1.11 as long as both +// formats are supported). +// +// Make sure this source is extended such that it exercises +// whatever export format change has taken place. + +package test + +// Any release before and including Go 1.7 didn't encode +// the package for a blank struct field. +type BlankField struct { + _ int +} diff --git a/src/go/internal/gcimporter/testdata/versions/test_go1.11_0i.a b/src/go/internal/gcimporter/testdata/versions/test_go1.11_0i.a new file mode 100644 index 0000000..b00fefe Binary files /dev/null and b/src/go/internal/gcimporter/testdata/versions/test_go1.11_0i.a differ diff --git a/src/go/internal/gcimporter/testdata/versions/test_go1.11_6b.a b/src/go/internal/gcimporter/testdata/versions/test_go1.11_6b.a new file mode 100644 index 0000000..c0a211e Binary files /dev/null and b/src/go/internal/gcimporter/testdata/versions/test_go1.11_6b.a differ diff --git a/src/go/internal/gcimporter/testdata/versions/test_go1.11_999b.a b/src/go/internal/gcimporter/testdata/versions/test_go1.11_999b.a new file mode 100644 index 0000000..c35d22d Binary files /dev/null and b/src/go/internal/gcimporter/testdata/versions/test_go1.11_999b.a differ diff --git a/src/go/internal/gcimporter/testdata/versions/test_go1.11_999i.a b/src/go/internal/gcimporter/testdata/versions/test_go1.11_999i.a new file mode 100644 index 0000000..99401d7 Binary files /dev/null and b/src/go/internal/gcimporter/testdata/versions/test_go1.11_999i.a differ diff --git a/src/go/internal/gcimporter/testdata/versions/test_go1.7_0.a b/src/go/internal/gcimporter/testdata/versions/test_go1.7_0.a new file mode 100644 index 0000000..edb6c3f Binary files /dev/null and b/src/go/internal/gcimporter/testdata/versions/test_go1.7_0.a differ diff --git a/src/go/internal/gcimporter/testdata/versions/test_go1.7_1.a b/src/go/internal/gcimporter/testdata/versions/test_go1.7_1.a new file mode 100644 index 0000000..554d04a Binary files /dev/null and b/src/go/internal/gcimporter/testdata/versions/test_go1.7_1.a differ diff --git a/src/go/internal/gcimporter/testdata/versions/test_go1.8_4.a b/src/go/internal/gcimporter/testdata/versions/test_go1.8_4.a new file mode 100644 index 0000000..26b8531 Binary files /dev/null and b/src/go/internal/gcimporter/testdata/versions/test_go1.8_4.a differ diff --git a/src/go/internal/gcimporter/testdata/versions/test_go1.8_5.a b/src/go/internal/gcimporter/testdata/versions/test_go1.8_5.a new file mode 100644 index 0000000..60e52ef Binary files /dev/null and b/src/go/internal/gcimporter/testdata/versions/test_go1.8_5.a differ diff --git a/src/go/internal/gcimporter/ureader.go b/src/go/internal/gcimporter/ureader.go new file mode 100644 index 0000000..ffd8402 --- /dev/null +++ b/src/go/internal/gcimporter/ureader.go @@ -0,0 +1,682 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gcimporter + +import ( + "go/token" + "go/types" + "internal/pkgbits" +) + +// A pkgReader holds the shared state for reading a unified IR package +// description. +type pkgReader struct { + pkgbits.PkgDecoder + + fake fakeFileSet + + ctxt *types.Context + imports map[string]*types.Package // previously imported packages, indexed by path + + // lazily initialized arrays corresponding to the unified IR + // PosBase, Pkg, and Type sections, respectively. + posBases []string // position bases (i.e., file names) + pkgs []*types.Package + typs []types.Type + + // laterFns holds functions that need to be invoked at the end of + // import reading. + laterFns []func() + + // ifaces holds a list of constructed Interfaces, which need to have + // Complete called after importing is done. + ifaces []*types.Interface +} + +// later adds a function to be invoked at the end of import reading. +func (pr *pkgReader) later(fn func()) { + pr.laterFns = append(pr.laterFns, fn) +} + +// readUnifiedPackage reads a package description from the given +// unified IR export data decoder. +func readUnifiedPackage(fset *token.FileSet, ctxt *types.Context, imports map[string]*types.Package, input pkgbits.PkgDecoder) *types.Package { + pr := pkgReader{ + PkgDecoder: input, + + fake: fakeFileSet{ + fset: fset, + files: make(map[string]*fileInfo), + }, + + ctxt: ctxt, + imports: imports, + + posBases: make([]string, input.NumElems(pkgbits.RelocPosBase)), + pkgs: make([]*types.Package, input.NumElems(pkgbits.RelocPkg)), + typs: make([]types.Type, input.NumElems(pkgbits.RelocType)), + } + defer pr.fake.setLines() + + r := pr.newReader(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic) + pkg := r.pkg() + r.Bool() // TODO(mdempsky): Remove; was "has init" + + for i, n := 0, r.Len(); i < n; i++ { + // As if r.obj(), but avoiding the Scope.Lookup call, + // to avoid eager loading of imports. + r.Sync(pkgbits.SyncObject) + assert(!r.Bool()) + r.p.objIdx(r.Reloc(pkgbits.RelocObj)) + assert(r.Len() == 0) + } + + r.Sync(pkgbits.SyncEOF) + + for _, fn := range pr.laterFns { + fn() + } + + for _, iface := range pr.ifaces { + iface.Complete() + } + + pkg.MarkComplete() + return pkg +} + +// A reader holds the state for reading a single unified IR element +// within a package. +type reader struct { + pkgbits.Decoder + + p *pkgReader + + dict *readerDict +} + +// A readerDict holds the state for type parameters that parameterize +// the current unified IR element. +type readerDict struct { + // bounds is a slice of typeInfos corresponding to the underlying + // bounds of the element's type parameters. + bounds []typeInfo + + // tparams is a slice of the constructed TypeParams for the element. + tparams []*types.TypeParam + + // devived is a slice of types derived from tparams, which may be + // instantiated while reading the current element. + derived []derivedInfo + derivedTypes []types.Type // lazily instantiated from derived +} + +func (pr *pkgReader) newReader(k pkgbits.RelocKind, idx pkgbits.Index, marker pkgbits.SyncMarker) *reader { + return &reader{ + Decoder: pr.NewDecoder(k, idx, marker), + p: pr, + } +} + +func (pr *pkgReader) tempReader(k pkgbits.RelocKind, idx pkgbits.Index, marker pkgbits.SyncMarker) *reader { + return &reader{ + Decoder: pr.TempDecoder(k, idx, marker), + p: pr, + } +} + +func (pr *pkgReader) retireReader(r *reader) { + pr.RetireDecoder(&r.Decoder) +} + +// @@@ Positions + +func (r *reader) pos() token.Pos { + r.Sync(pkgbits.SyncPos) + if !r.Bool() { + return token.NoPos + } + + // TODO(mdempsky): Delta encoding. + posBase := r.posBase() + line := r.Uint() + col := r.Uint() + return r.p.fake.pos(posBase, int(line), int(col)) +} + +func (r *reader) posBase() string { + return r.p.posBaseIdx(r.Reloc(pkgbits.RelocPosBase)) +} + +func (pr *pkgReader) posBaseIdx(idx pkgbits.Index) string { + if b := pr.posBases[idx]; b != "" { + return b + } + + var filename string + { + r := pr.tempReader(pkgbits.RelocPosBase, idx, pkgbits.SyncPosBase) + + // Within types2, position bases have a lot more details (e.g., + // keeping track of where //line directives appeared exactly). + // + // For go/types, we just track the file name. + + filename = r.String() + + if r.Bool() { // file base + // Was: "b = token.NewTrimmedFileBase(filename, true)" + } else { // line base + pos := r.pos() + line := r.Uint() + col := r.Uint() + + // Was: "b = token.NewLineBase(pos, filename, true, line, col)" + _, _, _ = pos, line, col + } + pr.retireReader(r) + } + b := filename + pr.posBases[idx] = b + return b +} + +// @@@ Packages + +func (r *reader) pkg() *types.Package { + r.Sync(pkgbits.SyncPkg) + return r.p.pkgIdx(r.Reloc(pkgbits.RelocPkg)) +} + +func (pr *pkgReader) pkgIdx(idx pkgbits.Index) *types.Package { + // TODO(mdempsky): Consider using some non-nil pointer to indicate + // the universe scope, so we don't need to keep re-reading it. + if pkg := pr.pkgs[idx]; pkg != nil { + return pkg + } + + pkg := pr.newReader(pkgbits.RelocPkg, idx, pkgbits.SyncPkgDef).doPkg() + pr.pkgs[idx] = pkg + return pkg +} + +func (r *reader) doPkg() *types.Package { + path := r.String() + switch path { + case "": + path = r.p.PkgPath() + case "builtin": + return nil // universe + case "unsafe": + return types.Unsafe + } + + if pkg := r.p.imports[path]; pkg != nil { + return pkg + } + + name := r.String() + + pkg := types.NewPackage(path, name) + r.p.imports[path] = pkg + + imports := make([]*types.Package, r.Len()) + for i := range imports { + imports[i] = r.pkg() + } + + // The documentation for (*types.Package).Imports requires + // flattening the import graph when reading from export data, as + // obviously incorrect as that is. + // + // TODO(mdempsky): Remove this if go.dev/issue/54096 is accepted. + pkg.SetImports(flattenImports(imports)) + + return pkg +} + +// flattenImports returns the transitive closure of all imported +// packages rooted from pkgs. +func flattenImports(pkgs []*types.Package) []*types.Package { + var res []*types.Package + seen := make(map[*types.Package]struct{}) + for _, pkg := range pkgs { + if _, ok := seen[pkg]; ok { + continue + } + seen[pkg] = struct{}{} + res = append(res, pkg) + + // pkg.Imports() is already flattened. + for _, pkg := range pkg.Imports() { + if _, ok := seen[pkg]; ok { + continue + } + seen[pkg] = struct{}{} + res = append(res, pkg) + } + } + return res +} + +// @@@ Types + +func (r *reader) typ() types.Type { + return r.p.typIdx(r.typInfo(), r.dict) +} + +func (r *reader) typInfo() typeInfo { + r.Sync(pkgbits.SyncType) + if r.Bool() { + return typeInfo{idx: pkgbits.Index(r.Len()), derived: true} + } + return typeInfo{idx: r.Reloc(pkgbits.RelocType), derived: false} +} + +func (pr *pkgReader) typIdx(info typeInfo, dict *readerDict) types.Type { + idx := info.idx + var where *types.Type + if info.derived { + where = &dict.derivedTypes[idx] + idx = dict.derived[idx].idx + } else { + where = &pr.typs[idx] + } + + if typ := *where; typ != nil { + return typ + } + + var typ types.Type + { + r := pr.tempReader(pkgbits.RelocType, idx, pkgbits.SyncTypeIdx) + r.dict = dict + + typ = r.doTyp() + assert(typ != nil) + pr.retireReader(r) + } + // See comment in pkgReader.typIdx explaining how this happens. + if prev := *where; prev != nil { + return prev + } + + *where = typ + return typ +} + +func (r *reader) doTyp() (res types.Type) { + switch tag := pkgbits.CodeType(r.Code(pkgbits.SyncType)); tag { + default: + errorf("unhandled type tag: %v", tag) + panic("unreachable") + + case pkgbits.TypeBasic: + return types.Typ[r.Len()] + + case pkgbits.TypeNamed: + obj, targs := r.obj() + name := obj.(*types.TypeName) + if len(targs) != 0 { + t, _ := types.Instantiate(r.p.ctxt, name.Type(), targs, false) + return t + } + return name.Type() + + case pkgbits.TypeTypeParam: + return r.dict.tparams[r.Len()] + + case pkgbits.TypeArray: + len := int64(r.Uint64()) + return types.NewArray(r.typ(), len) + case pkgbits.TypeChan: + dir := types.ChanDir(r.Len()) + return types.NewChan(dir, r.typ()) + case pkgbits.TypeMap: + return types.NewMap(r.typ(), r.typ()) + case pkgbits.TypePointer: + return types.NewPointer(r.typ()) + case pkgbits.TypeSignature: + return r.signature(nil, nil, nil) + case pkgbits.TypeSlice: + return types.NewSlice(r.typ()) + case pkgbits.TypeStruct: + return r.structType() + case pkgbits.TypeInterface: + return r.interfaceType() + case pkgbits.TypeUnion: + return r.unionType() + } +} + +func (r *reader) structType() *types.Struct { + fields := make([]*types.Var, r.Len()) + var tags []string + for i := range fields { + pos := r.pos() + pkg, name := r.selector() + ftyp := r.typ() + tag := r.String() + embedded := r.Bool() + + fields[i] = types.NewField(pos, pkg, name, ftyp, embedded) + if tag != "" { + for len(tags) < i { + tags = append(tags, "") + } + tags = append(tags, tag) + } + } + return types.NewStruct(fields, tags) +} + +func (r *reader) unionType() *types.Union { + terms := make([]*types.Term, r.Len()) + for i := range terms { + terms[i] = types.NewTerm(r.Bool(), r.typ()) + } + return types.NewUnion(terms) +} + +func (r *reader) interfaceType() *types.Interface { + methods := make([]*types.Func, r.Len()) + embeddeds := make([]types.Type, r.Len()) + implicit := len(methods) == 0 && len(embeddeds) == 1 && r.Bool() + + for i := range methods { + pos := r.pos() + pkg, name := r.selector() + mtyp := r.signature(nil, nil, nil) + methods[i] = types.NewFunc(pos, pkg, name, mtyp) + } + + for i := range embeddeds { + embeddeds[i] = r.typ() + } + + iface := types.NewInterfaceType(methods, embeddeds) + if implicit { + iface.MarkImplicit() + } + + // We need to call iface.Complete(), but if there are any embedded + // defined types, then we may not have set their underlying + // interface type yet. So we need to defer calling Complete until + // after we've called SetUnderlying everywhere. + // + // TODO(mdempsky): After CL 424876 lands, it should be safe to call + // iface.Complete() immediately. + r.p.ifaces = append(r.p.ifaces, iface) + + return iface +} + +func (r *reader) signature(recv *types.Var, rtparams, tparams []*types.TypeParam) *types.Signature { + r.Sync(pkgbits.SyncSignature) + + params := r.params() + results := r.params() + variadic := r.Bool() + + return types.NewSignatureType(recv, rtparams, tparams, params, results, variadic) +} + +func (r *reader) params() *types.Tuple { + r.Sync(pkgbits.SyncParams) + + params := make([]*types.Var, r.Len()) + for i := range params { + params[i] = r.param() + } + + return types.NewTuple(params...) +} + +func (r *reader) param() *types.Var { + r.Sync(pkgbits.SyncParam) + + pos := r.pos() + pkg, name := r.localIdent() + typ := r.typ() + + return types.NewParam(pos, pkg, name, typ) +} + +// @@@ Objects + +func (r *reader) obj() (types.Object, []types.Type) { + r.Sync(pkgbits.SyncObject) + + assert(!r.Bool()) + + pkg, name := r.p.objIdx(r.Reloc(pkgbits.RelocObj)) + obj := pkgScope(pkg).Lookup(name) + + targs := make([]types.Type, r.Len()) + for i := range targs { + targs[i] = r.typ() + } + + return obj, targs +} + +func (pr *pkgReader) objIdx(idx pkgbits.Index) (*types.Package, string) { + + var objPkg *types.Package + var objName string + var tag pkgbits.CodeObj + { + rname := pr.tempReader(pkgbits.RelocName, idx, pkgbits.SyncObject1) + + objPkg, objName = rname.qualifiedIdent() + assert(objName != "") + + tag = pkgbits.CodeObj(rname.Code(pkgbits.SyncCodeObj)) + pr.retireReader(rname) + } + + if tag == pkgbits.ObjStub { + assert(objPkg == nil || objPkg == types.Unsafe) + return objPkg, objName + } + + // Ignore local types promoted to global scope (#55110). + if _, suffix := splitVargenSuffix(objName); suffix != "" { + return objPkg, objName + } + + if objPkg.Scope().Lookup(objName) == nil { + dict := pr.objDictIdx(idx) + + r := pr.newReader(pkgbits.RelocObj, idx, pkgbits.SyncObject1) + r.dict = dict + + declare := func(obj types.Object) { + objPkg.Scope().Insert(obj) + } + + switch tag { + default: + panic("weird") + + case pkgbits.ObjAlias: + pos := r.pos() + typ := r.typ() + declare(types.NewTypeName(pos, objPkg, objName, typ)) + + case pkgbits.ObjConst: + pos := r.pos() + typ := r.typ() + val := r.Value() + declare(types.NewConst(pos, objPkg, objName, typ, val)) + + case pkgbits.ObjFunc: + pos := r.pos() + tparams := r.typeParamNames() + sig := r.signature(nil, nil, tparams) + declare(types.NewFunc(pos, objPkg, objName, sig)) + + case pkgbits.ObjType: + pos := r.pos() + + obj := types.NewTypeName(pos, objPkg, objName, nil) + named := types.NewNamed(obj, nil, nil) + declare(obj) + + named.SetTypeParams(r.typeParamNames()) + + underlying := r.typ().Underlying() + + // If the underlying type is an interface, we need to + // duplicate its methods so we can replace the receiver + // parameter's type (#49906). + if iface, ok := underlying.(*types.Interface); ok && iface.NumExplicitMethods() != 0 { + methods := make([]*types.Func, iface.NumExplicitMethods()) + for i := range methods { + fn := iface.ExplicitMethod(i) + sig := fn.Type().(*types.Signature) + + recv := types.NewVar(fn.Pos(), fn.Pkg(), "", named) + methods[i] = types.NewFunc(fn.Pos(), fn.Pkg(), fn.Name(), types.NewSignature(recv, sig.Params(), sig.Results(), sig.Variadic())) + } + + embeds := make([]types.Type, iface.NumEmbeddeds()) + for i := range embeds { + embeds[i] = iface.EmbeddedType(i) + } + + newIface := types.NewInterfaceType(methods, embeds) + r.p.ifaces = append(r.p.ifaces, newIface) + underlying = newIface + } + + named.SetUnderlying(underlying) + + for i, n := 0, r.Len(); i < n; i++ { + named.AddMethod(r.method()) + } + + case pkgbits.ObjVar: + pos := r.pos() + typ := r.typ() + declare(types.NewVar(pos, objPkg, objName, typ)) + } + } + + return objPkg, objName +} + +func (pr *pkgReader) objDictIdx(idx pkgbits.Index) *readerDict { + + var dict readerDict + + { + r := pr.tempReader(pkgbits.RelocObjDict, idx, pkgbits.SyncObject1) + if implicits := r.Len(); implicits != 0 { + errorf("unexpected object with %v implicit type parameter(s)", implicits) + } + + dict.bounds = make([]typeInfo, r.Len()) + for i := range dict.bounds { + dict.bounds[i] = r.typInfo() + } + + dict.derived = make([]derivedInfo, r.Len()) + dict.derivedTypes = make([]types.Type, len(dict.derived)) + for i := range dict.derived { + dict.derived[i] = derivedInfo{r.Reloc(pkgbits.RelocType), r.Bool()} + } + + pr.retireReader(r) + } + // function references follow, but reader doesn't need those + + return &dict +} + +func (r *reader) typeParamNames() []*types.TypeParam { + r.Sync(pkgbits.SyncTypeParamNames) + + // Note: This code assumes it only processes objects without + // implement type parameters. This is currently fine, because + // reader is only used to read in exported declarations, which are + // always package scoped. + + if len(r.dict.bounds) == 0 { + return nil + } + + // Careful: Type parameter lists may have cycles. To allow for this, + // we construct the type parameter list in two passes: first we + // create all the TypeNames and TypeParams, then we construct and + // set the bound type. + + r.dict.tparams = make([]*types.TypeParam, len(r.dict.bounds)) + for i := range r.dict.bounds { + pos := r.pos() + pkg, name := r.localIdent() + + tname := types.NewTypeName(pos, pkg, name, nil) + r.dict.tparams[i] = types.NewTypeParam(tname, nil) + } + + typs := make([]types.Type, len(r.dict.bounds)) + for i, bound := range r.dict.bounds { + typs[i] = r.p.typIdx(bound, r.dict) + } + + // TODO(mdempsky): This is subtle, elaborate further. + // + // We have to save tparams outside of the closure, because + // typeParamNames() can be called multiple times with the same + // dictionary instance. + // + // Also, this needs to happen later to make sure SetUnderlying has + // been called. + // + // TODO(mdempsky): Is it safe to have a single "later" slice or do + // we need to have multiple passes? See comments on CL 386002 and + // go.dev/issue/52104. + tparams := r.dict.tparams + r.p.later(func() { + for i, typ := range typs { + tparams[i].SetConstraint(typ) + } + }) + + return r.dict.tparams +} + +func (r *reader) method() *types.Func { + r.Sync(pkgbits.SyncMethod) + pos := r.pos() + pkg, name := r.selector() + + rparams := r.typeParamNames() + sig := r.signature(r.param(), rparams, nil) + + _ = r.pos() // TODO(mdempsky): Remove; this is a hacker for linker.go. + return types.NewFunc(pos, pkg, name, sig) +} + +func (r *reader) qualifiedIdent() (*types.Package, string) { return r.ident(pkgbits.SyncSym) } +func (r *reader) localIdent() (*types.Package, string) { return r.ident(pkgbits.SyncLocalIdent) } +func (r *reader) selector() (*types.Package, string) { return r.ident(pkgbits.SyncSelector) } + +func (r *reader) ident(marker pkgbits.SyncMarker) (*types.Package, string) { + r.Sync(marker) + return r.pkg(), r.String() +} + +// pkgScope returns pkg.Scope(). +// If pkg is nil, it returns types.Universe instead. +// +// TODO(mdempsky): Remove after x/tools can depend on Go 1.19. +func pkgScope(pkg *types.Package) *types.Scope { + if pkg != nil { + return pkg.Scope() + } + return types.Universe +} diff --git a/src/go/internal/srcimporter/srcimporter.go b/src/go/internal/srcimporter/srcimporter.go new file mode 100644 index 0000000..c964274 --- /dev/null +++ b/src/go/internal/srcimporter/srcimporter.go @@ -0,0 +1,269 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package srcimporter implements importing directly +// from source files rather than installed packages. +package srcimporter // import "go/internal/srcimporter" + +import ( + "fmt" + "go/ast" + "go/build" + "go/parser" + "go/token" + "go/types" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + _ "unsafe" // for go:linkname +) + +// An Importer provides the context for importing packages from source code. +type Importer struct { + ctxt *build.Context + fset *token.FileSet + sizes types.Sizes + packages map[string]*types.Package +} + +// New returns a new Importer for the given context, file set, and map +// of packages. The context is used to resolve import paths to package paths, +// and identifying the files belonging to the package. If the context provides +// non-nil file system functions, they are used instead of the regular package +// os functions. The file set is used to track position information of package +// files; and imported packages are added to the packages map. +func New(ctxt *build.Context, fset *token.FileSet, packages map[string]*types.Package) *Importer { + return &Importer{ + ctxt: ctxt, + fset: fset, + sizes: types.SizesFor(ctxt.Compiler, ctxt.GOARCH), // uses go/types default if GOARCH not found + packages: packages, + } +} + +// Importing is a sentinel taking the place in Importer.packages +// for a package that is in the process of being imported. +var importing types.Package + +// Import(path) is a shortcut for ImportFrom(path, ".", 0). +func (p *Importer) Import(path string) (*types.Package, error) { + return p.ImportFrom(path, ".", 0) // use "." rather than "" (see issue #24441) +} + +// ImportFrom imports the package with the given import path resolved from the given srcDir, +// adds the new package to the set of packages maintained by the importer, and returns the +// package. Package path resolution and file system operations are controlled by the context +// maintained with the importer. The import mode must be zero but is otherwise ignored. +// Packages that are not comprised entirely of pure Go files may fail to import because the +// type checker may not be able to determine all exported entities (e.g. due to cgo dependencies). +func (p *Importer) ImportFrom(path, srcDir string, mode types.ImportMode) (*types.Package, error) { + if mode != 0 { + panic("non-zero import mode") + } + + if abs, err := p.absPath(srcDir); err == nil { // see issue #14282 + srcDir = abs + } + bp, err := p.ctxt.Import(path, srcDir, 0) + if err != nil { + return nil, err // err may be *build.NoGoError - return as is + } + + // package unsafe is known to the type checker + if bp.ImportPath == "unsafe" { + return types.Unsafe, nil + } + + // no need to re-import if the package was imported completely before + pkg := p.packages[bp.ImportPath] + if pkg != nil { + if pkg == &importing { + return nil, fmt.Errorf("import cycle through package %q", bp.ImportPath) + } + if !pkg.Complete() { + // Package exists but is not complete - we cannot handle this + // at the moment since the source importer replaces the package + // wholesale rather than augmenting it (see #19337 for details). + // Return incomplete package with error (see #16088). + return pkg, fmt.Errorf("reimported partially imported package %q", bp.ImportPath) + } + return pkg, nil + } + + p.packages[bp.ImportPath] = &importing + defer func() { + // clean up in case of error + // TODO(gri) Eventually we may want to leave a (possibly empty) + // package in the map in all cases (and use that package to + // identify cycles). See also issue 16088. + if p.packages[bp.ImportPath] == &importing { + p.packages[bp.ImportPath] = nil + } + }() + + var filenames []string + filenames = append(filenames, bp.GoFiles...) + filenames = append(filenames, bp.CgoFiles...) + + files, err := p.parseFiles(bp.Dir, filenames) + if err != nil { + return nil, err + } + + // type-check package files + var firstHardErr error + conf := types.Config{ + IgnoreFuncBodies: true, + // continue type-checking after the first error + Error: func(err error) { + if firstHardErr == nil && !err.(types.Error).Soft { + firstHardErr = err + } + }, + Importer: p, + Sizes: p.sizes, + } + if len(bp.CgoFiles) > 0 { + if p.ctxt.OpenFile != nil { + // cgo, gcc, pkg-config, etc. do not support + // build.Context's VFS. + conf.FakeImportC = true + } else { + setUsesCgo(&conf) + file, err := p.cgo(bp) + if err != nil { + return nil, fmt.Errorf("error processing cgo for package %q: %w", bp.ImportPath, err) + } + files = append(files, file) + } + } + + pkg, err = conf.Check(bp.ImportPath, p.fset, files, nil) + if err != nil { + // If there was a hard error it is possibly unsafe + // to use the package as it may not be fully populated. + // Do not return it (see also #20837, #20855). + if firstHardErr != nil { + pkg = nil + err = firstHardErr // give preference to first hard error over any soft error + } + return pkg, fmt.Errorf("type-checking package %q failed (%v)", bp.ImportPath, err) + } + if firstHardErr != nil { + // this can only happen if we have a bug in go/types + panic("package is not safe yet no error was returned") + } + + p.packages[bp.ImportPath] = pkg + return pkg, nil +} + +func (p *Importer) parseFiles(dir string, filenames []string) ([]*ast.File, error) { + // use build.Context's OpenFile if there is one + open := p.ctxt.OpenFile + if open == nil { + open = func(name string) (io.ReadCloser, error) { return os.Open(name) } + } + + files := make([]*ast.File, len(filenames)) + errors := make([]error, len(filenames)) + + var wg sync.WaitGroup + wg.Add(len(filenames)) + for i, filename := range filenames { + go func(i int, filepath string) { + defer wg.Done() + src, err := open(filepath) + if err != nil { + errors[i] = err // open provides operation and filename in error + return + } + files[i], errors[i] = parser.ParseFile(p.fset, filepath, src, parser.SkipObjectResolution) + src.Close() // ignore Close error - parsing may have succeeded which is all we need + }(i, p.joinPath(dir, filename)) + } + wg.Wait() + + // if there are errors, return the first one for deterministic results + for _, err := range errors { + if err != nil { + return nil, err + } + } + + return files, nil +} + +func (p *Importer) cgo(bp *build.Package) (*ast.File, error) { + tmpdir, err := os.MkdirTemp("", "srcimporter") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmpdir) + + goCmd := "go" + if p.ctxt.GOROOT != "" { + goCmd = filepath.Join(p.ctxt.GOROOT, "bin", "go") + } + args := []string{goCmd, "tool", "cgo", "-objdir", tmpdir} + if bp.Goroot { + switch bp.ImportPath { + case "runtime/cgo": + args = append(args, "-import_runtime_cgo=false", "-import_syscall=false") + case "runtime/race": + args = append(args, "-import_syscall=false") + } + } + args = append(args, "--") + args = append(args, strings.Fields(os.Getenv("CGO_CPPFLAGS"))...) + args = append(args, bp.CgoCPPFLAGS...) + if len(bp.CgoPkgConfig) > 0 { + cmd := exec.Command("pkg-config", append([]string{"--cflags"}, bp.CgoPkgConfig...)...) + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("pkg-config --cflags: %w", err) + } + args = append(args, strings.Fields(string(out))...) + } + args = append(args, "-I", tmpdir) + args = append(args, strings.Fields(os.Getenv("CGO_CFLAGS"))...) + args = append(args, bp.CgoCFLAGS...) + args = append(args, bp.CgoFiles...) + + cmd := exec.Command(args[0], args[1:]...) + cmd.Dir = bp.Dir + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("go tool cgo: %w", err) + } + + return parser.ParseFile(p.fset, filepath.Join(tmpdir, "_cgo_gotypes.go"), nil, parser.SkipObjectResolution) +} + +// context-controlled file system operations + +func (p *Importer) absPath(path string) (string, error) { + // TODO(gri) This should be using p.ctxt.AbsPath which doesn't + // exist but probably should. See also issue #14282. + return filepath.Abs(path) +} + +func (p *Importer) isAbsPath(path string) bool { + if f := p.ctxt.IsAbsPath; f != nil { + return f(path) + } + return filepath.IsAbs(path) +} + +func (p *Importer) joinPath(elem ...string) string { + if f := p.ctxt.JoinPath; f != nil { + return f(elem...) + } + return filepath.Join(elem...) +} + +//go:linkname setUsesCgo go/types.srcimporter_setUsesCgo +func setUsesCgo(conf *types.Config) diff --git a/src/go/internal/srcimporter/srcimporter_test.go b/src/go/internal/srcimporter/srcimporter_test.go new file mode 100644 index 0000000..e877458 --- /dev/null +++ b/src/go/internal/srcimporter/srcimporter_test.go @@ -0,0 +1,253 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package srcimporter + +import ( + "flag" + "go/build" + "go/token" + "go/types" + "internal/testenv" + "os" + "path" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestMain(m *testing.M) { + flag.Parse() + build.Default.GOROOT = testenv.GOROOT(nil) + os.Exit(m.Run()) +} + +const maxTime = 2 * time.Second + +var importer = New(&build.Default, token.NewFileSet(), make(map[string]*types.Package)) + +func doImport(t *testing.T, path, srcDir string) { + t0 := time.Now() + if _, err := importer.ImportFrom(path, srcDir, 0); err != nil { + // don't report an error if there's no buildable Go files + if _, nogo := err.(*build.NoGoError); !nogo { + t.Errorf("import %q failed (%v)", path, err) + } + return + } + t.Logf("import %q: %v", path, time.Since(t0)) +} + +// walkDir imports the all the packages with the given path +// prefix recursively. It returns the number of packages +// imported and whether importing was aborted because time +// has passed endTime. +func walkDir(t *testing.T, path string, endTime time.Time) (int, bool) { + if time.Now().After(endTime) { + t.Log("testing time used up") + return 0, true + } + + // ignore fake packages and testdata directories + if path == "builtin" || path == "unsafe" || strings.HasSuffix(path, "testdata") { + return 0, false + } + + list, err := os.ReadDir(filepath.Join(testenv.GOROOT(t), "src", path)) + if err != nil { + t.Fatalf("walkDir %s failed (%v)", path, err) + } + + nimports := 0 + hasGoFiles := false + for _, f := range list { + if f.IsDir() { + n, abort := walkDir(t, filepath.Join(path, f.Name()), endTime) + nimports += n + if abort { + return nimports, true + } + } else if strings.HasSuffix(f.Name(), ".go") { + hasGoFiles = true + } + } + + if hasGoFiles { + doImport(t, path, "") + nimports++ + } + + return nimports, false +} + +func TestImportStdLib(t *testing.T) { + if !testenv.HasSrc() { + t.Skip("no source code available") + } + + if testing.Short() && testenv.Builder() == "" { + t.Skip("skipping in -short mode") + } + dt := maxTime + nimports, _ := walkDir(t, "", time.Now().Add(dt)) // installed packages + t.Logf("tested %d imports", nimports) +} + +var importedObjectTests = []struct { + name string + want string +}{ + {"flag.Bool", "func Bool(name string, value bool, usage string) *bool"}, + {"io.Reader", "type Reader interface{Read(p []byte) (n int, err error)}"}, + {"io.ReadWriter", "type ReadWriter interface{Reader; Writer}"}, // go/types.gcCompatibilityMode is off => interface not flattened + {"math.Pi", "const Pi untyped float"}, + {"math.Sin", "func Sin(x float64) float64"}, + {"math/big.Int", "type Int struct{neg bool; abs nat}"}, + {"golang.org/x/text/unicode/norm.MaxSegmentSize", "const MaxSegmentSize untyped int"}, +} + +func TestImportedTypes(t *testing.T) { + if !testenv.HasSrc() { + t.Skip("no source code available") + } + + for _, test := range importedObjectTests { + i := strings.LastIndex(test.name, ".") + if i < 0 { + t.Fatal("invalid test data format") + } + importPath := test.name[:i] + objName := test.name[i+1:] + + pkg, err := importer.ImportFrom(importPath, ".", 0) + if err != nil { + t.Error(err) + continue + } + + obj := pkg.Scope().Lookup(objName) + if obj == nil { + t.Errorf("%s: object not found", test.name) + continue + } + + got := types.ObjectString(obj, types.RelativeTo(pkg)) + if got != test.want { + t.Errorf("%s: got %q; want %q", test.name, got, test.want) + } + + if named, _ := obj.Type().(*types.Named); named != nil { + verifyInterfaceMethodRecvs(t, named, 0) + } + } +} + +// verifyInterfaceMethodRecvs verifies that method receiver types +// are named if the methods belong to a named interface type. +func verifyInterfaceMethodRecvs(t *testing.T, named *types.Named, level int) { + // avoid endless recursion in case of an embedding bug that lead to a cycle + if level > 10 { + t.Errorf("%s: embeds itself", named) + return + } + + iface, _ := named.Underlying().(*types.Interface) + if iface == nil { + return // not an interface + } + + // check explicitly declared methods + for i := 0; i < iface.NumExplicitMethods(); i++ { + m := iface.ExplicitMethod(i) + recv := m.Type().(*types.Signature).Recv() + if recv == nil { + t.Errorf("%s: missing receiver type", m) + continue + } + if recv.Type() != named { + t.Errorf("%s: got recv type %s; want %s", m, recv.Type(), named) + } + } + + // check embedded interfaces (they are named, too) + for i := 0; i < iface.NumEmbeddeds(); i++ { + // embedding of interfaces cannot have cycles; recursion will terminate + verifyInterfaceMethodRecvs(t, iface.Embedded(i), level+1) + } +} + +func TestReimport(t *testing.T) { + if !testenv.HasSrc() { + t.Skip("no source code available") + } + + // Reimporting a partially imported (incomplete) package is not supported (see issue #19337). + // Make sure we recognize the situation and report an error. + + mathPkg := types.NewPackage("math", "math") // incomplete package + importer := New(&build.Default, token.NewFileSet(), map[string]*types.Package{mathPkg.Path(): mathPkg}) + _, err := importer.ImportFrom("math", ".", 0) + if err == nil || !strings.HasPrefix(err.Error(), "reimport") { + t.Errorf("got %v; want reimport error", err) + } +} + +func TestIssue20855(t *testing.T) { + if !testenv.HasSrc() { + t.Skip("no source code available") + } + + pkg, err := importer.ImportFrom("go/internal/srcimporter/testdata/issue20855", ".", 0) + if err == nil || !strings.Contains(err.Error(), "missing function body") { + t.Fatalf("got unexpected or no error: %v", err) + } + if pkg == nil { + t.Error("got no package despite no hard errors") + } +} + +func testImportPath(t *testing.T, pkgPath string) { + if !testenv.HasSrc() { + t.Skip("no source code available") + } + + pkgName := path.Base(pkgPath) + + pkg, err := importer.Import(pkgPath) + if err != nil { + t.Fatal(err) + } + + if pkg.Name() != pkgName { + t.Errorf("got %q; want %q", pkg.Name(), pkgName) + } + + if pkg.Path() != pkgPath { + t.Errorf("got %q; want %q", pkg.Path(), pkgPath) + } +} + +// TestIssue23092 tests relative imports. +func TestIssue23092(t *testing.T) { + testImportPath(t, "./testdata/issue23092") +} + +// TestIssue24392 tests imports against a path containing 'testdata'. +func TestIssue24392(t *testing.T) { + testImportPath(t, "go/internal/srcimporter/testdata/issue24392") +} + +func TestCgo(t *testing.T) { + testenv.MustHaveGoBuild(t) + testenv.MustHaveCGO(t) + + buildCtx := build.Default + buildCtx.Dir = filepath.Join(testenv.GOROOT(t), "misc") + importer := New(&buildCtx, token.NewFileSet(), make(map[string]*types.Package)) + _, err := importer.ImportFrom("./cgo/test", buildCtx.Dir, 0) + if err != nil { + t.Fatalf("Import failed: %v", err) + } +} diff --git a/src/go/internal/srcimporter/testdata/issue20855/issue20855.go b/src/go/internal/srcimporter/testdata/issue20855/issue20855.go new file mode 100644 index 0000000..d55448b --- /dev/null +++ b/src/go/internal/srcimporter/testdata/issue20855/issue20855.go @@ -0,0 +1,7 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue20855 + +func init() // "missing function body" is a soft error diff --git a/src/go/internal/srcimporter/testdata/issue23092/issue23092.go b/src/go/internal/srcimporter/testdata/issue23092/issue23092.go new file mode 100644 index 0000000..608698b --- /dev/null +++ b/src/go/internal/srcimporter/testdata/issue23092/issue23092.go @@ -0,0 +1,5 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue23092 diff --git a/src/go/internal/srcimporter/testdata/issue24392/issue24392.go b/src/go/internal/srcimporter/testdata/issue24392/issue24392.go new file mode 100644 index 0000000..8ad5221 --- /dev/null +++ b/src/go/internal/srcimporter/testdata/issue24392/issue24392.go @@ -0,0 +1,5 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue24392 diff --git a/src/go/internal/typeparams/typeparams.go b/src/go/internal/typeparams/typeparams.go new file mode 100644 index 0000000..3f84f2f --- /dev/null +++ b/src/go/internal/typeparams/typeparams.go @@ -0,0 +1,54 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package typeparams + +import ( + "go/ast" + "go/token" +) + +func PackIndexExpr(x ast.Expr, lbrack token.Pos, exprs []ast.Expr, rbrack token.Pos) ast.Expr { + switch len(exprs) { + case 0: + panic("internal error: PackIndexExpr with empty expr slice") + case 1: + return &ast.IndexExpr{ + X: x, + Lbrack: lbrack, + Index: exprs[0], + Rbrack: rbrack, + } + default: + return &ast.IndexListExpr{ + X: x, + Lbrack: lbrack, + Indices: exprs, + Rbrack: rbrack, + } + } +} + +// IndexExpr wraps an ast.IndexExpr or ast.IndexListExpr. +// +// Orig holds the original ast.Expr from which this IndexExpr was derived. +type IndexExpr struct { + Orig ast.Expr // the wrapped expr, which may be distinct from the IndexListExpr below. + *ast.IndexListExpr +} + +func UnpackIndexExpr(n ast.Node) *IndexExpr { + switch e := n.(type) { + case *ast.IndexExpr: + return &IndexExpr{e, &ast.IndexListExpr{ + X: e.X, + Lbrack: e.Lbrack, + Indices: []ast.Expr{e.Index}, + Rbrack: e.Rbrack, + }} + case *ast.IndexListExpr: + return &IndexExpr{e, e} + } + return nil +} diff --git a/src/go/parser/error_test.go b/src/go/parser/error_test.go new file mode 100644 index 0000000..a4e17dd --- /dev/null +++ b/src/go/parser/error_test.go @@ -0,0 +1,202 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements a parser test harness. The files in the testdata +// directory are parsed and the errors reported are compared against the +// error messages expected in the test files. The test files must end in +// .src rather than .go so that they are not disturbed by gofmt runs. +// +// Expected errors are indicated in the test files by putting a comment +// of the form /* ERROR "rx" */ immediately following an offending token. +// The harness will verify that an error matching the regular expression +// rx is reported at that source position. +// +// For instance, the following test file indicates that a "not declared" +// error should be reported for the undeclared variable x: +// +// package p +// func f() { +// _ = x /* ERROR "not declared" */ + 1 +// } + +package parser + +import ( + "flag" + "go/scanner" + "go/token" + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +var traceErrs = flag.Bool("trace_errs", false, "whether to enable tracing for error tests") + +const testdata = "testdata" + +// getFile assumes that each filename occurs at most once +func getFile(fset *token.FileSet, filename string) (file *token.File) { + fset.Iterate(func(f *token.File) bool { + if f.Name() == filename { + if file != nil { + panic(filename + " used multiple times") + } + file = f + } + return true + }) + return file +} + +func getPos(fset *token.FileSet, filename string, offset int) token.Pos { + if f := getFile(fset, filename); f != nil { + return f.Pos(offset) + } + return token.NoPos +} + +// ERROR comments must be of the form /* ERROR "rx" */ and rx is +// a regular expression that matches the expected error message. +// The special form /* ERROR HERE "rx" */ must be used for error +// messages that appear immediately after a token, rather than at +// a token's position, and ERROR AFTER means after the comment +// (e.g. at end of line). +var errRx = regexp.MustCompile(`^/\* *ERROR *(HERE|AFTER)? *"([^"]*)" *\*/$`) + +// expectedErrors collects the regular expressions of ERROR comments found +// in files and returns them as a map of error positions to error messages. +func expectedErrors(fset *token.FileSet, filename string, src []byte) map[token.Pos]string { + errors := make(map[token.Pos]string) + + var s scanner.Scanner + // file was parsed already - do not add it again to the file + // set otherwise the position information returned here will + // not match the position information collected by the parser + s.Init(getFile(fset, filename), src, nil, scanner.ScanComments) + var prev token.Pos // position of last non-comment, non-semicolon token + var here token.Pos // position immediately after the token at position prev + + for { + pos, tok, lit := s.Scan() + switch tok { + case token.EOF: + return errors + case token.COMMENT: + s := errRx.FindStringSubmatch(lit) + if len(s) == 3 { + if s[1] == "HERE" { + pos = here // start of comment + } else if s[1] == "AFTER" { + pos += token.Pos(len(lit)) // end of comment + } else { + pos = prev // token prior to comment + } + errors[pos] = s[2] + } + case token.SEMICOLON: + // don't use the position of auto-inserted (invisible) semicolons + if lit != ";" { + break + } + fallthrough + default: + prev = pos + var l int // token length + if tok.IsLiteral() { + l = len(lit) + } else { + l = len(tok.String()) + } + here = prev + token.Pos(l) + } + } +} + +// compareErrors compares the map of expected error messages with the list +// of found errors and reports discrepancies. +func compareErrors(t *testing.T, fset *token.FileSet, expected map[token.Pos]string, found scanner.ErrorList) { + t.Helper() + for _, error := range found { + // error.Pos is a token.Position, but we want + // a token.Pos so we can do a map lookup + pos := getPos(fset, error.Pos.Filename, error.Pos.Offset) + if msg, found := expected[pos]; found { + // we expect a message at pos; check if it matches + rx, err := regexp.Compile(msg) + if err != nil { + t.Errorf("%s: %v", error.Pos, err) + continue + } + if match := rx.MatchString(error.Msg); !match { + t.Errorf("%s: %q does not match %q", error.Pos, error.Msg, msg) + continue + } + // we have a match - eliminate this error + delete(expected, pos) + } else { + // To keep in mind when analyzing failed test output: + // If the same error position occurs multiple times in errors, + // this message will be triggered (because the first error at + // the position removes this position from the expected errors). + t.Errorf("%s: unexpected error: %s", error.Pos, error.Msg) + } + } + + // there should be no expected errors left + if len(expected) > 0 { + t.Errorf("%d errors not reported:", len(expected)) + for pos, msg := range expected { + t.Errorf("%s: %s\n", fset.Position(pos), msg) + } + } +} + +func checkErrors(t *testing.T, filename string, input any, mode Mode, expectErrors bool) { + t.Helper() + src, err := readSource(filename, input) + if err != nil { + t.Error(err) + return + } + + fset := token.NewFileSet() + _, err = ParseFile(fset, filename, src, mode) + found, ok := err.(scanner.ErrorList) + if err != nil && !ok { + t.Error(err) + return + } + found.RemoveMultiples() + + expected := map[token.Pos]string{} + if expectErrors { + // we are expecting the following errors + // (collect these after parsing a file so that it is found in the file set) + expected = expectedErrors(fset, filename, src) + } + + // verify errors returned by the parser + compareErrors(t, fset, expected, found) +} + +func TestErrors(t *testing.T) { + list, err := os.ReadDir(testdata) + if err != nil { + t.Fatal(err) + } + for _, d := range list { + name := d.Name() + t.Run(name, func(t *testing.T) { + if !d.IsDir() && !strings.HasPrefix(name, ".") && (strings.HasSuffix(name, ".src") || strings.HasSuffix(name, ".go2")) { + mode := DeclarationErrors | AllErrors + if *traceErrs { + mode |= Trace + } + checkErrors(t, filepath.Join(testdata, name), nil, mode, true) + } + }) + } +} diff --git a/src/go/parser/example_test.go b/src/go/parser/example_test.go new file mode 100644 index 0000000..c2f7f29 --- /dev/null +++ b/src/go/parser/example_test.go @@ -0,0 +1,43 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parser_test + +import ( + "fmt" + "go/parser" + "go/token" +) + +func ExampleParseFile() { + fset := token.NewFileSet() // positions are relative to fset + + src := `package foo + +import ( + "fmt" + "time" +) + +func bar() { + fmt.Println(time.Now()) +}` + + // Parse src but stop after processing the imports. + f, err := parser.ParseFile(fset, "", src, parser.ImportsOnly) + if err != nil { + fmt.Println(err) + return + } + + // Print the imports from the file's AST. + for _, s := range f.Imports { + fmt.Println(s.Path.Value) + } + + // output: + // + // "fmt" + // "time" +} diff --git a/src/go/parser/interface.go b/src/go/parser/interface.go new file mode 100644 index 0000000..73cb162 --- /dev/null +++ b/src/go/parser/interface.go @@ -0,0 +1,238 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains the exported entry points for invoking the parser. + +package parser + +import ( + "bytes" + "errors" + "go/ast" + "go/token" + "io" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// If src != nil, readSource converts src to a []byte if possible; +// otherwise it returns an error. If src == nil, readSource returns +// the result of reading the file specified by filename. +func readSource(filename string, src any) ([]byte, error) { + if src != nil { + switch s := src.(type) { + case string: + return []byte(s), nil + case []byte: + return s, nil + case *bytes.Buffer: + // is io.Reader, but src is already available in []byte form + if s != nil { + return s.Bytes(), nil + } + case io.Reader: + return io.ReadAll(s) + } + return nil, errors.New("invalid source") + } + return os.ReadFile(filename) +} + +// A Mode value is a set of flags (or 0). +// They control the amount of source code parsed and other optional +// parser functionality. +type Mode uint + +const ( + PackageClauseOnly Mode = 1 << iota // stop parsing after package clause + ImportsOnly // stop parsing after import declarations + ParseComments // parse comments and add them to AST + Trace // print a trace of parsed productions + DeclarationErrors // report declaration errors + SpuriousErrors // same as AllErrors, for backward-compatibility + SkipObjectResolution // don't resolve identifiers to objects - see ParseFile + AllErrors = SpuriousErrors // report all errors (not just the first 10 on different lines) +) + +// ParseFile parses the source code of a single Go source file and returns +// the corresponding ast.File node. The source code may be provided via +// the filename of the source file, or via the src parameter. +// +// If src != nil, ParseFile parses the source from src and the filename is +// only used when recording position information. The type of the argument +// for the src parameter must be string, []byte, or io.Reader. +// If src == nil, ParseFile parses the file specified by filename. +// +// The mode parameter controls the amount of source text parsed and other +// optional parser functionality. If the SkipObjectResolution mode bit is set, +// the object resolution phase of parsing will be skipped, causing File.Scope, +// File.Unresolved, and all Ident.Obj fields to be nil. +// +// Position information is recorded in the file set fset, which must not be +// nil. +// +// If the source couldn't be read, the returned AST is nil and the error +// indicates the specific failure. If the source was read but syntax +// errors were found, the result is a partial AST (with ast.Bad* nodes +// representing the fragments of erroneous source code). Multiple errors +// are returned via a scanner.ErrorList which is sorted by source position. +func ParseFile(fset *token.FileSet, filename string, src any, mode Mode) (f *ast.File, err error) { + if fset == nil { + panic("parser.ParseFile: no token.FileSet provided (fset == nil)") + } + + // get source + text, err := readSource(filename, src) + if err != nil { + return nil, err + } + + var p parser + defer func() { + if e := recover(); e != nil { + // resume same panic if it's not a bailout + bail, ok := e.(bailout) + if !ok { + panic(e) + } else if bail.msg != "" { + p.errors.Add(p.file.Position(bail.pos), bail.msg) + } + } + + // set result values + if f == nil { + // source is not a valid Go source file - satisfy + // ParseFile API and return a valid (but) empty + // *ast.File + f = &ast.File{ + Name: new(ast.Ident), + Scope: ast.NewScope(nil), + } + } + + p.errors.Sort() + err = p.errors.Err() + }() + + // parse source + p.init(fset, filename, text, mode) + f = p.parseFile() + + return +} + +// ParseDir calls ParseFile for all files with names ending in ".go" in the +// directory specified by path and returns a map of package name -> package +// AST with all the packages found. +// +// If filter != nil, only the files with fs.FileInfo entries passing through +// the filter (and ending in ".go") are considered. The mode bits are passed +// to ParseFile unchanged. Position information is recorded in fset, which +// must not be nil. +// +// If the directory couldn't be read, a nil map and the respective error are +// returned. If a parse error occurred, a non-nil but incomplete map and the +// first error encountered are returned. +func ParseDir(fset *token.FileSet, path string, filter func(fs.FileInfo) bool, mode Mode) (pkgs map[string]*ast.Package, first error) { + list, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + pkgs = make(map[string]*ast.Package) + for _, d := range list { + if d.IsDir() || !strings.HasSuffix(d.Name(), ".go") { + continue + } + if filter != nil { + info, err := d.Info() + if err != nil { + return nil, err + } + if !filter(info) { + continue + } + } + filename := filepath.Join(path, d.Name()) + if src, err := ParseFile(fset, filename, nil, mode); err == nil { + name := src.Name.Name + pkg, found := pkgs[name] + if !found { + pkg = &ast.Package{ + Name: name, + Files: make(map[string]*ast.File), + } + pkgs[name] = pkg + } + pkg.Files[filename] = src + } else if first == nil { + first = err + } + } + + return +} + +// ParseExprFrom is a convenience function for parsing an expression. +// The arguments have the same meaning as for ParseFile, but the source must +// be a valid Go (type or value) expression. Specifically, fset must not +// be nil. +// +// If the source couldn't be read, the returned AST is nil and the error +// indicates the specific failure. If the source was read but syntax +// errors were found, the result is a partial AST (with ast.Bad* nodes +// representing the fragments of erroneous source code). Multiple errors +// are returned via a scanner.ErrorList which is sorted by source position. +func ParseExprFrom(fset *token.FileSet, filename string, src any, mode Mode) (expr ast.Expr, err error) { + if fset == nil { + panic("parser.ParseExprFrom: no token.FileSet provided (fset == nil)") + } + + // get source + text, err := readSource(filename, src) + if err != nil { + return nil, err + } + + var p parser + defer func() { + if e := recover(); e != nil { + // resume same panic if it's not a bailout + bail, ok := e.(bailout) + if !ok { + panic(e) + } else if bail.msg != "" { + p.errors.Add(p.file.Position(bail.pos), bail.msg) + } + } + p.errors.Sort() + err = p.errors.Err() + }() + + // parse expr + p.init(fset, filename, text, mode) + expr = p.parseRhs() + + // If a semicolon was inserted, consume it; + // report an error if there's more tokens. + if p.tok == token.SEMICOLON && p.lit == "\n" { + p.next() + } + p.expect(token.EOF) + + return +} + +// ParseExpr is a convenience function for obtaining the AST of an expression x. +// The position information recorded in the AST is undefined. The filename used +// in error messages is the empty string. +// +// If syntax errors were found, the result is a partial AST (with ast.Bad* nodes +// representing the fragments of erroneous source code). Multiple errors are +// returned via a scanner.ErrorList which is sorted by source position. +func ParseExpr(x string) (ast.Expr, error) { + return ParseExprFrom(token.NewFileSet(), "", []byte(x), 0) +} diff --git a/src/go/parser/parser.go b/src/go/parser/parser.go new file mode 100644 index 0000000..fac24df --- /dev/null +++ b/src/go/parser/parser.go @@ -0,0 +1,2864 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package parser implements a parser for Go source files. Input may be +// provided in a variety of forms (see the various Parse* functions); the +// output is an abstract syntax tree (AST) representing the Go source. The +// parser is invoked through one of the Parse* functions. +// +// The parser accepts a larger language than is syntactically permitted by +// the Go spec, for simplicity, and for improved robustness in the presence +// of syntax errors. For instance, in method declarations, the receiver is +// treated like an ordinary parameter list and thus may contain multiple +// entries where the spec permits exactly one. Consequently, the corresponding +// field in the AST (ast.FuncDecl.Recv) field is not restricted to one entry. +package parser + +import ( + "fmt" + "go/ast" + "go/internal/typeparams" + "go/scanner" + "go/token" +) + +// The parser structure holds the parser's internal state. +type parser struct { + file *token.File + errors scanner.ErrorList + scanner scanner.Scanner + + // Tracing/debugging + mode Mode // parsing mode + trace bool // == (mode&Trace != 0) + indent int // indentation used for tracing output + + // Comments + comments []*ast.CommentGroup + leadComment *ast.CommentGroup // last lead comment + lineComment *ast.CommentGroup // last line comment + + // Next token + pos token.Pos // token position + tok token.Token // one token look-ahead + lit string // token literal + + // Error recovery + // (used to limit the number of calls to parser.advance + // w/o making scanning progress - avoids potential endless + // loops across multiple parser functions during error recovery) + syncPos token.Pos // last synchronization position + syncCnt int // number of parser.advance calls without progress + + // Non-syntactic parser control + exprLev int // < 0: in control clause, >= 0: in expression + inRhs bool // if set, the parser is parsing a rhs expression + + imports []*ast.ImportSpec // list of imports + + // nestLev is used to track and limit the recursion depth + // during parsing. + nestLev int +} + +func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode Mode) { + p.file = fset.AddFile(filename, -1, len(src)) + var m scanner.Mode + if mode&ParseComments != 0 { + m = scanner.ScanComments + } + eh := func(pos token.Position, msg string) { p.errors.Add(pos, msg) } + p.scanner.Init(p.file, src, eh, m) + + p.mode = mode + p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently) + p.next() +} + +// ---------------------------------------------------------------------------- +// Parsing support + +func (p *parser) printTrace(a ...any) { + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + const n = len(dots) + pos := p.file.Position(p.pos) + fmt.Printf("%5d:%3d: ", pos.Line, pos.Column) + i := 2 * p.indent + for i > n { + fmt.Print(dots) + i -= n + } + // i <= n + fmt.Print(dots[0:i]) + fmt.Println(a...) +} + +func trace(p *parser, msg string) *parser { + p.printTrace(msg, "(") + p.indent++ + return p +} + +// Usage pattern: defer un(trace(p, "...")) +func un(p *parser) { + p.indent-- + p.printTrace(")") +} + +// maxNestLev is the deepest we're willing to recurse during parsing +const maxNestLev int = 1e5 + +func incNestLev(p *parser) *parser { + p.nestLev++ + if p.nestLev > maxNestLev { + p.error(p.pos, "exceeded max nesting depth") + panic(bailout{}) + } + return p +} + +// decNestLev is used to track nesting depth during parsing to prevent stack exhaustion. +// It is used along with incNestLev in a similar fashion to how un and trace are used. +func decNestLev(p *parser) { + p.nestLev-- +} + +// Advance to the next token. +func (p *parser) next0() { + // Because of one-token look-ahead, print the previous token + // when tracing as it provides a more readable output. The + // very first token (!p.pos.IsValid()) is not initialized + // (it is token.ILLEGAL), so don't print it. + if p.trace && p.pos.IsValid() { + s := p.tok.String() + switch { + case p.tok.IsLiteral(): + p.printTrace(s, p.lit) + case p.tok.IsOperator(), p.tok.IsKeyword(): + p.printTrace("\"" + s + "\"") + default: + p.printTrace(s) + } + } + + p.pos, p.tok, p.lit = p.scanner.Scan() +} + +// Consume a comment and return it and the line on which it ends. +func (p *parser) consumeComment() (comment *ast.Comment, endline int) { + // /*-style comments may end on a different line than where they start. + // Scan the comment for '\n' chars and adjust endline accordingly. + endline = p.file.Line(p.pos) + if p.lit[1] == '*' { + // don't use range here - no need to decode Unicode code points + for i := 0; i < len(p.lit); i++ { + if p.lit[i] == '\n' { + endline++ + } + } + } + + comment = &ast.Comment{Slash: p.pos, Text: p.lit} + p.next0() + + return +} + +// Consume a group of adjacent comments, add it to the parser's +// comments list, and return it together with the line at which +// the last comment in the group ends. A non-comment token or n +// empty lines terminate a comment group. +func (p *parser) consumeCommentGroup(n int) (comments *ast.CommentGroup, endline int) { + var list []*ast.Comment + endline = p.file.Line(p.pos) + for p.tok == token.COMMENT && p.file.Line(p.pos) <= endline+n { + var comment *ast.Comment + comment, endline = p.consumeComment() + list = append(list, comment) + } + + // add comment group to the comments list + comments = &ast.CommentGroup{List: list} + p.comments = append(p.comments, comments) + + return +} + +// Advance to the next non-comment token. In the process, collect +// any comment groups encountered, and remember the last lead and +// line comments. +// +// A lead comment is a comment group that starts and ends in a +// line without any other tokens and that is followed by a non-comment +// token on the line immediately after the comment group. +// +// A line comment is a comment group that follows a non-comment +// token on the same line, and that has no tokens after it on the line +// where it ends. +// +// Lead and line comments may be considered documentation that is +// stored in the AST. +func (p *parser) next() { + p.leadComment = nil + p.lineComment = nil + prev := p.pos + p.next0() + + if p.tok == token.COMMENT { + var comment *ast.CommentGroup + var endline int + + if p.file.Line(p.pos) == p.file.Line(prev) { + // The comment is on same line as the previous token; it + // cannot be a lead comment but may be a line comment. + comment, endline = p.consumeCommentGroup(0) + if p.file.Line(p.pos) != endline || p.tok == token.SEMICOLON || p.tok == token.EOF { + // The next token is on a different line, thus + // the last comment group is a line comment. + p.lineComment = comment + } + } + + // consume successor comments, if any + endline = -1 + for p.tok == token.COMMENT { + comment, endline = p.consumeCommentGroup(1) + } + + if endline+1 == p.file.Line(p.pos) { + // The next token is following on the line immediately after the + // comment group, thus the last comment group is a lead comment. + p.leadComment = comment + } + } +} + +// A bailout panic is raised to indicate early termination. pos and msg are +// only populated when bailing out of object resolution. +type bailout struct { + pos token.Pos + msg string +} + +func (p *parser) error(pos token.Pos, msg string) { + if p.trace { + defer un(trace(p, "error: "+msg)) + } + + epos := p.file.Position(pos) + + // If AllErrors is not set, discard errors reported on the same line + // as the last recorded error and stop parsing if there are more than + // 10 errors. + if p.mode&AllErrors == 0 { + n := len(p.errors) + if n > 0 && p.errors[n-1].Pos.Line == epos.Line { + return // discard - likely a spurious error + } + if n > 10 { + panic(bailout{}) + } + } + + p.errors.Add(epos, msg) +} + +func (p *parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + switch { + case p.tok == token.SEMICOLON && p.lit == "\n": + msg += ", found newline" + case p.tok.IsLiteral(): + // print 123 rather than 'INT', etc. + msg += ", found " + p.lit + default: + msg += ", found '" + p.tok.String() + "'" + } + } + p.error(pos, msg) +} + +func (p *parser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress + return pos +} + +// expect2 is like expect, but it returns an invalid position +// if the expected token is not found. +func (p *parser) expect2(tok token.Token) (pos token.Pos) { + if p.tok == tok { + pos = p.pos + } else { + p.errorExpected(p.pos, "'"+tok.String()+"'") + } + p.next() // make progress + return +} + +// expectClosing is like expect but provides a better error message +// for the common case of a missing comma before a newline. +func (p *parser) expectClosing(tok token.Token, context string) token.Pos { + if p.tok != tok && p.tok == token.SEMICOLON && p.lit == "\n" { + p.error(p.pos, "missing ',' before newline in "+context) + p.next() + } + return p.expect(tok) +} + +// expectSemi consumes a semicolon and returns the applicable line comment. +func (p *parser) expectSemi() (comment *ast.CommentGroup) { + // semicolon is optional before a closing ')' or '}' + if p.tok != token.RPAREN && p.tok != token.RBRACE { + switch p.tok { + case token.COMMA: + // permit a ',' instead of a ';' but complain + p.errorExpected(p.pos, "';'") + fallthrough + case token.SEMICOLON: + if p.lit == ";" { + // explicit semicolon + p.next() + comment = p.lineComment // use following comments + } else { + // artificial semicolon + comment = p.lineComment // use preceding comments + p.next() + } + return comment + default: + p.errorExpected(p.pos, "';'") + p.advance(stmtStart) + } + } + return nil +} + +func (p *parser) atComma(context string, follow token.Token) bool { + if p.tok == token.COMMA { + return true + } + if p.tok != follow { + msg := "missing ','" + if p.tok == token.SEMICOLON && p.lit == "\n" { + msg += " before newline" + } + p.error(p.pos, msg+" in "+context) + return true // "insert" comma and continue + } + return false +} + +func assert(cond bool, msg string) { + if !cond { + panic("go/parser internal error: " + msg) + } +} + +// advance consumes tokens until the current token p.tok +// is in the 'to' set, or token.EOF. For error recovery. +func (p *parser) advance(to map[token.Token]bool) { + for ; p.tok != token.EOF; p.next() { + if to[p.tok] { + // Return only if parser made some progress since last + // sync or if it has not reached 10 advance calls without + // progress. Otherwise consume at least one token to + // avoid an endless parser loop (it is possible that + // both parseOperand and parseStmt call advance and + // correctly do not advance, thus the need for the + // invocation limit p.syncCnt). + if p.pos == p.syncPos && p.syncCnt < 10 { + p.syncCnt++ + return + } + if p.pos > p.syncPos { + p.syncPos = p.pos + p.syncCnt = 0 + return + } + // Reaching here indicates a parser bug, likely an + // incorrect token list in this function, but it only + // leads to skipping of possibly correct code if a + // previous error is present, and thus is preferred + // over a non-terminating parse. + } + } +} + +var stmtStart = map[token.Token]bool{ + token.BREAK: true, + token.CONST: true, + token.CONTINUE: true, + token.DEFER: true, + token.FALLTHROUGH: true, + token.FOR: true, + token.GO: true, + token.GOTO: true, + token.IF: true, + token.RETURN: true, + token.SELECT: true, + token.SWITCH: true, + token.TYPE: true, + token.VAR: true, +} + +var declStart = map[token.Token]bool{ + token.IMPORT: true, + token.CONST: true, + token.TYPE: true, + token.VAR: true, +} + +var exprEnd = map[token.Token]bool{ + token.COMMA: true, + token.COLON: true, + token.SEMICOLON: true, + token.RPAREN: true, + token.RBRACK: true, + token.RBRACE: true, +} + +// safePos returns a valid file position for a given position: If pos +// is valid to begin with, safePos returns pos. If pos is out-of-range, +// safePos returns the EOF position. +// +// This is hack to work around "artificial" end positions in the AST which +// are computed by adding 1 to (presumably valid) token positions. If the +// token positions are invalid due to parse errors, the resulting end position +// may be past the file's EOF position, which would lead to panics if used +// later on. +func (p *parser) safePos(pos token.Pos) (res token.Pos) { + defer func() { + if recover() != nil { + res = token.Pos(p.file.Base() + p.file.Size()) // EOF position + } + }() + _ = p.file.Offset(pos) // trigger a panic if position is out-of-range + return pos +} + +// ---------------------------------------------------------------------------- +// Identifiers + +func (p *parser) parseIdent() *ast.Ident { + pos := p.pos + name := "_" + if p.tok == token.IDENT { + name = p.lit + p.next() + } else { + p.expect(token.IDENT) // use expect() error handling + } + return &ast.Ident{NamePos: pos, Name: name} +} + +func (p *parser) parseIdentList() (list []*ast.Ident) { + if p.trace { + defer un(trace(p, "IdentList")) + } + + list = append(list, p.parseIdent()) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseIdent()) + } + + return +} + +// ---------------------------------------------------------------------------- +// Common productions + +// If lhs is set, result list elements which are identifiers are not resolved. +func (p *parser) parseExprList() (list []ast.Expr) { + if p.trace { + defer un(trace(p, "ExpressionList")) + } + + list = append(list, p.parseExpr()) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseExpr()) + } + + return +} + +func (p *parser) parseList(inRhs bool) []ast.Expr { + old := p.inRhs + p.inRhs = inRhs + list := p.parseExprList() + p.inRhs = old + return list +} + +// ---------------------------------------------------------------------------- +// Types + +func (p *parser) parseType() ast.Expr { + if p.trace { + defer un(trace(p, "Type")) + } + + typ := p.tryIdentOrType() + + if typ == nil { + pos := p.pos + p.errorExpected(pos, "type") + p.advance(exprEnd) + return &ast.BadExpr{From: pos, To: p.pos} + } + + return typ +} + +func (p *parser) parseQualifiedIdent(ident *ast.Ident) ast.Expr { + if p.trace { + defer un(trace(p, "QualifiedIdent")) + } + + typ := p.parseTypeName(ident) + if p.tok == token.LBRACK { + typ = p.parseTypeInstance(typ) + } + + return typ +} + +// If the result is an identifier, it is not resolved. +func (p *parser) parseTypeName(ident *ast.Ident) ast.Expr { + if p.trace { + defer un(trace(p, "TypeName")) + } + + if ident == nil { + ident = p.parseIdent() + } + + if p.tok == token.PERIOD { + // ident is a package name + p.next() + sel := p.parseIdent() + return &ast.SelectorExpr{X: ident, Sel: sel} + } + + return ident +} + +// "[" has already been consumed, and lbrack is its position. +// If len != nil it is the already consumed array length. +func (p *parser) parseArrayType(lbrack token.Pos, len ast.Expr) *ast.ArrayType { + if p.trace { + defer un(trace(p, "ArrayType")) + } + + if len == nil { + p.exprLev++ + // always permit ellipsis for more fault-tolerant parsing + if p.tok == token.ELLIPSIS { + len = &ast.Ellipsis{Ellipsis: p.pos} + p.next() + } else if p.tok != token.RBRACK { + len = p.parseRhs() + } + p.exprLev-- + } + if p.tok == token.COMMA { + // Trailing commas are accepted in type parameter + // lists but not in array type declarations. + // Accept for better error handling but complain. + p.error(p.pos, "unexpected comma; expecting ]") + p.next() + } + p.expect(token.RBRACK) + elt := p.parseType() + return &ast.ArrayType{Lbrack: lbrack, Len: len, Elt: elt} +} + +func (p *parser) parseArrayFieldOrTypeInstance(x *ast.Ident) (*ast.Ident, ast.Expr) { + if p.trace { + defer un(trace(p, "ArrayFieldOrTypeInstance")) + } + + lbrack := p.expect(token.LBRACK) + trailingComma := token.NoPos // if valid, the position of a trailing comma preceding the ']' + var args []ast.Expr + if p.tok != token.RBRACK { + p.exprLev++ + args = append(args, p.parseRhs()) + for p.tok == token.COMMA { + comma := p.pos + p.next() + if p.tok == token.RBRACK { + trailingComma = comma + break + } + args = append(args, p.parseRhs()) + } + p.exprLev-- + } + rbrack := p.expect(token.RBRACK) + + if len(args) == 0 { + // x []E + elt := p.parseType() + return x, &ast.ArrayType{Lbrack: lbrack, Elt: elt} + } + + // x [P]E or x[P] + if len(args) == 1 { + elt := p.tryIdentOrType() + if elt != nil { + // x [P]E + if trailingComma.IsValid() { + // Trailing commas are invalid in array type fields. + p.error(trailingComma, "unexpected comma; expecting ]") + } + return x, &ast.ArrayType{Lbrack: lbrack, Len: args[0], Elt: elt} + } + } + + // x[P], x[P1, P2], ... + return nil, typeparams.PackIndexExpr(x, lbrack, args, rbrack) +} + +func (p *parser) parseFieldDecl() *ast.Field { + if p.trace { + defer un(trace(p, "FieldDecl")) + } + + doc := p.leadComment + + var names []*ast.Ident + var typ ast.Expr + switch p.tok { + case token.IDENT: + name := p.parseIdent() + if p.tok == token.PERIOD || p.tok == token.STRING || p.tok == token.SEMICOLON || p.tok == token.RBRACE { + // embedded type + typ = name + if p.tok == token.PERIOD { + typ = p.parseQualifiedIdent(name) + } + } else { + // name1, name2, ... T + names = []*ast.Ident{name} + for p.tok == token.COMMA { + p.next() + names = append(names, p.parseIdent()) + } + // Careful dance: We don't know if we have an embedded instantiated + // type T[P1, P2, ...] or a field T of array type []E or [P]E. + if len(names) == 1 && p.tok == token.LBRACK { + name, typ = p.parseArrayFieldOrTypeInstance(name) + if name == nil { + names = nil + } + } else { + // T P + typ = p.parseType() + } + } + case token.MUL: + star := p.pos + p.next() + if p.tok == token.LPAREN { + // *(T) + p.error(p.pos, "cannot parenthesize embedded type") + p.next() + typ = p.parseQualifiedIdent(nil) + // expect closing ')' but no need to complain if missing + if p.tok == token.RPAREN { + p.next() + } + } else { + // *T + typ = p.parseQualifiedIdent(nil) + } + typ = &ast.StarExpr{Star: star, X: typ} + + case token.LPAREN: + p.error(p.pos, "cannot parenthesize embedded type") + p.next() + if p.tok == token.MUL { + // (*T) + star := p.pos + p.next() + typ = &ast.StarExpr{Star: star, X: p.parseQualifiedIdent(nil)} + } else { + // (T) + typ = p.parseQualifiedIdent(nil) + } + // expect closing ')' but no need to complain if missing + if p.tok == token.RPAREN { + p.next() + } + + default: + pos := p.pos + p.errorExpected(pos, "field name or embedded type") + p.advance(exprEnd) + typ = &ast.BadExpr{From: pos, To: p.pos} + } + + var tag *ast.BasicLit + if p.tok == token.STRING { + tag = &ast.BasicLit{ValuePos: p.pos, Kind: p.tok, Value: p.lit} + p.next() + } + + comment := p.expectSemi() + + field := &ast.Field{Doc: doc, Names: names, Type: typ, Tag: tag, Comment: comment} + return field +} + +func (p *parser) parseStructType() *ast.StructType { + if p.trace { + defer un(trace(p, "StructType")) + } + + pos := p.expect(token.STRUCT) + lbrace := p.expect(token.LBRACE) + var list []*ast.Field + for p.tok == token.IDENT || p.tok == token.MUL || p.tok == token.LPAREN { + // a field declaration cannot start with a '(' but we accept + // it here for more robust parsing and better error messages + // (parseFieldDecl will check and complain if necessary) + list = append(list, p.parseFieldDecl()) + } + rbrace := p.expect(token.RBRACE) + + return &ast.StructType{ + Struct: pos, + Fields: &ast.FieldList{ + Opening: lbrace, + List: list, + Closing: rbrace, + }, + } +} + +func (p *parser) parsePointerType() *ast.StarExpr { + if p.trace { + defer un(trace(p, "PointerType")) + } + + star := p.expect(token.MUL) + base := p.parseType() + + return &ast.StarExpr{Star: star, X: base} +} + +func (p *parser) parseDotsType() *ast.Ellipsis { + if p.trace { + defer un(trace(p, "DotsType")) + } + + pos := p.expect(token.ELLIPSIS) + elt := p.parseType() + + return &ast.Ellipsis{Ellipsis: pos, Elt: elt} +} + +type field struct { + name *ast.Ident + typ ast.Expr +} + +func (p *parser) parseParamDecl(name *ast.Ident, typeSetsOK bool) (f field) { + // TODO(rFindley) refactor to be more similar to paramDeclOrNil in the syntax + // package + if p.trace { + defer un(trace(p, "ParamDeclOrNil")) + } + + ptok := p.tok + if name != nil { + p.tok = token.IDENT // force token.IDENT case in switch below + } else if typeSetsOK && p.tok == token.TILDE { + // "~" ... + return field{nil, p.embeddedElem(nil)} + } + + switch p.tok { + case token.IDENT: + // name + if name != nil { + f.name = name + p.tok = ptok + } else { + f.name = p.parseIdent() + } + switch p.tok { + case token.IDENT, token.MUL, token.ARROW, token.FUNC, token.CHAN, token.MAP, token.STRUCT, token.INTERFACE, token.LPAREN: + // name type + f.typ = p.parseType() + + case token.LBRACK: + // name "[" type1, ..., typeN "]" or name "[" n "]" type + f.name, f.typ = p.parseArrayFieldOrTypeInstance(f.name) + + case token.ELLIPSIS: + // name "..." type + f.typ = p.parseDotsType() + return // don't allow ...type "|" ... + + case token.PERIOD: + // name "." ... + f.typ = p.parseQualifiedIdent(f.name) + f.name = nil + + case token.TILDE: + if typeSetsOK { + f.typ = p.embeddedElem(nil) + return + } + + case token.OR: + if typeSetsOK { + // name "|" typeset + f.typ = p.embeddedElem(f.name) + f.name = nil + return + } + } + + case token.MUL, token.ARROW, token.FUNC, token.LBRACK, token.CHAN, token.MAP, token.STRUCT, token.INTERFACE, token.LPAREN: + // type + f.typ = p.parseType() + + case token.ELLIPSIS: + // "..." type + // (always accepted) + f.typ = p.parseDotsType() + return // don't allow ...type "|" ... + + default: + // TODO(rfindley): this is incorrect in the case of type parameter lists + // (should be "']'" in that case) + p.errorExpected(p.pos, "')'") + p.advance(exprEnd) + } + + // [name] type "|" + if typeSetsOK && p.tok == token.OR && f.typ != nil { + f.typ = p.embeddedElem(f.typ) + } + + return +} + +func (p *parser) parseParameterList(name0 *ast.Ident, typ0 ast.Expr, closing token.Token) (params []*ast.Field) { + if p.trace { + defer un(trace(p, "ParameterList")) + } + + // Type parameters are the only parameter list closed by ']'. + tparams := closing == token.RBRACK + // Type set notation is ok in type parameter lists. + typeSetsOK := tparams + + pos := p.pos + if name0 != nil { + pos = name0.Pos() + } + + var list []field + var named int // number of parameters that have an explicit name and type + + for name0 != nil || p.tok != closing && p.tok != token.EOF { + var par field + if typ0 != nil { + if typeSetsOK { + typ0 = p.embeddedElem(typ0) + } + par = field{name0, typ0} + } else { + par = p.parseParamDecl(name0, typeSetsOK) + } + name0 = nil // 1st name was consumed if present + typ0 = nil // 1st typ was consumed if present + if par.name != nil || par.typ != nil { + list = append(list, par) + if par.name != nil && par.typ != nil { + named++ + } + } + if !p.atComma("parameter list", closing) { + break + } + p.next() + } + + if len(list) == 0 { + return // not uncommon + } + + // TODO(gri) parameter distribution and conversion to []*ast.Field + // can be combined and made more efficient + + // distribute parameter types + if named == 0 { + // all unnamed => found names are type names + for i := 0; i < len(list); i++ { + par := &list[i] + if typ := par.name; typ != nil { + par.typ = typ + par.name = nil + } + } + if tparams { + p.error(pos, "type parameters must be named") + } + } else if named != len(list) { + // some named => all must be named + ok := true + var typ ast.Expr + missingName := pos + for i := len(list) - 1; i >= 0; i-- { + if par := &list[i]; par.typ != nil { + typ = par.typ + if par.name == nil { + ok = false + missingName = par.typ.Pos() + n := ast.NewIdent("_") + n.NamePos = typ.Pos() // correct position + par.name = n + } + } else if typ != nil { + par.typ = typ + } else { + // par.typ == nil && typ == nil => we only have a par.name + ok = false + missingName = par.name.Pos() + par.typ = &ast.BadExpr{From: par.name.Pos(), To: p.pos} + } + } + if !ok { + if tparams { + p.error(missingName, "type parameters must be named") + } else { + p.error(pos, "mixed named and unnamed parameters") + } + } + } + + // convert list []*ast.Field + if named == 0 { + // parameter list consists of types only + for _, par := range list { + assert(par.typ != nil, "nil type in unnamed parameter list") + params = append(params, &ast.Field{Type: par.typ}) + } + return + } + + // parameter list consists of named parameters with types + var names []*ast.Ident + var typ ast.Expr + addParams := func() { + assert(typ != nil, "nil type in named parameter list") + field := &ast.Field{Names: names, Type: typ} + params = append(params, field) + names = nil + } + for _, par := range list { + if par.typ != typ { + if len(names) > 0 { + addParams() + } + typ = par.typ + } + names = append(names, par.name) + } + if len(names) > 0 { + addParams() + } + return +} + +func (p *parser) parseParameters(acceptTParams bool) (tparams, params *ast.FieldList) { + if p.trace { + defer un(trace(p, "Parameters")) + } + + if acceptTParams && p.tok == token.LBRACK { + opening := p.pos + p.next() + // [T any](params) syntax + list := p.parseParameterList(nil, nil, token.RBRACK) + rbrack := p.expect(token.RBRACK) + tparams = &ast.FieldList{Opening: opening, List: list, Closing: rbrack} + // Type parameter lists must not be empty. + if tparams.NumFields() == 0 { + p.error(tparams.Closing, "empty type parameter list") + tparams = nil // avoid follow-on errors + } + } + + opening := p.expect(token.LPAREN) + + var fields []*ast.Field + if p.tok != token.RPAREN { + fields = p.parseParameterList(nil, nil, token.RPAREN) + } + + rparen := p.expect(token.RPAREN) + params = &ast.FieldList{Opening: opening, List: fields, Closing: rparen} + + return +} + +func (p *parser) parseResult() *ast.FieldList { + if p.trace { + defer un(trace(p, "Result")) + } + + if p.tok == token.LPAREN { + _, results := p.parseParameters(false) + return results + } + + typ := p.tryIdentOrType() + if typ != nil { + list := make([]*ast.Field, 1) + list[0] = &ast.Field{Type: typ} + return &ast.FieldList{List: list} + } + + return nil +} + +func (p *parser) parseFuncType() *ast.FuncType { + if p.trace { + defer un(trace(p, "FuncType")) + } + + pos := p.expect(token.FUNC) + tparams, params := p.parseParameters(true) + if tparams != nil { + p.error(tparams.Pos(), "function type must have no type parameters") + } + results := p.parseResult() + + return &ast.FuncType{Func: pos, Params: params, Results: results} +} + +func (p *parser) parseMethodSpec() *ast.Field { + if p.trace { + defer un(trace(p, "MethodSpec")) + } + + doc := p.leadComment + var idents []*ast.Ident + var typ ast.Expr + x := p.parseTypeName(nil) + if ident, _ := x.(*ast.Ident); ident != nil { + switch { + case p.tok == token.LBRACK: + // generic method or embedded instantiated type + lbrack := p.pos + p.next() + p.exprLev++ + x := p.parseExpr() + p.exprLev-- + if name0, _ := x.(*ast.Ident); name0 != nil && p.tok != token.COMMA && p.tok != token.RBRACK { + // generic method m[T any] + // + // Interface methods do not have type parameters. We parse them for a + // better error message and improved error recovery. + _ = p.parseParameterList(name0, nil, token.RBRACK) + _ = p.expect(token.RBRACK) + p.error(lbrack, "interface method must have no type parameters") + + // TODO(rfindley) refactor to share code with parseFuncType. + _, params := p.parseParameters(false) + results := p.parseResult() + idents = []*ast.Ident{ident} + typ = &ast.FuncType{ + Func: token.NoPos, + Params: params, + Results: results, + } + } else { + // embedded instantiated type + // TODO(rfindley) should resolve all identifiers in x. + list := []ast.Expr{x} + if p.atComma("type argument list", token.RBRACK) { + p.exprLev++ + p.next() + for p.tok != token.RBRACK && p.tok != token.EOF { + list = append(list, p.parseType()) + if !p.atComma("type argument list", token.RBRACK) { + break + } + p.next() + } + p.exprLev-- + } + rbrack := p.expectClosing(token.RBRACK, "type argument list") + typ = typeparams.PackIndexExpr(ident, lbrack, list, rbrack) + } + case p.tok == token.LPAREN: + // ordinary method + // TODO(rfindley) refactor to share code with parseFuncType. + _, params := p.parseParameters(false) + results := p.parseResult() + idents = []*ast.Ident{ident} + typ = &ast.FuncType{Func: token.NoPos, Params: params, Results: results} + default: + // embedded type + typ = x + } + } else { + // embedded, possibly instantiated type + typ = x + if p.tok == token.LBRACK { + // embedded instantiated interface + typ = p.parseTypeInstance(typ) + } + } + + // Comment is added at the callsite: the field below may joined with + // additional type specs using '|'. + // TODO(rfindley) this should be refactored. + // TODO(rfindley) add more tests for comment handling. + return &ast.Field{Doc: doc, Names: idents, Type: typ} +} + +func (p *parser) embeddedElem(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "EmbeddedElem")) + } + if x == nil { + x = p.embeddedTerm() + } + for p.tok == token.OR { + t := new(ast.BinaryExpr) + t.OpPos = p.pos + t.Op = token.OR + p.next() + t.X = x + t.Y = p.embeddedTerm() + x = t + } + return x +} + +func (p *parser) embeddedTerm() ast.Expr { + if p.trace { + defer un(trace(p, "EmbeddedTerm")) + } + if p.tok == token.TILDE { + t := new(ast.UnaryExpr) + t.OpPos = p.pos + t.Op = token.TILDE + p.next() + t.X = p.parseType() + return t + } + + t := p.tryIdentOrType() + if t == nil { + pos := p.pos + p.errorExpected(pos, "~ term or type") + p.advance(exprEnd) + return &ast.BadExpr{From: pos, To: p.pos} + } + + return t +} + +func (p *parser) parseInterfaceType() *ast.InterfaceType { + if p.trace { + defer un(trace(p, "InterfaceType")) + } + + pos := p.expect(token.INTERFACE) + lbrace := p.expect(token.LBRACE) + + var list []*ast.Field + +parseElements: + for { + switch { + case p.tok == token.IDENT: + f := p.parseMethodSpec() + if f.Names == nil { + f.Type = p.embeddedElem(f.Type) + } + f.Comment = p.expectSemi() + list = append(list, f) + case p.tok == token.TILDE: + typ := p.embeddedElem(nil) + comment := p.expectSemi() + list = append(list, &ast.Field{Type: typ, Comment: comment}) + default: + if t := p.tryIdentOrType(); t != nil { + typ := p.embeddedElem(t) + comment := p.expectSemi() + list = append(list, &ast.Field{Type: typ, Comment: comment}) + } else { + break parseElements + } + } + } + + // TODO(rfindley): the error produced here could be improved, since we could + // accept a identifier, 'type', or a '}' at this point. + rbrace := p.expect(token.RBRACE) + + return &ast.InterfaceType{ + Interface: pos, + Methods: &ast.FieldList{ + Opening: lbrace, + List: list, + Closing: rbrace, + }, + } +} + +func (p *parser) parseMapType() *ast.MapType { + if p.trace { + defer un(trace(p, "MapType")) + } + + pos := p.expect(token.MAP) + p.expect(token.LBRACK) + key := p.parseType() + p.expect(token.RBRACK) + value := p.parseType() + + return &ast.MapType{Map: pos, Key: key, Value: value} +} + +func (p *parser) parseChanType() *ast.ChanType { + if p.trace { + defer un(trace(p, "ChanType")) + } + + pos := p.pos + dir := ast.SEND | ast.RECV + var arrow token.Pos + if p.tok == token.CHAN { + p.next() + if p.tok == token.ARROW { + arrow = p.pos + p.next() + dir = ast.SEND + } + } else { + arrow = p.expect(token.ARROW) + p.expect(token.CHAN) + dir = ast.RECV + } + value := p.parseType() + + return &ast.ChanType{Begin: pos, Arrow: arrow, Dir: dir, Value: value} +} + +func (p *parser) parseTypeInstance(typ ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "TypeInstance")) + } + + opening := p.expect(token.LBRACK) + p.exprLev++ + var list []ast.Expr + for p.tok != token.RBRACK && p.tok != token.EOF { + list = append(list, p.parseType()) + if !p.atComma("type argument list", token.RBRACK) { + break + } + p.next() + } + p.exprLev-- + + closing := p.expectClosing(token.RBRACK, "type argument list") + + if len(list) == 0 { + p.errorExpected(closing, "type argument list") + return &ast.IndexExpr{ + X: typ, + Lbrack: opening, + Index: &ast.BadExpr{From: opening + 1, To: closing}, + Rbrack: closing, + } + } + + return typeparams.PackIndexExpr(typ, opening, list, closing) +} + +func (p *parser) tryIdentOrType() ast.Expr { + defer decNestLev(incNestLev(p)) + + switch p.tok { + case token.IDENT: + typ := p.parseTypeName(nil) + if p.tok == token.LBRACK { + typ = p.parseTypeInstance(typ) + } + return typ + case token.LBRACK: + lbrack := p.expect(token.LBRACK) + return p.parseArrayType(lbrack, nil) + case token.STRUCT: + return p.parseStructType() + case token.MUL: + return p.parsePointerType() + case token.FUNC: + return p.parseFuncType() + case token.INTERFACE: + return p.parseInterfaceType() + case token.MAP: + return p.parseMapType() + case token.CHAN, token.ARROW: + return p.parseChanType() + case token.LPAREN: + lparen := p.pos + p.next() + typ := p.parseType() + rparen := p.expect(token.RPAREN) + return &ast.ParenExpr{Lparen: lparen, X: typ, Rparen: rparen} + } + + // no type found + return nil +} + +// ---------------------------------------------------------------------------- +// Blocks + +func (p *parser) parseStmtList() (list []ast.Stmt) { + if p.trace { + defer un(trace(p, "StatementList")) + } + + for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF { + list = append(list, p.parseStmt()) + } + + return +} + +func (p *parser) parseBody() *ast.BlockStmt { + if p.trace { + defer un(trace(p, "Body")) + } + + lbrace := p.expect(token.LBRACE) + list := p.parseStmtList() + rbrace := p.expect2(token.RBRACE) + + return &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} +} + +func (p *parser) parseBlockStmt() *ast.BlockStmt { + if p.trace { + defer un(trace(p, "BlockStmt")) + } + + lbrace := p.expect(token.LBRACE) + list := p.parseStmtList() + rbrace := p.expect2(token.RBRACE) + + return &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} +} + +// ---------------------------------------------------------------------------- +// Expressions + +func (p *parser) parseFuncTypeOrLit() ast.Expr { + if p.trace { + defer un(trace(p, "FuncTypeOrLit")) + } + + typ := p.parseFuncType() + if p.tok != token.LBRACE { + // function type only + return typ + } + + p.exprLev++ + body := p.parseBody() + p.exprLev-- + + return &ast.FuncLit{Type: typ, Body: body} +} + +// parseOperand may return an expression or a raw type (incl. array +// types of the form [...]T). Callers must verify the result. +func (p *parser) parseOperand() ast.Expr { + if p.trace { + defer un(trace(p, "Operand")) + } + + switch p.tok { + case token.IDENT: + x := p.parseIdent() + return x + + case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING: + x := &ast.BasicLit{ValuePos: p.pos, Kind: p.tok, Value: p.lit} + p.next() + return x + + case token.LPAREN: + lparen := p.pos + p.next() + p.exprLev++ + x := p.parseRhs() // types may be parenthesized: (some type) + p.exprLev-- + rparen := p.expect(token.RPAREN) + return &ast.ParenExpr{Lparen: lparen, X: x, Rparen: rparen} + + case token.FUNC: + return p.parseFuncTypeOrLit() + } + + if typ := p.tryIdentOrType(); typ != nil { // do not consume trailing type parameters + // could be type for composite literal or conversion + _, isIdent := typ.(*ast.Ident) + assert(!isIdent, "type cannot be identifier") + return typ + } + + // we have an error + pos := p.pos + p.errorExpected(pos, "operand") + p.advance(stmtStart) + return &ast.BadExpr{From: pos, To: p.pos} +} + +func (p *parser) parseSelector(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "Selector")) + } + + sel := p.parseIdent() + + return &ast.SelectorExpr{X: x, Sel: sel} +} + +func (p *parser) parseTypeAssertion(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "TypeAssertion")) + } + + lparen := p.expect(token.LPAREN) + var typ ast.Expr + if p.tok == token.TYPE { + // type switch: typ == nil + p.next() + } else { + typ = p.parseType() + } + rparen := p.expect(token.RPAREN) + + return &ast.TypeAssertExpr{X: x, Type: typ, Lparen: lparen, Rparen: rparen} +} + +func (p *parser) parseIndexOrSliceOrInstance(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "parseIndexOrSliceOrInstance")) + } + + lbrack := p.expect(token.LBRACK) + if p.tok == token.RBRACK { + // empty index, slice or index expressions are not permitted; + // accept them for parsing tolerance, but complain + p.errorExpected(p.pos, "operand") + rbrack := p.pos + p.next() + return &ast.IndexExpr{ + X: x, + Lbrack: lbrack, + Index: &ast.BadExpr{From: rbrack, To: rbrack}, + Rbrack: rbrack, + } + } + p.exprLev++ + + const N = 3 // change the 3 to 2 to disable 3-index slices + var args []ast.Expr + var index [N]ast.Expr + var colons [N - 1]token.Pos + if p.tok != token.COLON { + // We can't know if we have an index expression or a type instantiation; + // so even if we see a (named) type we are not going to be in type context. + index[0] = p.parseRhs() + } + ncolons := 0 + switch p.tok { + case token.COLON: + // slice expression + for p.tok == token.COLON && ncolons < len(colons) { + colons[ncolons] = p.pos + ncolons++ + p.next() + if p.tok != token.COLON && p.tok != token.RBRACK && p.tok != token.EOF { + index[ncolons] = p.parseRhs() + } + } + case token.COMMA: + // instance expression + args = append(args, index[0]) + for p.tok == token.COMMA { + p.next() + if p.tok != token.RBRACK && p.tok != token.EOF { + args = append(args, p.parseType()) + } + } + } + + p.exprLev-- + rbrack := p.expect(token.RBRACK) + + if ncolons > 0 { + // slice expression + slice3 := false + if ncolons == 2 { + slice3 = true + // Check presence of middle and final index here rather than during type-checking + // to prevent erroneous programs from passing through gofmt (was issue 7305). + if index[1] == nil { + p.error(colons[0], "middle index required in 3-index slice") + index[1] = &ast.BadExpr{From: colons[0] + 1, To: colons[1]} + } + if index[2] == nil { + p.error(colons[1], "final index required in 3-index slice") + index[2] = &ast.BadExpr{From: colons[1] + 1, To: rbrack} + } + } + return &ast.SliceExpr{X: x, Lbrack: lbrack, Low: index[0], High: index[1], Max: index[2], Slice3: slice3, Rbrack: rbrack} + } + + if len(args) == 0 { + // index expression + return &ast.IndexExpr{X: x, Lbrack: lbrack, Index: index[0], Rbrack: rbrack} + } + + // instance expression + return typeparams.PackIndexExpr(x, lbrack, args, rbrack) +} + +func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr { + if p.trace { + defer un(trace(p, "CallOrConversion")) + } + + lparen := p.expect(token.LPAREN) + p.exprLev++ + var list []ast.Expr + var ellipsis token.Pos + for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() { + list = append(list, p.parseRhs()) // builtins may expect a type: make(some type, ...) + if p.tok == token.ELLIPSIS { + ellipsis = p.pos + p.next() + } + if !p.atComma("argument list", token.RPAREN) { + break + } + p.next() + } + p.exprLev-- + rparen := p.expectClosing(token.RPAREN, "argument list") + + return &ast.CallExpr{Fun: fun, Lparen: lparen, Args: list, Ellipsis: ellipsis, Rparen: rparen} +} + +func (p *parser) parseValue() ast.Expr { + if p.trace { + defer un(trace(p, "Element")) + } + + if p.tok == token.LBRACE { + return p.parseLiteralValue(nil) + } + + x := p.parseExpr() + + return x +} + +func (p *parser) parseElement() ast.Expr { + if p.trace { + defer un(trace(p, "Element")) + } + + x := p.parseValue() + if p.tok == token.COLON { + colon := p.pos + p.next() + x = &ast.KeyValueExpr{Key: x, Colon: colon, Value: p.parseValue()} + } + + return x +} + +func (p *parser) parseElementList() (list []ast.Expr) { + if p.trace { + defer un(trace(p, "ElementList")) + } + + for p.tok != token.RBRACE && p.tok != token.EOF { + list = append(list, p.parseElement()) + if !p.atComma("composite literal", token.RBRACE) { + break + } + p.next() + } + + return +} + +func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "LiteralValue")) + } + + lbrace := p.expect(token.LBRACE) + var elts []ast.Expr + p.exprLev++ + if p.tok != token.RBRACE { + elts = p.parseElementList() + } + p.exprLev-- + rbrace := p.expectClosing(token.RBRACE, "composite literal") + return &ast.CompositeLit{Type: typ, Lbrace: lbrace, Elts: elts, Rbrace: rbrace} +} + +// If x is of the form (T), unparen returns unparen(T), otherwise it returns x. +func unparen(x ast.Expr) ast.Expr { + if p, isParen := x.(*ast.ParenExpr); isParen { + x = unparen(p.X) + } + return x +} + +func (p *parser) parsePrimaryExpr(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "PrimaryExpr")) + } + + if x == nil { + x = p.parseOperand() + } + // We track the nesting here rather than at the entry for the function, + // since it can iteratively produce a nested output, and we want to + // limit how deep a structure we generate. + var n int + defer func() { p.nestLev -= n }() + for n = 1; ; n++ { + incNestLev(p) + switch p.tok { + case token.PERIOD: + p.next() + switch p.tok { + case token.IDENT: + x = p.parseSelector(x) + case token.LPAREN: + x = p.parseTypeAssertion(x) + default: + pos := p.pos + p.errorExpected(pos, "selector or type assertion") + // TODO(rFindley) The check for token.RBRACE below is a targeted fix + // to error recovery sufficient to make the x/tools tests to + // pass with the new parsing logic introduced for type + // parameters. Remove this once error recovery has been + // more generally reconsidered. + if p.tok != token.RBRACE { + p.next() // make progress + } + sel := &ast.Ident{NamePos: pos, Name: "_"} + x = &ast.SelectorExpr{X: x, Sel: sel} + } + case token.LBRACK: + x = p.parseIndexOrSliceOrInstance(x) + case token.LPAREN: + x = p.parseCallOrConversion(x) + case token.LBRACE: + // operand may have returned a parenthesized complit + // type; accept it but complain if we have a complit + t := unparen(x) + // determine if '{' belongs to a composite literal or a block statement + switch t.(type) { + case *ast.BadExpr, *ast.Ident, *ast.SelectorExpr: + if p.exprLev < 0 { + return x + } + // x is possibly a composite literal type + case *ast.IndexExpr, *ast.IndexListExpr: + if p.exprLev < 0 { + return x + } + // x is possibly a composite literal type + case *ast.ArrayType, *ast.StructType, *ast.MapType: + // x is a composite literal type + default: + return x + } + if t != x { + p.error(t.Pos(), "cannot parenthesize type in composite literal") + // already progressed, no need to advance + } + x = p.parseLiteralValue(x) + default: + return x + } + } +} + +func (p *parser) parseUnaryExpr() ast.Expr { + defer decNestLev(incNestLev(p)) + + if p.trace { + defer un(trace(p, "UnaryExpr")) + } + + switch p.tok { + case token.ADD, token.SUB, token.NOT, token.XOR, token.AND, token.TILDE: + pos, op := p.pos, p.tok + p.next() + x := p.parseUnaryExpr() + return &ast.UnaryExpr{OpPos: pos, Op: op, X: x} + + case token.ARROW: + // channel type or receive expression + arrow := p.pos + p.next() + + // If the next token is token.CHAN we still don't know if it + // is a channel type or a receive operation - we only know + // once we have found the end of the unary expression. There + // are two cases: + // + // <- type => (<-type) must be channel type + // <- expr => <-(expr) is a receive from an expression + // + // In the first case, the arrow must be re-associated with + // the channel type parsed already: + // + // <- (chan type) => (<-chan type) + // <- (chan<- type) => (<-chan (<-type)) + + x := p.parseUnaryExpr() + + // determine which case we have + if typ, ok := x.(*ast.ChanType); ok { + // (<-type) + + // re-associate position info and <- + dir := ast.SEND + for ok && dir == ast.SEND { + if typ.Dir == ast.RECV { + // error: (<-type) is (<-(<-chan T)) + p.errorExpected(typ.Arrow, "'chan'") + } + arrow, typ.Begin, typ.Arrow = typ.Arrow, arrow, arrow + dir, typ.Dir = typ.Dir, ast.RECV + typ, ok = typ.Value.(*ast.ChanType) + } + if dir == ast.SEND { + p.errorExpected(arrow, "channel type") + } + + return x + } + + // <-(expr) + return &ast.UnaryExpr{OpPos: arrow, Op: token.ARROW, X: x} + + case token.MUL: + // pointer type or unary "*" expression + pos := p.pos + p.next() + x := p.parseUnaryExpr() + return &ast.StarExpr{Star: pos, X: x} + } + + return p.parsePrimaryExpr(nil) +} + +func (p *parser) tokPrec() (token.Token, int) { + tok := p.tok + if p.inRhs && tok == token.ASSIGN { + tok = token.EQL + } + return tok, tok.Precedence() +} + +// parseBinaryExpr parses a (possibly) binary expression. +// If x is non-nil, it is used as the left operand. +// +// TODO(rfindley): parseBinaryExpr has become overloaded. Consider refactoring. +func (p *parser) parseBinaryExpr(x ast.Expr, prec1 int) ast.Expr { + if p.trace { + defer un(trace(p, "BinaryExpr")) + } + + if x == nil { + x = p.parseUnaryExpr() + } + // We track the nesting here rather than at the entry for the function, + // since it can iteratively produce a nested output, and we want to + // limit how deep a structure we generate. + var n int + defer func() { p.nestLev -= n }() + for n = 1; ; n++ { + incNestLev(p) + op, oprec := p.tokPrec() + if oprec < prec1 { + return x + } + pos := p.expect(op) + y := p.parseBinaryExpr(nil, oprec+1) + x = &ast.BinaryExpr{X: x, OpPos: pos, Op: op, Y: y} + } +} + +// The result may be a type or even a raw type ([...]int). +func (p *parser) parseExpr() ast.Expr { + if p.trace { + defer un(trace(p, "Expression")) + } + + return p.parseBinaryExpr(nil, token.LowestPrec+1) +} + +func (p *parser) parseRhs() ast.Expr { + old := p.inRhs + p.inRhs = true + x := p.parseExpr() + p.inRhs = old + return x +} + +// ---------------------------------------------------------------------------- +// Statements + +// Parsing modes for parseSimpleStmt. +const ( + basic = iota + labelOk + rangeOk +) + +// parseSimpleStmt returns true as 2nd result if it parsed the assignment +// of a range clause (with mode == rangeOk). The returned statement is an +// assignment with a right-hand side that is a single unary expression of +// the form "range x". No guarantees are given for the left-hand side. +func (p *parser) parseSimpleStmt(mode int) (ast.Stmt, bool) { + if p.trace { + defer un(trace(p, "SimpleStmt")) + } + + x := p.parseList(false) + + switch p.tok { + case + token.DEFINE, token.ASSIGN, token.ADD_ASSIGN, + token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN, + token.REM_ASSIGN, token.AND_ASSIGN, token.OR_ASSIGN, + token.XOR_ASSIGN, token.SHL_ASSIGN, token.SHR_ASSIGN, token.AND_NOT_ASSIGN: + // assignment statement, possibly part of a range clause + pos, tok := p.pos, p.tok + p.next() + var y []ast.Expr + isRange := false + if mode == rangeOk && p.tok == token.RANGE && (tok == token.DEFINE || tok == token.ASSIGN) { + pos := p.pos + p.next() + y = []ast.Expr{&ast.UnaryExpr{OpPos: pos, Op: token.RANGE, X: p.parseRhs()}} + isRange = true + } else { + y = p.parseList(true) + } + return &ast.AssignStmt{Lhs: x, TokPos: pos, Tok: tok, Rhs: y}, isRange + } + + if len(x) > 1 { + p.errorExpected(x[0].Pos(), "1 expression") + // continue with first expression + } + + switch p.tok { + case token.COLON: + // labeled statement + colon := p.pos + p.next() + if label, isIdent := x[0].(*ast.Ident); mode == labelOk && isIdent { + // Go spec: The scope of a label is the body of the function + // in which it is declared and excludes the body of any nested + // function. + stmt := &ast.LabeledStmt{Label: label, Colon: colon, Stmt: p.parseStmt()} + return stmt, false + } + // The label declaration typically starts at x[0].Pos(), but the label + // declaration may be erroneous due to a token after that position (and + // before the ':'). If SpuriousErrors is not set, the (only) error + // reported for the line is the illegal label error instead of the token + // before the ':' that caused the problem. Thus, use the (latest) colon + // position for error reporting. + p.error(colon, "illegal label declaration") + return &ast.BadStmt{From: x[0].Pos(), To: colon + 1}, false + + case token.ARROW: + // send statement + arrow := p.pos + p.next() + y := p.parseRhs() + return &ast.SendStmt{Chan: x[0], Arrow: arrow, Value: y}, false + + case token.INC, token.DEC: + // increment or decrement + s := &ast.IncDecStmt{X: x[0], TokPos: p.pos, Tok: p.tok} + p.next() + return s, false + } + + // expression + return &ast.ExprStmt{X: x[0]}, false +} + +func (p *parser) parseCallExpr(callType string) *ast.CallExpr { + x := p.parseRhs() // could be a conversion: (some type)(x) + if t := unparen(x); t != x { + p.error(x.Pos(), fmt.Sprintf("expression in %s must not be parenthesized", callType)) + x = t + } + if call, isCall := x.(*ast.CallExpr); isCall { + return call + } + if _, isBad := x.(*ast.BadExpr); !isBad { + // only report error if it's a new one + p.error(p.safePos(x.End()), fmt.Sprintf("expression in %s must be function call", callType)) + } + return nil +} + +func (p *parser) parseGoStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "GoStmt")) + } + + pos := p.expect(token.GO) + call := p.parseCallExpr("go") + p.expectSemi() + if call == nil { + return &ast.BadStmt{From: pos, To: pos + 2} // len("go") + } + + return &ast.GoStmt{Go: pos, Call: call} +} + +func (p *parser) parseDeferStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "DeferStmt")) + } + + pos := p.expect(token.DEFER) + call := p.parseCallExpr("defer") + p.expectSemi() + if call == nil { + return &ast.BadStmt{From: pos, To: pos + 5} // len("defer") + } + + return &ast.DeferStmt{Defer: pos, Call: call} +} + +func (p *parser) parseReturnStmt() *ast.ReturnStmt { + if p.trace { + defer un(trace(p, "ReturnStmt")) + } + + pos := p.pos + p.expect(token.RETURN) + var x []ast.Expr + if p.tok != token.SEMICOLON && p.tok != token.RBRACE { + x = p.parseList(true) + } + p.expectSemi() + + return &ast.ReturnStmt{Return: pos, Results: x} +} + +func (p *parser) parseBranchStmt(tok token.Token) *ast.BranchStmt { + if p.trace { + defer un(trace(p, "BranchStmt")) + } + + pos := p.expect(tok) + var label *ast.Ident + if tok != token.FALLTHROUGH && p.tok == token.IDENT { + label = p.parseIdent() + } + p.expectSemi() + + return &ast.BranchStmt{TokPos: pos, Tok: tok, Label: label} +} + +func (p *parser) makeExpr(s ast.Stmt, want string) ast.Expr { + if s == nil { + return nil + } + if es, isExpr := s.(*ast.ExprStmt); isExpr { + return es.X + } + found := "simple statement" + if _, isAss := s.(*ast.AssignStmt); isAss { + found = "assignment" + } + p.error(s.Pos(), fmt.Sprintf("expected %s, found %s (missing parentheses around composite literal?)", want, found)) + return &ast.BadExpr{From: s.Pos(), To: p.safePos(s.End())} +} + +// parseIfHeader is an adjusted version of parser.header +// in cmd/compile/internal/syntax/parser.go, which has +// been tuned for better error handling. +func (p *parser) parseIfHeader() (init ast.Stmt, cond ast.Expr) { + if p.tok == token.LBRACE { + p.error(p.pos, "missing condition in if statement") + cond = &ast.BadExpr{From: p.pos, To: p.pos} + return + } + // p.tok != token.LBRACE + + prevLev := p.exprLev + p.exprLev = -1 + + if p.tok != token.SEMICOLON { + // accept potential variable declaration but complain + if p.tok == token.VAR { + p.next() + p.error(p.pos, "var declaration not allowed in if initializer") + } + init, _ = p.parseSimpleStmt(basic) + } + + var condStmt ast.Stmt + var semi struct { + pos token.Pos + lit string // ";" or "\n"; valid if pos.IsValid() + } + if p.tok != token.LBRACE { + if p.tok == token.SEMICOLON { + semi.pos = p.pos + semi.lit = p.lit + p.next() + } else { + p.expect(token.SEMICOLON) + } + if p.tok != token.LBRACE { + condStmt, _ = p.parseSimpleStmt(basic) + } + } else { + condStmt = init + init = nil + } + + if condStmt != nil { + cond = p.makeExpr(condStmt, "boolean expression") + } else if semi.pos.IsValid() { + if semi.lit == "\n" { + p.error(semi.pos, "unexpected newline, expecting { after if clause") + } else { + p.error(semi.pos, "missing condition in if statement") + } + } + + // make sure we have a valid AST + if cond == nil { + cond = &ast.BadExpr{From: p.pos, To: p.pos} + } + + p.exprLev = prevLev + return +} + +func (p *parser) parseIfStmt() *ast.IfStmt { + defer decNestLev(incNestLev(p)) + + if p.trace { + defer un(trace(p, "IfStmt")) + } + + pos := p.expect(token.IF) + + init, cond := p.parseIfHeader() + body := p.parseBlockStmt() + + var else_ ast.Stmt + if p.tok == token.ELSE { + p.next() + switch p.tok { + case token.IF: + else_ = p.parseIfStmt() + case token.LBRACE: + else_ = p.parseBlockStmt() + p.expectSemi() + default: + p.errorExpected(p.pos, "if statement or block") + else_ = &ast.BadStmt{From: p.pos, To: p.pos} + } + } else { + p.expectSemi() + } + + return &ast.IfStmt{If: pos, Init: init, Cond: cond, Body: body, Else: else_} +} + +func (p *parser) parseCaseClause() *ast.CaseClause { + if p.trace { + defer un(trace(p, "CaseClause")) + } + + pos := p.pos + var list []ast.Expr + if p.tok == token.CASE { + p.next() + list = p.parseList(true) + } else { + p.expect(token.DEFAULT) + } + + colon := p.expect(token.COLON) + body := p.parseStmtList() + + return &ast.CaseClause{Case: pos, List: list, Colon: colon, Body: body} +} + +func isTypeSwitchAssert(x ast.Expr) bool { + a, ok := x.(*ast.TypeAssertExpr) + return ok && a.Type == nil +} + +func (p *parser) isTypeSwitchGuard(s ast.Stmt) bool { + switch t := s.(type) { + case *ast.ExprStmt: + // x.(type) + return isTypeSwitchAssert(t.X) + case *ast.AssignStmt: + // v := x.(type) + if len(t.Lhs) == 1 && len(t.Rhs) == 1 && isTypeSwitchAssert(t.Rhs[0]) { + switch t.Tok { + case token.ASSIGN: + // permit v = x.(type) but complain + p.error(t.TokPos, "expected ':=', found '='") + fallthrough + case token.DEFINE: + return true + } + } + } + return false +} + +func (p *parser) parseSwitchStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "SwitchStmt")) + } + + pos := p.expect(token.SWITCH) + + var s1, s2 ast.Stmt + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + s2, _ = p.parseSimpleStmt(basic) + } + if p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.LBRACE { + // A TypeSwitchGuard may declare a variable in addition + // to the variable declared in the initial SimpleStmt. + // Introduce extra scope to avoid redeclaration errors: + // + // switch t := 0; t := x.(T) { ... } + // + // (this code is not valid Go because the first t + // cannot be accessed and thus is never used, the extra + // scope is needed for the correct error message). + // + // If we don't have a type switch, s2 must be an expression. + // Having the extra nested but empty scope won't affect it. + s2, _ = p.parseSimpleStmt(basic) + } + } + p.exprLev = prevLev + } + + typeSwitch := p.isTypeSwitchGuard(s2) + lbrace := p.expect(token.LBRACE) + var list []ast.Stmt + for p.tok == token.CASE || p.tok == token.DEFAULT { + list = append(list, p.parseCaseClause()) + } + rbrace := p.expect(token.RBRACE) + p.expectSemi() + body := &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} + + if typeSwitch { + return &ast.TypeSwitchStmt{Switch: pos, Init: s1, Assign: s2, Body: body} + } + + return &ast.SwitchStmt{Switch: pos, Init: s1, Tag: p.makeExpr(s2, "switch expression"), Body: body} +} + +func (p *parser) parseCommClause() *ast.CommClause { + if p.trace { + defer un(trace(p, "CommClause")) + } + + pos := p.pos + var comm ast.Stmt + if p.tok == token.CASE { + p.next() + lhs := p.parseList(false) + if p.tok == token.ARROW { + // SendStmt + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression + } + arrow := p.pos + p.next() + rhs := p.parseRhs() + comm = &ast.SendStmt{Chan: lhs[0], Arrow: arrow, Value: rhs} + } else { + // RecvStmt + if tok := p.tok; tok == token.ASSIGN || tok == token.DEFINE { + // RecvStmt with assignment + if len(lhs) > 2 { + p.errorExpected(lhs[0].Pos(), "1 or 2 expressions") + // continue with first two expressions + lhs = lhs[0:2] + } + pos := p.pos + p.next() + rhs := p.parseRhs() + comm = &ast.AssignStmt{Lhs: lhs, TokPos: pos, Tok: tok, Rhs: []ast.Expr{rhs}} + } else { + // lhs must be single receive operation + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression + } + comm = &ast.ExprStmt{X: lhs[0]} + } + } + } else { + p.expect(token.DEFAULT) + } + + colon := p.expect(token.COLON) + body := p.parseStmtList() + + return &ast.CommClause{Case: pos, Comm: comm, Colon: colon, Body: body} +} + +func (p *parser) parseSelectStmt() *ast.SelectStmt { + if p.trace { + defer un(trace(p, "SelectStmt")) + } + + pos := p.expect(token.SELECT) + lbrace := p.expect(token.LBRACE) + var list []ast.Stmt + for p.tok == token.CASE || p.tok == token.DEFAULT { + list = append(list, p.parseCommClause()) + } + rbrace := p.expect(token.RBRACE) + p.expectSemi() + body := &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} + + return &ast.SelectStmt{Select: pos, Body: body} +} + +func (p *parser) parseForStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "ForStmt")) + } + + pos := p.expect(token.FOR) + + var s1, s2, s3 ast.Stmt + var isRange bool + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + if p.tok == token.RANGE { + // "for range x" (nil lhs in assignment) + pos := p.pos + p.next() + y := []ast.Expr{&ast.UnaryExpr{OpPos: pos, Op: token.RANGE, X: p.parseRhs()}} + s2 = &ast.AssignStmt{Rhs: y} + isRange = true + } else { + s2, isRange = p.parseSimpleStmt(rangeOk) + } + } + if !isRange && p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.SEMICOLON { + s2, _ = p.parseSimpleStmt(basic) + } + p.expectSemi() + if p.tok != token.LBRACE { + s3, _ = p.parseSimpleStmt(basic) + } + } + p.exprLev = prevLev + } + + body := p.parseBlockStmt() + p.expectSemi() + + if isRange { + as := s2.(*ast.AssignStmt) + // check lhs + var key, value ast.Expr + switch len(as.Lhs) { + case 0: + // nothing to do + case 1: + key = as.Lhs[0] + case 2: + key, value = as.Lhs[0], as.Lhs[1] + default: + p.errorExpected(as.Lhs[len(as.Lhs)-1].Pos(), "at most 2 expressions") + return &ast.BadStmt{From: pos, To: p.safePos(body.End())} + } + // parseSimpleStmt returned a right-hand side that + // is a single unary expression of the form "range x" + x := as.Rhs[0].(*ast.UnaryExpr).X + return &ast.RangeStmt{ + For: pos, + Key: key, + Value: value, + TokPos: as.TokPos, + Tok: as.Tok, + Range: as.Rhs[0].Pos(), + X: x, + Body: body, + } + } + + // regular for statement + return &ast.ForStmt{ + For: pos, + Init: s1, + Cond: p.makeExpr(s2, "boolean or range expression"), + Post: s3, + Body: body, + } +} + +func (p *parser) parseStmt() (s ast.Stmt) { + defer decNestLev(incNestLev(p)) + + if p.trace { + defer un(trace(p, "Statement")) + } + + switch p.tok { + case token.CONST, token.TYPE, token.VAR: + s = &ast.DeclStmt{Decl: p.parseDecl(stmtStart)} + case + // tokens that may start an expression + token.IDENT, token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operands + token.LBRACK, token.STRUCT, token.MAP, token.CHAN, token.INTERFACE, // composite types + token.ADD, token.SUB, token.MUL, token.AND, token.XOR, token.ARROW, token.NOT: // unary operators + s, _ = p.parseSimpleStmt(labelOk) + // because of the required look-ahead, labeled statements are + // parsed by parseSimpleStmt - don't expect a semicolon after + // them + if _, isLabeledStmt := s.(*ast.LabeledStmt); !isLabeledStmt { + p.expectSemi() + } + case token.GO: + s = p.parseGoStmt() + case token.DEFER: + s = p.parseDeferStmt() + case token.RETURN: + s = p.parseReturnStmt() + case token.BREAK, token.CONTINUE, token.GOTO, token.FALLTHROUGH: + s = p.parseBranchStmt(p.tok) + case token.LBRACE: + s = p.parseBlockStmt() + p.expectSemi() + case token.IF: + s = p.parseIfStmt() + case token.SWITCH: + s = p.parseSwitchStmt() + case token.SELECT: + s = p.parseSelectStmt() + case token.FOR: + s = p.parseForStmt() + case token.SEMICOLON: + // Is it ever possible to have an implicit semicolon + // producing an empty statement in a valid program? + // (handle correctly anyway) + s = &ast.EmptyStmt{Semicolon: p.pos, Implicit: p.lit == "\n"} + p.next() + case token.RBRACE: + // a semicolon may be omitted before a closing "}" + s = &ast.EmptyStmt{Semicolon: p.pos, Implicit: true} + default: + // no statement found + pos := p.pos + p.errorExpected(pos, "statement") + p.advance(stmtStart) + s = &ast.BadStmt{From: pos, To: p.pos} + } + + return +} + +// ---------------------------------------------------------------------------- +// Declarations + +type parseSpecFunction func(doc *ast.CommentGroup, keyword token.Token, iota int) ast.Spec + +func (p *parser) parseImportSpec(doc *ast.CommentGroup, _ token.Token, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "ImportSpec")) + } + + var ident *ast.Ident + switch p.tok { + case token.IDENT: + ident = p.parseIdent() + case token.PERIOD: + ident = &ast.Ident{NamePos: p.pos, Name: "."} + p.next() + } + + pos := p.pos + var path string + if p.tok == token.STRING { + path = p.lit + p.next() + } else if p.tok.IsLiteral() { + p.error(pos, "import path must be a string") + p.next() + } else { + p.error(pos, "missing import path") + p.advance(exprEnd) + } + comment := p.expectSemi() + + // collect imports + spec := &ast.ImportSpec{ + Doc: doc, + Name: ident, + Path: &ast.BasicLit{ValuePos: pos, Kind: token.STRING, Value: path}, + Comment: comment, + } + p.imports = append(p.imports, spec) + + return spec +} + +func (p *parser) parseValueSpec(doc *ast.CommentGroup, keyword token.Token, iota int) ast.Spec { + if p.trace { + defer un(trace(p, keyword.String()+"Spec")) + } + + idents := p.parseIdentList() + var typ ast.Expr + var values []ast.Expr + switch keyword { + case token.CONST: + // always permit optional type and initialization for more tolerant parsing + if p.tok != token.EOF && p.tok != token.SEMICOLON && p.tok != token.RPAREN { + typ = p.tryIdentOrType() + if p.tok == token.ASSIGN { + p.next() + values = p.parseList(true) + } + } + case token.VAR: + if p.tok != token.ASSIGN { + typ = p.parseType() + } + if p.tok == token.ASSIGN { + p.next() + values = p.parseList(true) + } + default: + panic("unreachable") + } + comment := p.expectSemi() + + spec := &ast.ValueSpec{ + Doc: doc, + Names: idents, + Type: typ, + Values: values, + Comment: comment, + } + return spec +} + +func (p *parser) parseGenericType(spec *ast.TypeSpec, openPos token.Pos, name0 *ast.Ident, typ0 ast.Expr) { + if p.trace { + defer un(trace(p, "parseGenericType")) + } + + list := p.parseParameterList(name0, typ0, token.RBRACK) + closePos := p.expect(token.RBRACK) + spec.TypeParams = &ast.FieldList{Opening: openPos, List: list, Closing: closePos} + // Let the type checker decide whether to accept type parameters on aliases: + // see issue #46477. + if p.tok == token.ASSIGN { + // type alias + spec.Assign = p.pos + p.next() + } + spec.Type = p.parseType() +} + +func (p *parser) parseTypeSpec(doc *ast.CommentGroup, _ token.Token, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "TypeSpec")) + } + + name := p.parseIdent() + spec := &ast.TypeSpec{Doc: doc, Name: name} + + if p.tok == token.LBRACK { + // spec.Name "[" ... + // array/slice type or type parameter list + lbrack := p.pos + p.next() + if p.tok == token.IDENT { + // We may have an array type or a type parameter list. + // In either case we expect an expression x (which may + // just be a name, or a more complex expression) which + // we can analyze further. + // + // A type parameter list may have a type bound starting + // with a "[" as in: P []E. In that case, simply parsing + // an expression would lead to an error: P[] is invalid. + // But since index or slice expressions are never constant + // and thus invalid array length expressions, if the name + // is followed by "[" it must be the start of an array or + // slice constraint. Only if we don't see a "[" do we + // need to parse a full expression. Notably, name <- x + // is not a concern because name <- x is a statement and + // not an expression. + var x ast.Expr = p.parseIdent() + if p.tok != token.LBRACK { + // To parse the expression starting with name, expand + // the call sequence we would get by passing in name + // to parser.expr, and pass in name to parsePrimaryExpr. + p.exprLev++ + lhs := p.parsePrimaryExpr(x) + x = p.parseBinaryExpr(lhs, token.LowestPrec+1) + p.exprLev-- + } + // Analyze expression x. If we can split x into a type parameter + // name, possibly followed by a type parameter type, we consider + // this the start of a type parameter list, with some caveats: + // a single name followed by "]" tilts the decision towards an + // array declaration; a type parameter type that could also be + // an ordinary expression but which is followed by a comma tilts + // the decision towards a type parameter list. + if pname, ptype := extractName(x, p.tok == token.COMMA); pname != nil && (ptype != nil || p.tok != token.RBRACK) { + // spec.Name "[" pname ... + // spec.Name "[" pname ptype ... + // spec.Name "[" pname ptype "," ... + p.parseGenericType(spec, lbrack, pname, ptype) // ptype may be nil + } else { + // spec.Name "[" pname "]" ... + // spec.Name "[" x ... + spec.Type = p.parseArrayType(lbrack, x) + } + } else { + // array type + spec.Type = p.parseArrayType(lbrack, nil) + } + } else { + // no type parameters + if p.tok == token.ASSIGN { + // type alias + spec.Assign = p.pos + p.next() + } + spec.Type = p.parseType() + } + + spec.Comment = p.expectSemi() + + return spec +} + +// extractName splits the expression x into (name, expr) if syntactically +// x can be written as name expr. The split only happens if expr is a type +// element (per the isTypeElem predicate) or if force is set. +// If x is just a name, the result is (name, nil). If the split succeeds, +// the result is (name, expr). Otherwise the result is (nil, x). +// Examples: +// +// x force name expr +// ------------------------------------ +// P*[]int T/F P *[]int +// P*E T P *E +// P*E F nil P*E +// P([]int) T/F P []int +// P(E) T P E +// P(E) F nil P(E) +// P*E|F|~G T/F P *E|F|~G +// P*E|F|G T P *E|F|G +// P*E|F|G F nil P*E|F|G +func extractName(x ast.Expr, force bool) (*ast.Ident, ast.Expr) { + switch x := x.(type) { + case *ast.Ident: + return x, nil + case *ast.BinaryExpr: + switch x.Op { + case token.MUL: + if name, _ := x.X.(*ast.Ident); name != nil && (force || isTypeElem(x.Y)) { + // x = name *x.Y + return name, &ast.StarExpr{Star: x.OpPos, X: x.Y} + } + case token.OR: + if name, lhs := extractName(x.X, force || isTypeElem(x.Y)); name != nil && lhs != nil { + // x = name lhs|x.Y + op := *x + op.X = lhs + return name, &op + } + } + case *ast.CallExpr: + if name, _ := x.Fun.(*ast.Ident); name != nil { + if len(x.Args) == 1 && x.Ellipsis == token.NoPos && (force || isTypeElem(x.Args[0])) { + // x = name "(" x.ArgList[0] ")" + return name, x.Args[0] + } + } + } + return nil, x +} + +// isTypeElem reports whether x is a (possibly parenthesized) type element expression. +// The result is false if x could be a type element OR an ordinary (value) expression. +func isTypeElem(x ast.Expr) bool { + switch x := x.(type) { + case *ast.ArrayType, *ast.StructType, *ast.FuncType, *ast.InterfaceType, *ast.MapType, *ast.ChanType: + return true + case *ast.BinaryExpr: + return isTypeElem(x.X) || isTypeElem(x.Y) + case *ast.UnaryExpr: + return x.Op == token.TILDE + case *ast.ParenExpr: + return isTypeElem(x.X) + } + return false +} + +func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.GenDecl { + if p.trace { + defer un(trace(p, "GenDecl("+keyword.String()+")")) + } + + doc := p.leadComment + pos := p.expect(keyword) + var lparen, rparen token.Pos + var list []ast.Spec + if p.tok == token.LPAREN { + lparen = p.pos + p.next() + for iota := 0; p.tok != token.RPAREN && p.tok != token.EOF; iota++ { + list = append(list, f(p.leadComment, keyword, iota)) + } + rparen = p.expect(token.RPAREN) + p.expectSemi() + } else { + list = append(list, f(nil, keyword, 0)) + } + + return &ast.GenDecl{ + Doc: doc, + TokPos: pos, + Tok: keyword, + Lparen: lparen, + Specs: list, + Rparen: rparen, + } +} + +func (p *parser) parseFuncDecl() *ast.FuncDecl { + if p.trace { + defer un(trace(p, "FunctionDecl")) + } + + doc := p.leadComment + pos := p.expect(token.FUNC) + + var recv *ast.FieldList + if p.tok == token.LPAREN { + _, recv = p.parseParameters(false) + } + + ident := p.parseIdent() + + tparams, params := p.parseParameters(true) + if recv != nil && tparams != nil { + // Method declarations do not have type parameters. We parse them for a + // better error message and improved error recovery. + p.error(tparams.Opening, "method must have no type parameters") + tparams = nil + } + results := p.parseResult() + + var body *ast.BlockStmt + switch p.tok { + case token.LBRACE: + body = p.parseBody() + p.expectSemi() + case token.SEMICOLON: + p.next() + if p.tok == token.LBRACE { + // opening { of function declaration on next line + p.error(p.pos, "unexpected semicolon or newline before {") + body = p.parseBody() + p.expectSemi() + } + default: + p.expectSemi() + } + + decl := &ast.FuncDecl{ + Doc: doc, + Recv: recv, + Name: ident, + Type: &ast.FuncType{ + Func: pos, + TypeParams: tparams, + Params: params, + Results: results, + }, + Body: body, + } + return decl +} + +func (p *parser) parseDecl(sync map[token.Token]bool) ast.Decl { + if p.trace { + defer un(trace(p, "Declaration")) + } + + var f parseSpecFunction + switch p.tok { + case token.IMPORT: + f = p.parseImportSpec + + case token.CONST, token.VAR: + f = p.parseValueSpec + + case token.TYPE: + f = p.parseTypeSpec + + case token.FUNC: + return p.parseFuncDecl() + + default: + pos := p.pos + p.errorExpected(pos, "declaration") + p.advance(sync) + return &ast.BadDecl{From: pos, To: p.pos} + } + + return p.parseGenDecl(p.tok, f) +} + +// ---------------------------------------------------------------------------- +// Source files + +func (p *parser) parseFile() *ast.File { + if p.trace { + defer un(trace(p, "File")) + } + + // Don't bother parsing the rest if we had errors scanning the first token. + // Likely not a Go source file at all. + if p.errors.Len() != 0 { + return nil + } + + // package clause + doc := p.leadComment + pos := p.expect(token.PACKAGE) + // Go spec: The package clause is not a declaration; + // the package name does not appear in any scope. + ident := p.parseIdent() + if ident.Name == "_" && p.mode&DeclarationErrors != 0 { + p.error(p.pos, "invalid package name _") + } + p.expectSemi() + + // Don't bother parsing the rest if we had errors parsing the package clause. + // Likely not a Go source file at all. + if p.errors.Len() != 0 { + return nil + } + + var decls []ast.Decl + if p.mode&PackageClauseOnly == 0 { + // import decls + for p.tok == token.IMPORT { + decls = append(decls, p.parseGenDecl(token.IMPORT, p.parseImportSpec)) + } + + if p.mode&ImportsOnly == 0 { + // rest of package body + prev := token.IMPORT + for p.tok != token.EOF { + // Continue to accept import declarations for error tolerance, but complain. + if p.tok == token.IMPORT && prev != token.IMPORT { + p.error(p.pos, "imports must appear before other declarations") + } + prev = p.tok + + decls = append(decls, p.parseDecl(declStart)) + } + } + } + + f := &ast.File{ + Doc: doc, + Package: pos, + Name: ident, + Decls: decls, + FileStart: token.Pos(p.file.Base()), + FileEnd: token.Pos(p.file.Base() + p.file.Size()), + Imports: p.imports, + Comments: p.comments, + } + var declErr func(token.Pos, string) + if p.mode&DeclarationErrors != 0 { + declErr = p.error + } + if p.mode&SkipObjectResolution == 0 { + resolveFile(f, p.file, declErr) + } + + return f +} diff --git a/src/go/parser/parser_test.go b/src/go/parser/parser_test.go new file mode 100644 index 0000000..22b11a0 --- /dev/null +++ b/src/go/parser/parser_test.go @@ -0,0 +1,782 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parser + +import ( + "fmt" + "go/ast" + "go/token" + "io/fs" + "strings" + "testing" +) + +var validFiles = []string{ + "parser.go", + "parser_test.go", + "error_test.go", + "short_test.go", +} + +func TestParse(t *testing.T) { + for _, filename := range validFiles { + _, err := ParseFile(token.NewFileSet(), filename, nil, DeclarationErrors) + if err != nil { + t.Fatalf("ParseFile(%s): %v", filename, err) + } + } +} + +func nameFilter(filename string) bool { + switch filename { + case "parser.go", "interface.go", "parser_test.go": + return true + case "parser.go.orig": + return true // permit but should be ignored by ParseDir + } + return false +} + +func dirFilter(f fs.FileInfo) bool { return nameFilter(f.Name()) } + +func TestParseFile(t *testing.T) { + src := "package p\nvar _=s[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]" + _, err := ParseFile(token.NewFileSet(), "", src, 0) + if err == nil { + t.Errorf("ParseFile(%s) succeeded unexpectedly", src) + } +} + +func TestParseExprFrom(t *testing.T) { + src := "s[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]+\ns[::]" + _, err := ParseExprFrom(token.NewFileSet(), "", src, 0) + if err == nil { + t.Errorf("ParseExprFrom(%s) succeeded unexpectedly", src) + } +} + +func TestParseDir(t *testing.T) { + path := "." + pkgs, err := ParseDir(token.NewFileSet(), path, dirFilter, 0) + if err != nil { + t.Fatalf("ParseDir(%s): %v", path, err) + } + if n := len(pkgs); n != 1 { + t.Errorf("got %d packages; want 1", n) + } + pkg := pkgs["parser"] + if pkg == nil { + t.Errorf(`package "parser" not found`) + return + } + if n := len(pkg.Files); n != 3 { + t.Errorf("got %d package files; want 3", n) + } + for filename := range pkg.Files { + if !nameFilter(filename) { + t.Errorf("unexpected package file: %s", filename) + } + } +} + +func TestIssue42951(t *testing.T) { + path := "./testdata/issue42951" + _, err := ParseDir(token.NewFileSet(), path, nil, 0) + if err != nil { + t.Errorf("ParseDir(%s): %v", path, err) + } +} + +func TestParseExpr(t *testing.T) { + // just kicking the tires: + // a valid arithmetic expression + src := "a + b" + x, err := ParseExpr(src) + if err != nil { + t.Errorf("ParseExpr(%q): %v", src, err) + } + // sanity check + if _, ok := x.(*ast.BinaryExpr); !ok { + t.Errorf("ParseExpr(%q): got %T, want *ast.BinaryExpr", src, x) + } + + // a valid type expression + src = "struct{x *int}" + x, err = ParseExpr(src) + if err != nil { + t.Errorf("ParseExpr(%q): %v", src, err) + } + // sanity check + if _, ok := x.(*ast.StructType); !ok { + t.Errorf("ParseExpr(%q): got %T, want *ast.StructType", src, x) + } + + // an invalid expression + src = "a + *" + x, err = ParseExpr(src) + if err == nil { + t.Errorf("ParseExpr(%q): got no error", src) + } + if x == nil { + t.Errorf("ParseExpr(%q): got no (partial) result", src) + } + if _, ok := x.(*ast.BinaryExpr); !ok { + t.Errorf("ParseExpr(%q): got %T, want *ast.BinaryExpr", src, x) + } + + // a valid expression followed by extra tokens is invalid + src = "a[i] := x" + if _, err := ParseExpr(src); err == nil { + t.Errorf("ParseExpr(%q): got no error", src) + } + + // a semicolon is not permitted unless automatically inserted + src = "a + b\n" + if _, err := ParseExpr(src); err != nil { + t.Errorf("ParseExpr(%q): got error %s", src, err) + } + src = "a + b;" + if _, err := ParseExpr(src); err == nil { + t.Errorf("ParseExpr(%q): got no error", src) + } + + // various other stuff following a valid expression + const validExpr = "a + b" + const anything = "dh3*#D)#_" + for _, c := range "!)]};," { + src := validExpr + string(c) + anything + if _, err := ParseExpr(src); err == nil { + t.Errorf("ParseExpr(%q): got no error", src) + } + } + + // ParseExpr must not crash + for _, src := range valids { + ParseExpr(src) + } +} + +func TestColonEqualsScope(t *testing.T) { + f, err := ParseFile(token.NewFileSet(), "", `package p; func f() { x, y, z := x, y, z }`, 0) + if err != nil { + t.Fatal(err) + } + + // RHS refers to undefined globals; LHS does not. + as := f.Decls[0].(*ast.FuncDecl).Body.List[0].(*ast.AssignStmt) + for _, v := range as.Rhs { + id := v.(*ast.Ident) + if id.Obj != nil { + t.Errorf("rhs %s has Obj, should not", id.Name) + } + } + for _, v := range as.Lhs { + id := v.(*ast.Ident) + if id.Obj == nil { + t.Errorf("lhs %s does not have Obj, should", id.Name) + } + } +} + +func TestVarScope(t *testing.T) { + f, err := ParseFile(token.NewFileSet(), "", `package p; func f() { var x, y, z = x, y, z }`, 0) + if err != nil { + t.Fatal(err) + } + + // RHS refers to undefined globals; LHS does not. + as := f.Decls[0].(*ast.FuncDecl).Body.List[0].(*ast.DeclStmt).Decl.(*ast.GenDecl).Specs[0].(*ast.ValueSpec) + for _, v := range as.Values { + id := v.(*ast.Ident) + if id.Obj != nil { + t.Errorf("rhs %s has Obj, should not", id.Name) + } + } + for _, id := range as.Names { + if id.Obj == nil { + t.Errorf("lhs %s does not have Obj, should", id.Name) + } + } +} + +func TestObjects(t *testing.T) { + const src = ` +package p +import fmt "fmt" +const pi = 3.14 +type T struct{} +var x int +func f() { L: } +` + + f, err := ParseFile(token.NewFileSet(), "", src, 0) + if err != nil { + t.Fatal(err) + } + + objects := map[string]ast.ObjKind{ + "p": ast.Bad, // not in a scope + "fmt": ast.Bad, // not resolved yet + "pi": ast.Con, + "T": ast.Typ, + "x": ast.Var, + "int": ast.Bad, // not resolved yet + "f": ast.Fun, + "L": ast.Lbl, + } + + ast.Inspect(f, func(n ast.Node) bool { + if ident, ok := n.(*ast.Ident); ok { + obj := ident.Obj + if obj == nil { + if objects[ident.Name] != ast.Bad { + t.Errorf("no object for %s", ident.Name) + } + return true + } + if obj.Name != ident.Name { + t.Errorf("names don't match: obj.Name = %s, ident.Name = %s", obj.Name, ident.Name) + } + kind := objects[ident.Name] + if obj.Kind != kind { + t.Errorf("%s: obj.Kind = %s; want %s", ident.Name, obj.Kind, kind) + } + } + return true + }) +} + +func TestUnresolved(t *testing.T) { + f, err := ParseFile(token.NewFileSet(), "", ` +package p +// +func f1a(int) +func f2a(byte, int, float) +func f3a(a, b int, c float) +func f4a(...complex) +func f5a(a s1a, b ...complex) +// +func f1b(*int) +func f2b([]byte, (int), *float) +func f3b(a, b *int, c []float) +func f4b(...*complex) +func f5b(a s1a, b ...[]complex) +// +type s1a struct { int } +type s2a struct { byte; int; s1a } +type s3a struct { a, b int; c float } +// +type s1b struct { *int } +type s2b struct { byte; int; *float } +type s3b struct { a, b *s3b; c []float } +`, 0) + if err != nil { + t.Fatal(err) + } + + want := "int " + // f1a + "byte int float " + // f2a + "int float " + // f3a + "complex " + // f4a + "complex " + // f5a + // + "int " + // f1b + "byte int float " + // f2b + "int float " + // f3b + "complex " + // f4b + "complex " + // f5b + // + "int " + // s1a + "byte int " + // s2a + "int float " + // s3a + // + "int " + // s1a + "byte int float " + // s2a + "float " // s3a + + // collect unresolved identifiers + var buf strings.Builder + for _, u := range f.Unresolved { + buf.WriteString(u.Name) + buf.WriteByte(' ') + } + got := buf.String() + + if got != want { + t.Errorf("\ngot: %s\nwant: %s", got, want) + } +} + +func TestCommentGroups(t *testing.T) { + f, err := ParseFile(token.NewFileSet(), "", ` +package p /* 1a */ /* 1b */ /* 1c */ // 1d +/* 2a +*/ +// 2b +const pi = 3.1415 +/* 3a */ // 3b +/* 3c */ const e = 2.7182 + +// Example from issue 3139 +func ExampleCount() { + fmt.Println(strings.Count("cheese", "e")) + fmt.Println(strings.Count("five", "")) // before & after each rune + // Output: + // 3 + // 5 +} +`, ParseComments) + if err != nil { + t.Fatal(err) + } + expected := [][]string{ + {"/* 1a */", "/* 1b */", "/* 1c */", "// 1d"}, + {"/* 2a\n*/", "// 2b"}, + {"/* 3a */", "// 3b", "/* 3c */"}, + {"// Example from issue 3139"}, + {"// before & after each rune"}, + {"// Output:", "// 3", "// 5"}, + } + if len(f.Comments) != len(expected) { + t.Fatalf("got %d comment groups; expected %d", len(f.Comments), len(expected)) + } + for i, exp := range expected { + got := f.Comments[i].List + if len(got) != len(exp) { + t.Errorf("got %d comments in group %d; expected %d", len(got), i, len(exp)) + continue + } + for j, exp := range exp { + got := got[j].Text + if got != exp { + t.Errorf("got %q in group %d; expected %q", got, i, exp) + } + } + } +} + +func getField(file *ast.File, fieldname string) *ast.Field { + parts := strings.Split(fieldname, ".") + for _, d := range file.Decls { + if d, ok := d.(*ast.GenDecl); ok && d.Tok == token.TYPE { + for _, s := range d.Specs { + if s, ok := s.(*ast.TypeSpec); ok && s.Name.Name == parts[0] { + if s, ok := s.Type.(*ast.StructType); ok { + for _, f := range s.Fields.List { + for _, name := range f.Names { + if name.Name == parts[1] { + return f + } + } + } + } + } + } + } + } + return nil +} + +// Don't use ast.CommentGroup.Text() - we want to see exact comment text. +func commentText(c *ast.CommentGroup) string { + var buf strings.Builder + if c != nil { + for _, c := range c.List { + buf.WriteString(c.Text) + } + } + return buf.String() +} + +func checkFieldComments(t *testing.T, file *ast.File, fieldname, lead, line string) { + f := getField(file, fieldname) + if f == nil { + t.Fatalf("field not found: %s", fieldname) + } + if got := commentText(f.Doc); got != lead { + t.Errorf("got lead comment %q; expected %q", got, lead) + } + if got := commentText(f.Comment); got != line { + t.Errorf("got line comment %q; expected %q", got, line) + } +} + +func TestLeadAndLineComments(t *testing.T) { + f, err := ParseFile(token.NewFileSet(), "", ` +package p +type T struct { + /* F1 lead comment */ + // + F1 int /* F1 */ // line comment + // F2 lead + // comment + F2 int // F2 line comment + // f3 lead comment + f3 int // f3 line comment + + f4 int /* not a line comment */ ; + f5 int ; // f5 line comment + f6 int ; /* f6 line comment */ + f7 int ; /*f7a*/ /*f7b*/ //f7c +} +`, ParseComments) + if err != nil { + t.Fatal(err) + } + checkFieldComments(t, f, "T.F1", "/* F1 lead comment *///", "/* F1 */// line comment") + checkFieldComments(t, f, "T.F2", "// F2 lead// comment", "// F2 line comment") + checkFieldComments(t, f, "T.f3", "// f3 lead comment", "// f3 line comment") + checkFieldComments(t, f, "T.f4", "", "") + checkFieldComments(t, f, "T.f5", "", "// f5 line comment") + checkFieldComments(t, f, "T.f6", "", "/* f6 line comment */") + checkFieldComments(t, f, "T.f7", "", "/*f7a*//*f7b*///f7c") + + ast.FileExports(f) + checkFieldComments(t, f, "T.F1", "/* F1 lead comment *///", "/* F1 */// line comment") + checkFieldComments(t, f, "T.F2", "// F2 lead// comment", "// F2 line comment") + if getField(f, "T.f3") != nil { + t.Error("not expected to find T.f3") + } +} + +// TestIssue9979 verifies that empty statements are contained within their enclosing blocks. +func TestIssue9979(t *testing.T) { + for _, src := range []string{ + "package p; func f() {;}", + "package p; func f() {L:}", + "package p; func f() {L:;}", + "package p; func f() {L:\n}", + "package p; func f() {L:\n;}", + "package p; func f() { ; }", + "package p; func f() { L: }", + "package p; func f() { L: ; }", + "package p; func f() { L: \n}", + "package p; func f() { L: \n; }", + } { + fset := token.NewFileSet() + f, err := ParseFile(fset, "", src, 0) + if err != nil { + t.Fatal(err) + } + + var pos, end token.Pos + ast.Inspect(f, func(x ast.Node) bool { + switch s := x.(type) { + case *ast.BlockStmt: + pos, end = s.Pos()+1, s.End()-1 // exclude "{", "}" + case *ast.LabeledStmt: + pos, end = s.Pos()+2, s.End() // exclude "L:" + case *ast.EmptyStmt: + // check containment + if s.Pos() < pos || s.End() > end { + t.Errorf("%s: %T[%d, %d] not inside [%d, %d]", src, s, s.Pos(), s.End(), pos, end) + } + // check semicolon + offs := fset.Position(s.Pos()).Offset + if ch := src[offs]; ch != ';' != s.Implicit { + want := "want ';'" + if s.Implicit { + want = "but ';' is implicit" + } + t.Errorf("%s: found %q at offset %d; %s", src, ch, offs, want) + } + } + return true + }) + } +} + +func TestFileStartEndPos(t *testing.T) { + const src = `// Copyright + +//+build tag + +// Package p doc comment. +package p + +var lastDecl int + +/* end of file */ +` + fset := token.NewFileSet() + f, err := ParseFile(fset, "file.go", src, 0) + if err != nil { + t.Fatal(err) + } + + // File{Start,End} spans the entire file, not just the declarations. + if got, want := fset.Position(f.FileStart).String(), "file.go:1:1"; got != want { + t.Errorf("for File.FileStart, got %s, want %s", got, want) + } + // The end position is the newline at the end of the /* end of file */ line. + if got, want := fset.Position(f.FileEnd).String(), "file.go:10:19"; got != want { + t.Errorf("for File.FileEnd, got %s, want %s", got, want) + } +} + +// TestIncompleteSelection ensures that an incomplete selector +// expression is parsed as a (blank) *ast.SelectorExpr, not a +// *ast.BadExpr. +func TestIncompleteSelection(t *testing.T) { + for _, src := range []string{ + "package p; var _ = fmt.", // at EOF + "package p; var _ = fmt.\ntype X int", // not at EOF + } { + fset := token.NewFileSet() + f, err := ParseFile(fset, "", src, 0) + if err == nil { + t.Errorf("ParseFile(%s) succeeded unexpectedly", src) + continue + } + + const wantErr = "expected selector or type assertion" + if !strings.Contains(err.Error(), wantErr) { + t.Errorf("ParseFile returned wrong error %q, want %q", err, wantErr) + } + + var sel *ast.SelectorExpr + ast.Inspect(f, func(n ast.Node) bool { + if n, ok := n.(*ast.SelectorExpr); ok { + sel = n + } + return true + }) + if sel == nil { + t.Error("found no *ast.SelectorExpr") + continue + } + const wantSel = "&{fmt _}" + if fmt.Sprint(sel) != wantSel { + t.Errorf("found selector %s, want %s", sel, wantSel) + continue + } + } +} + +func TestLastLineComment(t *testing.T) { + const src = `package main +type x int // comment +` + fset := token.NewFileSet() + f, err := ParseFile(fset, "", src, ParseComments) + if err != nil { + t.Fatal(err) + } + comment := f.Decls[0].(*ast.GenDecl).Specs[0].(*ast.TypeSpec).Comment.List[0].Text + if comment != "// comment" { + t.Errorf("got %q, want %q", comment, "// comment") + } +} + +var parseDepthTests = []struct { + name string + format string + // multipler is used when a single statement may result in more than one + // change in the depth level, for instance "1+(..." produces a BinaryExpr + // followed by a UnaryExpr, which increments the depth twice. The test + // case comment explains which nodes are triggering the multiple depth + // changes. + parseMultiplier int + // scope is true if we should also test the statement for the resolver scope + // depth limit. + scope bool + // scopeMultiplier does the same as parseMultiplier, but for the scope + // depths. + scopeMultiplier int +}{ + // The format expands the part inside Ā« Ā» many times. + // A second set of brackets nested inside the first stops the repetition, + // so that for example Ā«(Ā«1Ā»)Ā» expands to (((...((((1))))...))). + {name: "array", format: "package main; var x Ā«[1]Ā»int"}, + {name: "slice", format: "package main; var x Ā«[]Ā»int"}, + {name: "struct", format: "package main; var x Ā«struct { X Ā«intĀ» }Ā»", scope: true}, + {name: "pointer", format: "package main; var x Ā«*Ā»int"}, + {name: "func", format: "package main; var x Ā«func()Ā»int", scope: true}, + {name: "chan", format: "package main; var x Ā«chan Ā»int"}, + {name: "chan2", format: "package main; var x Ā«<-chan Ā»int"}, + {name: "interface", format: "package main; var x Ā«interface { M() Ā«intĀ» }Ā»", scope: true, scopeMultiplier: 2}, // Scopes: InterfaceType, FuncType + {name: "map", format: "package main; var x Ā«map[int]Ā»int"}, + {name: "slicelit", format: "package main; var x = Ā«[]any{Ā«Ā»}Ā»", parseMultiplier: 2}, // Parser nodes: UnaryExpr, CompositeLit + {name: "arraylit", format: "package main; var x = Ā«[1]any{Ā«nilĀ»}Ā»", parseMultiplier: 2}, // Parser nodes: UnaryExpr, CompositeLit + {name: "structlit", format: "package main; var x = Ā«struct{x any}{Ā«nilĀ»}Ā»", parseMultiplier: 2}, // Parser nodes: UnaryExpr, CompositeLit + {name: "maplit", format: "package main; var x = Ā«map[int]any{1:Ā«nilĀ»}Ā»", parseMultiplier: 2}, // Parser nodes: CompositeLit, KeyValueExpr + {name: "dot", format: "package main; var x = Ā«x.Ā»x"}, + {name: "index", format: "package main; var x = xĀ«[1]Ā»"}, + {name: "slice", format: "package main; var x = xĀ«[1:2]Ā»"}, + {name: "slice3", format: "package main; var x = xĀ«[1:2:3]Ā»"}, + {name: "dottype", format: "package main; var x = xĀ«.(any)Ā»"}, + {name: "callseq", format: "package main; var x = xĀ«()Ā»"}, + {name: "methseq", format: "package main; var x = xĀ«.m()Ā»", parseMultiplier: 2}, // Parser nodes: SelectorExpr, CallExpr + {name: "binary", format: "package main; var x = Ā«1+Ā»1"}, + {name: "binaryparen", format: "package main; var x = Ā«1+(Ā«1Ā»)Ā»", parseMultiplier: 2}, // Parser nodes: BinaryExpr, ParenExpr + {name: "unary", format: "package main; var x = Ā«^Ā»1"}, + {name: "addr", format: "package main; var x = Ā«& Ā»x"}, + {name: "star", format: "package main; var x = Ā«*Ā»x"}, + {name: "recv", format: "package main; var x = Ā«<-Ā»x"}, + {name: "call", format: "package main; var x = Ā«f(Ā«1Ā»)Ā»", parseMultiplier: 2}, // Parser nodes: Ident, CallExpr + {name: "conv", format: "package main; var x = Ā«(*T)(Ā«1Ā»)Ā»", parseMultiplier: 2}, // Parser nodes: ParenExpr, CallExpr + {name: "label", format: "package main; func main() { Ā«Label:Ā» }"}, + {name: "if", format: "package main; func main() { Ā«if true { Ā«Ā» }Ā»}", parseMultiplier: 2, scope: true, scopeMultiplier: 2}, // Parser nodes: IfStmt, BlockStmt. Scopes: IfStmt, BlockStmt + {name: "ifelse", format: "package main; func main() { Ā«if true {} else Ā» {} }", scope: true}, + {name: "switch", format: "package main; func main() { Ā«switch { default: Ā«Ā» }Ā»}", scope: true, scopeMultiplier: 2}, // Scopes: TypeSwitchStmt, CaseClause + {name: "typeswitch", format: "package main; func main() { Ā«switch x.(type) { default: Ā«Ā» }Ā» }", scope: true, scopeMultiplier: 2}, // Scopes: TypeSwitchStmt, CaseClause + {name: "for0", format: "package main; func main() { Ā«for { Ā«Ā» }Ā» }", scope: true, scopeMultiplier: 2}, // Scopes: ForStmt, BlockStmt + {name: "for1", format: "package main; func main() { Ā«for x { Ā«Ā» }Ā» }", scope: true, scopeMultiplier: 2}, // Scopes: ForStmt, BlockStmt + {name: "for3", format: "package main; func main() { Ā«for f(); g(); h() { Ā«Ā» }Ā» }", scope: true, scopeMultiplier: 2}, // Scopes: ForStmt, BlockStmt + {name: "forrange0", format: "package main; func main() { Ā«for range x { Ā«Ā» }Ā» }", scope: true, scopeMultiplier: 2}, // Scopes: RangeStmt, BlockStmt + {name: "forrange1", format: "package main; func main() { Ā«for x = range z { Ā«Ā» }Ā» }", scope: true, scopeMultiplier: 2}, // Scopes: RangeStmt, BlockStmt + {name: "forrange2", format: "package main; func main() { Ā«for x, y = range z { Ā«Ā» }Ā» }", scope: true, scopeMultiplier: 2}, // Scopes: RangeStmt, BlockStmt + {name: "go", format: "package main; func main() { Ā«go func() { Ā«Ā» }()Ā» }", parseMultiplier: 2, scope: true}, // Parser nodes: GoStmt, FuncLit + {name: "defer", format: "package main; func main() { Ā«defer func() { Ā«Ā» }()Ā» }", parseMultiplier: 2, scope: true}, // Parser nodes: DeferStmt, FuncLit + {name: "select", format: "package main; func main() { Ā«select { default: Ā«Ā» }Ā» }", scope: true}, +} + +// split splits preĀ«midĀ»post into pre, mid, post. +// If the string does not have that form, split returns x, "", "". +func split(x string) (pre, mid, post string) { + start, end := strings.Index(x, "Ā«"), strings.LastIndex(x, "Ā»") + if start < 0 || end < 0 { + return x, "", "" + } + return x[:start], x[start+len("Ā«") : end], x[end+len("Ā»"):] +} + +func TestParseDepthLimit(t *testing.T) { + if testing.Short() { + t.Skip("test requires significant memory") + } + for _, tt := range parseDepthTests { + for _, size := range []string{"small", "big"} { + t.Run(tt.name+"/"+size, func(t *testing.T) { + n := maxNestLev + 1 + if tt.parseMultiplier > 0 { + n /= tt.parseMultiplier + } + if size == "small" { + // Decrease the number of statements by 10, in order to check + // that we do not fail when under the limit. 10 is used to + // provide some wiggle room for cases where the surrounding + // scaffolding syntax adds some noise to the depth that changes + // on a per testcase basis. + n -= 10 + } + + pre, mid, post := split(tt.format) + if strings.Contains(mid, "Ā«") { + left, base, right := split(mid) + mid = strings.Repeat(left, n) + base + strings.Repeat(right, n) + } else { + mid = strings.Repeat(mid, n) + } + input := pre + mid + post + + fset := token.NewFileSet() + _, err := ParseFile(fset, "", input, ParseComments|SkipObjectResolution) + if size == "small" { + if err != nil { + t.Errorf("ParseFile(...): %v (want success)", err) + } + } else { + expected := "exceeded max nesting depth" + if err == nil || !strings.HasSuffix(err.Error(), expected) { + t.Errorf("ParseFile(...) = _, %v, want %q", err, expected) + } + } + }) + } + } +} + +func TestScopeDepthLimit(t *testing.T) { + for _, tt := range parseDepthTests { + if !tt.scope { + continue + } + for _, size := range []string{"small", "big"} { + t.Run(tt.name+"/"+size, func(t *testing.T) { + n := maxScopeDepth + 1 + if tt.scopeMultiplier > 0 { + n /= tt.scopeMultiplier + } + if size == "small" { + // Decrease the number of statements by 10, in order to check + // that we do not fail when under the limit. 10 is used to + // provide some wiggle room for cases where the surrounding + // scaffolding syntax adds some noise to the depth that changes + // on a per testcase basis. + n -= 10 + } + + pre, mid, post := split(tt.format) + if strings.Contains(mid, "Ā«") { + left, base, right := split(mid) + mid = strings.Repeat(left, n) + base + strings.Repeat(right, n) + } else { + mid = strings.Repeat(mid, n) + } + input := pre + mid + post + + fset := token.NewFileSet() + _, err := ParseFile(fset, "", input, DeclarationErrors) + if size == "small" { + if err != nil { + t.Errorf("ParseFile(...): %v (want success)", err) + } + } else { + expected := "exceeded max scope depth during object resolution" + if err == nil || !strings.HasSuffix(err.Error(), expected) { + t.Errorf("ParseFile(...) = _, %v, want %q", err, expected) + } + } + }) + } + } +} + +// proposal #50429 +func TestRangePos(t *testing.T) { + testcases := []string{ + "package p; func _() { for range x {} }", + "package p; func _() { for i = range x {} }", + "package p; func _() { for i := range x {} }", + "package p; func _() { for k, v = range x {} }", + "package p; func _() { for k, v := range x {} }", + } + + for _, src := range testcases { + fset := token.NewFileSet() + f, err := ParseFile(fset, src, src, 0) + if err != nil { + t.Fatal(err) + } + + ast.Inspect(f, func(x ast.Node) bool { + switch s := x.(type) { + case *ast.RangeStmt: + pos := fset.Position(s.Range) + if pos.Offset != strings.Index(src, "range") { + t.Errorf("%s: got offset %v, want %v", src, pos.Offset, strings.Index(src, "range")) + } + } + return true + }) + } +} + +// TestIssue59180 tests that line number overflow doesn't cause an infinite loop. +func TestIssue59180(t *testing.T) { + testcases := []string{ + "package p\n//line :9223372036854775806\n\n//", + "package p\n//line :1:9223372036854775806\n\n//", + "package p\n//line file:9223372036854775806\n\n//", + } + + for _, src := range testcases { + _, err := ParseFile(token.NewFileSet(), "", src, ParseComments) + if err == nil { + t.Errorf("ParseFile(%s) succeeded unexpectedly", src) + } + } +} diff --git a/src/go/parser/performance_test.go b/src/go/parser/performance_test.go new file mode 100644 index 0000000..1308f21 --- /dev/null +++ b/src/go/parser/performance_test.go @@ -0,0 +1,54 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parser + +import ( + "go/token" + "os" + "testing" +) + +var src = readFile("../printer/nodes.go") + +func readFile(filename string) []byte { + data, err := os.ReadFile(filename) + if err != nil { + panic(err) + } + return data +} + +func BenchmarkParse(b *testing.B) { + b.SetBytes(int64(len(src))) + for i := 0; i < b.N; i++ { + if _, err := ParseFile(token.NewFileSet(), "", src, ParseComments); err != nil { + b.Fatalf("benchmark failed due to parse error: %s", err) + } + } +} + +func BenchmarkParseOnly(b *testing.B) { + b.SetBytes(int64(len(src))) + for i := 0; i < b.N; i++ { + if _, err := ParseFile(token.NewFileSet(), "", src, ParseComments|SkipObjectResolution); err != nil { + b.Fatalf("benchmark failed due to parse error: %s", err) + } + } +} + +func BenchmarkResolve(b *testing.B) { + b.SetBytes(int64(len(src))) + for i := 0; i < b.N; i++ { + b.StopTimer() + fset := token.NewFileSet() + file, err := ParseFile(fset, "", src, SkipObjectResolution) + if err != nil { + b.Fatalf("benchmark failed due to parse error: %s", err) + } + b.StartTimer() + handle := fset.File(file.Package) + resolveFile(file, handle, nil) + } +} diff --git a/src/go/parser/resolver.go b/src/go/parser/resolver.go new file mode 100644 index 0000000..f8ff618 --- /dev/null +++ b/src/go/parser/resolver.go @@ -0,0 +1,612 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parser + +import ( + "fmt" + "go/ast" + "go/token" + "strings" +) + +const debugResolve = false + +// resolveFile walks the given file to resolve identifiers within the file +// scope, updating ast.Ident.Obj fields with declaration information. +// +// If declErr is non-nil, it is used to report declaration errors during +// resolution. tok is used to format position in error messages. +func resolveFile(file *ast.File, handle *token.File, declErr func(token.Pos, string)) { + pkgScope := ast.NewScope(nil) + r := &resolver{ + handle: handle, + declErr: declErr, + topScope: pkgScope, + pkgScope: pkgScope, + depth: 1, + } + + for _, decl := range file.Decls { + ast.Walk(r, decl) + } + + r.closeScope() + assert(r.topScope == nil, "unbalanced scopes") + assert(r.labelScope == nil, "unbalanced label scopes") + + // resolve global identifiers within the same file + i := 0 + for _, ident := range r.unresolved { + // i <= index for current ident + assert(ident.Obj == unresolved, "object already resolved") + ident.Obj = r.pkgScope.Lookup(ident.Name) // also removes unresolved sentinel + if ident.Obj == nil { + r.unresolved[i] = ident + i++ + } else if debugResolve { + pos := ident.Obj.Decl.(interface{ Pos() token.Pos }).Pos() + r.trace("resolved %s@%v to package object %v", ident.Name, ident.Pos(), pos) + } + } + file.Scope = r.pkgScope + file.Unresolved = r.unresolved[0:i] +} + +const maxScopeDepth int = 1e3 + +type resolver struct { + handle *token.File + declErr func(token.Pos, string) + + // Ordinary identifier scopes + pkgScope *ast.Scope // pkgScope.Outer == nil + topScope *ast.Scope // top-most scope; may be pkgScope + unresolved []*ast.Ident // unresolved identifiers + depth int // scope depth + + // Label scopes + // (maintained by open/close LabelScope) + labelScope *ast.Scope // label scope for current function + targetStack [][]*ast.Ident // stack of unresolved labels +} + +func (r *resolver) trace(format string, args ...any) { + fmt.Println(strings.Repeat(". ", r.depth) + r.sprintf(format, args...)) +} + +func (r *resolver) sprintf(format string, args ...any) string { + for i, arg := range args { + switch arg := arg.(type) { + case token.Pos: + args[i] = r.handle.Position(arg) + } + } + return fmt.Sprintf(format, args...) +} + +func (r *resolver) openScope(pos token.Pos) { + r.depth++ + if r.depth > maxScopeDepth { + panic(bailout{pos: pos, msg: "exceeded max scope depth during object resolution"}) + } + if debugResolve { + r.trace("opening scope @%v", pos) + } + r.topScope = ast.NewScope(r.topScope) +} + +func (r *resolver) closeScope() { + r.depth-- + if debugResolve { + r.trace("closing scope") + } + r.topScope = r.topScope.Outer +} + +func (r *resolver) openLabelScope() { + r.labelScope = ast.NewScope(r.labelScope) + r.targetStack = append(r.targetStack, nil) +} + +func (r *resolver) closeLabelScope() { + // resolve labels + n := len(r.targetStack) - 1 + scope := r.labelScope + for _, ident := range r.targetStack[n] { + ident.Obj = scope.Lookup(ident.Name) + if ident.Obj == nil && r.declErr != nil { + r.declErr(ident.Pos(), fmt.Sprintf("label %s undefined", ident.Name)) + } + } + // pop label scope + r.targetStack = r.targetStack[0:n] + r.labelScope = r.labelScope.Outer +} + +func (r *resolver) declare(decl, data any, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident) { + for _, ident := range idents { + if ident.Obj != nil { + panic(fmt.Sprintf("%v: identifier %s already declared or resolved", ident.Pos(), ident.Name)) + } + obj := ast.NewObj(kind, ident.Name) + // remember the corresponding declaration for redeclaration + // errors and global variable resolution/typechecking phase + obj.Decl = decl + obj.Data = data + // Identifiers (for receiver type parameters) are written to the scope, but + // never set as the resolved object. See issue #50956. + if _, ok := decl.(*ast.Ident); !ok { + ident.Obj = obj + } + if ident.Name != "_" { + if debugResolve { + r.trace("declaring %s@%v", ident.Name, ident.Pos()) + } + if alt := scope.Insert(obj); alt != nil && r.declErr != nil { + prevDecl := "" + if pos := alt.Pos(); pos.IsValid() { + prevDecl = r.sprintf("\n\tprevious declaration at %v", pos) + } + r.declErr(ident.Pos(), fmt.Sprintf("%s redeclared in this block%s", ident.Name, prevDecl)) + } + } + } +} + +func (r *resolver) shortVarDecl(decl *ast.AssignStmt) { + // Go spec: A short variable declaration may redeclare variables + // provided they were originally declared in the same block with + // the same type, and at least one of the non-blank variables is new. + n := 0 // number of new variables + for _, x := range decl.Lhs { + if ident, isIdent := x.(*ast.Ident); isIdent { + assert(ident.Obj == nil, "identifier already declared or resolved") + obj := ast.NewObj(ast.Var, ident.Name) + // remember corresponding assignment for other tools + obj.Decl = decl + ident.Obj = obj + if ident.Name != "_" { + if debugResolve { + r.trace("declaring %s@%v", ident.Name, ident.Pos()) + } + if alt := r.topScope.Insert(obj); alt != nil { + ident.Obj = alt // redeclaration + } else { + n++ // new declaration + } + } + } + } + if n == 0 && r.declErr != nil { + r.declErr(decl.Lhs[0].Pos(), "no new variables on left side of :=") + } +} + +// The unresolved object is a sentinel to mark identifiers that have been added +// to the list of unresolved identifiers. The sentinel is only used for verifying +// internal consistency. +var unresolved = new(ast.Object) + +// If x is an identifier, resolve attempts to resolve x by looking up +// the object it denotes. If no object is found and collectUnresolved is +// set, x is marked as unresolved and collected in the list of unresolved +// identifiers. +func (r *resolver) resolve(ident *ast.Ident, collectUnresolved bool) { + if ident.Obj != nil { + panic(r.sprintf("%v: identifier %s already declared or resolved", ident.Pos(), ident.Name)) + } + // '_' should never refer to existing declarations, because it has special + // handling in the spec. + if ident.Name == "_" { + return + } + for s := r.topScope; s != nil; s = s.Outer { + if obj := s.Lookup(ident.Name); obj != nil { + if debugResolve { + r.trace("resolved %v:%s to %v", ident.Pos(), ident.Name, obj) + } + assert(obj.Name != "", "obj with no name") + // Identifiers (for receiver type parameters) are written to the scope, + // but never set as the resolved object. See issue #50956. + if _, ok := obj.Decl.(*ast.Ident); !ok { + ident.Obj = obj + } + return + } + } + // all local scopes are known, so any unresolved identifier + // must be found either in the file scope, package scope + // (perhaps in another file), or universe scope --- collect + // them so that they can be resolved later + if collectUnresolved { + ident.Obj = unresolved + r.unresolved = append(r.unresolved, ident) + } +} + +func (r *resolver) walkExprs(list []ast.Expr) { + for _, node := range list { + ast.Walk(r, node) + } +} + +func (r *resolver) walkLHS(list []ast.Expr) { + for _, expr := range list { + expr := unparen(expr) + if _, ok := expr.(*ast.Ident); !ok && expr != nil { + ast.Walk(r, expr) + } + } +} + +func (r *resolver) walkStmts(list []ast.Stmt) { + for _, stmt := range list { + ast.Walk(r, stmt) + } +} + +func (r *resolver) Visit(node ast.Node) ast.Visitor { + if debugResolve && node != nil { + r.trace("node %T@%v", node, node.Pos()) + } + + switch n := node.(type) { + + // Expressions. + case *ast.Ident: + r.resolve(n, true) + + case *ast.FuncLit: + r.openScope(n.Pos()) + defer r.closeScope() + r.walkFuncType(n.Type) + r.walkBody(n.Body) + + case *ast.SelectorExpr: + ast.Walk(r, n.X) + // Note: don't try to resolve n.Sel, as we don't support qualified + // resolution. + + case *ast.StructType: + r.openScope(n.Pos()) + defer r.closeScope() + r.walkFieldList(n.Fields, ast.Var) + + case *ast.FuncType: + r.openScope(n.Pos()) + defer r.closeScope() + r.walkFuncType(n) + + case *ast.CompositeLit: + if n.Type != nil { + ast.Walk(r, n.Type) + } + for _, e := range n.Elts { + if kv, _ := e.(*ast.KeyValueExpr); kv != nil { + // See issue #45160: try to resolve composite lit keys, but don't + // collect them as unresolved if resolution failed. This replicates + // existing behavior when resolving during parsing. + if ident, _ := kv.Key.(*ast.Ident); ident != nil { + r.resolve(ident, false) + } else { + ast.Walk(r, kv.Key) + } + ast.Walk(r, kv.Value) + } else { + ast.Walk(r, e) + } + } + + case *ast.InterfaceType: + r.openScope(n.Pos()) + defer r.closeScope() + r.walkFieldList(n.Methods, ast.Fun) + + // Statements + case *ast.LabeledStmt: + r.declare(n, nil, r.labelScope, ast.Lbl, n.Label) + ast.Walk(r, n.Stmt) + + case *ast.AssignStmt: + r.walkExprs(n.Rhs) + if n.Tok == token.DEFINE { + r.shortVarDecl(n) + } else { + r.walkExprs(n.Lhs) + } + + case *ast.BranchStmt: + // add to list of unresolved targets + if n.Tok != token.FALLTHROUGH && n.Label != nil { + depth := len(r.targetStack) - 1 + r.targetStack[depth] = append(r.targetStack[depth], n.Label) + } + + case *ast.BlockStmt: + r.openScope(n.Pos()) + defer r.closeScope() + r.walkStmts(n.List) + + case *ast.IfStmt: + r.openScope(n.Pos()) + defer r.closeScope() + if n.Init != nil { + ast.Walk(r, n.Init) + } + ast.Walk(r, n.Cond) + ast.Walk(r, n.Body) + if n.Else != nil { + ast.Walk(r, n.Else) + } + + case *ast.CaseClause: + r.walkExprs(n.List) + r.openScope(n.Pos()) + defer r.closeScope() + r.walkStmts(n.Body) + + case *ast.SwitchStmt: + r.openScope(n.Pos()) + defer r.closeScope() + if n.Init != nil { + ast.Walk(r, n.Init) + } + if n.Tag != nil { + // The scope below reproduces some unnecessary behavior of the parser, + // opening an extra scope in case this is a type switch. It's not needed + // for expression switches. + // TODO: remove this once we've matched the parser resolution exactly. + if n.Init != nil { + r.openScope(n.Tag.Pos()) + defer r.closeScope() + } + ast.Walk(r, n.Tag) + } + if n.Body != nil { + r.walkStmts(n.Body.List) + } + + case *ast.TypeSwitchStmt: + if n.Init != nil { + r.openScope(n.Pos()) + defer r.closeScope() + ast.Walk(r, n.Init) + } + r.openScope(n.Assign.Pos()) + defer r.closeScope() + ast.Walk(r, n.Assign) + // s.Body consists only of case clauses, so does not get its own + // scope. + if n.Body != nil { + r.walkStmts(n.Body.List) + } + + case *ast.CommClause: + r.openScope(n.Pos()) + defer r.closeScope() + if n.Comm != nil { + ast.Walk(r, n.Comm) + } + r.walkStmts(n.Body) + + case *ast.SelectStmt: + // as for switch statements, select statement bodies don't get their own + // scope. + if n.Body != nil { + r.walkStmts(n.Body.List) + } + + case *ast.ForStmt: + r.openScope(n.Pos()) + defer r.closeScope() + if n.Init != nil { + ast.Walk(r, n.Init) + } + if n.Cond != nil { + ast.Walk(r, n.Cond) + } + if n.Post != nil { + ast.Walk(r, n.Post) + } + ast.Walk(r, n.Body) + + case *ast.RangeStmt: + r.openScope(n.Pos()) + defer r.closeScope() + ast.Walk(r, n.X) + var lhs []ast.Expr + if n.Key != nil { + lhs = append(lhs, n.Key) + } + if n.Value != nil { + lhs = append(lhs, n.Value) + } + if len(lhs) > 0 { + if n.Tok == token.DEFINE { + // Note: we can't exactly match the behavior of object resolution + // during the parsing pass here, as it uses the position of the RANGE + // token for the RHS OpPos. That information is not contained within + // the AST. + as := &ast.AssignStmt{ + Lhs: lhs, + Tok: token.DEFINE, + TokPos: n.TokPos, + Rhs: []ast.Expr{&ast.UnaryExpr{Op: token.RANGE, X: n.X}}, + } + // TODO(rFindley): this walkLHS reproduced the parser resolution, but + // is it necessary? By comparison, for a normal AssignStmt we don't + // walk the LHS in case there is an invalid identifier list. + r.walkLHS(lhs) + r.shortVarDecl(as) + } else { + r.walkExprs(lhs) + } + } + ast.Walk(r, n.Body) + + // Declarations + case *ast.GenDecl: + switch n.Tok { + case token.CONST, token.VAR: + for i, spec := range n.Specs { + spec := spec.(*ast.ValueSpec) + kind := ast.Con + if n.Tok == token.VAR { + kind = ast.Var + } + r.walkExprs(spec.Values) + if spec.Type != nil { + ast.Walk(r, spec.Type) + } + r.declare(spec, i, r.topScope, kind, spec.Names...) + } + case token.TYPE: + for _, spec := range n.Specs { + spec := spec.(*ast.TypeSpec) + // Go spec: The scope of a type identifier declared inside a function begins + // at the identifier in the TypeSpec and ends at the end of the innermost + // containing block. + r.declare(spec, nil, r.topScope, ast.Typ, spec.Name) + if spec.TypeParams != nil { + r.openScope(spec.Pos()) + defer r.closeScope() + r.walkTParams(spec.TypeParams) + } + ast.Walk(r, spec.Type) + } + } + + case *ast.FuncDecl: + // Open the function scope. + r.openScope(n.Pos()) + defer r.closeScope() + + r.walkRecv(n.Recv) + + // Type parameters are walked normally: they can reference each other, and + // can be referenced by normal parameters. + if n.Type.TypeParams != nil { + r.walkTParams(n.Type.TypeParams) + // TODO(rFindley): need to address receiver type parameters. + } + + // Resolve and declare parameters in a specific order to get duplicate + // declaration errors in the correct location. + r.resolveList(n.Type.Params) + r.resolveList(n.Type.Results) + r.declareList(n.Recv, ast.Var) + r.declareList(n.Type.Params, ast.Var) + r.declareList(n.Type.Results, ast.Var) + + r.walkBody(n.Body) + if n.Recv == nil && n.Name.Name != "init" { + r.declare(n, nil, r.pkgScope, ast.Fun, n.Name) + } + + default: + return r + } + + return nil +} + +func (r *resolver) walkFuncType(typ *ast.FuncType) { + // typ.TypeParams must be walked separately for FuncDecls. + r.resolveList(typ.Params) + r.resolveList(typ.Results) + r.declareList(typ.Params, ast.Var) + r.declareList(typ.Results, ast.Var) +} + +func (r *resolver) resolveList(list *ast.FieldList) { + if list == nil { + return + } + for _, f := range list.List { + if f.Type != nil { + ast.Walk(r, f.Type) + } + } +} + +func (r *resolver) declareList(list *ast.FieldList, kind ast.ObjKind) { + if list == nil { + return + } + for _, f := range list.List { + r.declare(f, nil, r.topScope, kind, f.Names...) + } +} + +func (r *resolver) walkRecv(recv *ast.FieldList) { + // If our receiver has receiver type parameters, we must declare them before + // trying to resolve the rest of the receiver, and avoid re-resolving the + // type parameter identifiers. + if recv == nil || len(recv.List) == 0 { + return // nothing to do + } + typ := recv.List[0].Type + if ptr, ok := typ.(*ast.StarExpr); ok { + typ = ptr.X + } + + var declareExprs []ast.Expr // exprs to declare + var resolveExprs []ast.Expr // exprs to resolve + switch typ := typ.(type) { + case *ast.IndexExpr: + declareExprs = []ast.Expr{typ.Index} + resolveExprs = append(resolveExprs, typ.X) + case *ast.IndexListExpr: + declareExprs = typ.Indices + resolveExprs = append(resolveExprs, typ.X) + default: + resolveExprs = append(resolveExprs, typ) + } + for _, expr := range declareExprs { + if id, _ := expr.(*ast.Ident); id != nil { + r.declare(expr, nil, r.topScope, ast.Typ, id) + } else { + // The receiver type parameter expression is invalid, but try to resolve + // it anyway for consistency. + resolveExprs = append(resolveExprs, expr) + } + } + for _, expr := range resolveExprs { + if expr != nil { + ast.Walk(r, expr) + } + } + // The receiver is invalid, but try to resolve it anyway for consistency. + for _, f := range recv.List[1:] { + if f.Type != nil { + ast.Walk(r, f.Type) + } + } +} + +func (r *resolver) walkFieldList(list *ast.FieldList, kind ast.ObjKind) { + if list == nil { + return + } + r.resolveList(list) + r.declareList(list, kind) +} + +// walkTParams is like walkFieldList, but declares type parameters eagerly so +// that they may be resolved in the constraint expressions held in the field +// Type. +func (r *resolver) walkTParams(list *ast.FieldList) { + r.declareList(list, ast.Typ) + r.resolveList(list) +} + +func (r *resolver) walkBody(body *ast.BlockStmt) { + if body == nil { + return + } + r.openLabelScope() + defer r.closeLabelScope() + r.walkStmts(body.List) +} diff --git a/src/go/parser/resolver_test.go b/src/go/parser/resolver_test.go new file mode 100644 index 0000000..38739a3 --- /dev/null +++ b/src/go/parser/resolver_test.go @@ -0,0 +1,172 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parser + +import ( + "fmt" + "go/ast" + "go/scanner" + "go/token" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestResolution checks that identifiers are resolved to the declarations +// annotated in the source, by comparing the positions of the resulting +// Ident.Obj.Decl to positions marked in the source via special comments. +// +// In the test source, any comment prefixed with '=' or '@' (or both) marks the +// previous token position as the declaration ('=') or a use ('@') of an +// identifier. The text following '=' and '@' in the comment string is the +// label to use for the location. Declaration labels must be unique within the +// file, and use labels must refer to an existing declaration label. It's OK +// for a comment to denote both the declaration and use of a label (e.g. +// '=@foo'). Leading and trailing whitespace is ignored. Any comment not +// beginning with '=' or '@' is ignored. +func TestResolution(t *testing.T) { + dir := filepath.Join("testdata", "resolution") + fis, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + for _, fi := range fis { + t.Run(fi.Name(), func(t *testing.T) { + fset := token.NewFileSet() + path := filepath.Join(dir, fi.Name()) + src := readFile(path) // panics on failure + var mode Mode + file, err := ParseFile(fset, path, src, mode) + if err != nil { + t.Fatal(err) + } + + // Compare the positions of objects resolved during parsing (fromParser) + // to those annotated in source comments (fromComments). + + handle := fset.File(file.Package) + fromParser := declsFromParser(file) + fromComments := declsFromComments(handle, src) + + pos := func(pos token.Pos) token.Position { + p := handle.Position(pos) + // The file name is implied by the subtest, so remove it to avoid + // clutter in error messages. + p.Filename = "" + return p + } + for k, want := range fromComments { + if got := fromParser[k]; got != want { + t.Errorf("%s resolved to %s, want %s", pos(k), pos(got), pos(want)) + } + delete(fromParser, k) + } + // What remains in fromParser are unexpected resolutions. + for k, got := range fromParser { + t.Errorf("%s resolved to %s, want no object", pos(k), pos(got)) + } + }) + } +} + +// declsFromParser walks the file and collects the map associating an +// identifier position with its declaration position. +func declsFromParser(file *ast.File) map[token.Pos]token.Pos { + objmap := map[token.Pos]token.Pos{} + ast.Inspect(file, func(node ast.Node) bool { + // Ignore blank identifiers to reduce noise. + if ident, _ := node.(*ast.Ident); ident != nil && ident.Obj != nil && ident.Name != "_" { + objmap[ident.Pos()] = ident.Obj.Pos() + } + return true + }) + return objmap +} + +// declsFromComments looks at comments annotating uses and declarations, and +// maps each identifier use to its corresponding declaration. See the +// description of these annotations in the documentation for TestResolution. +func declsFromComments(handle *token.File, src []byte) map[token.Pos]token.Pos { + decls, uses := positionMarkers(handle, src) + + objmap := make(map[token.Pos]token.Pos) + // Join decls and uses on name, to build the map of use->decl. + for name, posns := range uses { + declpos, ok := decls[name] + if !ok { + panic(fmt.Sprintf("missing declaration for %s", name)) + } + for _, pos := range posns { + objmap[pos] = declpos + } + } + return objmap +} + +// positionMarkers extracts named positions from the source denoted by comments +// prefixed with '=' (declarations) and '@' (uses): for example '@foo' or +// '=@bar'. It returns a map of name->position for declarations, and +// name->position(s) for uses. +func positionMarkers(handle *token.File, src []byte) (decls map[string]token.Pos, uses map[string][]token.Pos) { + var s scanner.Scanner + s.Init(handle, src, nil, scanner.ScanComments) + decls = make(map[string]token.Pos) + uses = make(map[string][]token.Pos) + var prev token.Pos // position of last non-comment, non-semicolon token + +scanFile: + for { + pos, tok, lit := s.Scan() + switch tok { + case token.EOF: + break scanFile + case token.COMMENT: + name, decl, use := annotatedObj(lit) + if len(name) > 0 { + if decl { + if _, ok := decls[name]; ok { + panic(fmt.Sprintf("duplicate declaration markers for %s", name)) + } + decls[name] = prev + } + if use { + uses[name] = append(uses[name], prev) + } + } + case token.SEMICOLON: + // ignore automatically inserted semicolon + if lit == "\n" { + continue scanFile + } + fallthrough + default: + prev = pos + } + } + return decls, uses +} + +func annotatedObj(lit string) (name string, decl, use bool) { + if lit[1] == '*' { + lit = lit[:len(lit)-2] // strip trailing */ + } + lit = strings.TrimSpace(lit[2:]) + +scanLit: + for idx, r := range lit { + switch r { + case '=': + decl = true + case '@': + use = true + default: + name = lit[idx:] + break scanLit + } + } + return +} diff --git a/src/go/parser/short_test.go b/src/go/parser/short_test.go new file mode 100644 index 0000000..f9575e1 --- /dev/null +++ b/src/go/parser/short_test.go @@ -0,0 +1,214 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains test cases for short valid and invalid programs. + +package parser + +import "testing" + +var valids = []string{ + "package p\n", + `package p;`, + `package p; import "fmt"; func f() { fmt.Println("Hello, World!") };`, + `package p; func f() { if f(T{}) {} };`, + `package p; func f() { _ = <-chan int(nil) };`, + `package p; func f() { _ = (<-chan int)(nil) };`, + `package p; func f() { _ = (<-chan <-chan int)(nil) };`, + `package p; func f() { _ = <-chan <-chan <-chan <-chan <-int(nil) };`, + `package p; func f(func() func() func());`, + `package p; func f(...T);`, + `package p; func f(float, ...int);`, + `package p; func f(x int, a ...int) { f(0, a...); f(1, a...,) };`, + `package p; func f(int,) {};`, + `package p; func f(...int,) {};`, + `package p; func f(x ...int,) {};`, + `package p; type T []int; var a []bool; func f() { if a[T{42}[0]] {} };`, + `package p; type T []int; func g(int) bool { return true }; func f() { if g(T{42}[0]) {} };`, + `package p; type T []int; func f() { for _ = range []int{T{42}[0]} {} };`, + `package p; var a = T{{1, 2}, {3, 4}}`, + `package p; func f() { select { case <- c: case c <- d: case c <- <- d: case <-c <- d: } };`, + `package p; func f() { select { case x := (<-c): } };`, + `package p; func f() { if ; true {} };`, + `package p; func f() { switch ; {} };`, + `package p; func f() { for _ = range "foo" + "bar" {} };`, + `package p; func f() { var s []int; g(s[:], s[i:], s[:j], s[i:j], s[i:j:k], s[:j:k]) };`, + `package p; var ( _ = (struct {*T}).m; _ = (interface {T}).m )`, + `package p; func ((T),) m() {}`, + `package p; func ((*T),) m() {}`, + `package p; func (*(T),) m() {}`, + `package p; func _(x []int) { for range x {} }`, + `package p; func _() { if [T{}.n]int{} {} }`, + `package p; func _() { map[int]int{}[0]++; map[int]int{}[0] += 1 }`, + `package p; func _(x interface{f()}) { interface{f()}(x).f() }`, + `package p; func _(x chan int) { chan int(x) <- 0 }`, + `package p; const (x = 0; y; z)`, // issue 9639 + `package p; var _ = map[P]int{P{}:0, {}:1}`, + `package p; var _ = map[*P]int{&P{}:0, {}:1}`, + `package p; type T = int`, + `package p; type (T = p.T; _ = struct{}; x = *T)`, + `package p; type T (*int)`, + `package p; type _ struct{ int }`, + `package p; type _ struct{ pkg.T }`, + `package p; type _ struct{ *pkg.T }`, + `package p; var _ = func()T(nil)`, + `package p; func _(T (P))`, + `package p; func _(T []E)`, + `package p; func _(T [P]E)`, + `package p; type _ [A+B]struct{}`, + `package p; func (R) _()`, + `package p; type _ struct{ f [n]E }`, + `package p; type _ struct{ f [a+b+c+d]E }`, + `package p; type I1 interface{}; type I2 interface{ I1 }`, + + // generic code + `package p; type _ []T[int]`, + `package p; type T[P any] struct { P }`, + `package p; type T[P comparable] struct { P }`, + `package p; type T[P comparable[P]] struct { P }`, + `package p; type T[P1, P2 any] struct { P1; f []P2 }`, + `package p; func _[T any]()()`, + `package p; func _(T (P))`, + `package p; func f[A, B any](); func _() { _ = f[int, int] }`, + `package p; func _(x T[P1, P2, P3])`, + `package p; func _(x p.T[Q])`, + `package p; func _(p.T[Q])`, + `package p; type _[A interface{},] struct{}`, + `package p; type _[A interface{}] struct{}`, + `package p; type _[A, B any,] struct{}`, + `package p; type _[A, B any] struct{}`, + `package p; type _[A any,] struct{}`, + `package p; type _[A any]struct{}`, + `package p; type _[A any] struct{ A }`, + `package p; func _[T any]()`, + `package p; func _[T any](x T)`, + `package p; func _[T1, T2 any](x T)`, + `package p; func _[A, B any](a A) B`, + `package p; func _[A, B C](a A) B`, + `package p; func _[A, B C[A, B]](a A) B`, + + `package p; type _[A, B any] interface { _(a A) B }`, + `package p; type _[A, B C[A, B]] interface { _(a A) B }`, + `package p; func _[T1, T2 interface{}](x T1) T2`, + `package p; func _[T1 interface{ m() }, T2, T3 interface{}](x T1, y T3) T2`, + `package p; var _ = []T[int]{}`, + `package p; var _ = [10]T[int]{}`, + `package p; var _ = func()T[int]{}`, + `package p; var _ = map[T[int]]T[int]{}`, + `package p; var _ = chan T[int](x)`, + `package p; func _(_ T[P], T P) T[P]`, + `package p; var _ T[chan int]`, + + `package p; func (_ R[P]) _(x T)`, + `package p; func (_ R[ P, Q]) _(x T)`, + + `package p; func (R[P]) _()`, + `package p; func _(T[P])`, + `package p; func _(T[P1, P2, P3 ])`, + `package p; func _(T[P]) T[P]`, + `package p; type _ struct{ T[P]}`, + `package p; type _ struct{ T[struct{a, b, c int}] }`, + `package p; type _ interface{int|float32; bool; m(); string;}`, + `package p; type I1[T any] interface{}; type I2 interface{ I1[int] }`, + `package p; type I1[T any] interface{}; type I2[T any] interface{ I1[T] }`, + `package p; type _ interface { N[T] }`, + `package p; type T[P any] = T0`, +} + +func TestValid(t *testing.T) { + for _, src := range valids { + checkErrors(t, src, src, DeclarationErrors|AllErrors, false) + } +} + +// TestSingle is useful to track down a problem with a single short test program. +func TestSingle(t *testing.T) { + const src = `package p; var _ = T{}` + checkErrors(t, src, src, DeclarationErrors|AllErrors, true) +} + +var invalids = []string{ + `foo /* ERROR "expected 'package'" */ !`, + `package p; func f() { if { /* ERROR "missing condition" */ } };`, + `package p; func f() { if ; /* ERROR "missing condition" */ {} };`, + `package p; func f() { if f(); /* ERROR "missing condition" */ {} };`, + `package p; func f() { if _ = range /* ERROR "expected operand" */ x; true {} };`, + `package p; func f() { switch _ /* ERROR "expected switch expression" */ = range x; true {} };`, + `package p; func f() { for _ = range x ; /* ERROR "expected '{'" */ ; {} };`, + `package p; func f() { for ; ; _ = range /* ERROR "expected operand" */ x {} };`, + `package p; func f() { for ; _ /* ERROR "expected boolean or range expression" */ = range x ; {} };`, + `package p; func f() { switch t = /* ERROR "expected ':=', found '='" */ t.(type) {} };`, + `package p; func f() { switch t /* ERROR "expected switch expression" */ , t = t.(type) {} };`, + `package p; func f() { switch t /* ERROR "expected switch expression" */ = t.(type), t {} };`, + `package p; func f() { _ = (<-<- /* ERROR "expected 'chan'" */ chan int)(nil) };`, + `package p; func f() { _ = (<-chan<-chan<-chan<-chan<-chan<- /* ERROR "expected channel type" */ int)(nil) };`, + `package p; func f() { if x := g(); x /* ERROR "expected boolean expression" */ = 0 {}};`, + `package p; func f() { _ = x = /* ERROR "expected '=='" */ 0 {}};`, + `package p; func f() { _ = 1 == func()int { var x bool; x = x = /* ERROR "expected '=='" */ true; return x }() };`, + `package p; func f() { var s []int; _ = s[] /* ERROR "expected operand" */ };`, + `package p; func f() { var s []int; _ = s[i:j: /* ERROR "final index required" */ ] };`, + `package p; func f() { var s []int; _ = s[i: /* ERROR "middle index required" */ :k] };`, + `package p; func f() { var s []int; _ = s[i: /* ERROR "middle index required" */ :] };`, + `package p; func f() { var s []int; _ = s[: /* ERROR "middle index required" */ :] };`, + `package p; func f() { var s []int; _ = s[: /* ERROR "middle index required" */ ::] };`, + `package p; func f() { var s []int; _ = s[i:j:k: /* ERROR "expected ']'" */ l] };`, + `package p; func f() { for x /* ERROR "boolean or range expression" */ = []string {} }`, + `package p; func f() { for x /* ERROR "boolean or range expression" */ := []string {} }`, + `package p; func f() { for i /* ERROR "boolean or range expression" */ , x = []string {} }`, + `package p; func f() { for i /* ERROR "boolean or range expression" */ , x := []string {} }`, + `package p; func f() { go f /* ERROR HERE "must be function call" */ }`, + `package p; func f() { go ( /* ERROR "must not be parenthesized" */ f()) }`, + `package p; func f() { defer func() {} /* ERROR HERE "must be function call" */ }`, + `package p; func f() { defer ( /* ERROR "must not be parenthesized" */ f()) }`, + `package p; func f() { go func() { func() { f(x func /* ERROR "missing ','" */ (){}) } } }`, + `package p; func _() (type /* ERROR "found 'type'" */ T)(T)`, + `package p; func (type /* ERROR "found 'type'" */ T)(T) _()`, + `package p; type _[A+B, /* ERROR "unexpected comma" */ ] int`, + + `package p; type _ struct{ [ /* ERROR "expected '}', found '\['" */ ]byte }`, + `package p; type _ struct{ ( /* ERROR "cannot parenthesize embedded type" */ int) }`, + `package p; type _ struct{ ( /* ERROR "cannot parenthesize embedded type" */ []byte) }`, + `package p; type _ struct{ *( /* ERROR "cannot parenthesize embedded type" */ int) }`, + `package p; type _ struct{ *( /* ERROR "cannot parenthesize embedded type" */ []byte) }`, + + // issue 8656 + `package p; func f() (a b string /* ERROR "missing ','" */ , ok bool)`, + + // issue 9639 + `package p; var x, y, z; /* ERROR "expected type" */`, + + // issue 12437 + `package p; var _ = struct { x int, /* ERROR "expected ';', found ','" */ }{};`, + `package p; var _ = struct { x int, /* ERROR "expected ';', found ','" */ y float }{};`, + + // issue 11611 + `package p; type _ struct { int, } /* ERROR "expected 'IDENT', found '}'" */ ;`, + `package p; type _ struct { int, float } /* ERROR "expected type, found '}'" */ ;`, + + // issue 13475 + `package p; func f() { if true {} else ; /* ERROR "expected if statement or block" */ }`, + `package p; func f() { if true {} else defer /* ERROR "expected if statement or block" */ f() }`, + + // generic code + `package p; type _[_ any] int; var _ = T[] /* ERROR "expected operand" */ {}`, + `package p; var _ func[ /* ERROR "must have no type parameters" */ T any](T)`, + `package p; func _[]/* ERROR "empty type parameter list" */()`, + + // TODO(rfindley) a better location would be after the ']' + `package p; type _[A /* ERROR "type parameters must be named" */ ,] struct{ A }`, + + `package p; func _[type /* ERROR "found 'type'" */ P, *Q interface{}]()`, + + `package p; func (T) _[ /* ERROR "must have no type parameters" */ A, B any](a A) B`, + `package p; func (T) _[ /* ERROR "must have no type parameters" */ A, B C](a A) B`, + `package p; func (T) _[ /* ERROR "must have no type parameters" */ A, B C[A, B]](a A) B`, + + `package p; func(*T[e, e /* ERROR "e redeclared" */ ]) _()`, +} + +func TestInvalid(t *testing.T) { + for _, src := range invalids { + checkErrors(t, src, src, DeclarationErrors|AllErrors, true) + } +} diff --git a/src/go/parser/testdata/chans.go2 b/src/go/parser/testdata/chans.go2 new file mode 100644 index 0000000..fad2bce --- /dev/null +++ b/src/go/parser/testdata/chans.go2 @@ -0,0 +1,62 @@ +package chans + +import "runtime" + +// Ranger returns a Sender and a Receiver. The Receiver provides a +// Next method to retrieve values. The Sender provides a Send method +// to send values and a Close method to stop sending values. The Next +// method indicates when the Sender has been closed, and the Send +// method indicates when the Receiver has been freed. +// +// This is a convenient way to exit a goroutine sending values when +// the receiver stops reading them. +func Ranger[T any]() (*Sender[T], *Receiver[T]) { + c := make(chan T) + d := make(chan bool) + s := &Sender[T]{values: c, done: d} + r := &Receiver[T]{values: c, done: d} + runtime.SetFinalizer(r, r.finalize) + return s, r +} + +// A sender is used to send values to a Receiver. +type Sender[T any] struct { + values chan<- T + done <-chan bool +} + +// Send sends a value to the receiver. It returns whether any more +// values may be sent; if it returns false the value was not sent. +func (s *Sender[T]) Send(v T) bool { + select { + case s.values <- v: + return true + case <-s.done: + return false + } +} + +// Close tells the receiver that no more values will arrive. +// After Close is called, the Sender may no longer be used. +func (s *Sender[T]) Close() { + close(s.values) +} + +// A Receiver receives values from a Sender. +type Receiver[T any] struct { + values <-chan T + done chan<- bool +} + +// Next returns the next value from the channel. The bool result +// indicates whether the value is valid, or whether the Sender has +// been closed and no more values will be received. +func (r *Receiver[T]) Next() (T, bool) { + v, ok := <-r.values + return v, ok +} + +// finalize is a finalizer for the receiver. +func (r *Receiver[T]) finalize() { + close(r.done) +} diff --git a/src/go/parser/testdata/commas.src b/src/go/parser/testdata/commas.src new file mode 100644 index 0000000..2c52ae5 --- /dev/null +++ b/src/go/parser/testdata/commas.src @@ -0,0 +1,19 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for error messages/parser synchronization +// after missing commas. + +package p + +var _ = []int{ + 0/* ERROR AFTER "missing ','" */ +} + +var _ = []int{ + 0, + 1, + 2, + 3/* ERROR AFTER "missing ','" */ +} diff --git a/src/go/parser/testdata/interface.go2 b/src/go/parser/testdata/interface.go2 new file mode 100644 index 0000000..2ed9339 --- /dev/null +++ b/src/go/parser/testdata/interface.go2 @@ -0,0 +1,76 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains test cases for interfaces containing +// constraint elements. + +package p + +type _ interface { + m() + ~int + ~int|string + E +} + +type _ interface { + m() + ~int + int | string + int | ~string + ~int | ~string +} + +type _ interface { + m() + ~int + T[int, string] | string + int | ~T[string, struct{}] + ~int | ~string +} + +type _ interface { + int + []byte + [10]int + struct{} + *int + func() + interface{} + map[string]int + chan T + chan<- T + <-chan T + T[int] +} + +type _ interface { + int | string + []byte | string + [10]int | string + struct{} | string + *int | string + func() | string + interface{} | string + map[string]int | string + chan T | string + chan<- T | string + <-chan T | string + T[int] | string +} + +type _ interface { + ~int | string + ~[]byte | string + ~[10]int | string + ~struct{} | string + ~*int | string + ~func() | string + ~interface{} | string + ~map[string]int | string + ~chan T | string + ~chan<- T | string + ~<-chan T | string + ~T[int] | string +} diff --git a/src/go/parser/testdata/issue11377.src b/src/go/parser/testdata/issue11377.src new file mode 100644 index 0000000..1c43800 --- /dev/null +++ b/src/go/parser/testdata/issue11377.src @@ -0,0 +1,27 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 11377: Better synchronization of +// parser after certain syntax errors. + +package p + +func bad1() { + if f()) /* ERROR "expected ';', found '\)'" */ { + return + } +} + +// There shouldn't be any errors down below. + +func F1() {} +func F2() {} +func F3() {} +func F4() {} +func F5() {} +func F6() {} +func F7() {} +func F8() {} +func F9() {} +func F10() {} diff --git a/src/go/parser/testdata/issue23434.src b/src/go/parser/testdata/issue23434.src new file mode 100644 index 0000000..24a0832 --- /dev/null +++ b/src/go/parser/testdata/issue23434.src @@ -0,0 +1,25 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 23434: Better synchronization of +// parser after missing type. There should be exactly +// one error each time, with now follow errors. + +package p + +func g() { + m := make(map[string]! /* ERROR "expected type, found '!'" */ ) + for { + x := 1 + print(x) + } +} + +func f() { + m := make(map[string]) /* ERROR "expected type, found '\)'" */ + for { + x := 1 + print(x) + } +} diff --git a/src/go/parser/testdata/issue3106.src b/src/go/parser/testdata/issue3106.src new file mode 100644 index 0000000..2db10be --- /dev/null +++ b/src/go/parser/testdata/issue3106.src @@ -0,0 +1,46 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 3106: Better synchronization of +// parser after certain syntax errors. + +package main + +func f() { + var m Mutex + c := MakeCond(&m) + percent := 0 + const step = 10 + for i := 0; i < 5; i++ { + go func() { + for { + // Emulates some useful work. + time.Sleep(1e8) + m.Lock() + defer + if /* ERROR "expected ';', found 'if'" */ percent == 100 { + m.Unlock() + break + } + percent++ + if percent % step == 0 { + //c.Signal() + } + m.Unlock() + } + }() + } + for { + m.Lock() + if percent == 0 || percent % step != 0 { + c.Wait() + } + fmt.Print(",") + if percent == 100 { + m.Unlock() + break + } + m.Unlock() + } +} diff --git a/src/go/parser/testdata/issue34946.src b/src/go/parser/testdata/issue34946.src new file mode 100644 index 0000000..6bb15e1 --- /dev/null +++ b/src/go/parser/testdata/issue34946.src @@ -0,0 +1,22 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 34946: Better synchronization of +// parser for function declarations that start their +// body's opening { on a new line. + +package p + +// accept Allman/BSD-style declaration but complain +// (implicit semicolon between signature and body) +func _() int +{ /* ERROR "unexpected semicolon or newline before {" */ + { return 0 } +} + +func _() {} + +func _(); { /* ERROR "unexpected semicolon or newline before {" */ } + +func _() {} diff --git a/src/go/parser/testdata/issue42951/not_a_file.go/invalid.go b/src/go/parser/testdata/issue42951/not_a_file.go/invalid.go new file mode 100644 index 0000000..bb698be --- /dev/null +++ b/src/go/parser/testdata/issue42951/not_a_file.go/invalid.go @@ -0,0 +1 @@ +This file should not be parsed by ParseDir. diff --git a/src/go/parser/testdata/issue44504.src b/src/go/parser/testdata/issue44504.src new file mode 100644 index 0000000..7791f4a --- /dev/null +++ b/src/go/parser/testdata/issue44504.src @@ -0,0 +1,13 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Test case for issue 44504: panic due to duplicate resolution of slice/index +// operands. We should not try to resolve a LHS expression with invalid syntax. + +package p + +func _() { + var items []bool + items[] /* ERROR "operand" */ = false +} diff --git a/src/go/parser/testdata/issue49174.go2 b/src/go/parser/testdata/issue49174.go2 new file mode 100644 index 0000000..77c1950 --- /dev/null +++ b/src/go/parser/testdata/issue49174.go2 @@ -0,0 +1,8 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func _[_ []int | int]() {} +func _[_ int | []int]() {} diff --git a/src/go/parser/testdata/issue49175.go2 b/src/go/parser/testdata/issue49175.go2 new file mode 100644 index 0000000..cf1c83c --- /dev/null +++ b/src/go/parser/testdata/issue49175.go2 @@ -0,0 +1,13 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type _[_ []t]t +type _[_ [1]t]t + +func _[_ []t]() {} +func _[_ [1]t]() {} + +type t [t /* ERROR "type parameters must be named" */ [0]]t diff --git a/src/go/parser/testdata/issue49482.go2 b/src/go/parser/testdata/issue49482.go2 new file mode 100644 index 0000000..d8385be --- /dev/null +++ b/src/go/parser/testdata/issue49482.go2 @@ -0,0 +1,34 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type ( + // these need a comma to disambiguate + _[P *T,] struct{} + _[P *T, _ any] struct{} + _[P (*T),] struct{} + _[P (*T), _ any] struct{} + _[P (T),] struct{} + _[P (T), _ any] struct{} + + // these parse as name followed by type + _[P *struct{}] struct{} + _[P (*struct{})] struct{} + _[P ([]int)] struct{} + + // array declarations + _ [P(T)]struct{} + _ [P((T))]struct{} + _ [P * *T]struct{} + _ [P * T]struct{} + _ [P(*T)]struct{} + _ [P(**T)]struct{} + _ [P * T - T]struct{} + _ [P*T-T, /* ERROR "unexpected comma" */ ]struct{} + _ [10, /* ERROR "unexpected comma" */ ]struct{} + + _[P *struct{}|int] struct{} + _[P *struct{}|int|string] struct{} +) diff --git a/src/go/parser/testdata/issue50427.go2 b/src/go/parser/testdata/issue50427.go2 new file mode 100644 index 0000000..1521459 --- /dev/null +++ b/src/go/parser/testdata/issue50427.go2 @@ -0,0 +1,19 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type T interface{ m[ /* ERROR "must have no type parameters" */ P any]() } + +func _(t T) { + var _ interface{ m[ /* ERROR "must have no type parameters" */ P any](); n() } = t +} + +type S struct{} + +func (S) m[ /* ERROR "must have no type parameters" */ P any]() {} + +func _(s S) { + var _ interface{ m[ /* ERROR "must have no type parameters" */ P any](); n() } = s +} diff --git a/src/go/parser/testdata/linalg.go2 b/src/go/parser/testdata/linalg.go2 new file mode 100644 index 0000000..7ccb19c --- /dev/null +++ b/src/go/parser/testdata/linalg.go2 @@ -0,0 +1,83 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package linalg + +import "math" + +// Numeric is type bound that matches any numeric type. +// It would likely be in a constraints package in the standard library. +type Numeric interface { + ~int|~int8|~int16|~int32|~int64| + ~uint|~uint8|~uint16|~uint32|~uint64|~uintptr| + ~float32|~float64| + ~complex64|~complex128 +} + +func DotProduct[T Numeric](s1, s2 []T) T { + if len(s1) != len(s2) { + panic("DotProduct: slices of unequal length") + } + var r T + for i := range s1 { + r += s1[i] * s2[i] + } + return r +} + +// NumericAbs matches numeric types with an Abs method. +type NumericAbs[T any] interface { + Numeric + + Abs() T +} + +// AbsDifference computes the absolute value of the difference of +// a and b, where the absolute value is determined by the Abs method. +func AbsDifference[T NumericAbs](a, b T) T { + d := a - b + return d.Abs() +} + +// OrderedNumeric is a type bound that matches numeric types that support the < operator. +type OrderedNumeric interface { + ~int|~int8|~int16|~int32|~int64| + ~uint|~uint8|~uint16|~uint32|~uint64|~uintptr| + ~float32|~float64 +} + +// Complex is a type bound that matches the two complex types, which do not have a < operator. +type Complex interface { + ~complex64|~complex128 +} + +// OrderedAbs is a helper type that defines an Abs method for +// ordered numeric types. +type OrderedAbs[T OrderedNumeric] T + +func (a OrderedAbs[T]) Abs() OrderedAbs[T] { + if a < 0 { + return -a + } + return a +} + +// ComplexAbs is a helper type that defines an Abs method for +// complex types. +type ComplexAbs[T Complex] T + +func (a ComplexAbs[T]) Abs() ComplexAbs[T] { + r := float64(real(a)) + i := float64(imag(a)) + d := math.Sqrt(r * r + i * i) + return ComplexAbs[T](complex(d, 0)) +} + +func OrderedAbsDifference[T OrderedNumeric](a, b T) T { + return T(AbsDifference(OrderedAbs[T](a), OrderedAbs[T](b))) +} + +func ComplexAbsDifference[T Complex](a, b T) T { + return T(AbsDifference(ComplexAbs[T](a), ComplexAbs[T](b))) +} diff --git a/src/go/parser/testdata/map.go2 b/src/go/parser/testdata/map.go2 new file mode 100644 index 0000000..74c79ae --- /dev/null +++ b/src/go/parser/testdata/map.go2 @@ -0,0 +1,109 @@ +// Package orderedmap provides an ordered map, implemented as a binary tree. +package orderedmap + +import "chans" + +// Map is an ordered map. +type Map[K, V any] struct { + root *node[K, V] + compare func(K, K) int +} + +// node is the type of a node in the binary tree. +type node[K, V any] struct { + key K + val V + left, right *node[K, V] +} + +// New returns a new map. +func New[K, V any](compare func(K, K) int) *Map[K, V] { + return &Map[K, V]{compare: compare} +} + +// find looks up key in the map, and returns either a pointer +// to the node holding key, or a pointer to the location where +// such a node would go. +func (m *Map[K, V]) find(key K) **node[K, V] { + pn := &m.root + for *pn != nil { + switch cmp := m.compare(key, (*pn).key); { + case cmp < 0: + pn = &(*pn).left + case cmp > 0: + pn = &(*pn).right + default: + return pn + } + } + return pn +} + +// Insert inserts a new key/value into the map. +// If the key is already present, the value is replaced. +// Returns true if this is a new key, false if already present. +func (m *Map[K, V]) Insert(key K, val V) bool { + pn := m.find(key) + if *pn != nil { + (*pn).val = val + return false + } + *pn = &node[K, V]{key: key, val: val} + return true +} + +// Find returns the value associated with a key, or zero if not present. +// The found result reports whether the key was found. +func (m *Map[K, V]) Find(key K) (V, bool) { + pn := m.find(key) + if *pn == nil { + var zero V // see the discussion of zero values, above + return zero, false + } + return (*pn).val, true +} + +// keyValue is a pair of key and value used when iterating. +type keyValue[K, V any] struct { + key K + val V +} + +// InOrder returns an iterator that does an in-order traversal of the map. +func (m *Map[K, V]) InOrder() *Iterator[K, V] { + sender, receiver := chans.Ranger[keyValue[K, V]]() + var f func(*node[K, V]) bool + f = func(n *node[K, V]) bool { + if n == nil { + return true + } + // Stop sending values if sender.Send returns false, + // meaning that nothing is listening at the receiver end. + return f(n.left) && + // TODO + // sender.Send(keyValue[K, V]{n.key, n.val}) && + f(n.right) + } + go func() { + f(m.root) + sender.Close() + }() + return &Iterator{receiver} +} + +// Iterator is used to iterate over the map. +type Iterator[K, V any] struct { + r *chans.Receiver[keyValue[K, V]] +} + +// Next returns the next key and value pair, and a boolean indicating +// whether they are valid or whether we have reached the end. +func (it *Iterator[K, V]) Next() (K, V, bool) { + keyval, ok := it.r.Next() + if !ok { + var zerok K + var zerov V + return zerok, zerov, false + } + return keyval.key, keyval.val, true +} diff --git a/src/go/parser/testdata/metrics.go2 b/src/go/parser/testdata/metrics.go2 new file mode 100644 index 0000000..ef1c66b --- /dev/null +++ b/src/go/parser/testdata/metrics.go2 @@ -0,0 +1,58 @@ +package metrics + +import "sync" + +type Metric1[T comparable] struct { + mu sync.Mutex + m map[T]int +} + +func (m *Metric1[T]) Add(v T) { + m.mu.Lock() + defer m.mu.Unlock() + if m.m == nil { + m.m = make(map[T]int) + } + m[v]++ +} + +type key2[T1, T2 comparable] struct { + f1 T1 + f2 T2 +} + +type Metric2[T1, T2 cmp2] struct { + mu sync.Mutex + m map[key2[T1, T2]]int +} + +func (m *Metric2[T1, T2]) Add(v1 T1, v2 T2) { + m.mu.Lock() + defer m.mu.Unlock() + if m.m == nil { + m.m = make(map[key2[T1, T2]]int) + } + m[key[T1, T2]{v1, v2}]++ +} + +type key3[T1, T2, T3 comparable] struct { + f1 T1 + f2 T2 + f3 T3 +} + +type Metric3[T1, T2, T3 comparable] struct { + mu sync.Mutex + m map[key3[T1, T2, T3]]int +} + +func (m *Metric3[T1, T2, T3]) Add(v1 T1, v2 T2, v3 T3) { + m.mu.Lock() + defer m.mu.Unlock() + if m.m == nil { + m.m = make(map[key3]int) + } + m[key[T1, T2, T3]{v1, v2, v3}]++ +} + +// Repeat for the maximum number of permitted arguments. diff --git a/src/go/parser/testdata/resolution/issue45136.src b/src/go/parser/testdata/resolution/issue45136.src new file mode 100644 index 0000000..e1d63d8 --- /dev/null +++ b/src/go/parser/testdata/resolution/issue45136.src @@ -0,0 +1,27 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue45136 + +type obj /* =@obj */ struct { + name /*=@name */ string +} + +func _() { + var foo /* =@foo */ = "foo" + obj /* @obj */ ["foo"] + obj /* @obj */ .run() + obj /* @obj */ { + name: foo /* @foo */, + } + obj /* @obj */ { + name: "bar", + }.run() + + var _ = File{key: obj /* @obj */ {}} + var _ = File{obj /* @obj */ {}} + + []obj /* @obj */ {foo /* @foo */} + x /* =@x1 */ := obj /* @obj */{} +} diff --git a/src/go/parser/testdata/resolution/issue45160.src b/src/go/parser/testdata/resolution/issue45160.src new file mode 100644 index 0000000..6be933b --- /dev/null +++ b/src/go/parser/testdata/resolution/issue45160.src @@ -0,0 +1,25 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package issue45160 + +func mklink1 /* =@mklink1func */() {} + +func _() { + var tests /* =@tests */ = []dirLinkTest /* @dirLinkTest */ { + { + mklink1 /* @mklink1func */: func() {}, + mklink2: func(link /* =@link */, target /* =@target */ string) error { + return nil + }, + }, + } +} + +type dirLinkTest /* =@dirLinkTest */ struct { + mklink1 /* =@mklink1field */ func(string, string) error + mklink2 /* =@mklink2field */ func(string, string) error +} + +func mklink2 /* =@mklink2func */() {} diff --git a/src/go/parser/testdata/resolution/resolution.src b/src/go/parser/testdata/resolution/resolution.src new file mode 100644 index 0000000..a880dd1 --- /dev/null +++ b/src/go/parser/testdata/resolution/resolution.src @@ -0,0 +1,63 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package resolution + +func f /* =@fdecl */(n /* =@narg */ ast.Node) bool { + if n /* =@ninit */, ok /* =@ok */ := n /* @narg */ .(*ast.SelectorExpr); ok /* @ok */ { + sel = n /* @ninit */ + } +} + +type c /* =@cdecl */ map[token.Pos]resolvedObj + +func (v /* =@vdecl */ c /* @cdecl */) Visit(node /* =@nodearg */ ast.Node) (w /* =@w */ ast.Visitor) {} + +const ( + basic /* =@basic */ = iota + labelOk // =@labelOk +) + +type T /* =@T */ int + +func _(count /* =@count */ T /* @T */) { + x /* =@x1 */ := c /* @cdecl */{} + switch x /* =@x2 */ := x /* @x1 */; x /* =@x3 */ := x /* @x2 */.(type) { + case c /* @cdecl */: + default: + } +loop /* =@loop */: + for { + if true { + break loop /* @loop */ + } + } + select { + case err /* =@err1 */ := <-_: + return err /* @err1 */ + case err /* =@err2 */ := <-_: + return err /* @err2 */ + } + + _ = func(p1 /* =@p1 */ int, p2 /* =@p2 */ p1) { + closed /* =@closed */ := p1 // @p1 + shadowed /* =@shadowed1 */ := p2 // @p2 + _ = func(shadowed /* =@shadowed2 */ p2 /* @p2 */) { + closed /* @closed */ = 1 + shadowed /* @shadowed2 */ = 2 + } + } +} + +func (r /* =@r */ c /* @cdecl */) m(_ r) c /* @cdecl */ { return r /* @r */ } + +var cycle /* =@cycle */ = cycle /* @cycle */ + 1 + +type chain /* =@chain */ struct { + next /* =@next */ *chain /* @chain */ +} + +func recursive /* =@recursive */() { + recursive /* @recursive */ () +} diff --git a/src/go/parser/testdata/resolution/typeparams.go2 b/src/go/parser/testdata/resolution/typeparams.go2 new file mode 100644 index 0000000..7395ca2 --- /dev/null +++ b/src/go/parser/testdata/resolution/typeparams.go2 @@ -0,0 +1,51 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package resolution + +type List /* =@List */ [E /* =@E */ any] []E // @E + +type Pair /* =@Pair */ [L /* =@L */, R /* =@R */ any] struct { + Left /* =@Left */ L // @L + Right /* =@Right */ R // @R + L /* =@Lfield */ int +} + +var _ = Pair /* @Pair */ [int, string]{} + +type Addable /* =@Addable */ interface { + ~int64|~float64 +} + +func Add /* =@AddDecl */[T /* =@T */ Addable /* @Addable */](l /* =@l */, r /* =@r */ T /* @T */) T /* @T */ { + var t /* =@t */ T /* @T */ + return l /* @l */ + r /* @r */ + t /* @t */ +} + +type Receiver /* =@Receiver */[P /* =@P */ any] struct {} + +type RP /* =@RP1 */ struct{} + +// TODO(rFindley): make a decision on how/whether to resolve identifiers that +// refer to receiver type parameters, as is the case for the 'P' result +// parameter below. +// +// For now, we ensure that types are not incorrectly resolved when receiver +// type parameters are in scope. +func (r /* =@recv */ Receiver /* @Receiver */ [RP]) m(RP) RP {} + +func f /* =@f */[T1 /* =@T1 */ interface{~[]T2 /* @T2 */}, T2 /* =@T2 */ any]( + x /* =@x */ T1 /* @T1 */, T1 /* =@T1_duplicate */ y, // Note that this is a bug: + // the duplicate T1 should + // not be allowed. + ){ + // Note that duplicate short var declarations resolve to their alt declaration. + x /* @x */ := 0 + y /* =@y */ := 0 + T1 /* @T1 */ := 0 + var t1var /* =@t1var */ T1 /* @T1 */ +} + +// From issue #39634 +func(*ph1[e, e])h(d) diff --git a/src/go/parser/testdata/set.go2 b/src/go/parser/testdata/set.go2 new file mode 100644 index 0000000..0da6377 --- /dev/null +++ b/src/go/parser/testdata/set.go2 @@ -0,0 +1,31 @@ +// Package set implements sets of any type. +package set + +type Set[Elem comparable] map[Elem]struct{} + +func Make[Elem comparable]() Set[Elem] { + return make(Set(Elem)) +} + +func (s Set[Elem]) Add(v Elem) { + s[v] = struct{}{} +} + +func (s Set[Elem]) Delete(v Elem) { + delete(s, v) +} + +func (s Set[Elem]) Contains(v Elem) bool { + _, ok := s[v] + return ok +} + +func (s Set[Elem]) Len() int { + return len(s) +} + +func (s Set[Elem]) Iterate(f func(Elem)) { + for v := range s { + f(v) + } +} diff --git a/src/go/parser/testdata/slices.go2 b/src/go/parser/testdata/slices.go2 new file mode 100644 index 0000000..e060212 --- /dev/null +++ b/src/go/parser/testdata/slices.go2 @@ -0,0 +1,31 @@ +// Package slices implements various slice algorithms. +package slices + +// Map turns a []T1 to a []T2 using a mapping function. +func Map[T1, T2 any](s []T1, f func(T1) T2) []T2 { + r := make([]T2, len(s)) + for i, v := range s { + r[i] = f(v) + } + return r +} + +// Reduce reduces a []T1 to a single value using a reduction function. +func Reduce[T1, T2 any](s []T1, initializer T2, f func(T2, T1) T2) T2 { + r := initializer + for _, v := range s { + r = f(r, v) + } + return r +} + +// Filter filters values from a slice using a filter function. +func Filter[T any](s []T, f func(T) bool) []T { + var r []T + for _, v := range s { + if f(v) { + r = append(r, v) + } + } + return r +} diff --git a/src/go/parser/testdata/sort.go2 b/src/go/parser/testdata/sort.go2 new file mode 100644 index 0000000..88be79f --- /dev/null +++ b/src/go/parser/testdata/sort.go2 @@ -0,0 +1,27 @@ +package sort + +type orderedSlice[Elem comparable] []Elem + +func (s orderedSlice[Elem]) Len() int { return len(s) } +func (s orderedSlice[Elem]) Less(i, j int) bool { return s[i] < s[j] } +func (s orderedSlice[Elem]) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// OrderedSlice sorts the slice s in ascending order. +// The elements of s must be ordered using the < operator. +func OrderedSlice[Elem comparable](s []Elem) { + sort.Sort(orderedSlice[Elem](s)) +} + +type sliceFn[Elem any] struct { + s []Elem + f func(Elem, Elem) bool +} + +func (s sliceFn[Elem]) Len() int { return len(s.s) } +func (s sliceFn[Elem]) Less(i, j int) bool { return s.f(s.s[i], s.s[j]) } +func (s sliceFn[Elem]) Swap(i, j int) { s.s[i], s.s[j] = s.s[j], s.s[i] } + +// SliceFn sorts the slice s according to the function f. +func SliceFn[Elem any](s []Elem, f func(Elem, Elem) bool) { + Sort(sliceFn[Elem]{s, f}) +} diff --git a/src/go/parser/testdata/tparams.go2 b/src/go/parser/testdata/tparams.go2 new file mode 100644 index 0000000..1a9a6c6 --- /dev/null +++ b/src/go/parser/testdata/tparams.go2 @@ -0,0 +1,54 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type _[a /* ERROR "type parameters must be named" */, b] struct{} +type _[a t, b t, c /* ERROR "type parameters must be named" */ ] struct{} +type _ struct { + t [n]byte + t[a] + t[a,] + t[a, b] + t[a, b,] +} +type _ struct { + t [n, /* ERROR "unexpected comma; expecting ]" */ ]byte +} +type _ interface { + t[a] + t[a,] + m[ /* ERROR "method must have no type parameters" */ _ _, /* ERROR mixed */ _]() + t[a, b] + t[a, b,] +} + +func _[] /* ERROR "empty type parameter list" */ () +func _[a /* ERROR "type parameters must be named" */, b ]() +func _[a t, b t, c /* ERROR "type parameters must be named" */ ]() + +// TODO(rfindley) incorrect error message (see existing TODO in parser) +func f[a b, 0 /* ERROR "expected '\)', found 0" */ ] () + +// issue #49482 +type ( + _[a *[]int] struct{} + _[a *t,] struct{} + _[a *t|[]int] struct{} + _[a *t|t,] struct{} + _[a *t|~t,] struct{} + _[a *struct{}|t] struct{} + _[a *t|struct{}] struct{} + _[a *struct{}|~t] struct{} +) + +// issue #51488 +type ( + _[a *t|t,] struct{} + _[a *t|t, b t] struct{} + _[a *t|t] struct{} + _[a *[]t|t] struct{} + _[a ([]t)] struct{} + _[a ([]t)|t] struct{} +) diff --git a/src/go/parser/testdata/typeset.go2 b/src/go/parser/testdata/typeset.go2 new file mode 100644 index 0000000..7844c22 --- /dev/null +++ b/src/go/parser/testdata/typeset.go2 @@ -0,0 +1,72 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains test cases for typeset-only constraint elements. +// TODO(gri) gofmt once/if gofmt supports this notation. + +package p + +type ( + _[_ t] t + _[_ ~t] t + _[_ t|t] t + _[_ ~t|t] t + _[_ t|~t] t + _[_ ~t|~t] t + + _[_ t, _, _ t|t] t + _[_ t, _, _ ~t|t] t + _[_ t, _, _ t|~t] t + _[_ t, _, _ ~t|~t] t + + _[_ t.t] t + _[_ ~t.t] t + _[_ t.t|t.t] t + _[_ ~t.t|t.t] t + _[_ t.t|~t.t] t + _[_ ~t.t|~t.t] t + + _[_ t, _, _ t.t|t.t] t + _[_ t, _, _ ~t.t|t.t] t + _[_ t, _, _ t.t|~t.t] t + _[_ t, _, _ ~t.t|~t.t] t + + _[_ struct{}] t + _[_ ~struct{}] t + + _[_ struct{}|t] t + _[_ ~struct{}|t] t + _[_ struct{}|~t] t + _[_ ~struct{}|~t] t + + _[_ t|struct{}] t + _[_ ~t|struct{}] t + _[_ t|~struct{}] t + _[_ ~t|~struct{}] t +) + +// Single-expression type parameter lists and those that don't start +// with a (type parameter) name are considered array sizes. +// The term must be a valid expression (it could be a type incl. a +// tilde term) but the type-checker will complain. +type ( + _[t] t + _[t|t] t + + // These are invalid and the type-checker will complain. + _[~t] t + _[~t|t] t + _[t|~t] t + _[~t|~t] t +) + +type _[_ t, t /* ERROR "type parameters must be named" */ ] t +type _[_ ~t, t /* ERROR "type parameters must be named" */ ] t +type _[_ t, ~ /* ERROR "type parameters must be named" */ t] t +type _[_ ~t, ~ /* ERROR "type parameters must be named" */ t] t + +type _[_ t|t, t /* ERROR "type parameters must be named" */ |t] t +type _[_ ~t|t, t /* ERROR "type parameters must be named" */ |t] t +type _[_ t|t, ~ /* ERROR "type parameters must be named" */ t|t] t +type _[_ ~t|t, ~ /* ERROR "type parameters must be named" */ t|t] t diff --git a/src/go/printer/comment.go b/src/go/printer/comment.go new file mode 100644 index 0000000..9012714 --- /dev/null +++ b/src/go/printer/comment.go @@ -0,0 +1,155 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package printer + +import ( + "go/ast" + "go/doc/comment" + "strings" +) + +// formatDocComment reformats the doc comment list, +// returning the canonical formatting. +func formatDocComment(list []*ast.Comment) []*ast.Comment { + // Extract comment text (removing comment markers). + var kind, text string + var directives []*ast.Comment + if len(list) == 1 && strings.HasPrefix(list[0].Text, "/*") { + kind = "/*" + text = list[0].Text + if !strings.Contains(text, "\n") || allStars(text) { + // Single-line /* .. */ comment in doc comment position, + // or multiline old-style comment like + // /* + // * Comment + // * text here. + // */ + // Should not happen, since it will not work well as a + // doc comment, but if it does, just ignore: + // reformatting it will only make the situation worse. + return list + } + text = text[2 : len(text)-2] // cut /* and */ + } else if strings.HasPrefix(list[0].Text, "//") { + kind = "//" + var b strings.Builder + for _, c := range list { + after, found := strings.CutPrefix(c.Text, "//") + if !found { + return list + } + // Accumulate //go:build etc lines separately. + if isDirective(after) { + directives = append(directives, c) + continue + } + b.WriteString(strings.TrimPrefix(after, " ")) + b.WriteString("\n") + } + text = b.String() + } else { + // Not sure what this is, so leave alone. + return list + } + + if text == "" { + return list + } + + // Parse comment and reformat as text. + var p comment.Parser + d := p.Parse(text) + + var pr comment.Printer + text = string(pr.Comment(d)) + + // For /* */ comment, return one big comment with text inside. + slash := list[0].Slash + if kind == "/*" { + c := &ast.Comment{ + Slash: slash, + Text: "/*\n" + text + "*/", + } + return []*ast.Comment{c} + } + + // For // comment, return sequence of // lines. + var out []*ast.Comment + for text != "" { + var line string + line, text, _ = strings.Cut(text, "\n") + if line == "" { + line = "//" + } else if strings.HasPrefix(line, "\t") { + line = "//" + line + } else { + line = "// " + line + } + out = append(out, &ast.Comment{ + Slash: slash, + Text: line, + }) + } + if len(directives) > 0 { + out = append(out, &ast.Comment{ + Slash: slash, + Text: "//", + }) + for _, c := range directives { + out = append(out, &ast.Comment{ + Slash: slash, + Text: c.Text, + }) + } + } + return out +} + +// isDirective reports whether c is a comment directive. +// See go.dev/issue/37974. +// This code is also in go/ast. +func isDirective(c string) bool { + // "//line " is a line directive. + // "//extern " is for gccgo. + // "//export " is for cgo. + // (The // has been removed.) + if strings.HasPrefix(c, "line ") || strings.HasPrefix(c, "extern ") || strings.HasPrefix(c, "export ") { + return true + } + + // "//[a-z0-9]+:[a-z0-9]" + // (The // has been removed.) + colon := strings.Index(c, ":") + if colon <= 0 || colon+1 >= len(c) { + return false + } + for i := 0; i <= colon+1; i++ { + if i == colon { + continue + } + b := c[i] + if !('a' <= b && b <= 'z' || '0' <= b && b <= '9') { + return false + } + } + return true +} + +// allStars reports whether text is the interior of an +// old-style /* */ comment with a star at the start of each line. +func allStars(text string) bool { + for i := 0; i < len(text); i++ { + if text[i] == '\n' { + j := i + 1 + for j < len(text) && (text[j] == ' ' || text[j] == '\t') { + j++ + } + if j < len(text) && text[j] != '*' { + return false + } + } + } + return true +} diff --git a/src/go/printer/example_test.go b/src/go/printer/example_test.go new file mode 100644 index 0000000..f7d72d1 --- /dev/null +++ b/src/go/printer/example_test.go @@ -0,0 +1,67 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package printer_test + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "strings" +) + +func parseFunc(filename, functionname string) (fun *ast.FuncDecl, fset *token.FileSet) { + fset = token.NewFileSet() + if file, err := parser.ParseFile(fset, filename, nil, 0); err == nil { + for _, d := range file.Decls { + if f, ok := d.(*ast.FuncDecl); ok && f.Name.Name == functionname { + fun = f + return + } + } + } + panic("function not found") +} + +func printSelf() { + // Parse source file and extract the AST without comments for + // this function, with position information referring to the + // file set fset. + funcAST, fset := parseFunc("example_test.go", "printSelf") + + // Print the function body into buffer buf. + // The file set is provided to the printer so that it knows + // about the original source formatting and can add additional + // line breaks where they were present in the source. + var buf bytes.Buffer + printer.Fprint(&buf, fset, funcAST.Body) + + // Remove braces {} enclosing the function body, unindent, + // and trim leading and trailing white space. + s := buf.String() + s = s[1 : len(s)-1] + s = strings.TrimSpace(strings.ReplaceAll(s, "\n\t", "\n")) + + // Print the cleaned-up body text to stdout. + fmt.Println(s) +} + +func ExampleFprint() { + printSelf() + + // Output: + // funcAST, fset := parseFunc("example_test.go", "printSelf") + // + // var buf bytes.Buffer + // printer.Fprint(&buf, fset, funcAST.Body) + // + // s := buf.String() + // s = s[1 : len(s)-1] + // s = strings.TrimSpace(strings.ReplaceAll(s, "\n\t", "\n")) + // + // fmt.Println(s) +} diff --git a/src/go/printer/gobuild.go b/src/go/printer/gobuild.go new file mode 100644 index 0000000..f00492d --- /dev/null +++ b/src/go/printer/gobuild.go @@ -0,0 +1,170 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package printer + +import ( + "go/build/constraint" + "sort" + "text/tabwriter" +) + +func (p *printer) fixGoBuildLines() { + if len(p.goBuild)+len(p.plusBuild) == 0 { + return + } + + // Find latest possible placement of //go:build and // +build comments. + // That's just after the last blank line before we find a non-comment. + // (We'll add another blank line after our comment block.) + // When we start dropping // +build comments, we can skip over /* */ comments too. + // Note that we are processing tabwriter input, so every comment + // begins and ends with a tabwriter.Escape byte. + // And some newlines have turned into \f bytes. + insert := 0 + for pos := 0; ; { + // Skip leading space at beginning of line. + blank := true + for pos < len(p.output) && (p.output[pos] == ' ' || p.output[pos] == '\t') { + pos++ + } + // Skip over // comment if any. + if pos+3 < len(p.output) && p.output[pos] == tabwriter.Escape && p.output[pos+1] == '/' && p.output[pos+2] == '/' { + blank = false + for pos < len(p.output) && !isNL(p.output[pos]) { + pos++ + } + } + // Skip over \n at end of line. + if pos >= len(p.output) || !isNL(p.output[pos]) { + break + } + pos++ + + if blank { + insert = pos + } + } + + // If there is a //go:build comment before the place we identified, + // use that point instead. (Earlier in the file is always fine.) + if len(p.goBuild) > 0 && p.goBuild[0] < insert { + insert = p.goBuild[0] + } else if len(p.plusBuild) > 0 && p.plusBuild[0] < insert { + insert = p.plusBuild[0] + } + + var x constraint.Expr + switch len(p.goBuild) { + case 0: + // Synthesize //go:build expression from // +build lines. + for _, pos := range p.plusBuild { + y, err := constraint.Parse(p.commentTextAt(pos)) + if err != nil { + x = nil + break + } + if x == nil { + x = y + } else { + x = &constraint.AndExpr{X: x, Y: y} + } + } + case 1: + // Parse //go:build expression. + x, _ = constraint.Parse(p.commentTextAt(p.goBuild[0])) + } + + var block []byte + if x == nil { + // Don't have a valid //go:build expression to treat as truth. + // Bring all the lines together but leave them alone. + // Note that these are already tabwriter-escaped. + for _, pos := range p.goBuild { + block = append(block, p.lineAt(pos)...) + } + for _, pos := range p.plusBuild { + block = append(block, p.lineAt(pos)...) + } + } else { + block = append(block, tabwriter.Escape) + block = append(block, "//go:build "...) + block = append(block, x.String()...) + block = append(block, tabwriter.Escape, '\n') + if len(p.plusBuild) > 0 { + lines, err := constraint.PlusBuildLines(x) + if err != nil { + lines = []string{"// +build error: " + err.Error()} + } + for _, line := range lines { + block = append(block, tabwriter.Escape) + block = append(block, line...) + block = append(block, tabwriter.Escape, '\n') + } + } + } + block = append(block, '\n') + + // Build sorted list of lines to delete from remainder of output. + toDelete := append(p.goBuild, p.plusBuild...) + sort.Ints(toDelete) + + // Collect output after insertion point, with lines deleted, into after. + var after []byte + start := insert + for _, end := range toDelete { + if end < start { + continue + } + after = appendLines(after, p.output[start:end]) + start = end + len(p.lineAt(end)) + } + after = appendLines(after, p.output[start:]) + if n := len(after); n >= 2 && isNL(after[n-1]) && isNL(after[n-2]) { + after = after[:n-1] + } + + p.output = p.output[:insert] + p.output = append(p.output, block...) + p.output = append(p.output, after...) +} + +// appendLines is like append(x, y...) +// but it avoids creating doubled blank lines, +// which would not be gofmt-standard output. +// It assumes that only whole blocks of lines are being appended, +// not line fragments. +func appendLines(x, y []byte) []byte { + if len(y) > 0 && isNL(y[0]) && // y starts in blank line + (len(x) == 0 || len(x) >= 2 && isNL(x[len(x)-1]) && isNL(x[len(x)-2])) { // x is empty or ends in blank line + y = y[1:] // delete y's leading blank line + } + return append(x, y...) +} + +func (p *printer) lineAt(start int) []byte { + pos := start + for pos < len(p.output) && !isNL(p.output[pos]) { + pos++ + } + if pos < len(p.output) { + pos++ + } + return p.output[start:pos] +} + +func (p *printer) commentTextAt(start int) string { + if start < len(p.output) && p.output[start] == tabwriter.Escape { + start++ + } + pos := start + for pos < len(p.output) && p.output[pos] != tabwriter.Escape && !isNL(p.output[pos]) { + pos++ + } + return string(p.output[start:pos]) +} + +func isNL(b byte) bool { + return b == '\n' || b == '\f' +} diff --git a/src/go/printer/nodes.go b/src/go/printer/nodes.go new file mode 100644 index 0000000..e41ffc1 --- /dev/null +++ b/src/go/printer/nodes.go @@ -0,0 +1,2001 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements printing of AST nodes; specifically +// expressions, statements, declarations, and files. It uses +// the print functionality implemented in printer.go. + +package printer + +import ( + "go/ast" + "go/token" + "math" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// Formatting issues: +// - better comment formatting for /*-style comments at the end of a line (e.g. a declaration) +// when the comment spans multiple lines; if such a comment is just two lines, formatting is +// not idempotent +// - formatting of expression lists +// - should use blank instead of tab to separate one-line function bodies from +// the function header unless there is a group of consecutive one-liners + +// ---------------------------------------------------------------------------- +// Common AST nodes. + +// Print as many newlines as necessary (but at least min newlines) to get to +// the current line. ws is printed before the first line break. If newSection +// is set, the first line break is printed as formfeed. Returns 0 if no line +// breaks were printed, returns 1 if there was exactly one newline printed, +// and returns a value > 1 if there was a formfeed or more than one newline +// printed. +// +// TODO(gri): linebreak may add too many lines if the next statement at "line" +// is preceded by comments because the computation of n assumes +// the current position before the comment and the target position +// after the comment. Thus, after interspersing such comments, the +// space taken up by them is not considered to reduce the number of +// linebreaks. At the moment there is no easy way to know about +// future (not yet interspersed) comments in this function. +func (p *printer) linebreak(line, min int, ws whiteSpace, newSection bool) (nbreaks int) { + n := nlimit(line - p.pos.Line) + if n < min { + n = min + } + if n > 0 { + p.print(ws) + if newSection { + p.print(formfeed) + n-- + nbreaks = 2 + } + nbreaks += n + for ; n > 0; n-- { + p.print(newline) + } + } + return +} + +// setComment sets g as the next comment if g != nil and if node comments +// are enabled - this mode is used when printing source code fragments such +// as exports only. It assumes that there is no pending comment in p.comments +// and at most one pending comment in the p.comment cache. +func (p *printer) setComment(g *ast.CommentGroup) { + if g == nil || !p.useNodeComments { + return + } + if p.comments == nil { + // initialize p.comments lazily + p.comments = make([]*ast.CommentGroup, 1) + } else if p.cindex < len(p.comments) { + // for some reason there are pending comments; this + // should never happen - handle gracefully and flush + // all comments up to g, ignore anything after that + p.flush(p.posFor(g.List[0].Pos()), token.ILLEGAL) + p.comments = p.comments[0:1] + // in debug mode, report error + p.internalError("setComment found pending comments") + } + p.comments[0] = g + p.cindex = 0 + // don't overwrite any pending comment in the p.comment cache + // (there may be a pending comment when a line comment is + // immediately followed by a lead comment with no other + // tokens between) + if p.commentOffset == infinity { + p.nextComment() // get comment ready for use + } +} + +type exprListMode uint + +const ( + commaTerm exprListMode = 1 << iota // list is optionally terminated by a comma + noIndent // no extra indentation in multi-line lists +) + +// If indent is set, a multi-line identifier list is indented after the +// first linebreak encountered. +func (p *printer) identList(list []*ast.Ident, indent bool) { + // convert into an expression list so we can re-use exprList formatting + xlist := make([]ast.Expr, len(list)) + for i, x := range list { + xlist[i] = x + } + var mode exprListMode + if !indent { + mode = noIndent + } + p.exprList(token.NoPos, xlist, 1, mode, token.NoPos, false) +} + +const filteredMsg = "contains filtered or unexported fields" + +// Print a list of expressions. If the list spans multiple +// source lines, the original line breaks are respected between +// expressions. +// +// TODO(gri) Consider rewriting this to be independent of []ast.Expr +// so that we can use the algorithm for any kind of list +// +// (e.g., pass list via a channel over which to range). +func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exprListMode, next0 token.Pos, isIncomplete bool) { + if len(list) == 0 { + if isIncomplete { + prev := p.posFor(prev0) + next := p.posFor(next0) + if prev.IsValid() && prev.Line == next.Line { + p.print("/* " + filteredMsg + " */") + } else { + p.print(newline) + p.print(indent, "// "+filteredMsg, unindent, newline) + } + } + return + } + + prev := p.posFor(prev0) + next := p.posFor(next0) + line := p.lineFor(list[0].Pos()) + endLine := p.lineFor(list[len(list)-1].End()) + + if prev.IsValid() && prev.Line == line && line == endLine { + // all list entries on a single line + for i, x := range list { + if i > 0 { + // use position of expression following the comma as + // comma position for correct comment placement + p.setPos(x.Pos()) + p.print(token.COMMA, blank) + } + p.expr0(x, depth) + } + if isIncomplete { + p.print(token.COMMA, blank, "/* "+filteredMsg+" */") + } + return + } + + // list entries span multiple lines; + // use source code positions to guide line breaks + + // Don't add extra indentation if noIndent is set; + // i.e., pretend that the first line is already indented. + ws := ignore + if mode&noIndent == 0 { + ws = indent + } + + // The first linebreak is always a formfeed since this section must not + // depend on any previous formatting. + prevBreak := -1 // index of last expression that was followed by a linebreak + if prev.IsValid() && prev.Line < line && p.linebreak(line, 0, ws, true) > 0 { + ws = ignore + prevBreak = 0 + } + + // initialize expression/key size: a zero value indicates expr/key doesn't fit on a single line + size := 0 + + // We use the ratio between the geometric mean of the previous key sizes and + // the current size to determine if there should be a break in the alignment. + // To compute the geometric mean we accumulate the ln(size) values (lnsum) + // and the number of sizes included (count). + lnsum := 0.0 + count := 0 + + // print all list elements + prevLine := prev.Line + for i, x := range list { + line = p.lineFor(x.Pos()) + + // Determine if the next linebreak, if any, needs to use formfeed: + // in general, use the entire node size to make the decision; for + // key:value expressions, use the key size. + // TODO(gri) for a better result, should probably incorporate both + // the key and the node size into the decision process + useFF := true + + // Determine element size: All bets are off if we don't have + // position information for the previous and next token (likely + // generated code - simply ignore the size in this case by setting + // it to 0). + prevSize := size + const infinity = 1e6 // larger than any source line + size = p.nodeSize(x, infinity) + pair, isPair := x.(*ast.KeyValueExpr) + if size <= infinity && prev.IsValid() && next.IsValid() { + // x fits on a single line + if isPair { + size = p.nodeSize(pair.Key, infinity) // size <= infinity + } + } else { + // size too large or we don't have good layout information + size = 0 + } + + // If the previous line and the current line had single- + // line-expressions and the key sizes are small or the + // ratio between the current key and the geometric mean + // if the previous key sizes does not exceed a threshold, + // align columns and do not use formfeed. + if prevSize > 0 && size > 0 { + const smallSize = 40 + if count == 0 || prevSize <= smallSize && size <= smallSize { + useFF = false + } else { + const r = 2.5 // threshold + geomean := math.Exp(lnsum / float64(count)) // count > 0 + ratio := float64(size) / geomean + useFF = r*ratio <= 1 || r <= ratio + } + } + + needsLinebreak := 0 < prevLine && prevLine < line + if i > 0 { + // Use position of expression following the comma as + // comma position for correct comment placement, but + // only if the expression is on the same line. + if !needsLinebreak { + p.setPos(x.Pos()) + } + p.print(token.COMMA) + needsBlank := true + if needsLinebreak { + // Lines are broken using newlines so comments remain aligned + // unless useFF is set or there are multiple expressions on + // the same line in which case formfeed is used. + nbreaks := p.linebreak(line, 0, ws, useFF || prevBreak+1 < i) + if nbreaks > 0 { + ws = ignore + prevBreak = i + needsBlank = false // we got a line break instead + } + // If there was a new section or more than one new line + // (which means that the tabwriter will implicitly break + // the section), reset the geomean variables since we are + // starting a new group of elements with the next element. + if nbreaks > 1 { + lnsum = 0 + count = 0 + } + } + if needsBlank { + p.print(blank) + } + } + + if len(list) > 1 && isPair && size > 0 && needsLinebreak { + // We have a key:value expression that fits onto one line + // and it's not on the same line as the prior expression: + // Use a column for the key such that consecutive entries + // can align if possible. + // (needsLinebreak is set if we started a new line before) + p.expr(pair.Key) + p.setPos(pair.Colon) + p.print(token.COLON, vtab) + p.expr(pair.Value) + } else { + p.expr0(x, depth) + } + + if size > 0 { + lnsum += math.Log(float64(size)) + count++ + } + + prevLine = line + } + + if mode&commaTerm != 0 && next.IsValid() && p.pos.Line < next.Line { + // Print a terminating comma if the next token is on a new line. + p.print(token.COMMA) + if isIncomplete { + p.print(newline) + p.print("// " + filteredMsg) + } + if ws == ignore && mode&noIndent == 0 { + // unindent if we indented + p.print(unindent) + } + p.print(formfeed) // terminating comma needs a line break to look good + return + } + + if isIncomplete { + p.print(token.COMMA, newline) + p.print("// "+filteredMsg, newline) + } + + if ws == ignore && mode&noIndent == 0 { + // unindent if we indented + p.print(unindent) + } +} + +type paramMode int + +const ( + funcParam paramMode = iota + funcTParam + typeTParam +) + +func (p *printer) parameters(fields *ast.FieldList, mode paramMode) { + openTok, closeTok := token.LPAREN, token.RPAREN + if mode != funcParam { + openTok, closeTok = token.LBRACK, token.RBRACK + } + p.setPos(fields.Opening) + p.print(openTok) + if len(fields.List) > 0 { + prevLine := p.lineFor(fields.Opening) + ws := indent + for i, par := range fields.List { + // determine par begin and end line (may be different + // if there are multiple parameter names for this par + // or the type is on a separate line) + parLineBeg := p.lineFor(par.Pos()) + parLineEnd := p.lineFor(par.End()) + // separating "," if needed + needsLinebreak := 0 < prevLine && prevLine < parLineBeg + if i > 0 { + // use position of parameter following the comma as + // comma position for correct comma placement, but + // only if the next parameter is on the same line + if !needsLinebreak { + p.setPos(par.Pos()) + } + p.print(token.COMMA) + } + // separator if needed (linebreak or blank) + if needsLinebreak && p.linebreak(parLineBeg, 0, ws, true) > 0 { + // break line if the opening "(" or previous parameter ended on a different line + ws = ignore + } else if i > 0 { + p.print(blank) + } + // parameter names + if len(par.Names) > 0 { + // Very subtle: If we indented before (ws == ignore), identList + // won't indent again. If we didn't (ws == indent), identList will + // indent if the identList spans multiple lines, and it will outdent + // again at the end (and still ws == indent). Thus, a subsequent indent + // by a linebreak call after a type, or in the next multi-line identList + // will do the right thing. + p.identList(par.Names, ws == indent) + p.print(blank) + } + // parameter type + p.expr(stripParensAlways(par.Type)) + prevLine = parLineEnd + } + + // if the closing ")" is on a separate line from the last parameter, + // print an additional "," and line break + if closing := p.lineFor(fields.Closing); 0 < prevLine && prevLine < closing { + p.print(token.COMMA) + p.linebreak(closing, 0, ignore, true) + } else if mode == typeTParam && fields.NumFields() == 1 && combinesWithName(fields.List[0].Type) { + // A type parameter list [P T] where the name P and the type expression T syntactically + // combine to another valid (value) expression requires a trailing comma, as in [P *T,] + // (or an enclosing interface as in [P interface(*T)]), so that the type parameter list + // is not parsed as an array length [P*T]. + p.print(token.COMMA) + } + + // unindent if we indented + if ws == ignore { + p.print(unindent) + } + } + + p.setPos(fields.Closing) + p.print(closeTok) +} + +// combinesWithName reports whether a name followed by the expression x +// syntactically combines to another valid (value) expression. For instance +// using *T for x, "name *T" syntactically appears as the expression x*T. +// On the other hand, using P|Q or *P|~Q for x, "name P|Q" or name *P|~Q" +// cannot be combined into a valid (value) expression. +func combinesWithName(x ast.Expr) bool { + switch x := x.(type) { + case *ast.StarExpr: + // name *x.X combines to name*x.X if x.X is not a type element + return !isTypeElem(x.X) + case *ast.BinaryExpr: + return combinesWithName(x.X) && !isTypeElem(x.Y) + case *ast.ParenExpr: + // name(x) combines but we are making sure at + // the call site that x is never parenthesized. + panic("unexpected parenthesized expression") + } + return false +} + +// isTypeElem reports whether x is a (possibly parenthesized) type element expression. +// The result is false if x could be a type element OR an ordinary (value) expression. +func isTypeElem(x ast.Expr) bool { + switch x := x.(type) { + case *ast.ArrayType, *ast.StructType, *ast.FuncType, *ast.InterfaceType, *ast.MapType, *ast.ChanType: + return true + case *ast.UnaryExpr: + return x.Op == token.TILDE + case *ast.BinaryExpr: + return isTypeElem(x.X) || isTypeElem(x.Y) + case *ast.ParenExpr: + return isTypeElem(x.X) + } + return false +} + +func (p *printer) signature(sig *ast.FuncType) { + if sig.TypeParams != nil { + p.parameters(sig.TypeParams, funcTParam) + } + if sig.Params != nil { + p.parameters(sig.Params, funcParam) + } else { + p.print(token.LPAREN, token.RPAREN) + } + res := sig.Results + n := res.NumFields() + if n > 0 { + // res != nil + p.print(blank) + if n == 1 && res.List[0].Names == nil { + // single anonymous res; no ()'s + p.expr(stripParensAlways(res.List[0].Type)) + return + } + p.parameters(res, funcParam) + } +} + +func identListSize(list []*ast.Ident, maxSize int) (size int) { + for i, x := range list { + if i > 0 { + size += len(", ") + } + size += utf8.RuneCountInString(x.Name) + if size >= maxSize { + break + } + } + return +} + +func (p *printer) isOneLineFieldList(list []*ast.Field) bool { + if len(list) != 1 { + return false // allow only one field + } + f := list[0] + if f.Tag != nil || f.Comment != nil { + return false // don't allow tags or comments + } + // only name(s) and type + const maxSize = 30 // adjust as appropriate, this is an approximate value + namesSize := identListSize(f.Names, maxSize) + if namesSize > 0 { + namesSize = 1 // blank between names and types + } + typeSize := p.nodeSize(f.Type, maxSize) + return namesSize+typeSize <= maxSize +} + +func (p *printer) setLineComment(text string) { + p.setComment(&ast.CommentGroup{List: []*ast.Comment{{Slash: token.NoPos, Text: text}}}) +} + +func (p *printer) fieldList(fields *ast.FieldList, isStruct, isIncomplete bool) { + lbrace := fields.Opening + list := fields.List + rbrace := fields.Closing + hasComments := isIncomplete || p.commentBefore(p.posFor(rbrace)) + srcIsOneLine := lbrace.IsValid() && rbrace.IsValid() && p.lineFor(lbrace) == p.lineFor(rbrace) + + if !hasComments && srcIsOneLine { + // possibly a one-line struct/interface + if len(list) == 0 { + // no blank between keyword and {} in this case + p.setPos(lbrace) + p.print(token.LBRACE) + p.setPos(rbrace) + p.print(token.RBRACE) + return + } else if p.isOneLineFieldList(list) { + // small enough - print on one line + // (don't use identList and ignore source line breaks) + p.setPos(lbrace) + p.print(token.LBRACE, blank) + f := list[0] + if isStruct { + for i, x := range f.Names { + if i > 0 { + // no comments so no need for comma position + p.print(token.COMMA, blank) + } + p.expr(x) + } + if len(f.Names) > 0 { + p.print(blank) + } + p.expr(f.Type) + } else { // interface + if len(f.Names) > 0 { + name := f.Names[0] // method name + p.expr(name) + p.signature(f.Type.(*ast.FuncType)) // don't print "func" + } else { + // embedded interface + p.expr(f.Type) + } + } + p.print(blank) + p.setPos(rbrace) + p.print(token.RBRACE) + return + } + } + // hasComments || !srcIsOneLine + + p.print(blank) + p.setPos(lbrace) + p.print(token.LBRACE, indent) + if hasComments || len(list) > 0 { + p.print(formfeed) + } + + if isStruct { + + sep := vtab + if len(list) == 1 { + sep = blank + } + var line int + for i, f := range list { + if i > 0 { + p.linebreak(p.lineFor(f.Pos()), 1, ignore, p.linesFrom(line) > 0) + } + extraTabs := 0 + p.setComment(f.Doc) + p.recordLine(&line) + if len(f.Names) > 0 { + // named fields + p.identList(f.Names, false) + p.print(sep) + p.expr(f.Type) + extraTabs = 1 + } else { + // anonymous field + p.expr(f.Type) + extraTabs = 2 + } + if f.Tag != nil { + if len(f.Names) > 0 && sep == vtab { + p.print(sep) + } + p.print(sep) + p.expr(f.Tag) + extraTabs = 0 + } + if f.Comment != nil { + for ; extraTabs > 0; extraTabs-- { + p.print(sep) + } + p.setComment(f.Comment) + } + } + if isIncomplete { + if len(list) > 0 { + p.print(formfeed) + } + p.flush(p.posFor(rbrace), token.RBRACE) // make sure we don't lose the last line comment + p.setLineComment("// " + filteredMsg) + } + + } else { // interface + + var line int + var prev *ast.Ident // previous "type" identifier + for i, f := range list { + var name *ast.Ident // first name, or nil + if len(f.Names) > 0 { + name = f.Names[0] + } + if i > 0 { + // don't do a line break (min == 0) if we are printing a list of types + // TODO(gri) this doesn't work quite right if the list of types is + // spread across multiple lines + min := 1 + if prev != nil && name == prev { + min = 0 + } + p.linebreak(p.lineFor(f.Pos()), min, ignore, p.linesFrom(line) > 0) + } + p.setComment(f.Doc) + p.recordLine(&line) + if name != nil { + // method + p.expr(name) + p.signature(f.Type.(*ast.FuncType)) // don't print "func" + prev = nil + } else { + // embedded interface + p.expr(f.Type) + prev = nil + } + p.setComment(f.Comment) + } + if isIncomplete { + if len(list) > 0 { + p.print(formfeed) + } + p.flush(p.posFor(rbrace), token.RBRACE) // make sure we don't lose the last line comment + p.setLineComment("// contains filtered or unexported methods") + } + + } + p.print(unindent, formfeed) + p.setPos(rbrace) + p.print(token.RBRACE) +} + +// ---------------------------------------------------------------------------- +// Expressions + +func walkBinary(e *ast.BinaryExpr) (has4, has5 bool, maxProblem int) { + switch e.Op.Precedence() { + case 4: + has4 = true + case 5: + has5 = true + } + + switch l := e.X.(type) { + case *ast.BinaryExpr: + if l.Op.Precedence() < e.Op.Precedence() { + // parens will be inserted. + // pretend this is an *ast.ParenExpr and do nothing. + break + } + h4, h5, mp := walkBinary(l) + has4 = has4 || h4 + has5 = has5 || h5 + if maxProblem < mp { + maxProblem = mp + } + } + + switch r := e.Y.(type) { + case *ast.BinaryExpr: + if r.Op.Precedence() <= e.Op.Precedence() { + // parens will be inserted. + // pretend this is an *ast.ParenExpr and do nothing. + break + } + h4, h5, mp := walkBinary(r) + has4 = has4 || h4 + has5 = has5 || h5 + if maxProblem < mp { + maxProblem = mp + } + + case *ast.StarExpr: + if e.Op == token.QUO { // `*/` + maxProblem = 5 + } + + case *ast.UnaryExpr: + switch e.Op.String() + r.Op.String() { + case "/*", "&&", "&^": + maxProblem = 5 + case "++", "--": + if maxProblem < 4 { + maxProblem = 4 + } + } + } + return +} + +func cutoff(e *ast.BinaryExpr, depth int) int { + has4, has5, maxProblem := walkBinary(e) + if maxProblem > 0 { + return maxProblem + 1 + } + if has4 && has5 { + if depth == 1 { + return 5 + } + return 4 + } + if depth == 1 { + return 6 + } + return 4 +} + +func diffPrec(expr ast.Expr, prec int) int { + x, ok := expr.(*ast.BinaryExpr) + if !ok || prec != x.Op.Precedence() { + return 1 + } + return 0 +} + +func reduceDepth(depth int) int { + depth-- + if depth < 1 { + depth = 1 + } + return depth +} + +// Format the binary expression: decide the cutoff and then format. +// Let's call depth == 1 Normal mode, and depth > 1 Compact mode. +// (Algorithm suggestion by Russ Cox.) +// +// The precedences are: +// +// 5 * / % << >> & &^ +// 4 + - | ^ +// 3 == != < <= > >= +// 2 && +// 1 || +// +// The only decision is whether there will be spaces around levels 4 and 5. +// There are never spaces at level 6 (unary), and always spaces at levels 3 and below. +// +// To choose the cutoff, look at the whole expression but excluding primary +// expressions (function calls, parenthesized exprs), and apply these rules: +// +// 1. If there is a binary operator with a right side unary operand +// that would clash without a space, the cutoff must be (in order): +// +// /* 6 +// && 6 +// &^ 6 +// ++ 5 +// -- 5 +// +// (Comparison operators always have spaces around them.) +// +// 2. If there is a mix of level 5 and level 4 operators, then the cutoff +// is 5 (use spaces to distinguish precedence) in Normal mode +// and 4 (never use spaces) in Compact mode. +// +// 3. If there are no level 4 operators or no level 5 operators, then the +// cutoff is 6 (always use spaces) in Normal mode +// and 4 (never use spaces) in Compact mode. +func (p *printer) binaryExpr(x *ast.BinaryExpr, prec1, cutoff, depth int) { + prec := x.Op.Precedence() + if prec < prec1 { + // parenthesis needed + // Note: The parser inserts an ast.ParenExpr node; thus this case + // can only occur if the AST is created in a different way. + p.print(token.LPAREN) + p.expr0(x, reduceDepth(depth)) // parentheses undo one level of depth + p.print(token.RPAREN) + return + } + + printBlank := prec < cutoff + + ws := indent + p.expr1(x.X, prec, depth+diffPrec(x.X, prec)) + if printBlank { + p.print(blank) + } + xline := p.pos.Line // before the operator (it may be on the next line!) + yline := p.lineFor(x.Y.Pos()) + p.setPos(x.OpPos) + p.print(x.Op) + if xline != yline && xline > 0 && yline > 0 { + // at least one line break, but respect an extra empty line + // in the source + if p.linebreak(yline, 1, ws, true) > 0 { + ws = ignore + printBlank = false // no blank after line break + } + } + if printBlank { + p.print(blank) + } + p.expr1(x.Y, prec+1, depth+1) + if ws == ignore { + p.print(unindent) + } +} + +func isBinary(expr ast.Expr) bool { + _, ok := expr.(*ast.BinaryExpr) + return ok +} + +func (p *printer) expr1(expr ast.Expr, prec1, depth int) { + p.setPos(expr.Pos()) + + switch x := expr.(type) { + case *ast.BadExpr: + p.print("BadExpr") + + case *ast.Ident: + p.print(x) + + case *ast.BinaryExpr: + if depth < 1 { + p.internalError("depth < 1:", depth) + depth = 1 + } + p.binaryExpr(x, prec1, cutoff(x, depth), depth) + + case *ast.KeyValueExpr: + p.expr(x.Key) + p.setPos(x.Colon) + p.print(token.COLON, blank) + p.expr(x.Value) + + case *ast.StarExpr: + const prec = token.UnaryPrec + if prec < prec1 { + // parenthesis needed + p.print(token.LPAREN) + p.print(token.MUL) + p.expr(x.X) + p.print(token.RPAREN) + } else { + // no parenthesis needed + p.print(token.MUL) + p.expr(x.X) + } + + case *ast.UnaryExpr: + const prec = token.UnaryPrec + if prec < prec1 { + // parenthesis needed + p.print(token.LPAREN) + p.expr(x) + p.print(token.RPAREN) + } else { + // no parenthesis needed + p.print(x.Op) + if x.Op == token.RANGE { + // TODO(gri) Remove this code if it cannot be reached. + p.print(blank) + } + p.expr1(x.X, prec, depth) + } + + case *ast.BasicLit: + if p.Config.Mode&normalizeNumbers != 0 { + x = normalizedNumber(x) + } + p.print(x) + + case *ast.FuncLit: + p.setPos(x.Type.Pos()) + p.print(token.FUNC) + // See the comment in funcDecl about how the header size is computed. + startCol := p.out.Column - len("func") + p.signature(x.Type) + p.funcBody(p.distanceFrom(x.Type.Pos(), startCol), blank, x.Body) + + case *ast.ParenExpr: + if _, hasParens := x.X.(*ast.ParenExpr); hasParens { + // don't print parentheses around an already parenthesized expression + // TODO(gri) consider making this more general and incorporate precedence levels + p.expr0(x.X, depth) + } else { + p.print(token.LPAREN) + p.expr0(x.X, reduceDepth(depth)) // parentheses undo one level of depth + p.setPos(x.Rparen) + p.print(token.RPAREN) + } + + case *ast.SelectorExpr: + p.selectorExpr(x, depth, false) + + case *ast.TypeAssertExpr: + p.expr1(x.X, token.HighestPrec, depth) + p.print(token.PERIOD) + p.setPos(x.Lparen) + p.print(token.LPAREN) + if x.Type != nil { + p.expr(x.Type) + } else { + p.print(token.TYPE) + } + p.setPos(x.Rparen) + p.print(token.RPAREN) + + case *ast.IndexExpr: + // TODO(gri): should treat[] like parentheses and undo one level of depth + p.expr1(x.X, token.HighestPrec, 1) + p.setPos(x.Lbrack) + p.print(token.LBRACK) + p.expr0(x.Index, depth+1) + p.setPos(x.Rbrack) + p.print(token.RBRACK) + + case *ast.IndexListExpr: + // TODO(gri): as for IndexExpr, should treat [] like parentheses and undo + // one level of depth + p.expr1(x.X, token.HighestPrec, 1) + p.setPos(x.Lbrack) + p.print(token.LBRACK) + p.exprList(x.Lbrack, x.Indices, depth+1, commaTerm, x.Rbrack, false) + p.setPos(x.Rbrack) + p.print(token.RBRACK) + + case *ast.SliceExpr: + // TODO(gri): should treat[] like parentheses and undo one level of depth + p.expr1(x.X, token.HighestPrec, 1) + p.setPos(x.Lbrack) + p.print(token.LBRACK) + indices := []ast.Expr{x.Low, x.High} + if x.Max != nil { + indices = append(indices, x.Max) + } + // determine if we need extra blanks around ':' + var needsBlanks bool + if depth <= 1 { + var indexCount int + var hasBinaries bool + for _, x := range indices { + if x != nil { + indexCount++ + if isBinary(x) { + hasBinaries = true + } + } + } + if indexCount > 1 && hasBinaries { + needsBlanks = true + } + } + for i, x := range indices { + if i > 0 { + if indices[i-1] != nil && needsBlanks { + p.print(blank) + } + p.print(token.COLON) + if x != nil && needsBlanks { + p.print(blank) + } + } + if x != nil { + p.expr0(x, depth+1) + } + } + p.setPos(x.Rbrack) + p.print(token.RBRACK) + + case *ast.CallExpr: + if len(x.Args) > 1 { + depth++ + } + var wasIndented bool + if _, ok := x.Fun.(*ast.FuncType); ok { + // conversions to literal function types require parentheses around the type + p.print(token.LPAREN) + wasIndented = p.possibleSelectorExpr(x.Fun, token.HighestPrec, depth) + p.print(token.RPAREN) + } else { + wasIndented = p.possibleSelectorExpr(x.Fun, token.HighestPrec, depth) + } + p.setPos(x.Lparen) + p.print(token.LPAREN) + if x.Ellipsis.IsValid() { + p.exprList(x.Lparen, x.Args, depth, 0, x.Ellipsis, false) + p.setPos(x.Ellipsis) + p.print(token.ELLIPSIS) + if x.Rparen.IsValid() && p.lineFor(x.Ellipsis) < p.lineFor(x.Rparen) { + p.print(token.COMMA, formfeed) + } + } else { + p.exprList(x.Lparen, x.Args, depth, commaTerm, x.Rparen, false) + } + p.setPos(x.Rparen) + p.print(token.RPAREN) + if wasIndented { + p.print(unindent) + } + + case *ast.CompositeLit: + // composite literal elements that are composite literals themselves may have the type omitted + if x.Type != nil { + p.expr1(x.Type, token.HighestPrec, depth) + } + p.level++ + p.setPos(x.Lbrace) + p.print(token.LBRACE) + p.exprList(x.Lbrace, x.Elts, 1, commaTerm, x.Rbrace, x.Incomplete) + // do not insert extra line break following a /*-style comment + // before the closing '}' as it might break the code if there + // is no trailing ',' + mode := noExtraLinebreak + // do not insert extra blank following a /*-style comment + // before the closing '}' unless the literal is empty + if len(x.Elts) > 0 { + mode |= noExtraBlank + } + // need the initial indent to print lone comments with + // the proper level of indentation + p.print(indent, unindent, mode) + p.setPos(x.Rbrace) + p.print(token.RBRACE, mode) + p.level-- + + case *ast.Ellipsis: + p.print(token.ELLIPSIS) + if x.Elt != nil { + p.expr(x.Elt) + } + + case *ast.ArrayType: + p.print(token.LBRACK) + if x.Len != nil { + p.expr(x.Len) + } + p.print(token.RBRACK) + p.expr(x.Elt) + + case *ast.StructType: + p.print(token.STRUCT) + p.fieldList(x.Fields, true, x.Incomplete) + + case *ast.FuncType: + p.print(token.FUNC) + p.signature(x) + + case *ast.InterfaceType: + p.print(token.INTERFACE) + p.fieldList(x.Methods, false, x.Incomplete) + + case *ast.MapType: + p.print(token.MAP, token.LBRACK) + p.expr(x.Key) + p.print(token.RBRACK) + p.expr(x.Value) + + case *ast.ChanType: + switch x.Dir { + case ast.SEND | ast.RECV: + p.print(token.CHAN) + case ast.RECV: + p.print(token.ARROW, token.CHAN) // x.Arrow and x.Pos() are the same + case ast.SEND: + p.print(token.CHAN) + p.setPos(x.Arrow) + p.print(token.ARROW) + } + p.print(blank) + p.expr(x.Value) + + default: + panic("unreachable") + } +} + +// normalizedNumber rewrites base prefixes and exponents +// of numbers to use lower-case letters (0X123 to 0x123 and 1.2E3 to 1.2e3), +// and removes leading 0's from integer imaginary literals (0765i to 765i). +// It leaves hexadecimal digits alone. +// +// normalizedNumber doesn't modify the ast.BasicLit value lit points to. +// If lit is not a number or a number in canonical format already, +// lit is returned as is. Otherwise a new ast.BasicLit is created. +func normalizedNumber(lit *ast.BasicLit) *ast.BasicLit { + if lit.Kind != token.INT && lit.Kind != token.FLOAT && lit.Kind != token.IMAG { + return lit // not a number - nothing to do + } + if len(lit.Value) < 2 { + return lit // only one digit (common case) - nothing to do + } + // len(lit.Value) >= 2 + + // We ignore lit.Kind because for lit.Kind == token.IMAG the literal may be an integer + // or floating-point value, decimal or not. Instead, just consider the literal pattern. + x := lit.Value + switch x[:2] { + default: + // 0-prefix octal, decimal int, or float (possibly with 'i' suffix) + if i := strings.LastIndexByte(x, 'E'); i >= 0 { + x = x[:i] + "e" + x[i+1:] + break + } + // remove leading 0's from integer (but not floating-point) imaginary literals + if x[len(x)-1] == 'i' && !strings.ContainsAny(x, ".e") { + x = strings.TrimLeft(x, "0_") + if x == "i" { + x = "0i" + } + } + case "0X": + x = "0x" + x[2:] + // possibly a hexadecimal float + if i := strings.LastIndexByte(x, 'P'); i >= 0 { + x = x[:i] + "p" + x[i+1:] + } + case "0x": + // possibly a hexadecimal float + i := strings.LastIndexByte(x, 'P') + if i == -1 { + return lit // nothing to do + } + x = x[:i] + "p" + x[i+1:] + case "0O": + x = "0o" + x[2:] + case "0o": + return lit // nothing to do + case "0B": + x = "0b" + x[2:] + case "0b": + return lit // nothing to do + } + + return &ast.BasicLit{ValuePos: lit.ValuePos, Kind: lit.Kind, Value: x} +} + +func (p *printer) possibleSelectorExpr(expr ast.Expr, prec1, depth int) bool { + if x, ok := expr.(*ast.SelectorExpr); ok { + return p.selectorExpr(x, depth, true) + } + p.expr1(expr, prec1, depth) + return false +} + +// selectorExpr handles an *ast.SelectorExpr node and reports whether x spans +// multiple lines. +func (p *printer) selectorExpr(x *ast.SelectorExpr, depth int, isMethod bool) bool { + p.expr1(x.X, token.HighestPrec, depth) + p.print(token.PERIOD) + if line := p.lineFor(x.Sel.Pos()); p.pos.IsValid() && p.pos.Line < line { + p.print(indent, newline) + p.setPos(x.Sel.Pos()) + p.print(x.Sel) + if !isMethod { + p.print(unindent) + } + return true + } + p.setPos(x.Sel.Pos()) + p.print(x.Sel) + return false +} + +func (p *printer) expr0(x ast.Expr, depth int) { + p.expr1(x, token.LowestPrec, depth) +} + +func (p *printer) expr(x ast.Expr) { + const depth = 1 + p.expr1(x, token.LowestPrec, depth) +} + +// ---------------------------------------------------------------------------- +// Statements + +// Print the statement list indented, but without a newline after the last statement. +// Extra line breaks between statements in the source are respected but at most one +// empty line is printed between statements. +func (p *printer) stmtList(list []ast.Stmt, nindent int, nextIsRBrace bool) { + if nindent > 0 { + p.print(indent) + } + var line int + i := 0 + for _, s := range list { + // ignore empty statements (was issue 3466) + if _, isEmpty := s.(*ast.EmptyStmt); !isEmpty { + // nindent == 0 only for lists of switch/select case clauses; + // in those cases each clause is a new section + if len(p.output) > 0 { + // only print line break if we are not at the beginning of the output + // (i.e., we are not printing only a partial program) + p.linebreak(p.lineFor(s.Pos()), 1, ignore, i == 0 || nindent == 0 || p.linesFrom(line) > 0) + } + p.recordLine(&line) + p.stmt(s, nextIsRBrace && i == len(list)-1) + // labeled statements put labels on a separate line, but here + // we only care about the start line of the actual statement + // without label - correct line for each label + for t := s; ; { + lt, _ := t.(*ast.LabeledStmt) + if lt == nil { + break + } + line++ + t = lt.Stmt + } + i++ + } + } + if nindent > 0 { + p.print(unindent) + } +} + +// block prints an *ast.BlockStmt; it always spans at least two lines. +func (p *printer) block(b *ast.BlockStmt, nindent int) { + p.setPos(b.Lbrace) + p.print(token.LBRACE) + p.stmtList(b.List, nindent, true) + p.linebreak(p.lineFor(b.Rbrace), 1, ignore, true) + p.setPos(b.Rbrace) + p.print(token.RBRACE) +} + +func isTypeName(x ast.Expr) bool { + switch t := x.(type) { + case *ast.Ident: + return true + case *ast.SelectorExpr: + return isTypeName(t.X) + } + return false +} + +func stripParens(x ast.Expr) ast.Expr { + if px, strip := x.(*ast.ParenExpr); strip { + // parentheses must not be stripped if there are any + // unparenthesized composite literals starting with + // a type name + ast.Inspect(px.X, func(node ast.Node) bool { + switch x := node.(type) { + case *ast.ParenExpr: + // parentheses protect enclosed composite literals + return false + case *ast.CompositeLit: + if isTypeName(x.Type) { + strip = false // do not strip parentheses + } + return false + } + // in all other cases, keep inspecting + return true + }) + if strip { + return stripParens(px.X) + } + } + return x +} + +func stripParensAlways(x ast.Expr) ast.Expr { + if x, ok := x.(*ast.ParenExpr); ok { + return stripParensAlways(x.X) + } + return x +} + +func (p *printer) controlClause(isForStmt bool, init ast.Stmt, expr ast.Expr, post ast.Stmt) { + p.print(blank) + needsBlank := false + if init == nil && post == nil { + // no semicolons required + if expr != nil { + p.expr(stripParens(expr)) + needsBlank = true + } + } else { + // all semicolons required + // (they are not separators, print them explicitly) + if init != nil { + p.stmt(init, false) + } + p.print(token.SEMICOLON, blank) + if expr != nil { + p.expr(stripParens(expr)) + needsBlank = true + } + if isForStmt { + p.print(token.SEMICOLON, blank) + needsBlank = false + if post != nil { + p.stmt(post, false) + needsBlank = true + } + } + } + if needsBlank { + p.print(blank) + } +} + +// indentList reports whether an expression list would look better if it +// were indented wholesale (starting with the very first element, rather +// than starting at the first line break). +func (p *printer) indentList(list []ast.Expr) bool { + // Heuristic: indentList reports whether there are more than one multi- + // line element in the list, or if there is any element that is not + // starting on the same line as the previous one ends. + if len(list) >= 2 { + var b = p.lineFor(list[0].Pos()) + var e = p.lineFor(list[len(list)-1].End()) + if 0 < b && b < e { + // list spans multiple lines + n := 0 // multi-line element count + line := b + for _, x := range list { + xb := p.lineFor(x.Pos()) + xe := p.lineFor(x.End()) + if line < xb { + // x is not starting on the same + // line as the previous one ended + return true + } + if xb < xe { + // x is a multi-line element + n++ + } + line = xe + } + return n > 1 + } + } + return false +} + +func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool) { + p.setPos(stmt.Pos()) + + switch s := stmt.(type) { + case *ast.BadStmt: + p.print("BadStmt") + + case *ast.DeclStmt: + p.decl(s.Decl) + + case *ast.EmptyStmt: + // nothing to do + + case *ast.LabeledStmt: + // a "correcting" unindent immediately following a line break + // is applied before the line break if there is no comment + // between (see writeWhitespace) + p.print(unindent) + p.expr(s.Label) + p.setPos(s.Colon) + p.print(token.COLON, indent) + if e, isEmpty := s.Stmt.(*ast.EmptyStmt); isEmpty { + if !nextIsRBrace { + p.print(newline) + p.setPos(e.Pos()) + p.print(token.SEMICOLON) + break + } + } else { + p.linebreak(p.lineFor(s.Stmt.Pos()), 1, ignore, true) + } + p.stmt(s.Stmt, nextIsRBrace) + + case *ast.ExprStmt: + const depth = 1 + p.expr0(s.X, depth) + + case *ast.SendStmt: + const depth = 1 + p.expr0(s.Chan, depth) + p.print(blank) + p.setPos(s.Arrow) + p.print(token.ARROW, blank) + p.expr0(s.Value, depth) + + case *ast.IncDecStmt: + const depth = 1 + p.expr0(s.X, depth+1) + p.setPos(s.TokPos) + p.print(s.Tok) + + case *ast.AssignStmt: + var depth = 1 + if len(s.Lhs) > 1 && len(s.Rhs) > 1 { + depth++ + } + p.exprList(s.Pos(), s.Lhs, depth, 0, s.TokPos, false) + p.print(blank) + p.setPos(s.TokPos) + p.print(s.Tok, blank) + p.exprList(s.TokPos, s.Rhs, depth, 0, token.NoPos, false) + + case *ast.GoStmt: + p.print(token.GO, blank) + p.expr(s.Call) + + case *ast.DeferStmt: + p.print(token.DEFER, blank) + p.expr(s.Call) + + case *ast.ReturnStmt: + p.print(token.RETURN) + if s.Results != nil { + p.print(blank) + // Use indentList heuristic to make corner cases look + // better (issue 1207). A more systematic approach would + // always indent, but this would cause significant + // reformatting of the code base and not necessarily + // lead to more nicely formatted code in general. + if p.indentList(s.Results) { + p.print(indent) + // Use NoPos so that a newline never goes before + // the results (see issue #32854). + p.exprList(token.NoPos, s.Results, 1, noIndent, token.NoPos, false) + p.print(unindent) + } else { + p.exprList(token.NoPos, s.Results, 1, 0, token.NoPos, false) + } + } + + case *ast.BranchStmt: + p.print(s.Tok) + if s.Label != nil { + p.print(blank) + p.expr(s.Label) + } + + case *ast.BlockStmt: + p.block(s, 1) + + case *ast.IfStmt: + p.print(token.IF) + p.controlClause(false, s.Init, s.Cond, nil) + p.block(s.Body, 1) + if s.Else != nil { + p.print(blank, token.ELSE, blank) + switch s.Else.(type) { + case *ast.BlockStmt, *ast.IfStmt: + p.stmt(s.Else, nextIsRBrace) + default: + // This can only happen with an incorrectly + // constructed AST. Permit it but print so + // that it can be parsed without errors. + p.print(token.LBRACE, indent, formfeed) + p.stmt(s.Else, true) + p.print(unindent, formfeed, token.RBRACE) + } + } + + case *ast.CaseClause: + if s.List != nil { + p.print(token.CASE, blank) + p.exprList(s.Pos(), s.List, 1, 0, s.Colon, false) + } else { + p.print(token.DEFAULT) + } + p.setPos(s.Colon) + p.print(token.COLON) + p.stmtList(s.Body, 1, nextIsRBrace) + + case *ast.SwitchStmt: + p.print(token.SWITCH) + p.controlClause(false, s.Init, s.Tag, nil) + p.block(s.Body, 0) + + case *ast.TypeSwitchStmt: + p.print(token.SWITCH) + if s.Init != nil { + p.print(blank) + p.stmt(s.Init, false) + p.print(token.SEMICOLON) + } + p.print(blank) + p.stmt(s.Assign, false) + p.print(blank) + p.block(s.Body, 0) + + case *ast.CommClause: + if s.Comm != nil { + p.print(token.CASE, blank) + p.stmt(s.Comm, false) + } else { + p.print(token.DEFAULT) + } + p.setPos(s.Colon) + p.print(token.COLON) + p.stmtList(s.Body, 1, nextIsRBrace) + + case *ast.SelectStmt: + p.print(token.SELECT, blank) + body := s.Body + if len(body.List) == 0 && !p.commentBefore(p.posFor(body.Rbrace)) { + // print empty select statement w/o comments on one line + p.setPos(body.Lbrace) + p.print(token.LBRACE) + p.setPos(body.Rbrace) + p.print(token.RBRACE) + } else { + p.block(body, 0) + } + + case *ast.ForStmt: + p.print(token.FOR) + p.controlClause(true, s.Init, s.Cond, s.Post) + p.block(s.Body, 1) + + case *ast.RangeStmt: + p.print(token.FOR, blank) + if s.Key != nil { + p.expr(s.Key) + if s.Value != nil { + // use position of value following the comma as + // comma position for correct comment placement + p.setPos(s.Value.Pos()) + p.print(token.COMMA, blank) + p.expr(s.Value) + } + p.print(blank) + p.setPos(s.TokPos) + p.print(s.Tok, blank) + } + p.print(token.RANGE, blank) + p.expr(stripParens(s.X)) + p.print(blank) + p.block(s.Body, 1) + + default: + panic("unreachable") + } +} + +// ---------------------------------------------------------------------------- +// Declarations + +// The keepTypeColumn function determines if the type column of a series of +// consecutive const or var declarations must be kept, or if initialization +// values (V) can be placed in the type column (T) instead. The i'th entry +// in the result slice is true if the type column in spec[i] must be kept. +// +// For example, the declaration: +// +// const ( +// foobar int = 42 // comment +// x = 7 // comment +// foo +// bar = 991 +// ) +// +// leads to the type/values matrix below. A run of value columns (V) can +// be moved into the type column if there is no type for any of the values +// in that column (we only move entire columns so that they align properly). +// +// matrix formatted result +// matrix +// T V -> T V -> true there is a T and so the type +// - V - V true column must be kept +// - - - - false +// - V V - false V is moved into T column +func keepTypeColumn(specs []ast.Spec) []bool { + m := make([]bool, len(specs)) + + populate := func(i, j int, keepType bool) { + if keepType { + for ; i < j; i++ { + m[i] = true + } + } + } + + i0 := -1 // if i0 >= 0 we are in a run and i0 is the start of the run + var keepType bool + for i, s := range specs { + t := s.(*ast.ValueSpec) + if t.Values != nil { + if i0 < 0 { + // start of a run of ValueSpecs with non-nil Values + i0 = i + keepType = false + } + } else { + if i0 >= 0 { + // end of a run + populate(i0, i, keepType) + i0 = -1 + } + } + if t.Type != nil { + keepType = true + } + } + if i0 >= 0 { + // end of a run + populate(i0, len(specs), keepType) + } + + return m +} + +func (p *printer) valueSpec(s *ast.ValueSpec, keepType bool) { + p.setComment(s.Doc) + p.identList(s.Names, false) // always present + extraTabs := 3 + if s.Type != nil || keepType { + p.print(vtab) + extraTabs-- + } + if s.Type != nil { + p.expr(s.Type) + } + if s.Values != nil { + p.print(vtab, token.ASSIGN, blank) + p.exprList(token.NoPos, s.Values, 1, 0, token.NoPos, false) + extraTabs-- + } + if s.Comment != nil { + for ; extraTabs > 0; extraTabs-- { + p.print(vtab) + } + p.setComment(s.Comment) + } +} + +func sanitizeImportPath(lit *ast.BasicLit) *ast.BasicLit { + // Note: An unmodified AST generated by go/parser will already + // contain a backward- or double-quoted path string that does + // not contain any invalid characters, and most of the work + // here is not needed. However, a modified or generated AST + // may possibly contain non-canonical paths. Do the work in + // all cases since it's not too hard and not speed-critical. + + // if we don't have a proper string, be conservative and return whatever we have + if lit.Kind != token.STRING { + return lit + } + s, err := strconv.Unquote(lit.Value) + if err != nil { + return lit + } + + // if the string is an invalid path, return whatever we have + // + // spec: "Implementation restriction: A compiler may restrict + // ImportPaths to non-empty strings using only characters belonging + // to Unicode's L, M, N, P, and S general categories (the Graphic + // characters without spaces) and may also exclude the characters + // !"#$%&'()*,:;<=>?[\]^`{|} and the Unicode replacement character + // U+FFFD." + if s == "" { + return lit + } + const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" + for _, r := range s { + if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { + return lit + } + } + + // otherwise, return the double-quoted path + s = strconv.Quote(s) + if s == lit.Value { + return lit // nothing wrong with lit + } + return &ast.BasicLit{ValuePos: lit.ValuePos, Kind: token.STRING, Value: s} +} + +// The parameter n is the number of specs in the group. If doIndent is set, +// multi-line identifier lists in the spec are indented when the first +// linebreak is encountered. +func (p *printer) spec(spec ast.Spec, n int, doIndent bool) { + switch s := spec.(type) { + case *ast.ImportSpec: + p.setComment(s.Doc) + if s.Name != nil { + p.expr(s.Name) + p.print(blank) + } + p.expr(sanitizeImportPath(s.Path)) + p.setComment(s.Comment) + p.setPos(s.EndPos) + + case *ast.ValueSpec: + if n != 1 { + p.internalError("expected n = 1; got", n) + } + p.setComment(s.Doc) + p.identList(s.Names, doIndent) // always present + if s.Type != nil { + p.print(blank) + p.expr(s.Type) + } + if s.Values != nil { + p.print(blank, token.ASSIGN, blank) + p.exprList(token.NoPos, s.Values, 1, 0, token.NoPos, false) + } + p.setComment(s.Comment) + + case *ast.TypeSpec: + p.setComment(s.Doc) + p.expr(s.Name) + if s.TypeParams != nil { + p.parameters(s.TypeParams, typeTParam) + } + if n == 1 { + p.print(blank) + } else { + p.print(vtab) + } + if s.Assign.IsValid() { + p.print(token.ASSIGN, blank) + } + p.expr(s.Type) + p.setComment(s.Comment) + + default: + panic("unreachable") + } +} + +func (p *printer) genDecl(d *ast.GenDecl) { + p.setComment(d.Doc) + p.setPos(d.Pos()) + p.print(d.Tok, blank) + + if d.Lparen.IsValid() || len(d.Specs) > 1 { + // group of parenthesized declarations + p.setPos(d.Lparen) + p.print(token.LPAREN) + if n := len(d.Specs); n > 0 { + p.print(indent, formfeed) + if n > 1 && (d.Tok == token.CONST || d.Tok == token.VAR) { + // two or more grouped const/var declarations: + // determine if the type column must be kept + keepType := keepTypeColumn(d.Specs) + var line int + for i, s := range d.Specs { + if i > 0 { + p.linebreak(p.lineFor(s.Pos()), 1, ignore, p.linesFrom(line) > 0) + } + p.recordLine(&line) + p.valueSpec(s.(*ast.ValueSpec), keepType[i]) + } + } else { + var line int + for i, s := range d.Specs { + if i > 0 { + p.linebreak(p.lineFor(s.Pos()), 1, ignore, p.linesFrom(line) > 0) + } + p.recordLine(&line) + p.spec(s, n, false) + } + } + p.print(unindent, formfeed) + } + p.setPos(d.Rparen) + p.print(token.RPAREN) + + } else if len(d.Specs) > 0 { + // single declaration + p.spec(d.Specs[0], 1, true) + } +} + +// sizeCounter is an io.Writer which counts the number of bytes written, +// as well as whether a newline character was seen. +type sizeCounter struct { + hasNewline bool + size int +} + +func (c *sizeCounter) Write(p []byte) (int, error) { + if !c.hasNewline { + for _, b := range p { + if b == '\n' || b == '\f' { + c.hasNewline = true + break + } + } + } + c.size += len(p) + return len(p), nil +} + +// nodeSize determines the size of n in chars after formatting. +// The result is <= maxSize if the node fits on one line with at +// most maxSize chars and the formatted output doesn't contain +// any control chars. Otherwise, the result is > maxSize. +func (p *printer) nodeSize(n ast.Node, maxSize int) (size int) { + // nodeSize invokes the printer, which may invoke nodeSize + // recursively. For deep composite literal nests, this can + // lead to an exponential algorithm. Remember previous + // results to prune the recursion (was issue 1628). + if size, found := p.nodeSizes[n]; found { + return size + } + + size = maxSize + 1 // assume n doesn't fit + p.nodeSizes[n] = size + + // nodeSize computation must be independent of particular + // style so that we always get the same decision; print + // in RawFormat + cfg := Config{Mode: RawFormat} + var counter sizeCounter + if err := cfg.fprint(&counter, p.fset, n, p.nodeSizes); err != nil { + return + } + if counter.size <= maxSize && !counter.hasNewline { + // n fits in a single line + size = counter.size + p.nodeSizes[n] = size + } + return +} + +// numLines returns the number of lines spanned by node n in the original source. +func (p *printer) numLines(n ast.Node) int { + if from := n.Pos(); from.IsValid() { + if to := n.End(); to.IsValid() { + return p.lineFor(to) - p.lineFor(from) + 1 + } + } + return infinity +} + +// bodySize is like nodeSize but it is specialized for *ast.BlockStmt's. +func (p *printer) bodySize(b *ast.BlockStmt, maxSize int) int { + pos1 := b.Pos() + pos2 := b.Rbrace + if pos1.IsValid() && pos2.IsValid() && p.lineFor(pos1) != p.lineFor(pos2) { + // opening and closing brace are on different lines - don't make it a one-liner + return maxSize + 1 + } + if len(b.List) > 5 { + // too many statements - don't make it a one-liner + return maxSize + 1 + } + // otherwise, estimate body size + bodySize := p.commentSizeBefore(p.posFor(pos2)) + for i, s := range b.List { + if bodySize > maxSize { + break // no need to continue + } + if i > 0 { + bodySize += 2 // space for a semicolon and blank + } + bodySize += p.nodeSize(s, maxSize) + } + return bodySize +} + +// funcBody prints a function body following a function header of given headerSize. +// If the header's and block's size are "small enough" and the block is "simple enough", +// the block is printed on the current line, without line breaks, spaced from the header +// by sep. Otherwise the block's opening "{" is printed on the current line, followed by +// lines for the block's statements and its closing "}". +func (p *printer) funcBody(headerSize int, sep whiteSpace, b *ast.BlockStmt) { + if b == nil { + return + } + + // save/restore composite literal nesting level + defer func(level int) { + p.level = level + }(p.level) + p.level = 0 + + const maxSize = 100 + if headerSize+p.bodySize(b, maxSize) <= maxSize { + p.print(sep) + p.setPos(b.Lbrace) + p.print(token.LBRACE) + if len(b.List) > 0 { + p.print(blank) + for i, s := range b.List { + if i > 0 { + p.print(token.SEMICOLON, blank) + } + p.stmt(s, i == len(b.List)-1) + } + p.print(blank) + } + p.print(noExtraLinebreak) + p.setPos(b.Rbrace) + p.print(token.RBRACE, noExtraLinebreak) + return + } + + if sep != ignore { + p.print(blank) // always use blank + } + p.block(b, 1) +} + +// distanceFrom returns the column difference between p.out (the current output +// position) and startOutCol. If the start position is on a different line from +// the current position (or either is unknown), the result is infinity. +func (p *printer) distanceFrom(startPos token.Pos, startOutCol int) int { + if startPos.IsValid() && p.pos.IsValid() && p.posFor(startPos).Line == p.pos.Line { + return p.out.Column - startOutCol + } + return infinity +} + +func (p *printer) funcDecl(d *ast.FuncDecl) { + p.setComment(d.Doc) + p.setPos(d.Pos()) + p.print(token.FUNC, blank) + // We have to save startCol only after emitting FUNC; otherwise it can be on a + // different line (all whitespace preceding the FUNC is emitted only when the + // FUNC is emitted). + startCol := p.out.Column - len("func ") + if d.Recv != nil { + p.parameters(d.Recv, funcParam) // method: print receiver + p.print(blank) + } + p.expr(d.Name) + p.signature(d.Type) + p.funcBody(p.distanceFrom(d.Pos(), startCol), vtab, d.Body) +} + +func (p *printer) decl(decl ast.Decl) { + switch d := decl.(type) { + case *ast.BadDecl: + p.setPos(d.Pos()) + p.print("BadDecl") + case *ast.GenDecl: + p.genDecl(d) + case *ast.FuncDecl: + p.funcDecl(d) + default: + panic("unreachable") + } +} + +// ---------------------------------------------------------------------------- +// Files + +func declToken(decl ast.Decl) (tok token.Token) { + tok = token.ILLEGAL + switch d := decl.(type) { + case *ast.GenDecl: + tok = d.Tok + case *ast.FuncDecl: + tok = token.FUNC + } + return +} + +func (p *printer) declList(list []ast.Decl) { + tok := token.ILLEGAL + for _, d := range list { + prev := tok + tok = declToken(d) + // If the declaration token changed (e.g., from CONST to TYPE) + // or the next declaration has documentation associated with it, + // print an empty line between top-level declarations. + // (because p.linebreak is called with the position of d, which + // is past any documentation, the minimum requirement is satisfied + // even w/o the extra getDoc(d) nil-check - leave it in case the + // linebreak logic improves - there's already a TODO). + if len(p.output) > 0 { + // only print line break if we are not at the beginning of the output + // (i.e., we are not printing only a partial program) + min := 1 + if prev != tok || getDoc(d) != nil { + min = 2 + } + // start a new section if the next declaration is a function + // that spans multiple lines (see also issue #19544) + p.linebreak(p.lineFor(d.Pos()), min, ignore, tok == token.FUNC && p.numLines(d) > 1) + } + p.decl(d) + } +} + +func (p *printer) file(src *ast.File) { + p.setComment(src.Doc) + p.setPos(src.Pos()) + p.print(token.PACKAGE, blank) + p.expr(src.Name) + p.declList(src.Decls) + p.print(newline) +} diff --git a/src/go/printer/performance_test.go b/src/go/printer/performance_test.go new file mode 100644 index 0000000..c58f6d4 --- /dev/null +++ b/src/go/printer/performance_test.go @@ -0,0 +1,91 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements a simple printer performance benchmark: +// go test -bench=BenchmarkPrint + +package printer + +import ( + "bytes" + "go/ast" + "go/parser" + "go/token" + "io" + "log" + "os" + "testing" +) + +var ( + fileNode *ast.File + fileSize int64 + + declNode ast.Decl + declSize int64 +) + +func testprint(out io.Writer, node ast.Node) { + if err := (&Config{TabIndent | UseSpaces | normalizeNumbers, 8, 0}).Fprint(out, fset, node); err != nil { + log.Fatalf("print error: %s", err) + } +} + +// cannot initialize in init because (printer) Fprint launches goroutines. +func initialize() { + const filename = "testdata/parser.go" + + src, err := os.ReadFile(filename) + if err != nil { + log.Fatalf("%s", err) + } + + file, err := parser.ParseFile(fset, filename, src, parser.ParseComments) + if err != nil { + log.Fatalf("%s", err) + } + + var buf bytes.Buffer + testprint(&buf, file) + if !bytes.Equal(buf.Bytes(), src) { + log.Fatalf("print error: %s not idempotent", filename) + } + + fileNode = file + fileSize = int64(len(src)) + + for _, decl := range file.Decls { + // The first global variable, which is pretty short: + // + // var unresolved = new(ast.Object) + if decl, ok := decl.(*ast.GenDecl); ok && decl.Tok == token.VAR { + declNode = decl + declSize = int64(fset.Position(decl.End()).Offset - fset.Position(decl.Pos()).Offset) + break + } + + } +} + +func BenchmarkPrintFile(b *testing.B) { + if fileNode == nil { + initialize() + } + b.ReportAllocs() + b.SetBytes(fileSize) + for i := 0; i < b.N; i++ { + testprint(io.Discard, fileNode) + } +} + +func BenchmarkPrintDecl(b *testing.B) { + if declNode == nil { + initialize() + } + b.ReportAllocs() + b.SetBytes(declSize) + for i := 0; i < b.N; i++ { + testprint(io.Discard, declNode) + } +} diff --git a/src/go/printer/printer.go b/src/go/printer/printer.go new file mode 100644 index 0000000..46131c6 --- /dev/null +++ b/src/go/printer/printer.go @@ -0,0 +1,1436 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package printer implements printing of AST nodes. +package printer + +import ( + "fmt" + "go/ast" + "go/build/constraint" + "go/token" + "io" + "os" + "strings" + "sync" + "text/tabwriter" + "unicode" +) + +const ( + maxNewlines = 2 // max. number of newlines between source text + debug = false // enable for debugging + infinity = 1 << 30 +) + +type whiteSpace byte + +const ( + ignore = whiteSpace(0) + blank = whiteSpace(' ') + vtab = whiteSpace('\v') + newline = whiteSpace('\n') + formfeed = whiteSpace('\f') + indent = whiteSpace('>') + unindent = whiteSpace('<') +) + +// A pmode value represents the current printer mode. +type pmode int + +const ( + noExtraBlank pmode = 1 << iota // disables extra blank after /*-style comment + noExtraLinebreak // disables extra line break after /*-style comment +) + +type commentInfo struct { + cindex int // current comment index + comment *ast.CommentGroup // = printer.comments[cindex]; or nil + commentOffset int // = printer.posFor(printer.comments[cindex].List[0].Pos()).Offset; or infinity + commentNewline bool // true if the comment group contains newlines +} + +type printer struct { + // Configuration (does not change after initialization) + Config + fset *token.FileSet + + // Current state + output []byte // raw printer result + indent int // current indentation + level int // level == 0: outside composite literal; level > 0: inside composite literal + mode pmode // current printer mode + endAlignment bool // if set, terminate alignment immediately + impliedSemi bool // if set, a linebreak implies a semicolon + lastTok token.Token // last token printed (token.ILLEGAL if it's whitespace) + prevOpen token.Token // previous non-brace "open" token (, [, or token.ILLEGAL + wsbuf []whiteSpace // delayed white space + goBuild []int // start index of all //go:build comments in output + plusBuild []int // start index of all // +build comments in output + + // Positions + // The out position differs from the pos position when the result + // formatting differs from the source formatting (in the amount of + // white space). If there's a difference and SourcePos is set in + // ConfigMode, //line directives are used in the output to restore + // original source positions for a reader. + pos token.Position // current position in AST (source) space + out token.Position // current position in output space + last token.Position // value of pos after calling writeString + linePtr *int // if set, record out.Line for the next token in *linePtr + sourcePosErr error // if non-nil, the first error emitting a //line directive + + // The list of all source comments, in order of appearance. + comments []*ast.CommentGroup // may be nil + useNodeComments bool // if not set, ignore lead and line comments of nodes + + // Information about p.comments[p.cindex]; set up by nextComment. + commentInfo + + // Cache of already computed node sizes. + nodeSizes map[ast.Node]int + + // Cache of most recently computed line position. + cachedPos token.Pos + cachedLine int // line corresponding to cachedPos +} + +func (p *printer) internalError(msg ...any) { + if debug { + fmt.Print(p.pos.String() + ": ") + fmt.Println(msg...) + panic("go/printer") + } +} + +// commentsHaveNewline reports whether a list of comments belonging to +// an *ast.CommentGroup contains newlines. Because the position information +// may only be partially correct, we also have to read the comment text. +func (p *printer) commentsHaveNewline(list []*ast.Comment) bool { + // len(list) > 0 + line := p.lineFor(list[0].Pos()) + for i, c := range list { + if i > 0 && p.lineFor(list[i].Pos()) != line { + // not all comments on the same line + return true + } + if t := c.Text; len(t) >= 2 && (t[1] == '/' || strings.Contains(t, "\n")) { + return true + } + } + _ = line + return false +} + +func (p *printer) nextComment() { + for p.cindex < len(p.comments) { + c := p.comments[p.cindex] + p.cindex++ + if list := c.List; len(list) > 0 { + p.comment = c + p.commentOffset = p.posFor(list[0].Pos()).Offset + p.commentNewline = p.commentsHaveNewline(list) + return + } + // we should not reach here (correct ASTs don't have empty + // ast.CommentGroup nodes), but be conservative and try again + } + // no more comments + p.commentOffset = infinity +} + +// commentBefore reports whether the current comment group occurs +// before the next position in the source code and printing it does +// not introduce implicit semicolons. +func (p *printer) commentBefore(next token.Position) bool { + return p.commentOffset < next.Offset && (!p.impliedSemi || !p.commentNewline) +} + +// commentSizeBefore returns the estimated size of the +// comments on the same line before the next position. +func (p *printer) commentSizeBefore(next token.Position) int { + // save/restore current p.commentInfo (p.nextComment() modifies it) + defer func(info commentInfo) { + p.commentInfo = info + }(p.commentInfo) + + size := 0 + for p.commentBefore(next) { + for _, c := range p.comment.List { + size += len(c.Text) + } + p.nextComment() + } + return size +} + +// recordLine records the output line number for the next non-whitespace +// token in *linePtr. It is used to compute an accurate line number for a +// formatted construct, independent of pending (not yet emitted) whitespace +// or comments. +func (p *printer) recordLine(linePtr *int) { + p.linePtr = linePtr +} + +// linesFrom returns the number of output lines between the current +// output line and the line argument, ignoring any pending (not yet +// emitted) whitespace or comments. It is used to compute an accurate +// size (in number of lines) for a formatted construct. +func (p *printer) linesFrom(line int) int { + return p.out.Line - line +} + +func (p *printer) posFor(pos token.Pos) token.Position { + // not used frequently enough to cache entire token.Position + return p.fset.PositionFor(pos, false /* absolute position */) +} + +func (p *printer) lineFor(pos token.Pos) int { + if pos != p.cachedPos { + p.cachedPos = pos + p.cachedLine = p.fset.PositionFor(pos, false /* absolute position */).Line + } + return p.cachedLine +} + +// writeLineDirective writes a //line directive if necessary. +func (p *printer) writeLineDirective(pos token.Position) { + if pos.IsValid() && (p.out.Line != pos.Line || p.out.Filename != pos.Filename) { + if strings.ContainsAny(pos.Filename, "\r\n") { + if p.sourcePosErr == nil { + p.sourcePosErr = fmt.Errorf("go/printer: source filename contains unexpected newline character: %q", pos.Filename) + } + return + } + + p.output = append(p.output, tabwriter.Escape) // protect '\n' in //line from tabwriter interpretation + p.output = append(p.output, fmt.Sprintf("//line %s:%d\n", pos.Filename, pos.Line)...) + p.output = append(p.output, tabwriter.Escape) + // p.out must match the //line directive + p.out.Filename = pos.Filename + p.out.Line = pos.Line + } +} + +// writeIndent writes indentation. +func (p *printer) writeIndent() { + // use "hard" htabs - indentation columns + // must not be discarded by the tabwriter + n := p.Config.Indent + p.indent // include base indentation + for i := 0; i < n; i++ { + p.output = append(p.output, '\t') + } + + // update positions + p.pos.Offset += n + p.pos.Column += n + p.out.Column += n +} + +// writeByte writes ch n times to p.output and updates p.pos. +// Only used to write formatting (white space) characters. +func (p *printer) writeByte(ch byte, n int) { + if p.endAlignment { + // Ignore any alignment control character; + // and at the end of the line, break with + // a formfeed to indicate termination of + // existing columns. + switch ch { + case '\t', '\v': + ch = ' ' + case '\n', '\f': + ch = '\f' + p.endAlignment = false + } + } + + if p.out.Column == 1 { + // no need to write line directives before white space + p.writeIndent() + } + + for i := 0; i < n; i++ { + p.output = append(p.output, ch) + } + + // update positions + p.pos.Offset += n + if ch == '\n' || ch == '\f' { + p.pos.Line += n + p.out.Line += n + p.pos.Column = 1 + p.out.Column = 1 + return + } + p.pos.Column += n + p.out.Column += n +} + +// writeString writes the string s to p.output and updates p.pos, p.out, +// and p.last. If isLit is set, s is escaped w/ tabwriter.Escape characters +// to protect s from being interpreted by the tabwriter. +// +// Note: writeString is only used to write Go tokens, literals, and +// comments, all of which must be written literally. Thus, it is correct +// to always set isLit = true. However, setting it explicitly only when +// needed (i.e., when we don't know that s contains no tabs or line breaks) +// avoids processing extra escape characters and reduces run time of the +// printer benchmark by up to 10%. +func (p *printer) writeString(pos token.Position, s string, isLit bool) { + if p.out.Column == 1 { + if p.Config.Mode&SourcePos != 0 { + p.writeLineDirective(pos) + } + p.writeIndent() + } + + if pos.IsValid() { + // update p.pos (if pos is invalid, continue with existing p.pos) + // Note: Must do this after handling line beginnings because + // writeIndent updates p.pos if there's indentation, but p.pos + // is the position of s. + p.pos = pos + } + + if isLit { + // Protect s such that is passes through the tabwriter + // unchanged. Note that valid Go programs cannot contain + // tabwriter.Escape bytes since they do not appear in legal + // UTF-8 sequences. + p.output = append(p.output, tabwriter.Escape) + } + + if debug { + p.output = append(p.output, fmt.Sprintf("/*%s*/", pos)...) // do not update p.pos! + } + p.output = append(p.output, s...) + + // update positions + nlines := 0 + var li int // index of last newline; valid if nlines > 0 + for i := 0; i < len(s); i++ { + // Raw string literals may contain any character except back quote (`). + if ch := s[i]; ch == '\n' || ch == '\f' { + // account for line break + nlines++ + li = i + // A line break inside a literal will break whatever column + // formatting is in place; ignore any further alignment through + // the end of the line. + p.endAlignment = true + } + } + p.pos.Offset += len(s) + if nlines > 0 { + p.pos.Line += nlines + p.out.Line += nlines + c := len(s) - li + p.pos.Column = c + p.out.Column = c + } else { + p.pos.Column += len(s) + p.out.Column += len(s) + } + + if isLit { + p.output = append(p.output, tabwriter.Escape) + } + + p.last = p.pos +} + +// writeCommentPrefix writes the whitespace before a comment. +// If there is any pending whitespace, it consumes as much of +// it as is likely to help position the comment nicely. +// pos is the comment position, next the position of the item +// after all pending comments, prev is the previous comment in +// a group of comments (or nil), and tok is the next token. +func (p *printer) writeCommentPrefix(pos, next token.Position, prev *ast.Comment, tok token.Token) { + if len(p.output) == 0 { + // the comment is the first item to be printed - don't write any whitespace + return + } + + if pos.IsValid() && pos.Filename != p.last.Filename { + // comment in a different file - separate with newlines + p.writeByte('\f', maxNewlines) + return + } + + if pos.Line == p.last.Line && (prev == nil || prev.Text[1] != '/') { + // comment on the same line as last item: + // separate with at least one separator + hasSep := false + if prev == nil { + // first comment of a comment group + j := 0 + for i, ch := range p.wsbuf { + switch ch { + case blank: + // ignore any blanks before a comment + p.wsbuf[i] = ignore + continue + case vtab: + // respect existing tabs - important + // for proper formatting of commented structs + hasSep = true + continue + case indent: + // apply pending indentation + continue + } + j = i + break + } + p.writeWhitespace(j) + } + // make sure there is at least one separator + if !hasSep { + sep := byte('\t') + if pos.Line == next.Line { + // next item is on the same line as the comment + // (which must be a /*-style comment): separate + // with a blank instead of a tab + sep = ' ' + } + p.writeByte(sep, 1) + } + + } else { + // comment on a different line: + // separate with at least one line break + droppedLinebreak := false + j := 0 + for i, ch := range p.wsbuf { + switch ch { + case blank, vtab: + // ignore any horizontal whitespace before line breaks + p.wsbuf[i] = ignore + continue + case indent: + // apply pending indentation + continue + case unindent: + // if this is not the last unindent, apply it + // as it is (likely) belonging to the last + // construct (e.g., a multi-line expression list) + // and is not part of closing a block + if i+1 < len(p.wsbuf) && p.wsbuf[i+1] == unindent { + continue + } + // if the next token is not a closing }, apply the unindent + // if it appears that the comment is aligned with the + // token; otherwise assume the unindent is part of a + // closing block and stop (this scenario appears with + // comments before a case label where the comments + // apply to the next case instead of the current one) + if tok != token.RBRACE && pos.Column == next.Column { + continue + } + case newline, formfeed: + p.wsbuf[i] = ignore + droppedLinebreak = prev == nil // record only if first comment of a group + } + j = i + break + } + p.writeWhitespace(j) + + // determine number of linebreaks before the comment + n := 0 + if pos.IsValid() && p.last.IsValid() { + n = pos.Line - p.last.Line + if n < 0 { // should never happen + n = 0 + } + } + + // at the package scope level only (p.indent == 0), + // add an extra newline if we dropped one before: + // this preserves a blank line before documentation + // comments at the package scope level (issue 2570) + if p.indent == 0 && droppedLinebreak { + n++ + } + + // make sure there is at least one line break + // if the previous comment was a line comment + if n == 0 && prev != nil && prev.Text[1] == '/' { + n = 1 + } + + if n > 0 { + // use formfeeds to break columns before a comment; + // this is analogous to using formfeeds to separate + // individual lines of /*-style comments + p.writeByte('\f', nlimit(n)) + } + } +} + +// Returns true if s contains only white space +// (only tabs and blanks can appear in the printer's context). +func isBlank(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] > ' ' { + return false + } + } + return true +} + +// commonPrefix returns the common prefix of a and b. +func commonPrefix(a, b string) string { + i := 0 + for i < len(a) && i < len(b) && a[i] == b[i] && (a[i] <= ' ' || a[i] == '*') { + i++ + } + return a[0:i] +} + +// trimRight returns s with trailing whitespace removed. +func trimRight(s string) string { + return strings.TrimRightFunc(s, unicode.IsSpace) +} + +// stripCommonPrefix removes a common prefix from /*-style comment lines (unless no +// comment line is indented, all but the first line have some form of space prefix). +// The prefix is computed using heuristics such that is likely that the comment +// contents are nicely laid out after re-printing each line using the printer's +// current indentation. +func stripCommonPrefix(lines []string) { + if len(lines) <= 1 { + return // at most one line - nothing to do + } + // len(lines) > 1 + + // The heuristic in this function tries to handle a few + // common patterns of /*-style comments: Comments where + // the opening /* and closing */ are aligned and the + // rest of the comment text is aligned and indented with + // blanks or tabs, cases with a vertical "line of stars" + // on the left, and cases where the closing */ is on the + // same line as the last comment text. + + // Compute maximum common white prefix of all but the first, + // last, and blank lines, and replace blank lines with empty + // lines (the first line starts with /* and has no prefix). + // In cases where only the first and last lines are not blank, + // such as two-line comments, or comments where all inner lines + // are blank, consider the last line for the prefix computation + // since otherwise the prefix would be empty. + // + // Note that the first and last line are never empty (they + // contain the opening /* and closing */ respectively) and + // thus they can be ignored by the blank line check. + prefix := "" + prefixSet := false + if len(lines) > 2 { + for i, line := range lines[1 : len(lines)-1] { + if isBlank(line) { + lines[1+i] = "" // range starts with lines[1] + } else { + if !prefixSet { + prefix = line + prefixSet = true + } + prefix = commonPrefix(prefix, line) + } + + } + } + // If we don't have a prefix yet, consider the last line. + if !prefixSet { + line := lines[len(lines)-1] + prefix = commonPrefix(line, line) + } + + /* + * Check for vertical "line of stars" and correct prefix accordingly. + */ + lineOfStars := false + if p, _, ok := strings.Cut(prefix, "*"); ok { + // remove trailing blank from prefix so stars remain aligned + prefix = strings.TrimSuffix(p, " ") + lineOfStars = true + } else { + // No line of stars present. + // Determine the white space on the first line after the /* + // and before the beginning of the comment text, assume two + // blanks instead of the /* unless the first character after + // the /* is a tab. If the first comment line is empty but + // for the opening /*, assume up to 3 blanks or a tab. This + // whitespace may be found as suffix in the common prefix. + first := lines[0] + if isBlank(first[2:]) { + // no comment text on the first line: + // reduce prefix by up to 3 blanks or a tab + // if present - this keeps comment text indented + // relative to the /* and */'s if it was indented + // in the first place + i := len(prefix) + for n := 0; n < 3 && i > 0 && prefix[i-1] == ' '; n++ { + i-- + } + if i == len(prefix) && i > 0 && prefix[i-1] == '\t' { + i-- + } + prefix = prefix[0:i] + } else { + // comment text on the first line + suffix := make([]byte, len(first)) + n := 2 // start after opening /* + for n < len(first) && first[n] <= ' ' { + suffix[n] = first[n] + n++ + } + if n > 2 && suffix[2] == '\t' { + // assume the '\t' compensates for the /* + suffix = suffix[2:n] + } else { + // otherwise assume two blanks + suffix[0], suffix[1] = ' ', ' ' + suffix = suffix[0:n] + } + // Shorten the computed common prefix by the length of + // suffix, if it is found as suffix of the prefix. + prefix = strings.TrimSuffix(prefix, string(suffix)) + } + } + + // Handle last line: If it only contains a closing */, align it + // with the opening /*, otherwise align the text with the other + // lines. + last := lines[len(lines)-1] + closing := "*/" + before, _, _ := strings.Cut(last, closing) // closing always present + if isBlank(before) { + // last line only contains closing */ + if lineOfStars { + closing = " */" // add blank to align final star + } + lines[len(lines)-1] = prefix + closing + } else { + // last line contains more comment text - assume + // it is aligned like the other lines and include + // in prefix computation + prefix = commonPrefix(prefix, last) + } + + // Remove the common prefix from all but the first and empty lines. + for i, line := range lines { + if i > 0 && line != "" { + lines[i] = line[len(prefix):] + } + } +} + +func (p *printer) writeComment(comment *ast.Comment) { + text := comment.Text + pos := p.posFor(comment.Pos()) + + const linePrefix = "//line " + if strings.HasPrefix(text, linePrefix) && (!pos.IsValid() || pos.Column == 1) { + // Possibly a //-style line directive. + // Suspend indentation temporarily to keep line directive valid. + defer func(indent int) { p.indent = indent }(p.indent) + p.indent = 0 + } + + // shortcut common case of //-style comments + if text[1] == '/' { + if constraint.IsGoBuild(text) { + p.goBuild = append(p.goBuild, len(p.output)) + } else if constraint.IsPlusBuild(text) { + p.plusBuild = append(p.plusBuild, len(p.output)) + } + p.writeString(pos, trimRight(text), true) + return + } + + // for /*-style comments, print line by line and let the + // write function take care of the proper indentation + lines := strings.Split(text, "\n") + + // The comment started in the first column but is going + // to be indented. For an idempotent result, add indentation + // to all lines such that they look like they were indented + // before - this will make sure the common prefix computation + // is the same independent of how many times formatting is + // applied (was issue 1835). + if pos.IsValid() && pos.Column == 1 && p.indent > 0 { + for i, line := range lines[1:] { + lines[1+i] = " " + line + } + } + + stripCommonPrefix(lines) + + // write comment lines, separated by formfeed, + // without a line break after the last line + for i, line := range lines { + if i > 0 { + p.writeByte('\f', 1) + pos = p.pos + } + if len(line) > 0 { + p.writeString(pos, trimRight(line), true) + } + } +} + +// writeCommentSuffix writes a line break after a comment if indicated +// and processes any leftover indentation information. If a line break +// is needed, the kind of break (newline vs formfeed) depends on the +// pending whitespace. The writeCommentSuffix result indicates if a +// newline was written or if a formfeed was dropped from the whitespace +// buffer. +func (p *printer) writeCommentSuffix(needsLinebreak bool) (wroteNewline, droppedFF bool) { + for i, ch := range p.wsbuf { + switch ch { + case blank, vtab: + // ignore trailing whitespace + p.wsbuf[i] = ignore + case indent, unindent: + // don't lose indentation information + case newline, formfeed: + // if we need a line break, keep exactly one + // but remember if we dropped any formfeeds + if needsLinebreak { + needsLinebreak = false + wroteNewline = true + } else { + if ch == formfeed { + droppedFF = true + } + p.wsbuf[i] = ignore + } + } + } + p.writeWhitespace(len(p.wsbuf)) + + // make sure we have a line break + if needsLinebreak { + p.writeByte('\n', 1) + wroteNewline = true + } + + return +} + +// containsLinebreak reports whether the whitespace buffer contains any line breaks. +func (p *printer) containsLinebreak() bool { + for _, ch := range p.wsbuf { + if ch == newline || ch == formfeed { + return true + } + } + return false +} + +// intersperseComments consumes all comments that appear before the next token +// tok and prints it together with the buffered whitespace (i.e., the whitespace +// that needs to be written before the next token). A heuristic is used to mix +// the comments and whitespace. The intersperseComments result indicates if a +// newline was written or if a formfeed was dropped from the whitespace buffer. +func (p *printer) intersperseComments(next token.Position, tok token.Token) (wroteNewline, droppedFF bool) { + var last *ast.Comment + for p.commentBefore(next) { + list := p.comment.List + changed := false + if p.lastTok != token.IMPORT && // do not rewrite cgo's import "C" comments + p.posFor(p.comment.Pos()).Column == 1 && + p.posFor(p.comment.End()+1) == next { + // Unindented comment abutting next token position: + // a top-level doc comment. + list = formatDocComment(list) + changed = true + + if len(p.comment.List) > 0 && len(list) == 0 { + // The doc comment was removed entirely. + // Keep preceding whitespace. + p.writeCommentPrefix(p.posFor(p.comment.Pos()), next, last, tok) + // Change print state to continue at next. + p.pos = next + p.last = next + // There can't be any more comments. + p.nextComment() + return p.writeCommentSuffix(false) + } + } + for _, c := range list { + p.writeCommentPrefix(p.posFor(c.Pos()), next, last, tok) + p.writeComment(c) + last = c + } + // In case list was rewritten, change print state to where + // the original list would have ended. + if len(p.comment.List) > 0 && changed { + last = p.comment.List[len(p.comment.List)-1] + p.pos = p.posFor(last.End()) + p.last = p.pos + } + p.nextComment() + } + + if last != nil { + // If the last comment is a /*-style comment and the next item + // follows on the same line but is not a comma, and not a "closing" + // token immediately following its corresponding "opening" token, + // add an extra separator unless explicitly disabled. Use a blank + // as separator unless we have pending linebreaks, they are not + // disabled, and we are outside a composite literal, in which case + // we want a linebreak (issue 15137). + // TODO(gri) This has become overly complicated. We should be able + // to track whether we're inside an expression or statement and + // use that information to decide more directly. + needsLinebreak := false + if p.mode&noExtraBlank == 0 && + last.Text[1] == '*' && p.lineFor(last.Pos()) == next.Line && + tok != token.COMMA && + (tok != token.RPAREN || p.prevOpen == token.LPAREN) && + (tok != token.RBRACK || p.prevOpen == token.LBRACK) { + if p.containsLinebreak() && p.mode&noExtraLinebreak == 0 && p.level == 0 { + needsLinebreak = true + } else { + p.writeByte(' ', 1) + } + } + // Ensure that there is a line break after a //-style comment, + // before EOF, and before a closing '}' unless explicitly disabled. + if last.Text[1] == '/' || + tok == token.EOF || + tok == token.RBRACE && p.mode&noExtraLinebreak == 0 { + needsLinebreak = true + } + return p.writeCommentSuffix(needsLinebreak) + } + + // no comment was written - we should never reach here since + // intersperseComments should not be called in that case + p.internalError("intersperseComments called without pending comments") + return +} + +// whiteWhitespace writes the first n whitespace entries. +func (p *printer) writeWhitespace(n int) { + // write entries + for i := 0; i < n; i++ { + switch ch := p.wsbuf[i]; ch { + case ignore: + // ignore! + case indent: + p.indent++ + case unindent: + p.indent-- + if p.indent < 0 { + p.internalError("negative indentation:", p.indent) + p.indent = 0 + } + case newline, formfeed: + // A line break immediately followed by a "correcting" + // unindent is swapped with the unindent - this permits + // proper label positioning. If a comment is between + // the line break and the label, the unindent is not + // part of the comment whitespace prefix and the comment + // will be positioned correctly indented. + if i+1 < n && p.wsbuf[i+1] == unindent { + // Use a formfeed to terminate the current section. + // Otherwise, a long label name on the next line leading + // to a wide column may increase the indentation column + // of lines before the label; effectively leading to wrong + // indentation. + p.wsbuf[i], p.wsbuf[i+1] = unindent, formfeed + i-- // do it again + continue + } + fallthrough + default: + p.writeByte(byte(ch), 1) + } + } + + // shift remaining entries down + l := copy(p.wsbuf, p.wsbuf[n:]) + p.wsbuf = p.wsbuf[:l] +} + +// ---------------------------------------------------------------------------- +// Printing interface + +// nlimit limits n to maxNewlines. +func nlimit(n int) int { + if n > maxNewlines { + n = maxNewlines + } + return n +} + +func mayCombine(prev token.Token, next byte) (b bool) { + switch prev { + case token.INT: + b = next == '.' // 1. + case token.ADD: + b = next == '+' // ++ + case token.SUB: + b = next == '-' // -- + case token.QUO: + b = next == '*' // /* + case token.LSS: + b = next == '-' || next == '<' // <- or << + case token.AND: + b = next == '&' || next == '^' // && or &^ + } + return +} + +func (p *printer) setPos(pos token.Pos) { + if pos.IsValid() { + p.pos = p.posFor(pos) // accurate position of next item + } +} + +// print prints a list of "items" (roughly corresponding to syntactic +// tokens, but also including whitespace and formatting information). +// It is the only print function that should be called directly from +// any of the AST printing functions in nodes.go. +// +// Whitespace is accumulated until a non-whitespace token appears. Any +// comments that need to appear before that token are printed first, +// taking into account the amount and structure of any pending white- +// space for best comment placement. Then, any leftover whitespace is +// printed, followed by the actual token. +func (p *printer) print(args ...any) { + for _, arg := range args { + // information about the current arg + var data string + var isLit bool + var impliedSemi bool // value for p.impliedSemi after this arg + + // record previous opening token, if any + switch p.lastTok { + case token.ILLEGAL: + // ignore (white space) + case token.LPAREN, token.LBRACK: + p.prevOpen = p.lastTok + default: + // other tokens followed any opening token + p.prevOpen = token.ILLEGAL + } + + switch x := arg.(type) { + case pmode: + // toggle printer mode + p.mode ^= x + continue + + case whiteSpace: + if x == ignore { + // don't add ignore's to the buffer; they + // may screw up "correcting" unindents (see + // LabeledStmt) + continue + } + i := len(p.wsbuf) + if i == cap(p.wsbuf) { + // Whitespace sequences are very short so this should + // never happen. Handle gracefully (but possibly with + // bad comment placement) if it does happen. + p.writeWhitespace(i) + i = 0 + } + p.wsbuf = p.wsbuf[0 : i+1] + p.wsbuf[i] = x + if x == newline || x == formfeed { + // newlines affect the current state (p.impliedSemi) + // and not the state after printing arg (impliedSemi) + // because comments can be interspersed before the arg + // in this case + p.impliedSemi = false + } + p.lastTok = token.ILLEGAL + continue + + case *ast.Ident: + data = x.Name + impliedSemi = true + p.lastTok = token.IDENT + + case *ast.BasicLit: + data = x.Value + isLit = true + impliedSemi = true + p.lastTok = x.Kind + + case token.Token: + s := x.String() + if mayCombine(p.lastTok, s[0]) { + // the previous and the current token must be + // separated by a blank otherwise they combine + // into a different incorrect token sequence + // (except for token.INT followed by a '.' this + // should never happen because it is taken care + // of via binary expression formatting) + if len(p.wsbuf) != 0 { + p.internalError("whitespace buffer not empty") + } + p.wsbuf = p.wsbuf[0:1] + p.wsbuf[0] = ' ' + } + data = s + // some keywords followed by a newline imply a semicolon + switch x { + case token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN, + token.INC, token.DEC, token.RPAREN, token.RBRACK, token.RBRACE: + impliedSemi = true + } + p.lastTok = x + + case string: + // incorrect AST - print error message + data = x + isLit = true + impliedSemi = true + p.lastTok = token.STRING + + default: + fmt.Fprintf(os.Stderr, "print: unsupported argument %v (%T)\n", arg, arg) + panic("go/printer type") + } + // data != "" + + next := p.pos // estimated/accurate position of next item + wroteNewline, droppedFF := p.flush(next, p.lastTok) + + // intersperse extra newlines if present in the source and + // if they don't cause extra semicolons (don't do this in + // flush as it will cause extra newlines at the end of a file) + if !p.impliedSemi { + n := nlimit(next.Line - p.pos.Line) + // don't exceed maxNewlines if we already wrote one + if wroteNewline && n == maxNewlines { + n = maxNewlines - 1 + } + if n > 0 { + ch := byte('\n') + if droppedFF { + ch = '\f' // use formfeed since we dropped one before + } + p.writeByte(ch, n) + impliedSemi = false + } + } + + // the next token starts now - record its line number if requested + if p.linePtr != nil { + *p.linePtr = p.out.Line + p.linePtr = nil + } + + p.writeString(next, data, isLit) + p.impliedSemi = impliedSemi + } +} + +// flush prints any pending comments and whitespace occurring textually +// before the position of the next token tok. The flush result indicates +// if a newline was written or if a formfeed was dropped from the whitespace +// buffer. +func (p *printer) flush(next token.Position, tok token.Token) (wroteNewline, droppedFF bool) { + if p.commentBefore(next) { + // if there are comments before the next item, intersperse them + wroteNewline, droppedFF = p.intersperseComments(next, tok) + } else { + // otherwise, write any leftover whitespace + p.writeWhitespace(len(p.wsbuf)) + } + return +} + +// getDoc returns the ast.CommentGroup associated with n, if any. +func getDoc(n ast.Node) *ast.CommentGroup { + switch n := n.(type) { + case *ast.Field: + return n.Doc + case *ast.ImportSpec: + return n.Doc + case *ast.ValueSpec: + return n.Doc + case *ast.TypeSpec: + return n.Doc + case *ast.GenDecl: + return n.Doc + case *ast.FuncDecl: + return n.Doc + case *ast.File: + return n.Doc + } + return nil +} + +func getLastComment(n ast.Node) *ast.CommentGroup { + switch n := n.(type) { + case *ast.Field: + return n.Comment + case *ast.ImportSpec: + return n.Comment + case *ast.ValueSpec: + return n.Comment + case *ast.TypeSpec: + return n.Comment + case *ast.GenDecl: + if len(n.Specs) > 0 { + return getLastComment(n.Specs[len(n.Specs)-1]) + } + case *ast.File: + if len(n.Comments) > 0 { + return n.Comments[len(n.Comments)-1] + } + } + return nil +} + +func (p *printer) printNode(node any) error { + // unpack *CommentedNode, if any + var comments []*ast.CommentGroup + if cnode, ok := node.(*CommentedNode); ok { + node = cnode.Node + comments = cnode.Comments + } + + if comments != nil { + // commented node - restrict comment list to relevant range + n, ok := node.(ast.Node) + if !ok { + goto unsupported + } + beg := n.Pos() + end := n.End() + // if the node has associated documentation, + // include that commentgroup in the range + // (the comment list is sorted in the order + // of the comment appearance in the source code) + if doc := getDoc(n); doc != nil { + beg = doc.Pos() + } + if com := getLastComment(n); com != nil { + if e := com.End(); e > end { + end = e + } + } + // token.Pos values are global offsets, we can + // compare them directly + i := 0 + for i < len(comments) && comments[i].End() < beg { + i++ + } + j := i + for j < len(comments) && comments[j].Pos() < end { + j++ + } + if i < j { + p.comments = comments[i:j] + } + } else if n, ok := node.(*ast.File); ok { + // use ast.File comments, if any + p.comments = n.Comments + } + + // if there are no comments, use node comments + p.useNodeComments = p.comments == nil + + // get comments ready for use + p.nextComment() + + p.print(pmode(0)) + + // format node + switch n := node.(type) { + case ast.Expr: + p.expr(n) + case ast.Stmt: + // A labeled statement will un-indent to position the label. + // Set p.indent to 1 so we don't get indent "underflow". + if _, ok := n.(*ast.LabeledStmt); ok { + p.indent = 1 + } + p.stmt(n, false) + case ast.Decl: + p.decl(n) + case ast.Spec: + p.spec(n, 1, false) + case []ast.Stmt: + // A labeled statement will un-indent to position the label. + // Set p.indent to 1 so we don't get indent "underflow". + for _, s := range n { + if _, ok := s.(*ast.LabeledStmt); ok { + p.indent = 1 + } + } + p.stmtList(n, 0, false) + case []ast.Decl: + p.declList(n) + case *ast.File: + p.file(n) + default: + goto unsupported + } + + return p.sourcePosErr + +unsupported: + return fmt.Errorf("go/printer: unsupported node type %T", node) +} + +// ---------------------------------------------------------------------------- +// Trimmer + +// A trimmer is an io.Writer filter for stripping tabwriter.Escape +// characters, trailing blanks and tabs, and for converting formfeed +// and vtab characters into newlines and htabs (in case no tabwriter +// is used). Text bracketed by tabwriter.Escape characters is passed +// through unchanged. +type trimmer struct { + output io.Writer + state int + space []byte +} + +// trimmer is implemented as a state machine. +// It can be in one of the following states: +const ( + inSpace = iota // inside space + inEscape // inside text bracketed by tabwriter.Escapes + inText // inside text +) + +func (p *trimmer) resetSpace() { + p.state = inSpace + p.space = p.space[0:0] +} + +// Design note: It is tempting to eliminate extra blanks occurring in +// whitespace in this function as it could simplify some +// of the blanks logic in the node printing functions. +// However, this would mess up any formatting done by +// the tabwriter. + +var aNewline = []byte("\n") + +func (p *trimmer) Write(data []byte) (n int, err error) { + // invariants: + // p.state == inSpace: + // p.space is unwritten + // p.state == inEscape, inText: + // data[m:n] is unwritten + m := 0 + var b byte + for n, b = range data { + if b == '\v' { + b = '\t' // convert to htab + } + switch p.state { + case inSpace: + switch b { + case '\t', ' ': + p.space = append(p.space, b) + case '\n', '\f': + p.resetSpace() // discard trailing space + _, err = p.output.Write(aNewline) + case tabwriter.Escape: + _, err = p.output.Write(p.space) + p.state = inEscape + m = n + 1 // +1: skip tabwriter.Escape + default: + _, err = p.output.Write(p.space) + p.state = inText + m = n + } + case inEscape: + if b == tabwriter.Escape { + _, err = p.output.Write(data[m:n]) + p.resetSpace() + } + case inText: + switch b { + case '\t', ' ': + _, err = p.output.Write(data[m:n]) + p.resetSpace() + p.space = append(p.space, b) + case '\n', '\f': + _, err = p.output.Write(data[m:n]) + p.resetSpace() + if err == nil { + _, err = p.output.Write(aNewline) + } + case tabwriter.Escape: + _, err = p.output.Write(data[m:n]) + p.state = inEscape + m = n + 1 // +1: skip tabwriter.Escape + } + default: + panic("unreachable") + } + if err != nil { + return + } + } + n = len(data) + + switch p.state { + case inEscape, inText: + _, err = p.output.Write(data[m:n]) + p.resetSpace() + } + + return +} + +// ---------------------------------------------------------------------------- +// Public interface + +// A Mode value is a set of flags (or 0). They control printing. +type Mode uint + +const ( + RawFormat Mode = 1 << iota // do not use a tabwriter; if set, UseSpaces is ignored + TabIndent // use tabs for indentation independent of UseSpaces + UseSpaces // use spaces instead of tabs for alignment + SourcePos // emit //line directives to preserve original source positions +) + +// The mode below is not included in printer's public API because +// editing code text is deemed out of scope. Because this mode is +// unexported, it's also possible to modify or remove it based on +// the evolving needs of go/format and cmd/gofmt without breaking +// users. See discussion in CL 240683. +const ( + // normalizeNumbers means to canonicalize number + // literal prefixes and exponents while printing. + // + // This value is known in and used by go/format and cmd/gofmt. + // It is currently more convenient and performant for those + // packages to apply number normalization during printing, + // rather than by modifying the AST in advance. + normalizeNumbers Mode = 1 << 30 +) + +// A Config node controls the output of Fprint. +type Config struct { + Mode Mode // default: 0 + Tabwidth int // default: 8 + Indent int // default: 0 (all code is indented at least by this much) +} + +var printerPool = sync.Pool{ + New: func() any { + return &printer{ + // Whitespace sequences are short. + wsbuf: make([]whiteSpace, 0, 16), + // We start the printer with a 16K output buffer, which is currently + // larger than about 80% of Go files in the standard library. + output: make([]byte, 0, 16<<10), + } + }, +} + +func newPrinter(cfg *Config, fset *token.FileSet, nodeSizes map[ast.Node]int) *printer { + p := printerPool.Get().(*printer) + *p = printer{ + Config: *cfg, + fset: fset, + pos: token.Position{Line: 1, Column: 1}, + out: token.Position{Line: 1, Column: 1}, + wsbuf: p.wsbuf[:0], + nodeSizes: nodeSizes, + cachedPos: -1, + output: p.output[:0], + } + return p +} + +func (p *printer) free() { + // Hard limit on buffer size; see https://golang.org/issue/23199. + if cap(p.output) > 64<<10 { + return + } + + printerPool.Put(p) +} + +// fprint implements Fprint and takes a nodesSizes map for setting up the printer state. +func (cfg *Config) fprint(output io.Writer, fset *token.FileSet, node any, nodeSizes map[ast.Node]int) (err error) { + // print node + p := newPrinter(cfg, fset, nodeSizes) + defer p.free() + if err = p.printNode(node); err != nil { + return + } + // print outstanding comments + p.impliedSemi = false // EOF acts like a newline + p.flush(token.Position{Offset: infinity, Line: infinity}, token.EOF) + + // output is buffered in p.output now. + // fix //go:build and // +build comments if needed. + p.fixGoBuildLines() + + // redirect output through a trimmer to eliminate trailing whitespace + // (Input to a tabwriter must be untrimmed since trailing tabs provide + // formatting information. The tabwriter could provide trimming + // functionality but no tabwriter is used when RawFormat is set.) + output = &trimmer{output: output} + + // redirect output through a tabwriter if necessary + if cfg.Mode&RawFormat == 0 { + minwidth := cfg.Tabwidth + + padchar := byte('\t') + if cfg.Mode&UseSpaces != 0 { + padchar = ' ' + } + + twmode := tabwriter.DiscardEmptyColumns + if cfg.Mode&TabIndent != 0 { + minwidth = 0 + twmode |= tabwriter.TabIndent + } + + output = tabwriter.NewWriter(output, minwidth, cfg.Tabwidth, 1, padchar, twmode) + } + + // write printer result via tabwriter/trimmer to output + if _, err = output.Write(p.output); err != nil { + return + } + + // flush tabwriter, if any + if tw, _ := output.(*tabwriter.Writer); tw != nil { + err = tw.Flush() + } + + return +} + +// A CommentedNode bundles an AST node and corresponding comments. +// It may be provided as argument to any of the Fprint functions. +type CommentedNode struct { + Node any // *ast.File, or ast.Expr, ast.Decl, ast.Spec, or ast.Stmt + Comments []*ast.CommentGroup +} + +// Fprint "pretty-prints" an AST node to output for a given configuration cfg. +// Position information is interpreted relative to the file set fset. +// The node type must be *ast.File, *CommentedNode, []ast.Decl, []ast.Stmt, +// or assignment-compatible to ast.Expr, ast.Decl, ast.Spec, or ast.Stmt. +func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node any) error { + return cfg.fprint(output, fset, node, make(map[ast.Node]int)) +} + +// Fprint "pretty-prints" an AST node to output. +// It calls Config.Fprint with default settings. +// Note that gofmt uses tabs for indentation but spaces for alignment; +// use format.Node (package go/format) for output that matches gofmt. +func Fprint(output io.Writer, fset *token.FileSet, node any) error { + return (&Config{Tabwidth: 8}).Fprint(output, fset, node) +} diff --git a/src/go/printer/printer_test.go b/src/go/printer/printer_test.go new file mode 100644 index 0000000..3a8ce60 --- /dev/null +++ b/src/go/printer/printer_test.go @@ -0,0 +1,827 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package printer + +import ( + "bytes" + "errors" + "flag" + "fmt" + "go/ast" + "go/parser" + "go/token" + "internal/diff" + "io" + "os" + "path/filepath" + "testing" + "time" +) + +const ( + dataDir = "testdata" + tabwidth = 8 +) + +var update = flag.Bool("update", false, "update golden files") + +var fset = token.NewFileSet() + +type checkMode uint + +const ( + export checkMode = 1 << iota + rawFormat + normNumber + idempotent + allowTypeParams +) + +// format parses src, prints the corresponding AST, verifies the resulting +// src is syntactically correct, and returns the resulting src or an error +// if any. +func format(src []byte, mode checkMode) ([]byte, error) { + // parse src + f, err := parser.ParseFile(fset, "", src, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("parse: %s\n%s", err, src) + } + + // filter exports if necessary + if mode&export != 0 { + ast.FileExports(f) // ignore result + f.Comments = nil // don't print comments that are not in AST + } + + // determine printer configuration + cfg := Config{Tabwidth: tabwidth} + if mode&rawFormat != 0 { + cfg.Mode |= RawFormat + } + if mode&normNumber != 0 { + cfg.Mode |= normalizeNumbers + } + + // print AST + var buf bytes.Buffer + if err := cfg.Fprint(&buf, fset, f); err != nil { + return nil, fmt.Errorf("print: %s", err) + } + + // make sure formatted output is syntactically correct + res := buf.Bytes() + if _, err := parser.ParseFile(fset, "", res, parser.ParseComments); err != nil { + return nil, fmt.Errorf("re-parse: %s\n%s", err, buf.Bytes()) + } + + return res, nil +} + +// lineAt returns the line in text starting at offset offs. +func lineAt(text []byte, offs int) []byte { + i := offs + for i < len(text) && text[i] != '\n' { + i++ + } + return text[offs:i] +} + +// checkEqual compares a and b. +func checkEqual(aname, bname string, a, b []byte) error { + if bytes.Equal(a, b) { + return nil + } + return errors.New(string(diff.Diff(aname, a, bname, b))) +} + +func runcheck(t *testing.T, source, golden string, mode checkMode) { + src, err := os.ReadFile(source) + if err != nil { + t.Error(err) + return + } + + res, err := format(src, mode) + if err != nil { + t.Error(err) + return + } + + // update golden files if necessary + if *update { + if err := os.WriteFile(golden, res, 0644); err != nil { + t.Error(err) + } + return + } + + // get golden + gld, err := os.ReadFile(golden) + if err != nil { + t.Error(err) + return + } + + // formatted source and golden must be the same + if err := checkEqual(source, golden, res, gld); err != nil { + t.Error(err) + return + } + + if mode&idempotent != 0 { + // formatting golden must be idempotent + // (This is very difficult to achieve in general and for now + // it is only checked for files explicitly marked as such.) + res, err = format(gld, mode) + if err != nil { + t.Error(err) + return + } + if err := checkEqual(golden, fmt.Sprintf("format(%s)", golden), gld, res); err != nil { + t.Errorf("golden is not idempotent: %s", err) + } + } +} + +func check(t *testing.T, source, golden string, mode checkMode) { + // run the test + cc := make(chan int, 1) + go func() { + runcheck(t, source, golden, mode) + cc <- 0 + }() + + // wait with timeout + select { + case <-time.After(10 * time.Second): // plenty of a safety margin, even for very slow machines + // test running past time out + t.Errorf("%s: running too slowly", source) + case <-cc: + // test finished within allotted time margin + } +} + +type entry struct { + source, golden string + mode checkMode +} + +// Use go test -update to create/update the respective golden files. +var data = []entry{ + {"empty.input", "empty.golden", idempotent}, + {"comments.input", "comments.golden", 0}, + {"comments.input", "comments.x", export}, + {"comments2.input", "comments2.golden", idempotent}, + {"alignment.input", "alignment.golden", idempotent}, + {"linebreaks.input", "linebreaks.golden", idempotent}, + {"expressions.input", "expressions.golden", idempotent}, + {"expressions.input", "expressions.raw", rawFormat | idempotent}, + {"declarations.input", "declarations.golden", 0}, + {"statements.input", "statements.golden", 0}, + {"slow.input", "slow.golden", idempotent}, + {"complit.input", "complit.x", export}, + {"go2numbers.input", "go2numbers.golden", idempotent}, + {"go2numbers.input", "go2numbers.norm", normNumber | idempotent}, + {"generics.input", "generics.golden", idempotent | allowTypeParams}, + {"gobuild1.input", "gobuild1.golden", idempotent}, + {"gobuild2.input", "gobuild2.golden", idempotent}, + {"gobuild3.input", "gobuild3.golden", idempotent}, + {"gobuild4.input", "gobuild4.golden", idempotent}, + {"gobuild5.input", "gobuild5.golden", idempotent}, + {"gobuild6.input", "gobuild6.golden", idempotent}, + {"gobuild7.input", "gobuild7.golden", idempotent}, +} + +func TestFiles(t *testing.T) { + t.Parallel() + for _, e := range data { + source := filepath.Join(dataDir, e.source) + golden := filepath.Join(dataDir, e.golden) + mode := e.mode + t.Run(e.source, func(t *testing.T) { + t.Parallel() + check(t, source, golden, mode) + // TODO(gri) check that golden is idempotent + //check(t, golden, golden, e.mode) + }) + } +} + +// TestLineComments, using a simple test case, checks that consecutive line +// comments are properly terminated with a newline even if the AST position +// information is incorrect. +func TestLineComments(t *testing.T) { + const src = `// comment 1 + // comment 2 + // comment 3 + package main + ` + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "", src, parser.ParseComments) + if err != nil { + panic(err) // error in test + } + + var buf bytes.Buffer + fset = token.NewFileSet() // use the wrong file set + Fprint(&buf, fset, f) + + nlines := 0 + for _, ch := range buf.Bytes() { + if ch == '\n' { + nlines++ + } + } + + const expected = 3 + if nlines < expected { + t.Errorf("got %d, expected %d\n", nlines, expected) + t.Errorf("result:\n%s", buf.Bytes()) + } +} + +// Verify that the printer can be invoked during initialization. +func init() { + const name = "foobar" + var buf bytes.Buffer + if err := Fprint(&buf, fset, &ast.Ident{Name: name}); err != nil { + panic(err) // error in test + } + // in debug mode, the result contains additional information; + // ignore it + if s := buf.String(); !debug && s != name { + panic("got " + s + ", want " + name) + } +} + +// Verify that the printer doesn't crash if the AST contains BadXXX nodes. +func TestBadNodes(t *testing.T) { + const src = "package p\n(" + const res = "package p\nBadDecl\n" + f, err := parser.ParseFile(fset, "", src, parser.ParseComments) + if err == nil { + t.Error("expected illegal program") // error in test + } + var buf bytes.Buffer + Fprint(&buf, fset, f) + if buf.String() != res { + t.Errorf("got %q, expected %q", buf.String(), res) + } +} + +// testComment verifies that f can be parsed again after printing it +// with its first comment set to comment at any possible source offset. +func testComment(t *testing.T, f *ast.File, srclen int, comment *ast.Comment) { + f.Comments[0].List[0] = comment + var buf bytes.Buffer + for offs := 0; offs <= srclen; offs++ { + buf.Reset() + // Printing f should result in a correct program no + // matter what the (incorrect) comment position is. + if err := Fprint(&buf, fset, f); err != nil { + t.Error(err) + } + if _, err := parser.ParseFile(fset, "", buf.Bytes(), 0); err != nil { + t.Fatalf("incorrect program for pos = %d:\n%s", comment.Slash, buf.String()) + } + // Position information is just an offset. + // Move comment one byte down in the source. + comment.Slash++ + } +} + +// Verify that the printer produces a correct program +// even if the position information of comments introducing newlines +// is incorrect. +func TestBadComments(t *testing.T) { + t.Parallel() + const src = ` +// first comment - text and position changed by test +package p +import "fmt" +const pi = 3.14 // rough circle +var ( + x, y, z int = 1, 2, 3 + u, v float64 +) +func fibo(n int) { + if n < 2 { + return n /* seed values */ + } + return fibo(n-1) + fibo(n-2) +} +` + + f, err := parser.ParseFile(fset, "", src, parser.ParseComments) + if err != nil { + t.Error(err) // error in test + } + + comment := f.Comments[0].List[0] + pos := comment.Pos() + if fset.PositionFor(pos, false /* absolute position */).Offset != 1 { + t.Error("expected offset 1") // error in test + } + + testComment(t, f, len(src), &ast.Comment{Slash: pos, Text: "//-style comment"}) + testComment(t, f, len(src), &ast.Comment{Slash: pos, Text: "/*-style comment */"}) + testComment(t, f, len(src), &ast.Comment{Slash: pos, Text: "/*-style \n comment */"}) + testComment(t, f, len(src), &ast.Comment{Slash: pos, Text: "/*-style comment \n\n\n */"}) +} + +type visitor chan *ast.Ident + +func (v visitor) Visit(n ast.Node) (w ast.Visitor) { + if ident, ok := n.(*ast.Ident); ok { + v <- ident + } + return v +} + +// idents is an iterator that returns all idents in f via the result channel. +func idents(f *ast.File) <-chan *ast.Ident { + v := make(visitor) + go func() { + ast.Walk(v, f) + close(v) + }() + return v +} + +// identCount returns the number of identifiers found in f. +func identCount(f *ast.File) int { + n := 0 + for range idents(f) { + n++ + } + return n +} + +// Verify that the SourcePos mode emits correct //line directives +// by testing that position information for matching identifiers +// is maintained. +func TestSourcePos(t *testing.T) { + const src = ` +package p +import ( "go/printer"; "math" ) +const pi = 3.14; var x = 0 +type t struct{ x, y, z int; u, v, w float32 } +func (t *t) foo(a, b, c int) int { + return a*t.x + b*t.y + + // two extra lines here + // ... + c*t.z +} +` + + // parse original + f1, err := parser.ParseFile(fset, "src", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + + // pretty-print original + var buf bytes.Buffer + err = (&Config{Mode: UseSpaces | SourcePos, Tabwidth: 8}).Fprint(&buf, fset, f1) + if err != nil { + t.Fatal(err) + } + + // parse pretty printed original + // (//line directives must be interpreted even w/o parser.ParseComments set) + f2, err := parser.ParseFile(fset, "", buf.Bytes(), 0) + if err != nil { + t.Fatalf("%s\n%s", err, buf.Bytes()) + } + + // At this point the position information of identifiers in f2 should + // match the position information of corresponding identifiers in f1. + + // number of identifiers must be > 0 (test should run) and must match + n1 := identCount(f1) + n2 := identCount(f2) + if n1 == 0 { + t.Fatal("got no idents") + } + if n2 != n1 { + t.Errorf("got %d idents; want %d", n2, n1) + } + + // verify that all identifiers have correct line information + i2range := idents(f2) + for i1 := range idents(f1) { + i2 := <-i2range + + if i2.Name != i1.Name { + t.Errorf("got ident %s; want %s", i2.Name, i1.Name) + } + + // here we care about the relative (line-directive adjusted) positions + l1 := fset.Position(i1.Pos()).Line + l2 := fset.Position(i2.Pos()).Line + if l2 != l1 { + t.Errorf("got line %d; want %d for %s", l2, l1, i1.Name) + } + } + + if t.Failed() { + t.Logf("\n%s", buf.Bytes()) + } +} + +// Verify that the SourcePos mode doesn't emit unnecessary //line directives +// before empty lines. +func TestIssue5945(t *testing.T) { + const orig = ` +package p // line 2 +func f() {} // line 3 + +var x, y, z int + + +func g() { // line 8 +} +` + + const want = `//line src.go:2 +package p + +//line src.go:3 +func f() {} + +var x, y, z int + +//line src.go:8 +func g() { +} +` + + // parse original + f1, err := parser.ParseFile(fset, "src.go", orig, 0) + if err != nil { + t.Fatal(err) + } + + // pretty-print original + var buf bytes.Buffer + err = (&Config{Mode: UseSpaces | SourcePos, Tabwidth: 8}).Fprint(&buf, fset, f1) + if err != nil { + t.Fatal(err) + } + got := buf.String() + + // compare original with desired output + if got != want { + t.Errorf("got:\n%s\nwant:\n%s\n", got, want) + } +} + +var decls = []string{ + `import "fmt"`, + "const pi = 3.1415\nconst e = 2.71828\n\nvar x = pi", + "func sum(x, y int) int\t{ return x + y }", +} + +func TestDeclLists(t *testing.T) { + for _, src := range decls { + file, err := parser.ParseFile(fset, "", "package p;"+src, parser.ParseComments) + if err != nil { + panic(err) // error in test + } + + var buf bytes.Buffer + err = Fprint(&buf, fset, file.Decls) // only print declarations + if err != nil { + panic(err) // error in test + } + + out := buf.String() + if out != src { + t.Errorf("\ngot : %q\nwant: %q\n", out, src) + } + } +} + +var stmts = []string{ + "i := 0", + "select {}\nvar a, b = 1, 2\nreturn a + b", + "go f()\ndefer func() {}()", +} + +func TestStmtLists(t *testing.T) { + for _, src := range stmts { + file, err := parser.ParseFile(fset, "", "package p; func _() {"+src+"}", parser.ParseComments) + if err != nil { + panic(err) // error in test + } + + var buf bytes.Buffer + err = Fprint(&buf, fset, file.Decls[0].(*ast.FuncDecl).Body.List) // only print statements + if err != nil { + panic(err) // error in test + } + + out := buf.String() + if out != src { + t.Errorf("\ngot : %q\nwant: %q\n", out, src) + } + } +} + +func TestBaseIndent(t *testing.T) { + t.Parallel() + // The testfile must not contain multi-line raw strings since those + // are not indented (because their values must not change) and make + // this test fail. + const filename = "printer.go" + src, err := os.ReadFile(filename) + if err != nil { + panic(err) // error in test + } + + file, err := parser.ParseFile(fset, filename, src, 0) + if err != nil { + panic(err) // error in test + } + + for indent := 0; indent < 4; indent++ { + indent := indent + t.Run(fmt.Sprint(indent), func(t *testing.T) { + t.Parallel() + var buf bytes.Buffer + (&Config{Tabwidth: tabwidth, Indent: indent}).Fprint(&buf, fset, file) + // all code must be indented by at least 'indent' tabs + lines := bytes.Split(buf.Bytes(), []byte{'\n'}) + for i, line := range lines { + if len(line) == 0 { + continue // empty lines don't have indentation + } + n := 0 + for j, b := range line { + if b != '\t' { + // end of indentation + n = j + break + } + } + if n < indent { + t.Errorf("line %d: got only %d tabs; want at least %d: %q", i, n, indent, line) + } + } + }) + } +} + +// TestFuncType tests that an ast.FuncType with a nil Params field +// can be printed (per go/ast specification). Test case for issue 3870. +func TestFuncType(t *testing.T) { + src := &ast.File{ + Name: &ast.Ident{Name: "p"}, + Decls: []ast.Decl{ + &ast.FuncDecl{ + Name: &ast.Ident{Name: "f"}, + Type: &ast.FuncType{}, + }, + }, + } + + var buf bytes.Buffer + if err := Fprint(&buf, fset, src); err != nil { + t.Fatal(err) + } + got := buf.String() + + const want = `package p + +func f() +` + + if got != want { + t.Fatalf("got:\n%s\nwant:\n%s\n", got, want) + } +} + +type limitWriter struct { + remaining int + errCount int +} + +func (l *limitWriter) Write(buf []byte) (n int, err error) { + n = len(buf) + if n >= l.remaining { + n = l.remaining + err = io.EOF + l.errCount++ + } + l.remaining -= n + return n, err +} + +// Test whether the printer stops writing after the first error +func TestWriteErrors(t *testing.T) { + t.Parallel() + const filename = "printer.go" + src, err := os.ReadFile(filename) + if err != nil { + panic(err) // error in test + } + file, err := parser.ParseFile(fset, filename, src, 0) + if err != nil { + panic(err) // error in test + } + for i := 0; i < 20; i++ { + lw := &limitWriter{remaining: i} + err := (&Config{Mode: RawFormat}).Fprint(lw, fset, file) + if lw.errCount > 1 { + t.Fatal("Writes continued after first error returned") + } + // We expect errCount be 1 iff err is set + if (lw.errCount != 0) != (err != nil) { + t.Fatal("Expected err when errCount != 0") + } + } +} + +// TextX is a skeleton test that can be filled in for debugging one-off cases. +// Do not remove. +func TestX(t *testing.T) { + const src = ` +package p +func _() {} +` + _, err := format([]byte(src), 0) + if err != nil { + t.Error(err) + } +} + +func TestCommentedNode(t *testing.T) { + const ( + input = `package main + +func foo() { + // comment inside func +} + +// leading comment +type bar int // comment2 + +` + + foo = `func foo() { + // comment inside func +}` + + bar = `// leading comment +type bar int // comment2 +` + ) + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "input.go", input, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + + err = Fprint(&buf, fset, &CommentedNode{Node: f.Decls[0], Comments: f.Comments}) + if err != nil { + t.Fatal(err) + } + + if buf.String() != foo { + t.Errorf("got %q, want %q", buf.String(), foo) + } + + buf.Reset() + + err = Fprint(&buf, fset, &CommentedNode{Node: f.Decls[1], Comments: f.Comments}) + if err != nil { + t.Fatal(err) + } + + if buf.String() != bar { + t.Errorf("got %q, want %q", buf.String(), bar) + } +} + +func TestIssue11151(t *testing.T) { + const src = "package p\t/*\r/1\r*\r/2*\r\r\r\r/3*\r\r+\r\r/4*/\n" + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + Fprint(&buf, fset, f) + got := buf.String() + const want = "package p\t/*/1*\r/2*\r/3*+/4*/\n" // \r following opening /* should be stripped + if got != want { + t.Errorf("\ngot : %q\nwant: %q", got, want) + } + + // the resulting program must be valid + _, err = parser.ParseFile(fset, "", got, 0) + if err != nil { + t.Errorf("%v\norig: %q\ngot : %q", err, src, got) + } +} + +// If a declaration has multiple specifications, a parenthesized +// declaration must be printed even if Lparen is token.NoPos. +func TestParenthesizedDecl(t *testing.T) { + // a package with multiple specs in a single declaration + const src = "package p; var ( a float64; b int )" + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "", src, 0) + if err != nil { + t.Fatal(err) + } + + // print the original package + var buf bytes.Buffer + err = Fprint(&buf, fset, f) + if err != nil { + t.Fatal(err) + } + original := buf.String() + + // now remove parentheses from the declaration + for i := 0; i != len(f.Decls); i++ { + f.Decls[i].(*ast.GenDecl).Lparen = token.NoPos + } + buf.Reset() + err = Fprint(&buf, fset, f) + if err != nil { + t.Fatal(err) + } + noparen := buf.String() + + if noparen != original { + t.Errorf("got %q, want %q", noparen, original) + } +} + +// Verify that we don't print a newline between "return" and its results, as +// that would incorrectly cause a naked return. +func TestIssue32854(t *testing.T) { + src := `package foo + +func f() { + return Composite{ + call(), + } +}` + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "", src, 0) + if err != nil { + panic(err) + } + + // Replace the result with call(), which is on the next line. + fd := file.Decls[0].(*ast.FuncDecl) + ret := fd.Body.List[0].(*ast.ReturnStmt) + ret.Results[0] = ret.Results[0].(*ast.CompositeLit).Elts[0] + + var buf bytes.Buffer + if err := Fprint(&buf, fset, ret); err != nil { + t.Fatal(err) + } + want := "return call()" + if got := buf.String(); got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func TestSourcePosNewline(t *testing.T) { + // We don't provide a syntax for escaping or unescaping characters in line + // directives (see https://go.dev/issue/24183#issuecomment-372449628). + // As a result, we cannot write a line directive with the correct path for a + // filename containing newlines. We should return an error rather than + // silently dropping or mangling it. + + fname := "foo\nbar/bar.go" + src := `package bar` + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, fname, src, parser.ParseComments|parser.AllErrors|parser.SkipObjectResolution) + if err != nil { + t.Fatal(err) + } + + cfg := &Config{ + Mode: SourcePos, // emit line comments + Tabwidth: 8, + } + var buf bytes.Buffer + if err := cfg.Fprint(&buf, fset, f); err == nil { + t.Errorf("Fprint did not error for source file path containing newline") + } + if buf.Len() != 0 { + t.Errorf("unexpected Fprint output:\n%s", buf.Bytes()) + } +} diff --git a/src/go/printer/testdata/alignment.golden b/src/go/printer/testdata/alignment.golden new file mode 100644 index 0000000..96086ed --- /dev/null +++ b/src/go/printer/testdata/alignment.golden @@ -0,0 +1,172 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package alignment + +// ---------------------------------------------------------------------------- +// Examples from issue #7335. + +func main() { + z := MyStruct{ + Foo: "foo", + Bar: "bar", + Name: "name", + LongName: "longname", + Baz: "baz", + } + y := MyStruct{ + Foo: "foo", + Bar: "bar", + NameXX: "name", + LongNameXXXXXXXXXXXXX: "longname", + Baz: "baz", + } + z := MyStruct{ + Foo: "foo", + Bar: "bar", + Name: "name", + LongNameXXXXXXXXXXXXX: "longname", + Baz: "baz", + } +} + +// ---------------------------------------------------------------------------- +// Examples from issue #10392. + +var kcfg = KubeletConfig{ + Address: s.Address, + AllowPrivileged: s.AllowPrivileged, + HostNetworkSources: hostNetworkSources, + HostnameOverride: s.HostnameOverride, + RootDirectory: s.RootDirectory, + ConfigFile: s.Config, + ManifestURL: s.ManifestURL, + FileCheckFrequency: s.FileCheckFrequency, + HTTPCheckFrequency: s.HTTPCheckFrequency, + PodInfraContainerImage: s.PodInfraContainerImage, + SyncFrequency: s.SyncFrequency, + RegistryPullQPS: s.RegistryPullQPS, + RegistryBurst: s.RegistryBurst, + MinimumGCAge: s.MinimumGCAge, + MaxPerPodContainerCount: s.MaxPerPodContainerCount, + MaxContainerCount: s.MaxContainerCount, + ClusterDomain: s.ClusterDomain, + ClusterDNS: s.ClusterDNS, + Runonce: s.RunOnce, + Port: s.Port, + ReadOnlyPort: s.ReadOnlyPort, + CadvisorInterface: cadvisorInterface, + EnableServer: s.EnableServer, + EnableDebuggingHandlers: s.EnableDebuggingHandlers, + DockerClient: dockertools.ConnectToDockerOrDie(s.DockerEndpoint), + KubeClient: client, + MasterServiceNamespace: s.MasterServiceNamespace, + VolumePlugins: ProbeVolumePlugins(), + NetworkPlugins: ProbeNetworkPlugins(), + NetworkPluginName: s.NetworkPluginName, + StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout, + TLSOptions: tlsOptions, + ImageGCPolicy: imageGCPolicy, imageGCPolicy, + Cloud: cloud, + NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency, +} + +var a = A{ + Long: 1, + LongLong: 1, + LongLongLong: 1, + LongLongLongLong: 1, + LongLongLongLongLong: 1, + LongLongLongLongLongLong: 1, + LongLongLongLongLongLongLong: 1, + LongLongLongLongLongLongLongLong: 1, + Short: 1, + LongLongLongLongLongLongLongLongLong: 3, +} + +// ---------------------------------------------------------------------------- +// Examples from issue #22852. + +var fmtMap = map[string]string{ + "1": "123", + "12": "123", + "123": "123", + "1234": "123", + "12345": "123", + "123456": "123", + "12345678901234567890123456789": "123", + "abcde": "123", + "123456789012345678901234567890": "123", + "1234567": "123", + "abcdefghijklmnopqrstuvwxyzabcd": "123", + "abcd": "123", +} + +type Fmt struct { + abcdefghijklmnopqrstuvwx string + abcdefghijklmnopqrstuvwxy string + abcdefghijklmnopqrstuvwxyz string + abcdefghijklmnopqrstuvwxyza string + abcdefghijklmnopqrstuvwxyzab string + abcdefghijklmnopqrstuvwxyzabc string + abcde string + abcdefghijklmnopqrstuvwxyzabcde string + abcdefg string +} + +func main() { + _ := Fmt{ + abcdefghijklmnopqrstuvwx: "foo", + abcdefghijklmnopqrstuvwxyza: "foo", + abcdefghijklmnopqrstuvwxyzab: "foo", + abcdefghijklmnopqrstuvwxyzabc: "foo", + abcde: "foo", + abcdefghijklmnopqrstuvwxyzabcde: "foo", + abcdefg: "foo", + abcdefghijklmnopqrstuvwxy: "foo", + abcdefghijklmnopqrstuvwxyz: "foo", + } +} + +// ---------------------------------------------------------------------------- +// Examples from issue #26352. + +var _ = map[int]string{ + 1: "", + + 12345678901234567890123456789: "", + 12345678901234567890123456789012345678: "", +} + +func f() { + _ = map[int]string{ + 1: "", + + 12345678901234567: "", + 12345678901234567890123456789012345678901: "", + } +} + +// ---------------------------------------------------------------------------- +// Examples from issue #26930. + +var _ = S{ + F1: []string{}, + F2____: []string{}, +} + +var _ = S{ + F1: []string{}, + F2____: []string{}, +} + +var _ = S{ + F1____: []string{}, + F2: []string{}, +} + +var _ = S{ + F1____: []string{}, + F2: []string{}, +} diff --git a/src/go/printer/testdata/alignment.input b/src/go/printer/testdata/alignment.input new file mode 100644 index 0000000..323d268 --- /dev/null +++ b/src/go/printer/testdata/alignment.input @@ -0,0 +1,179 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package alignment + +// ---------------------------------------------------------------------------- +// Examples from issue #7335. + +func main() { + z := MyStruct{ + Foo: "foo", + Bar: "bar", + Name: "name", + LongName: "longname", + Baz: "baz", + } + y := MyStruct{ + Foo: "foo", + Bar: "bar", + NameXX: "name", + LongNameXXXXXXXXXXXXX: "longname", + Baz: "baz", + } + z := MyStruct{ + Foo: "foo", + Bar: "bar", + Name: "name", + LongNameXXXXXXXXXXXXX: "longname", + Baz: "baz", + } +} + +// ---------------------------------------------------------------------------- +// Examples from issue #10392. + +var kcfg = KubeletConfig{ + Address: s.Address, + AllowPrivileged: s.AllowPrivileged, + HostNetworkSources: hostNetworkSources, + HostnameOverride: s.HostnameOverride, + RootDirectory: s.RootDirectory, + ConfigFile: s.Config, + ManifestURL: s.ManifestURL, + FileCheckFrequency: s.FileCheckFrequency, + HTTPCheckFrequency: s.HTTPCheckFrequency, + PodInfraContainerImage: s.PodInfraContainerImage, + SyncFrequency: s.SyncFrequency, + RegistryPullQPS: s.RegistryPullQPS, + RegistryBurst: s.RegistryBurst, + MinimumGCAge: s.MinimumGCAge, + MaxPerPodContainerCount: s.MaxPerPodContainerCount, + MaxContainerCount: s.MaxContainerCount, + ClusterDomain: s.ClusterDomain, + ClusterDNS: s.ClusterDNS, + Runonce: s.RunOnce, + Port: s.Port, + ReadOnlyPort: s.ReadOnlyPort, + CadvisorInterface: cadvisorInterface, + EnableServer: s.EnableServer, + EnableDebuggingHandlers: s.EnableDebuggingHandlers, + DockerClient: dockertools.ConnectToDockerOrDie(s.DockerEndpoint), + KubeClient: client, + MasterServiceNamespace: s.MasterServiceNamespace, + VolumePlugins: ProbeVolumePlugins(), + NetworkPlugins: ProbeNetworkPlugins(), + NetworkPluginName: s.NetworkPluginName, + StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout, + TLSOptions: tlsOptions, + ImageGCPolicy: imageGCPolicy,imageGCPolicy, + Cloud: cloud, + NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency, +} + +var a = A{ + Long: 1, + LongLong: 1, + LongLongLong: 1, + LongLongLongLong: 1, + LongLongLongLongLong: 1, + LongLongLongLongLongLong: 1, + LongLongLongLongLongLongLong: 1, + LongLongLongLongLongLongLongLong: 1, + Short: 1, + LongLongLongLongLongLongLongLongLong: 3, +} + +// ---------------------------------------------------------------------------- +// Examples from issue #22852. + +var fmtMap = map[string]string{ + "1": "123", + "12": "123", + "123": "123", + "1234": "123", + "12345": "123", + "123456": "123", + "12345678901234567890123456789": "123", + "abcde": "123", + "123456789012345678901234567890": "123", + "1234567": "123", + "abcdefghijklmnopqrstuvwxyzabcd": "123", + "abcd": "123", +} + +type Fmt struct { + abcdefghijklmnopqrstuvwx string + abcdefghijklmnopqrstuvwxy string + abcdefghijklmnopqrstuvwxyz string + abcdefghijklmnopqrstuvwxyza string + abcdefghijklmnopqrstuvwxyzab string + abcdefghijklmnopqrstuvwxyzabc string + abcde string + abcdefghijklmnopqrstuvwxyzabcde string + abcdefg string +} + +func main() { + _ := Fmt{ + abcdefghijklmnopqrstuvwx: "foo", + abcdefghijklmnopqrstuvwxyza: "foo", + abcdefghijklmnopqrstuvwxyzab: "foo", + abcdefghijklmnopqrstuvwxyzabc: "foo", + abcde: "foo", + abcdefghijklmnopqrstuvwxyzabcde: "foo", + abcdefg: "foo", + abcdefghijklmnopqrstuvwxy: "foo", + abcdefghijklmnopqrstuvwxyz: "foo", + } +} + +// ---------------------------------------------------------------------------- +// Examples from issue #26352. + +var _ = map[int]string{ + 1: "", + + 12345678901234567890123456789: "", + 12345678901234567890123456789012345678: "", +} + +func f() { + _ = map[int]string{ + 1: "", + + 12345678901234567: "", + 12345678901234567890123456789012345678901: "", + } +} + +// ---------------------------------------------------------------------------- +// Examples from issue #26930. + +var _ = S{ + F1: []string{ + }, + F2____: []string{}, +} + +var _ = S{ + F1: []string{ + + + }, + F2____: []string{}, +} + +var _ = S{ + F1____: []string{ + }, + F2: []string{}, +} + +var _ = S{ + F1____: []string{ + + }, + F2: []string{}, +} diff --git a/src/go/printer/testdata/comments.golden b/src/go/printer/testdata/comments.golden new file mode 100644 index 0000000..1e5d17b --- /dev/null +++ b/src/go/printer/testdata/comments.golden @@ -0,0 +1,774 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This is a package for testing comment placement by go/printer. +package main + +import "fmt" // fmt + +const c0 = 0 // zero +const ( + c1 = iota // c1 + c2 // c2 +) + +// Alignment of comments in declarations> +const ( + _ T = iota // comment + _ // comment + _ // comment + _ = iota + 10 + _ // comments + + _ = 10 // comment + _ T = 20 // comment +) + +const ( + _____ = iota // foo + _ // bar + _ = 0 // bal + _ // bat +) + +const ( + _ T = iota // comment + _ // comment + _ // comment + _ = iota + 10 + _ // comment + _ = 10 + _ = 20 // comment + _ T = 0 // comment +) + +// The SZ struct; it is empty. +type SZ struct{} + +// The S0 struct; no field is exported. +type S0 struct { + int + x, y, z int // 3 unexported fields +} + +// The S1 struct; some fields are not exported. +type S1 struct { + S0 + A, B, C float // 3 exported fields + D, b, c int // 2 unexported fields +} + +// The S2 struct; all fields are exported. +type S2 struct { + S1 + A, B, C float // 3 exported fields +} + +// The IZ interface; it is empty. +type SZ interface{} + +// The I0 interface; no method is exported. +type I0 interface { + f(x int) int // unexported method +} + +// The I1 interface; some methods are not exported. +type I1 interface { + I0 + F(x float) float // exported methods + g(x int) int // unexported method +} + +// The I2 interface; all methods are exported. +type I2 interface { + I0 + F(x float) float // exported method + G(x float) float // exported method +} + +// The S3 struct; all comments except for the last one must appear in the export. +type S3 struct { + // lead comment for F1 + F1 int // line comment for F1 + // lead comment for F2 + F2 int // line comment for F2 + f3 int // f3 is not exported +} + +// Here is a comment. +// Here is an accidentally unindented line. +// More comment. +// +//dir:ect ive +type directiveCheck struct{} + +// This comment group should be separated +// with a newline from the next comment +// group. + +// This comment should NOT be associated with the next declaration. + +var x int // x +var () + +// This comment SHOULD be associated with f0. +func f0() { + const pi = 3.14 // pi + var s1 struct{} /* an empty struct */ /* foo */ + // a struct constructor + // -------------------- + var s2 struct{} = struct{}{} + x := pi +} + +// This comment should be associated with f1, with one blank line before the comment. +func f1() { + f0() + /* 1 */ + // 2 + /* 3 */ + /* 4 */ + f0() +} + +func _() { + // this comment should be properly indented +} + +func _(x int) int { + if x < 0 { // the tab printed before this comment's // must not affect the remaining lines + return -x // this statement should be properly indented + } + if x < 0 { /* the tab printed before this comment's /* must not affect the remaining lines */ + return -x // this statement should be properly indented + } + return x +} + +func typeswitch(x interface{}) { + switch v := x.(type) { + case bool, int, float: + case string: + default: + } + + switch x.(type) { + } + + switch v0, ok := x.(int); v := x.(type) { + } + + switch v0, ok := x.(int); x.(type) { + case byte: // this comment should be on the same line as the keyword + // this comment should be normally indented + _ = 0 + case bool, int, float: + // this comment should be indented + case string: + default: + // this comment should be indented + } + // this comment should not be indented +} + +// +// Indentation of comments after possibly indented multi-line constructs +// (test cases for issue 3147). +// + +func _() { + s := 1 + + 2 + // should be indented like s +} + +func _() { + s := 1 + + 2 // comment + // should be indented like s +} + +func _() { + s := 1 + + 2 // comment + // should be indented like s + _ = 0 +} + +func _() { + s := 1 + + 2 + // should be indented like s + _ = 0 +} + +func _() { + s := 1 + + 2 + + // should be indented like s +} + +func _() { + s := 1 + + 2 // comment + + // should be indented like s +} + +func _() { + s := 1 + + 2 // comment + + // should be indented like s + _ = 0 +} + +func _() { + s := 1 + + 2 + + // should be indented like s + _ = 0 +} + +// Test case from issue 3147. +func f() { + templateText := "a" + // A + "b" + // B + "c" // C + + // should be aligned with f() + f() +} + +// Modified test case from issue 3147. +func f() { + templateText := "a" + // A + "b" + // B + "c" // C + + // may not be aligned with f() (source is not aligned) + f() +} + +// +// Test cases for alignment of lines in general comments. +// + +func _() { + /* freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + aligned line */ +} + +func _() { + /* freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + aligned line */ +} + +func _() { + /* + freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + aligned line */ +} + +func _() { + /* + freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + aligned line */ +} + +func _() { + /* freestanding comment + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line */ +} + +func _() { + /* freestanding comment + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line */ +} + +func _() { + /* + freestanding comment + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line */ +} + +func _() { + /* + freestanding comment + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line */ +} + +// Issue 9751. +func _() { + /*a string + + b string*/ + + /*A string + + + + Z string*/ + + /*a string + + b string + + c string*/ + + { + /*a string + b string*/ + + /*a string + + b string*/ + + /*a string + + b string + + c string*/ + } + + { + /*a string + b string*/ + + /*a string + + b string*/ + + /*a string + + b string + + c string*/ + } + + /* + */ + + /* + + */ + + /* + + * line + + */ +} + +/* + * line + * of + * stars + */ + +/* another line + * of + * stars */ + +/* and another line + * of + * stars */ + +/* a line of + * stars */ + +/* and another line of + * stars */ + +/* a line of stars + */ + +/* and another line of + */ + +/* a line of stars + */ + +/* and another line of + */ + +/* +aligned in middle +here + not here +*/ + +/* +blank line in middle: + +with no leading spaces on blank line. +*/ + +/* + aligned in middle + here + not here +*/ + +/* + blank line in middle: + + with no leading spaces on blank line. +*/ + +func _() { + /* + * line + * of + * stars + */ + + /* + aligned in middle + here + not here + */ + + /* + blank line in middle: + + with no leading spaces on blank line. + */ +} + +// Some interesting interspersed comments. +// See below for more common cases. +func _( /* this */ x /* is */ /* an */ int) { +} + +func _( /* no params - extra blank before and after comment */ ) {} +func _(a, b int /* params - no extra blank after comment */) {} + +func _() { f( /* no args - extra blank before and after comment */ ) } +func _() { f(a, b /* args - no extra blank after comment */) } + +func _() { + f( /* no args - extra blank before and after comment */ ) + f(a, b /* args - no extra blank after comment */) +} + +func ( /* comment1 */ T /* comment2 */) _() {} + +func _() { /* "short-ish one-line functions with comments are formatted as multi-line functions */ } +func _() { x := 0; /* comment */ y = x /* comment */ } + +func _() { + _ = 0 + /* closing curly brace should be on new line */ +} + +func _() { + _ = []int{0, 1 /* don't introduce a newline after this comment - was issue 1365 */} +} + +// Test cases from issue 1542: +// Comments must not be placed before commas and cause invalid programs. +func _() { + var a = []int{1, 2 /*jasldf*/} + _ = a +} + +func _() { + var a = []int{1, 2}/*jasldf + */ + + _ = a +} + +func _() { + var a = []int{1, 2}// jasldf + + _ = a +} + +// Test cases from issues 11274, 15137: +// Semicolon must not be lost when multiple statements are on the same line with a comment. +func _() { + x := 0 /**/ + y := 1 +} + +func _() { + f() + f() + f() /* comment */ + f() + f() /* comment */ + f() + f() /* a */ /* b */ + f() + f() /* a */ /* b */ + f() + f() /* a */ /* b */ + f() +} + +func _() { + f() /* a */ /* b */ +} + +// Comments immediately adjacent to punctuation followed by a newline +// remain after the punctuation (looks better and permits alignment of +// comments). +func _() { + _ = T{ + 1, // comment after comma + 2, /* comment after comma */ + 3, // comment after comma + } + _ = T{ + 1, // comment after comma + 2, /* comment after comma */ + 3, // comment after comma + } + _ = T{ + /* comment before literal */ 1, + 2, /* comment before comma - ok to move after comma */ + 3, /* comment before comma - ok to move after comma */ + } + + for i = 0; // comment after semicolon + i < 9; /* comment after semicolon */ + i++ { // comment after opening curly brace + } + + // TODO(gri) the last comment in this example should be aligned */ + for i = 0; // comment after semicolon + i < 9; /* comment before semicolon - ok to move after semicolon */ + i++ /* comment before opening curly brace */ { + } +} + +// If there is no newline following punctuation, commas move before the punctuation. +// This way, commas interspersed in lists stay with the respective expression. +func f(x /* comment */, y int, z int /* comment */, u, v, w int /* comment */) { + f(x /* comment */, y) + f(x, /* comment */ + y) + f( + x, /* comment */ + ) +} + +func g( + x int, /* comment */ +) { +} + +type _ struct { + a, b /* comment */, c int +} + +type _ struct { + a, b /* comment */, c int +} + +func _() { + for a /* comment */, b := range x { + } +} + +//extern foo +func foo() {} + +//export bar +func bar() {} + +// Print line directives correctly. + +// The following is a legal line directive. +// +//line foo:1 +func _() { + _ = 0 + // The following is a legal line directive. It must not be indented: +//line foo:2 + _ = 1 + + // The following is not a legal line directive (it doesn't start in column 1): + //line foo:2 + _ = 2 + + // The following is not a legal line directive (missing colon): +//line foo -3 + _ = 3 +} + +// Line comments with tabs +func _() { + var finput *bufio.Reader // input file + var stderr *bufio.Writer + var ftable *bufio.Writer // y.go file + var foutput *bufio.Writer // y.output file + + var oflag string // -o [y.go] - y.go file + var vflag string // -v [y.output] - y.output file + var lflag bool // -l - disable line directives +} + +// Trailing white space in comments should be trimmed +func _() { + // This comment has 4 blanks following that should be trimmed: + /* Each line of this comment has blanks or tabs following that should be trimmed: + line 2: + line 3: + */ +} + +var _ = []T{ /* lone comment */ } + +var _ = []T{ + /* lone comment */ +} + +var _ = []T{ + // lone comments + // in composite lit +} + +var _ = [][]T{ + { + // lone comments + // in composite lit + }, +} + +// TODO: gofmt doesn't add these tabs; make it so that these golden +// tests run the printer in a way that it's exactly like gofmt. + +var _ = []T{ // lone comment +} + +var _ = []T{ // lone comments + // in composite lit +} + +func _() {} + +func _() {} + +/* This comment is the last entry in this file. It must be printed and should be followed by a newline */ diff --git a/src/go/printer/testdata/comments.input b/src/go/printer/testdata/comments.input new file mode 100644 index 0000000..40aa55b --- /dev/null +++ b/src/go/printer/testdata/comments.input @@ -0,0 +1,773 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This is a package for testing comment placement by go/printer. +// +package main + +import "fmt" // fmt + +const c0 = 0 // zero +const ( + c1 = iota // c1 + c2 // c2 +) + +// Alignment of comments in declarations> +const ( + _ T = iota // comment + _ // comment + _ // comment + _ = iota+10 + _ // comments + + _ = 10 // comment + _ T = 20 // comment +) + +const ( + _____ = iota // foo + _ // bar + _ = 0 // bal + _ // bat +) + +const ( + _ T = iota // comment + _ // comment + _ // comment + _ = iota + 10 + _ // comment + _ = 10 + _ = 20 // comment + _ T = 0 // comment +) + +// The SZ struct; it is empty. +type SZ struct {} + +// The S0 struct; no field is exported. +type S0 struct { + int + x, y, z int // 3 unexported fields +} + +// The S1 struct; some fields are not exported. +type S1 struct { + S0 + A, B, C float // 3 exported fields + D, b, c int // 2 unexported fields +} + +// The S2 struct; all fields are exported. +type S2 struct { + S1 + A, B, C float // 3 exported fields +} + +// The IZ interface; it is empty. +type SZ interface {} + +// The I0 interface; no method is exported. +type I0 interface { + f(x int) int // unexported method +} + +// The I1 interface; some methods are not exported. +type I1 interface { + I0 + F(x float) float // exported methods + g(x int) int // unexported method +} + +// The I2 interface; all methods are exported. +type I2 interface { + I0 + F(x float) float // exported method + G(x float) float // exported method +} + +// The S3 struct; all comments except for the last one must appear in the export. +type S3 struct { + // lead comment for F1 + F1 int // line comment for F1 + // lead comment for F2 + F2 int // line comment for F2 + f3 int // f3 is not exported +} + +// Here is a comment. +//Here is an accidentally unindented line. +//dir:ect ive +// More comment. +type directiveCheck struct{} + +// This comment group should be separated +// with a newline from the next comment +// group. + +// This comment should NOT be associated with the next declaration. + +var x int // x +var () + + +// This comment SHOULD be associated with f0. +func f0() { + const pi = 3.14 // pi + var s1 struct {} /* an empty struct */ /* foo */ + // a struct constructor + // -------------------- + var s2 struct {} = struct {}{} + x := pi +} +// +// This comment should be associated with f1, with one blank line before the comment. +// +func f1() { + f0() + /* 1 */ + // 2 + /* 3 */ + /* 4 */ + f0() +} + + +func _() { +// this comment should be properly indented +} + + +func _(x int) int { + if x < 0 { // the tab printed before this comment's // must not affect the remaining lines + return -x // this statement should be properly indented + } + if x < 0 { /* the tab printed before this comment's /* must not affect the remaining lines */ + return -x // this statement should be properly indented + } + return x +} + + +func typeswitch(x interface{}) { + switch v := x.(type) { + case bool, int, float: + case string: + default: + } + + switch x.(type) { + } + + switch v0, ok := x.(int); v := x.(type) { + } + + switch v0, ok := x.(int); x.(type) { + case byte: // this comment should be on the same line as the keyword + // this comment should be normally indented + _ = 0 + case bool, int, float: + // this comment should be indented + case string: + default: + // this comment should be indented + } + // this comment should not be indented +} + +// +// Indentation of comments after possibly indented multi-line constructs +// (test cases for issue 3147). +// + +func _() { + s := 1 + + 2 +// should be indented like s +} + +func _() { + s := 1 + + 2 // comment + // should be indented like s +} + +func _() { + s := 1 + + 2 // comment + // should be indented like s + _ = 0 +} + +func _() { + s := 1 + + 2 + // should be indented like s + _ = 0 +} + +func _() { + s := 1 + + 2 + +// should be indented like s +} + +func _() { + s := 1 + + 2 // comment + + // should be indented like s +} + +func _() { + s := 1 + + 2 // comment + + // should be indented like s + _ = 0 +} + +func _() { + s := 1 + + 2 + + // should be indented like s + _ = 0 +} + +// Test case from issue 3147. +func f() { + templateText := "a" + // A + "b" + // B + "c" // C + + // should be aligned with f() + f() +} + +// Modified test case from issue 3147. +func f() { + templateText := "a" + // A + "b" + // B + "c" // C + + // may not be aligned with f() (source is not aligned) + f() +} + +// +// Test cases for alignment of lines in general comments. +// + +func _() { + /* freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + aligned line */ +} + +func _() { + /* freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + aligned line */ +} + + +func _() { + /* + freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + aligned line */ +} + +func _() { + /* + freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + aligned line */ +} + +func _() { + /* freestanding comment + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line */ +} + +func _() { + /* freestanding comment + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line + */ +} + +func _() { + /* freestanding comment + aligned line */ +} + + +func _() { + /* + freestanding comment + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line */ +} + +func _() { + /* + freestanding comment + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line + */ +} + +func _() { + /* + freestanding comment + aligned line */ +} + +// Issue 9751. +func _() { + /*a string + + b string*/ + + /*A string + + + + Z string*/ + + /*a string + + b string + + c string*/ + + { + /*a string +b string*/ + + /*a string + +b string*/ + + /*a string + +b string + +c string*/ + } + + { + /*a string + b string*/ + + /*a string + + b string*/ + + /*a string + + b string + + c string*/ + } + + /* + */ + + /* + + */ + + /* + + * line + + */ +} + +/* + * line + * of + * stars + */ + +/* another line + * of + * stars */ + +/* and another line + * of + * stars */ + +/* a line of + * stars */ + +/* and another line of + * stars */ + +/* a line of stars +*/ + +/* and another line of +*/ + +/* a line of stars + */ + +/* and another line of + */ + +/* +aligned in middle +here + not here +*/ + +/* +blank line in middle: + +with no leading spaces on blank line. +*/ + +/* + aligned in middle + here + not here +*/ + +/* + blank line in middle: + + with no leading spaces on blank line. +*/ + +func _() { + /* + * line + * of + * stars + */ + + /* + aligned in middle + here + not here + */ + + /* + blank line in middle: + + with no leading spaces on blank line. +*/ +} + + +// Some interesting interspersed comments. +// See below for more common cases. +func _(/* this */x/* is *//* an */ int) { +} + +func _(/* no params - extra blank before and after comment */) {} +func _(a, b int /* params - no extra blank after comment */) {} + +func _() { f(/* no args - extra blank before and after comment */) } +func _() { f(a, b /* args - no extra blank after comment */) } + +func _() { + f(/* no args - extra blank before and after comment */) + f(a, b /* args - no extra blank after comment */) +} + +func (/* comment1 */ T /* comment2 */) _() {} + +func _() { /* "short-ish one-line functions with comments are formatted as multi-line functions */ } +func _() { x := 0; /* comment */ y = x /* comment */ } + +func _() { + _ = 0 + /* closing curly brace should be on new line */ } + +func _() { + _ = []int{0, 1 /* don't introduce a newline after this comment - was issue 1365 */} +} + +// Test cases from issue 1542: +// Comments must not be placed before commas and cause invalid programs. +func _() { + var a = []int{1, 2, /*jasldf*/ + } + _ = a +} + +func _() { + var a = []int{1, 2, /*jasldf + */ + } + _ = a +} + +func _() { + var a = []int{1, 2, // jasldf + } + _ = a +} + +// Test cases from issues 11274, 15137: +// Semicolon must not be lost when multiple statements are on the same line with a comment. +func _() { + x := 0 /**/; y := 1 +} + +func _() { + f(); f() + f(); /* comment */ f() + f() /* comment */; f() + f(); /* a */ /* b */ f() + f() /* a */ /* b */; f() + f() /* a */; /* b */ f() +} + +func _() { + f() /* a */ /* b */ } + +// Comments immediately adjacent to punctuation followed by a newline +// remain after the punctuation (looks better and permits alignment of +// comments). +func _() { + _ = T{ + 1, // comment after comma + 2, /* comment after comma */ + 3 , // comment after comma + } + _ = T{ + 1 ,// comment after comma + 2 ,/* comment after comma */ + 3,// comment after comma + } + _ = T{ + /* comment before literal */1, + 2/* comment before comma - ok to move after comma */, + 3 /* comment before comma - ok to move after comma */ , + } + + for + i=0;// comment after semicolon + i<9;/* comment after semicolon */ + i++{// comment after opening curly brace + } + + // TODO(gri) the last comment in this example should be aligned */ + for + i=0;// comment after semicolon + i<9/* comment before semicolon - ok to move after semicolon */; + i++ /* comment before opening curly brace */ { + } +} + +// If there is no newline following punctuation, commas move before the punctuation. +// This way, commas interspersed in lists stay with the respective expression. +func f(x/* comment */, y int, z int /* comment */, u, v, w int /* comment */) { + f(x /* comment */, y) + f(x /* comment */, + y) + f( + x /* comment */, + ) +} + +func g( + x int /* comment */, +) {} + +type _ struct { + a, b /* comment */, c int +} + +type _ struct { a, b /* comment */, c int } + +func _() { + for a /* comment */, b := range x { + } +} + +//extern foo +func foo() {} + +//export bar +func bar() {} + +// Print line directives correctly. + +// The following is a legal line directive. +//line foo:1 +func _() { + _ = 0 +// The following is a legal line directive. It must not be indented: +//line foo:2 + _ = 1 + +// The following is not a legal line directive (it doesn't start in column 1): + //line foo:2 + _ = 2 + +// The following is not a legal line directive (missing colon): +//line foo -3 + _ = 3 +} + +// Line comments with tabs +func _() { +var finput *bufio.Reader // input file +var stderr *bufio.Writer +var ftable *bufio.Writer // y.go file +var foutput *bufio.Writer // y.output file + +var oflag string // -o [y.go] - y.go file +var vflag string // -v [y.output] - y.output file +var lflag bool // -l - disable line directives +} + +// Trailing white space in comments should be trimmed +func _() { +// This comment has 4 blanks following that should be trimmed: +/* Each line of this comment has blanks or tabs following that should be trimmed: + line 2: + line 3: +*/ +} + +var _ = []T{/* lone comment */} + +var _ = []T{ +/* lone comment */ +} + +var _ = []T{ +// lone comments +// in composite lit +} + +var _ = [][]T{ + { + // lone comments + // in composite lit + }, +} + +// TODO: gofmt doesn't add these tabs; make it so that these golden +// tests run the printer in a way that it's exactly like gofmt. + +var _ = []T{// lone comment +} + +var _ = []T{// lone comments +// in composite lit +} + +func _() {} + +// +func _() {} + +/* This comment is the last entry in this file. It must be printed and should be followed by a newline */ diff --git a/src/go/printer/testdata/comments.x b/src/go/printer/testdata/comments.x new file mode 100644 index 0000000..5d088ab --- /dev/null +++ b/src/go/printer/testdata/comments.x @@ -0,0 +1,55 @@ +// This is a package for testing comment placement by go/printer. +package main + +// The SZ struct; it is empty. +type SZ struct{} + +// The S0 struct; no field is exported. +type S0 struct { + // contains filtered or unexported fields +} + +// The S1 struct; some fields are not exported. +type S1 struct { + S0 + A, B, C float // 3 exported fields + D int // 2 unexported fields + // contains filtered or unexported fields +} + +// The S2 struct; all fields are exported. +type S2 struct { + S1 + A, B, C float // 3 exported fields +} + +// The IZ interface; it is empty. +type SZ interface{} + +// The I0 interface; no method is exported. +type I0 interface { + // contains filtered or unexported methods +} + +// The I1 interface; some methods are not exported. +type I1 interface { + I0 + F(x float) float // exported methods + // contains filtered or unexported methods +} + +// The I2 interface; all methods are exported. +type I2 interface { + I0 + F(x float) float // exported method + G(x float) float // exported method +} + +// The S3 struct; all comments except for the last one must appear in the export. +type S3 struct { + // lead comment for F1 + F1 int // line comment for F1 + // lead comment for F2 + F2 int // line comment for F2 + // contains filtered or unexported fields +} diff --git a/src/go/printer/testdata/comments2.golden b/src/go/printer/testdata/comments2.golden new file mode 100644 index 0000000..83213d1 --- /dev/null +++ b/src/go/printer/testdata/comments2.golden @@ -0,0 +1,163 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This is a package for testing comment placement by go/printer. +package main + +// Test cases for idempotent comment formatting (was issue 1835). +/* +c1a +*/ +/* + c1b +*/ +/* foo +c1c +*/ +/* foo + c1d +*/ +/* +c1e +foo */ +/* + c1f + foo */ + +func f() { + /* + c2a + */ + /* + c2b + */ + /* foo + c2c + */ + /* foo + c2d + */ + /* + c2e + foo */ + /* + c2f + foo */ +} + +func g() { + /* + c3a + */ + /* + c3b + */ + /* foo + c3c + */ + /* foo + c3d + */ + /* + c3e + foo */ + /* + c3f + foo */ +} + +// Test case taken literally from issue 1835. +func main() { + /* + prints test 5 times + */ + for i := 0; i < 5; i++ { + println("test") + } +} + +func issue5623() { +L: + _ = yyyyyyyyyyyyyyyy // comment - should be aligned + _ = xxxxxxxxxxxxxxxxxxxxxxxxxxxx /* comment */ + + _ = yyyyyyyyyyyyyyyy /* comment - should be aligned */ + _ = xxxxxxxxxxxxxxxxxxxxxxxxxxxx // comment + +LLLLLLL: + _ = yyyyyyyyyyyyyyyy // comment - should be aligned + _ = xxxxxxxxxxxxxxxxxxxxxxxxxxxx // comment + +LL: +LLLLL: + _ = xxxxxxxxxxxxxxxxxxxxxxxxxxxx /* comment */ + _ = yyyyyyyyyyyyyyyy /* comment - should be aligned */ + + _ = xxxxxxxxxxxxxxxxxxxxxxxxxxxx // comment + _ = yyyyyyyyyyyyyyyy // comment - should be aligned + + // test case from issue +label: + mask := uint64(1)<>4 + _ = "foo" + s + _ = s + "foo" + _ = 'a' + 'b' + _ = len(s) / 2 + _ = len(t0.x) / a + + // spaces around expressions of different precedence or expressions containing spaces + _ = a + -b + _ = a - ^b + _ = a / *p + _ = a + b*c + _ = 1 + b*c + _ = a + 2*c + _ = a + c*2 + _ = 1 + 2*3 + _ = s[1 : 2*3] + _ = s[a : b-c] + _ = s[0:] + _ = s[a+b] + _ = s[:b-c] + _ = s[a+b:] + _ = a[a< b + _ = a >= b + _ = a < b + _ = a <= b + _ = a < b && c > d + _ = a < b || c > d + + // spaces around "long" operands + _ = a + longIdentifier1 + _ = longIdentifier1 + a + _ = longIdentifier1 + longIdentifier2*longIdentifier3 + _ = s + "a longer string" + + // some selected cases + _ = a + t0.x + _ = a + t0.x + t1.x*t2.x + _ = a + b + c + d + e + 2*3 + _ = a + b + c + 2*3 + d + e + _ = (a + b + c) * 2 + _ = a - b + c - d + (a + b + c) + d&e + _ = under_bar - 1 + _ = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666) + _ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx) + + // test case for issue 8021 + // want: + // ([]bool{})[([]int{})[((1)+(((1)+((((1)*(((1)+(1))+(1)))+(1))*(1)))+(1)))]] + _ = ([]bool{})[([]int{})[((1)+(((1)+((((1)*(((1)+(1))+(1)))+(1))*(1)))+(1)))]] + + // the parser does not restrict expressions that may appear as statements + true + 42 + "foo" + x + (x) + a + b + a + b + c + a + (b * c) + a + (b / c) + 1 + a + a + 1 + s[a] + x << 1 + (s[0] << 1) & 0xf + "foo" + s + x == y + x < y || z > 42 +} + +// slice expressions with cap +func _() { + _ = x[a:b:c] + _ = x[a : b : c+d] + _ = x[a : b+d : c] + _ = x[a : b+d : c+d] + _ = x[a+d : b : c] + _ = x[a+d : b : c+d] + _ = x[a+d : b+d : c] + _ = x[a+d : b+d : c+d] + + _ = x[:b:c] + _ = x[: b : c+d] + _ = x[: b+d : c] + _ = x[: b+d : c+d] +} + +func issue22111() { + _ = x[:] + + _ = x[:b] + _ = x[:b+1] + + _ = x[a:] + _ = x[a+1:] + + _ = x[a:b] + _ = x[a+1 : b] + _ = x[a : b+1] + _ = x[a+1 : b+1] + + _ = x[:b:c] + _ = x[: b+1 : c] + _ = x[: b : c+1] + _ = x[: b+1 : c+1] + + _ = x[a:b:c] + _ = x[a+1 : b : c] + _ = x[a : b+1 : c] + _ = x[a+1 : b+1 : c] + _ = x[a : b : c+1] + _ = x[a+1 : b : c+1] + _ = x[a : b+1 : c+1] + _ = x[a+1 : b+1 : c+1] +} + +func _() { + _ = a + b + _ = a + b + c + _ = a + b*c + _ = a + (b * c) + _ = (a + b) * c + _ = a + (b * c * d) + _ = a + (b*c + d) + + _ = 1 << x + _ = -1 << x + _ = 1<>4 + + b.buf = b.buf[0 : b.off+m+n] + b.buf = b.buf[0 : b.off+m*n] + f(b.buf[0 : b.off+m+n]) + + signed += ' ' * 8 + tw.octal(header[148:155], chksum) + + _ = x > 0 && i >= 0 + + x1, x0 := x>>w2, x&m2 + z0 = t1<>w2) >> w2 + q1, r1 := x1/d1, x1%d1 + r1 = r1*b2 | x0>>w2 + x1 = (x1 << z) | (x0 >> (uint(w) - z)) + x1 = x1<>(uint(w)-z) + + _ = buf[0 : len(buf)+1] + _ = buf[0 : n+1] + + a, b = b, a + a = b + c + a = b*c + d + _ = a*b + c + _ = a - b - c + _ = a - (b - c) + _ = a - b*c + _ = a - (b * c) + _ = a * b / c + _ = a / *b + _ = x[a|^b] + _ = x[a / *b] + _ = a & ^b + _ = a + +b + _ = a - -b + _ = x[a*-b] + _ = x[a + +b] + _ = x ^ y ^ z + _ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF] + _ = len(longVariableName) * 2 + + _ = token(matchType + xlength<> 4 + _ = "foo"+s + _ = s+"foo" + _ = 'a'+'b' + _ = len(s)/2 + _ = len(t0.x)/a + + // spaces around expressions of different precedence or expressions containing spaces + _ = a + -b + _ = a - ^b + _ = a / *p + _ = a + b*c + _ = 1 + b*c + _ = a + 2*c + _ = a + c*2 + _ = 1 + 2*3 + _ = s[1 : 2*3] + _ = s[a : b-c] + _ = s[0:] + _ = s[a+b] + _ = s[: b-c] + _ = s[a+b :] + _ = a[a< b + _ = a >= b + _ = a < b + _ = a <= b + _ = a < b && c > d + _ = a < b || c > d + + // spaces around "long" operands + _ = a + longIdentifier1 + _ = longIdentifier1 + a + _ = longIdentifier1 + longIdentifier2 * longIdentifier3 + _ = s + "a longer string" + + // some selected cases + _ = a + t0.x + _ = a + t0.x + t1.x * t2.x + _ = a + b + c + d + e + 2*3 + _ = a + b + c + 2*3 + d + e + _ = (a+b+c)*2 + _ = a - b + c - d + (a+b+c) + d&e + _ = under_bar-1 + _ = Open(dpath + "/file", O_WRONLY | O_CREAT, 0666) + _ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx) + + // test case for issue 8021 + // want: + // ([]bool{})[([]int{})[((1)+(((1)+((((1)*(((1)+(1))+(1)))+(1))*(1)))+(1)))]] + _ = ([]bool{})[([]int{})[((1) + (((((1) + (((((((1) * (((((1) + (1))) + (1))))) + (1))) * (1))))) + (1))))]] + + // the parser does not restrict expressions that may appear as statements + true + 42 + "foo" + x + (x) + a+b + a+b+c + a+(b*c) + a+(b/c) + 1+a + a+1 + s[a] + x<<1 + (s[0]<<1)&0xf + "foo"+s + x == y + x < y || z > 42 +} + + +// slice expressions with cap +func _() { + _ = x[a:b:c] + _ = x[a:b:c+d] + _ = x[a:b+d:c] + _ = x[a:b+d:c+d] + _ = x[a+d:b:c] + _ = x[a+d:b:c+d] + _ = x[a+d:b+d:c] + _ = x[a+d:b+d:c+d] + + _ = x[:b:c] + _ = x[:b:c+d] + _ = x[:b+d:c] + _ = x[:b+d:c+d] +} + +func issue22111() { + _ = x[:] + + _ = x[:b] + _ = x[:b+1] + + _ = x[a:] + _ = x[a+1:] + + _ = x[a:b] + _ = x[a+1:b] + _ = x[a:b+1] + _ = x[a+1:b+1] + + _ = x[:b:c] + _ = x[:b+1:c] + _ = x[:b:c+1] + _ = x[:b+1:c+1] + + _ = x[a:b:c] + _ = x[a+1:b:c] + _ = x[a:b+1:c] + _ = x[a+1:b+1:c] + _ = x[a:b:c+1] + _ = x[a+1:b:c+1] + _ = x[a:b+1:c+1] + _ = x[a+1:b+1:c+1] +} + +func _() { + _ = a+b + _ = a+b+c + _ = a+b*c + _ = a+(b*c) + _ = (a+b)*c + _ = a+(b*c*d) + _ = a+(b*c+d) + + _ = 1<>4 + + b.buf = b.buf[0:b.off+m+n] + b.buf = b.buf[0:b.off+m*n] + f(b.buf[0:b.off+m+n]) + + signed += ' '*8 + tw.octal(header[148:155], chksum) + + _ = x > 0 && i >= 0 + + x1, x0 := x>>w2, x&m2 + z0 = t1<>w2)>>w2 + q1, r1 := x1/d1, x1%d1 + r1 = r1*b2 | x0>>w2 + x1 = (x1<>(uint(w)-z)) + x1 = x1<>(uint(w)-z) + + _ = buf[0:len(buf)+1] + _ = buf[0:n+1] + + a,b = b,a + a = b+c + a = b*c+d + _ = a*b+c + _ = a-b-c + _ = a-(b-c) + _ = a-b*c + _ = a-(b*c) + _ = a*b/c + _ = a/ *b + _ = x[a|^b] + _ = x[a/ *b] + _ = a& ^b + _ = a+ +b + _ = a- -b + _ = x[a*-b] + _ = x[a+ +b] + _ = x^y^z + _ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF] + _ = len(longVariableName)*2 + + _ = token(matchType + xlength<>4 + _ = "foo" + s + _ = s + "foo" + _ = 'a' + 'b' + _ = len(s) / 2 + _ = len(t0.x) / a + + // spaces around expressions of different precedence or expressions containing spaces + _ = a + -b + _ = a - ^b + _ = a / *p + _ = a + b*c + _ = 1 + b*c + _ = a + 2*c + _ = a + c*2 + _ = 1 + 2*3 + _ = s[1 : 2*3] + _ = s[a : b-c] + _ = s[0:] + _ = s[a+b] + _ = s[:b-c] + _ = s[a+b:] + _ = a[a< b + _ = a >= b + _ = a < b + _ = a <= b + _ = a < b && c > d + _ = a < b || c > d + + // spaces around "long" operands + _ = a + longIdentifier1 + _ = longIdentifier1 + a + _ = longIdentifier1 + longIdentifier2*longIdentifier3 + _ = s + "a longer string" + + // some selected cases + _ = a + t0.x + _ = a + t0.x + t1.x*t2.x + _ = a + b + c + d + e + 2*3 + _ = a + b + c + 2*3 + d + e + _ = (a + b + c) * 2 + _ = a - b + c - d + (a + b + c) + d&e + _ = under_bar - 1 + _ = Open(dpath+"/file", O_WRONLY|O_CREAT, 0666) + _ = int(c0&_Mask4)<<18 | int(c1&_Maskx)<<12 | int(c2&_Maskx)<<6 | int(c3&_Maskx) + + // test case for issue 8021 + // want: + // ([]bool{})[([]int{})[((1)+(((1)+((((1)*(((1)+(1))+(1)))+(1))*(1)))+(1)))]] + _ = ([]bool{})[([]int{})[((1)+(((1)+((((1)*(((1)+(1))+(1)))+(1))*(1)))+(1)))]] + + // the parser does not restrict expressions that may appear as statements + true + 42 + "foo" + x + (x) + a + b + a + b + c + a + (b * c) + a + (b / c) + 1 + a + a + 1 + s[a] + x << 1 + (s[0] << 1) & 0xf + "foo" + s + x == y + x < y || z > 42 +} + +// slice expressions with cap +func _() { + _ = x[a:b:c] + _ = x[a : b : c+d] + _ = x[a : b+d : c] + _ = x[a : b+d : c+d] + _ = x[a+d : b : c] + _ = x[a+d : b : c+d] + _ = x[a+d : b+d : c] + _ = x[a+d : b+d : c+d] + + _ = x[:b:c] + _ = x[: b : c+d] + _ = x[: b+d : c] + _ = x[: b+d : c+d] +} + +func issue22111() { + _ = x[:] + + _ = x[:b] + _ = x[:b+1] + + _ = x[a:] + _ = x[a+1:] + + _ = x[a:b] + _ = x[a+1 : b] + _ = x[a : b+1] + _ = x[a+1 : b+1] + + _ = x[:b:c] + _ = x[: b+1 : c] + _ = x[: b : c+1] + _ = x[: b+1 : c+1] + + _ = x[a:b:c] + _ = x[a+1 : b : c] + _ = x[a : b+1 : c] + _ = x[a+1 : b+1 : c] + _ = x[a : b : c+1] + _ = x[a+1 : b : c+1] + _ = x[a : b+1 : c+1] + _ = x[a+1 : b+1 : c+1] +} + +func _() { + _ = a + b + _ = a + b + c + _ = a + b*c + _ = a + (b * c) + _ = (a + b) * c + _ = a + (b * c * d) + _ = a + (b*c + d) + + _ = 1 << x + _ = -1 << x + _ = 1<>4 + + b.buf = b.buf[0 : b.off+m+n] + b.buf = b.buf[0 : b.off+m*n] + f(b.buf[0 : b.off+m+n]) + + signed += ' ' * 8 + tw.octal(header[148:155], chksum) + + _ = x > 0 && i >= 0 + + x1, x0 := x>>w2, x&m2 + z0 = t1<>w2) >> w2 + q1, r1 := x1/d1, x1%d1 + r1 = r1*b2 | x0>>w2 + x1 = (x1 << z) | (x0 >> (uint(w) - z)) + x1 = x1<>(uint(w)-z) + + _ = buf[0 : len(buf)+1] + _ = buf[0 : n+1] + + a, b = b, a + a = b + c + a = b*c + d + _ = a*b + c + _ = a - b - c + _ = a - (b - c) + _ = a - b*c + _ = a - (b * c) + _ = a * b / c + _ = a / *b + _ = x[a|^b] + _ = x[a / *b] + _ = a & ^b + _ = a + +b + _ = a - -b + _ = x[a*-b] + _ = x[a + +b] + _ = x ^ y ^ z + _ = b[a>>24] ^ b[(a>>16)&0xFF] ^ b[(a>>8)&0xFF] ^ b[a&0xFF] + _ = len(longVariableName) * 2 + + _ = token(matchType + xlength< /tmp/16gig.txt + // tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar + &writerTest{ + file: "testdata/writer-big.tar", + entries: []*writerTestEntry{ + &writerTestEntry{ + header: &Header{ + Name: "tmp/16gig.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 16 << 30, + Mtime: 1254699560, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + // no contents + }, + }, + }, +} + +type untarTest struct { + file string + headers []*Header +} + +var untarTests = []*untarTest{ + &untarTest{ + file: "testdata/gnu.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244428340, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244436044, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + }, + }, + &untarTest{ + file: "testdata/star.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244592783, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + Atime: 1244592783, + Ctime: 1244592783, + }, + &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244592783, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + Atime: 1244592783, + Ctime: 1244592783, + }, + }, + }, + &untarTest{ + file: "testdata/v7.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0444, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244593104, + Typeflag: '\x00', + }, + &Header{ + Name: "small2.txt", + Mode: 0444, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244593104, + Typeflag: '\x00', + }, + }, + }, +} + +var facts = map[int]string{ + 0: "1", + 1: "1", + 2: "2", + 10: "3628800", + 20: "2432902008176640000", + 100: "933262154439441526816992388562667004907159682643816214685929" + + "638952175999932299156089414639761565182862536979208272237582" + + "51185210916864000000000000000000000000", +} + +func usage() { + fmt.Fprintf(os.Stderr, + // TODO(gri): the 2nd string of this string list should not be indented + "usage: godoc package [name ...]\n"+ + " godoc -http=:6060\n") + flag.PrintDefaults() + os.Exit(2) +} + +func TestReader(t *testing.T) { +testLoop: + for i, test := range untarTests { + f, err := os.Open(test.file, os.O_RDONLY, 0444) + if err != nil { + t.Errorf("test %d: Unexpected error: %v", i, err) + continue + } + tr := NewReader(f) + for j, header := range test.headers { + hdr, err := tr.Next() + if err != nil || hdr == nil { + t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err) + f.Close() + continue testLoop + } + if !reflect.DeepEqual(hdr, header) { + t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v", + i, j, *hdr, *header) + } + } + hdr, err := tr.Next() + if hdr != nil || err != nil { + t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, err) + } + f.Close() + } +} + +// Respect line breaks in function calls. +func _() { + f(x) + f(x, + x) + f(x, + x, + ) + f( + x, + x) + f( + x, + x, + ) +} + +// Respect line breaks in function declarations. +func _(x T) {} +func _(x T, + y T) { +} +func _(x T, + y T, +) { +} +func _( + x T, + y T) { +} +func _( + x T, + y T, +) { +} + +// Example from issue #2597. +func ManageStatus0( + in <-chan *Status, + req <-chan Request, + stat chan<- *TargetInfo, + TargetHistorySize int) { +} + +func ManageStatus1( + in <-chan *Status, + req <-chan Request, + stat chan<- *TargetInfo, + TargetHistorySize int, +) { +} + +// Example from issue #9064. +func (y *y) xerrors() error { + _ = "xerror.test" //TODO- + _ = []byte(` +foo bar foo bar foo bar +`) //TODO- +} + +func _() { + _ = "abc" // foo + _ = `abc_0123456789_` // foo +} + +func _() { + _ = "abc" // foo + _ = `abc +0123456789 +` // foo +} + +// There should be exactly one linebreak after this comment. diff --git a/src/go/printer/testdata/linebreaks.input b/src/go/printer/testdata/linebreaks.input new file mode 100644 index 0000000..9e714f3 --- /dev/null +++ b/src/go/printer/testdata/linebreaks.input @@ -0,0 +1,291 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package linebreaks + +import ( + "bytes" + "fmt" + "io" + "os" + "reflect" + "strings" + "testing" +) + +type writerTestEntry struct { + header *Header + contents string +} + +type writerTest struct { + file string // filename of expected output + entries []*writerTestEntry +} + +var writerTests = []*writerTest{ + &writerTest{ + file: "testdata/writer.tar", + entries: []*writerTestEntry{ + &writerTestEntry{ + header: &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1246508266, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + contents: "Kilts", + }, + &writerTestEntry{ + header: &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1245217492, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + contents: "Google.com\n", + }, + }, + }, + // The truncated test file was produced using these commands: + // dd if=/dev/zero bs=1048576 count=16384 > /tmp/16gig.txt + // tar -b 1 -c -f- /tmp/16gig.txt | dd bs=512 count=8 > writer-big.tar + &writerTest{ + file: "testdata/writer-big.tar", + entries: []*writerTestEntry{ + &writerTestEntry{ + header: &Header{ + Name: "tmp/16gig.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 16 << 30, + Mtime: 1254699560, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + // no contents + }, + }, + }, +} + +type untarTest struct { + file string + headers []*Header +} + +var untarTests = []*untarTest{ + &untarTest{ + file: "testdata/gnu.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244428340, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244436044, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + }, + }, + }, + &untarTest{ + file: "testdata/star.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244592783, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + Atime: 1244592783, + Ctime: 1244592783, + }, + &Header{ + Name: "small2.txt", + Mode: 0640, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244592783, + Typeflag: '0', + Uname: "dsymonds", + Gname: "eng", + Atime: 1244592783, + Ctime: 1244592783, + }, + }, + }, + &untarTest{ + file: "testdata/v7.tar", + headers: []*Header{ + &Header{ + Name: "small.txt", + Mode: 0444, + Uid: 73025, + Gid: 5000, + Size: 5, + Mtime: 1244593104, + Typeflag: '\x00', + }, + &Header{ + Name: "small2.txt", + Mode: 0444, + Uid: 73025, + Gid: 5000, + Size: 11, + Mtime: 1244593104, + Typeflag: '\x00', + }, + }, + }, +} + +var facts = map[int] string { + 0: "1", + 1: "1", + 2: "2", + 10: "3628800", + 20: "2432902008176640000", + 100: "933262154439441526816992388562667004907159682643816214685929" + + "638952175999932299156089414639761565182862536979208272237582" + + "51185210916864000000000000000000000000", +} + +func usage() { + fmt.Fprintf(os.Stderr, + // TODO(gri): the 2nd string of this string list should not be indented + "usage: godoc package [name ...]\n" + + " godoc -http=:6060\n") + flag.PrintDefaults() + os.Exit(2) +} + +func TestReader(t *testing.T) { +testLoop: + for i, test := range untarTests { + f, err := os.Open(test.file, os.O_RDONLY, 0444) + if err != nil { + t.Errorf("test %d: Unexpected error: %v", i, err) + continue + } + tr := NewReader(f) + for j, header := range test.headers { + hdr, err := tr.Next() + if err != nil || hdr == nil { + t.Errorf("test %d, entry %d: Didn't get entry: %v", i, j, err) + f.Close() + continue testLoop + } + if !reflect.DeepEqual(hdr, header) { + t.Errorf("test %d, entry %d: Incorrect header:\nhave %+v\nwant %+v", + i, j, *hdr, *header) + } + } + hdr, err := tr.Next() + if hdr != nil || err != nil { + t.Errorf("test %d: Unexpected entry or error: hdr=%v err=%v", i, err) + } + f.Close() + } +} + +// Respect line breaks in function calls. +func _() { + f(x) + f(x, + x) + f(x, + x, + ) + f( + x, + x) + f( + x, + x, + ) +} + +// Respect line breaks in function declarations. +func _(x T) {} +func _(x T, + y T) {} +func _(x T, + y T, +) {} +func _( + x T, + y T) {} +func _( + x T, + y T, +) {} + +// Example from issue #2597. +func ManageStatus0( + in <-chan *Status, + req <-chan Request, + stat chan<- *TargetInfo, + TargetHistorySize int) { +} + +func ManageStatus1( + in <-chan *Status, + req <-chan Request, + stat chan<- *TargetInfo, + TargetHistorySize int, +) { +} + +// Example from issue #9064. +func (y *y) xerrors() error { + _ = "xerror.test" //TODO- + _ = []byte(` +foo bar foo bar foo bar +`) //TODO- +} + +func _() { + _ = "abc" // foo + _ = `abc_0123456789_` // foo +} + +func _() { + _ = "abc" // foo + _ = `abc +0123456789 +` // foo +} + +// There should be exactly one linebreak after this comment. diff --git a/src/go/printer/testdata/parser.go b/src/go/printer/testdata/parser.go new file mode 100644 index 0000000..bb06c8d --- /dev/null +++ b/src/go/printer/testdata/parser.go @@ -0,0 +1,2148 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package parser implements a parser for Go source files. Input may be +// provided in a variety of forms (see the various Parse* functions); the +// output is an abstract syntax tree (AST) representing the Go source. The +// parser is invoked through one of the Parse* functions. + +package parser + +import ( + "fmt" + "go/ast" + "go/scanner" + "go/token" +) + +// The mode parameter to the Parse* functions is a set of flags (or 0). +// They control the amount of source code parsed and other optional +// parser functionality. +const ( + PackageClauseOnly uint = 1 << iota // parsing stops after package clause + ImportsOnly // parsing stops after import declarations + ParseComments // parse comments and add them to AST + Trace // print a trace of parsed productions + DeclarationErrors // report declaration errors +) + +// The parser structure holds the parser's internal state. +type parser struct { + file *token.File + scanner.ErrorVector + scanner scanner.Scanner + + // Tracing/debugging + mode uint // parsing mode + trace bool // == (mode & Trace != 0) + indent uint // indentation used for tracing output + + // Comments + comments []*ast.CommentGroup + leadComment *ast.CommentGroup // last lead comment + lineComment *ast.CommentGroup // last line comment + + // Next token + pos token.Pos // token position + tok token.Token // one token look-ahead + lit string // token literal + + // Non-syntactic parser control + exprLev int // < 0: in control clause, >= 0: in expression + + // Ordinary identifier scopes + pkgScope *ast.Scope // pkgScope.Outer == nil + topScope *ast.Scope // top-most scope; may be pkgScope + unresolved []*ast.Ident // unresolved identifiers + imports []*ast.ImportSpec // list of imports + + // Label scope + // (maintained by open/close LabelScope) + labelScope *ast.Scope // label scope for current function + targetStack [][]*ast.Ident // stack of unresolved labels +} + +// scannerMode returns the scanner mode bits given the parser's mode bits. +func scannerMode(mode uint) uint { + var m uint = scanner.InsertSemis + if mode&ParseComments != 0 { + m |= scanner.ScanComments + } + return m +} + +func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uint) { + p.file = fset.AddFile(filename, fset.Base(), len(src)) + p.scanner.Init(p.file, src, p, scannerMode(mode)) + + p.mode = mode + p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently) + + p.next() + + // set up the pkgScope here (as opposed to in parseFile) because + // there are other parser entry points (ParseExpr, etc.) + p.openScope() + p.pkgScope = p.topScope + + // for the same reason, set up a label scope + p.openLabelScope() +} + +// ---------------------------------------------------------------------------- +// Scoping support + +func (p *parser) openScope() { + p.topScope = ast.NewScope(p.topScope) +} + +func (p *parser) closeScope() { + p.topScope = p.topScope.Outer +} + +func (p *parser) openLabelScope() { + p.labelScope = ast.NewScope(p.labelScope) + p.targetStack = append(p.targetStack, nil) +} + +func (p *parser) closeLabelScope() { + // resolve labels + n := len(p.targetStack) - 1 + scope := p.labelScope + for _, ident := range p.targetStack[n] { + ident.Obj = scope.Lookup(ident.Name) + if ident.Obj == nil && p.mode&DeclarationErrors != 0 { + p.error(ident.Pos(), fmt.Sprintf("label %s undefined", ident.Name)) + } + } + // pop label scope + p.targetStack = p.targetStack[0:n] + p.labelScope = p.labelScope.Outer +} + +func (p *parser) declare(decl any, scope *ast.Scope, kind ast.ObjKind, idents ...*ast.Ident) { + for _, ident := range idents { + assert(ident.Obj == nil, "identifier already declared or resolved") + if ident.Name != "_" { + obj := ast.NewObj(kind, ident.Name) + // remember the corresponding declaration for redeclaration + // errors and global variable resolution/typechecking phase + obj.Decl = decl + if alt := scope.Insert(obj); alt != nil && p.mode&DeclarationErrors != 0 { + prevDecl := "" + if pos := alt.Pos(); pos.IsValid() { + prevDecl = fmt.Sprintf("\n\tprevious declaration at %s", p.file.Position(pos)) + } + p.error(ident.Pos(), fmt.Sprintf("%s redeclared in this block%s", ident.Name, prevDecl)) + } + ident.Obj = obj + } + } +} + +func (p *parser) shortVarDecl(idents []*ast.Ident) { + // Go spec: A short variable declaration may redeclare variables + // provided they were originally declared in the same block with + // the same type, and at least one of the non-blank variables is new. + n := 0 // number of new variables + for _, ident := range idents { + assert(ident.Obj == nil, "identifier already declared or resolved") + if ident.Name != "_" { + obj := ast.NewObj(ast.Var, ident.Name) + // short var declarations cannot have redeclaration errors + // and are not global => no need to remember the respective + // declaration + alt := p.topScope.Insert(obj) + if alt == nil { + n++ // new declaration + alt = obj + } + ident.Obj = alt + } + } + if n == 0 && p.mode&DeclarationErrors != 0 { + p.error(idents[0].Pos(), "no new variables on left side of :=") + } +} + +// The unresolved object is a sentinel to mark identifiers that have been added +// to the list of unresolved identifiers. The sentinel is only used for verifying +// internal consistency. +var unresolved = new(ast.Object) + +func (p *parser) resolve(x ast.Expr) { + // nothing to do if x is not an identifier or the blank identifier + ident, _ := x.(*ast.Ident) + if ident == nil { + return + } + assert(ident.Obj == nil, "identifier already declared or resolved") + if ident.Name == "_" { + return + } + // try to resolve the identifier + for s := p.topScope; s != nil; s = s.Outer { + if obj := s.Lookup(ident.Name); obj != nil { + ident.Obj = obj + return + } + } + // all local scopes are known, so any unresolved identifier + // must be found either in the file scope, package scope + // (perhaps in another file), or universe scope --- collect + // them so that they can be resolved later + ident.Obj = unresolved + p.unresolved = append(p.unresolved, ident) +} + +// ---------------------------------------------------------------------------- +// Parsing support + +func (p *parser) printTrace(a ...any) { + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + + ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + const n = uint(len(dots)) + pos := p.file.Position(p.pos) + fmt.Printf("%5d:%3d: ", pos.Line, pos.Column) + i := 2 * p.indent + for ; i > n; i -= n { + fmt.Print(dots) + } + fmt.Print(dots[0:i]) + fmt.Println(a...) +} + +func trace(p *parser, msg string) *parser { + p.printTrace(msg, "(") + p.indent++ + return p +} + +// Usage pattern: defer un(trace(p, "...")); +func un(p *parser) { + p.indent-- + p.printTrace(")") +} + +// Advance to the next token. +func (p *parser) next0() { + // Because of one-token look-ahead, print the previous token + // when tracing as it provides a more readable output. The + // very first token (!p.pos.IsValid()) is not initialized + // (it is token.ILLEGAL), so don't print it. + if p.trace && p.pos.IsValid() { + s := p.tok.String() + switch { + case p.tok.IsLiteral(): + p.printTrace(s, p.lit) + case p.tok.IsOperator(), p.tok.IsKeyword(): + p.printTrace("\"" + s + "\"") + default: + p.printTrace(s) + } + } + + p.pos, p.tok, p.lit = p.scanner.Scan() +} + +// Consume a comment and return it and the line on which it ends. +func (p *parser) consumeComment() (comment *ast.Comment, endline int) { + // /*-style comments may end on a different line than where they start. + // Scan the comment for '\n' chars and adjust endline accordingly. + endline = p.file.Line(p.pos) + if p.lit[1] == '*' { + // don't use range here - no need to decode Unicode code points + for i := 0; i < len(p.lit); i++ { + if p.lit[i] == '\n' { + endline++ + } + } + } + + comment = &ast.Comment{p.pos, p.lit} + p.next0() + + return +} + +// Consume a group of adjacent comments, add it to the parser's +// comments list, and return it together with the line at which +// the last comment in the group ends. An empty line or non-comment +// token terminates a comment group. +func (p *parser) consumeCommentGroup() (comments *ast.CommentGroup, endline int) { + var list []*ast.Comment + endline = p.file.Line(p.pos) + for p.tok == token.COMMENT && endline+1 >= p.file.Line(p.pos) { + var comment *ast.Comment + comment, endline = p.consumeComment() + list = append(list, comment) + } + + // add comment group to the comments list + comments = &ast.CommentGroup{list} + p.comments = append(p.comments, comments) + + return +} + +// Advance to the next non-comment token. In the process, collect +// any comment groups encountered, and remember the last lead and +// line comments. +// +// A lead comment is a comment group that starts and ends in a +// line without any other tokens and that is followed by a non-comment +// token on the line immediately after the comment group. +// +// A line comment is a comment group that follows a non-comment +// token on the same line, and that has no tokens after it on the line +// where it ends. +// +// Lead and line comments may be considered documentation that is +// stored in the AST. +func (p *parser) next() { + p.leadComment = nil + p.lineComment = nil + line := p.file.Line(p.pos) // current line + p.next0() + + if p.tok == token.COMMENT { + var comment *ast.CommentGroup + var endline int + + if p.file.Line(p.pos) == line { + // The comment is on same line as the previous token; it + // cannot be a lead comment but may be a line comment. + comment, endline = p.consumeCommentGroup() + if p.file.Line(p.pos) != endline { + // The next token is on a different line, thus + // the last comment group is a line comment. + p.lineComment = comment + } + } + + // consume successor comments, if any + endline = -1 + for p.tok == token.COMMENT { + comment, endline = p.consumeCommentGroup() + } + + if endline+1 == p.file.Line(p.pos) { + // The next token is following on the line immediately after the + // comment group, thus the last comment group is a lead comment. + p.leadComment = comment + } + } +} + +func (p *parser) error(pos token.Pos, msg string) { + p.Error(p.file.Position(pos), msg) +} + +func (p *parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + if p.tok == token.SEMICOLON && p.lit[0] == '\n' { + msg += ", found newline" + } else { + msg += ", found '" + p.tok.String() + "'" + if p.tok.IsLiteral() { + msg += " " + p.lit + } + } + } + p.error(pos, msg) +} + +func (p *parser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress + return pos +} + +func (p *parser) expectSemi() { + if p.tok != token.RPAREN && p.tok != token.RBRACE { + p.expect(token.SEMICOLON) + } +} + +func assert(cond bool, msg string) { + if !cond { + panic("go/parser internal error: " + msg) + } +} + +// ---------------------------------------------------------------------------- +// Identifiers + +func (p *parser) parseIdent() *ast.Ident { + pos := p.pos + name := "_" + if p.tok == token.IDENT { + name = p.lit + p.next() + } else { + p.expect(token.IDENT) // use expect() error handling + } + return &ast.Ident{pos, name, nil} +} + +func (p *parser) parseIdentList() (list []*ast.Ident) { + if p.trace { + defer un(trace(p, "IdentList")) + } + + list = append(list, p.parseIdent()) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseIdent()) + } + + return +} + +// ---------------------------------------------------------------------------- +// Common productions + +// If lhs is set, result list elements which are identifiers are not resolved. +func (p *parser) parseExprList(lhs bool) (list []ast.Expr) { + if p.trace { + defer un(trace(p, "ExpressionList")) + } + + list = append(list, p.parseExpr(lhs)) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseExpr(lhs)) + } + + return +} + +func (p *parser) parseLhsList() []ast.Expr { + list := p.parseExprList(true) + switch p.tok { + case token.DEFINE: + // lhs of a short variable declaration + p.shortVarDecl(p.makeIdentList(list)) + case token.COLON: + // lhs of a label declaration or a communication clause of a select + // statement (parseLhsList is not called when parsing the case clause + // of a switch statement): + // - labels are declared by the caller of parseLhsList + // - for communication clauses, if there is a stand-alone identifier + // followed by a colon, we have a syntax error; there is no need + // to resolve the identifier in that case + default: + // identifiers must be declared elsewhere + for _, x := range list { + p.resolve(x) + } + } + return list +} + +func (p *parser) parseRhsList() []ast.Expr { + return p.parseExprList(false) +} + +// ---------------------------------------------------------------------------- +// Types + +func (p *parser) parseType() ast.Expr { + if p.trace { + defer un(trace(p, "Type")) + } + + typ := p.tryType() + + if typ == nil { + pos := p.pos + p.errorExpected(pos, "type") + p.next() // make progress + return &ast.BadExpr{pos, p.pos} + } + + return typ +} + +// If the result is an identifier, it is not resolved. +func (p *parser) parseTypeName() ast.Expr { + if p.trace { + defer un(trace(p, "TypeName")) + } + + ident := p.parseIdent() + // don't resolve ident yet - it may be a parameter or field name + + if p.tok == token.PERIOD { + // ident is a package name + p.next() + p.resolve(ident) + sel := p.parseIdent() + return &ast.SelectorExpr{ident, sel} + } + + return ident +} + +func (p *parser) parseArrayType(ellipsisOk bool) ast.Expr { + if p.trace { + defer un(trace(p, "ArrayType")) + } + + lbrack := p.expect(token.LBRACK) + var len ast.Expr + if ellipsisOk && p.tok == token.ELLIPSIS { + len = &ast.Ellipsis{p.pos, nil} + p.next() + } else if p.tok != token.RBRACK { + len = p.parseRhs() + } + p.expect(token.RBRACK) + elt := p.parseType() + + return &ast.ArrayType{lbrack, len, elt} +} + +func (p *parser) makeIdentList(list []ast.Expr) []*ast.Ident { + idents := make([]*ast.Ident, len(list)) + for i, x := range list { + ident, isIdent := x.(*ast.Ident) + if !isIdent { + pos := x.(ast.Expr).Pos() + p.errorExpected(pos, "identifier") + ident = &ast.Ident{pos, "_", nil} + } + idents[i] = ident + } + return idents +} + +func (p *parser) parseFieldDecl(scope *ast.Scope) *ast.Field { + if p.trace { + defer un(trace(p, "FieldDecl")) + } + + doc := p.leadComment + + // fields + list, typ := p.parseVarList(false) + + // optional tag + var tag *ast.BasicLit + if p.tok == token.STRING { + tag = &ast.BasicLit{p.pos, p.tok, p.lit} + p.next() + } + + // analyze case + var idents []*ast.Ident + if typ != nil { + // IdentifierList Type + idents = p.makeIdentList(list) + } else { + // ["*"] TypeName (AnonymousField) + typ = list[0] // we always have at least one element + p.resolve(typ) + if n := len(list); n > 1 || !isTypeName(deref(typ)) { + pos := typ.Pos() + p.errorExpected(pos, "anonymous field") + typ = &ast.BadExpr{pos, list[n-1].End()} + } + } + + p.expectSemi() // call before accessing p.linecomment + + field := &ast.Field{doc, idents, typ, tag, p.lineComment} + p.declare(field, scope, ast.Var, idents...) + + return field +} + +func (p *parser) parseStructType() *ast.StructType { + if p.trace { + defer un(trace(p, "StructType")) + } + + pos := p.expect(token.STRUCT) + lbrace := p.expect(token.LBRACE) + scope := ast.NewScope(nil) // struct scope + var list []*ast.Field + for p.tok == token.IDENT || p.tok == token.MUL || p.tok == token.LPAREN { + // a field declaration cannot start with a '(' but we accept + // it here for more robust parsing and better error messages + // (parseFieldDecl will check and complain if necessary) + list = append(list, p.parseFieldDecl(scope)) + } + rbrace := p.expect(token.RBRACE) + + // TODO(gri): store struct scope in AST + return &ast.StructType{pos, &ast.FieldList{lbrace, list, rbrace}, false} +} + +func (p *parser) parsePointerType() *ast.StarExpr { + if p.trace { + defer un(trace(p, "PointerType")) + } + + star := p.expect(token.MUL) + base := p.parseType() + + return &ast.StarExpr{star, base} +} + +func (p *parser) tryVarType(isParam bool) ast.Expr { + if isParam && p.tok == token.ELLIPSIS { + pos := p.pos + p.next() + typ := p.tryIdentOrType(isParam) // don't use parseType so we can provide better error message + if typ == nil { + p.error(pos, "'...' parameter is missing type") + typ = &ast.BadExpr{pos, p.pos} + } + if p.tok != token.RPAREN { + p.error(pos, "can use '...' with last parameter type only") + } + return &ast.Ellipsis{pos, typ} + } + return p.tryIdentOrType(false) +} + +func (p *parser) parseVarType(isParam bool) ast.Expr { + typ := p.tryVarType(isParam) + if typ == nil { + pos := p.pos + p.errorExpected(pos, "type") + p.next() // make progress + typ = &ast.BadExpr{pos, p.pos} + } + return typ +} + +func (p *parser) parseVarList(isParam bool) (list []ast.Expr, typ ast.Expr) { + if p.trace { + defer un(trace(p, "VarList")) + } + + // a list of identifiers looks like a list of type names + for { + // parseVarType accepts any type (including parenthesized ones) + // even though the syntax does not permit them here: we + // accept them all for more robust parsing and complain + // afterwards + list = append(list, p.parseVarType(isParam)) + if p.tok != token.COMMA { + break + } + p.next() + } + + // if we had a list of identifiers, it must be followed by a type + typ = p.tryVarType(isParam) + if typ != nil { + p.resolve(typ) + } + + return +} + +func (p *parser) parseParameterList(scope *ast.Scope, ellipsisOk bool) (params []*ast.Field) { + if p.trace { + defer un(trace(p, "ParameterList")) + } + + list, typ := p.parseVarList(ellipsisOk) + if typ != nil { + // IdentifierList Type + idents := p.makeIdentList(list) + field := &ast.Field{nil, idents, typ, nil, nil} + params = append(params, field) + // Go spec: The scope of an identifier denoting a function + // parameter or result variable is the function body. + p.declare(field, scope, ast.Var, idents...) + if p.tok == token.COMMA { + p.next() + } + + for p.tok != token.RPAREN && p.tok != token.EOF { + idents := p.parseIdentList() + typ := p.parseVarType(ellipsisOk) + field := &ast.Field{nil, idents, typ, nil, nil} + params = append(params, field) + // Go spec: The scope of an identifier denoting a function + // parameter or result variable is the function body. + p.declare(field, scope, ast.Var, idents...) + if p.tok != token.COMMA { + break + } + p.next() + } + + } else { + // Type { "," Type } (anonymous parameters) + params = make([]*ast.Field, len(list)) + for i, x := range list { + p.resolve(x) + params[i] = &ast.Field{Type: x} + } + } + + return +} + +func (p *parser) parseParameters(scope *ast.Scope, ellipsisOk bool) *ast.FieldList { + if p.trace { + defer un(trace(p, "Parameters")) + } + + var params []*ast.Field + lparen := p.expect(token.LPAREN) + if p.tok != token.RPAREN { + params = p.parseParameterList(scope, ellipsisOk) + } + rparen := p.expect(token.RPAREN) + + return &ast.FieldList{lparen, params, rparen} +} + +func (p *parser) parseResult(scope *ast.Scope) *ast.FieldList { + if p.trace { + defer un(trace(p, "Result")) + } + + if p.tok == token.LPAREN { + return p.parseParameters(scope, false) + } + + typ := p.tryType() + if typ != nil { + list := make([]*ast.Field, 1) + list[0] = &ast.Field{Type: typ} + return &ast.FieldList{List: list} + } + + return nil +} + +func (p *parser) parseSignature(scope *ast.Scope) (params, results *ast.FieldList) { + if p.trace { + defer un(trace(p, "Signature")) + } + + params = p.parseParameters(scope, true) + results = p.parseResult(scope) + + return +} + +func (p *parser) parseFuncType() (*ast.FuncType, *ast.Scope) { + if p.trace { + defer un(trace(p, "FuncType")) + } + + pos := p.expect(token.FUNC) + scope := ast.NewScope(p.topScope) // function scope + params, results := p.parseSignature(scope) + + return &ast.FuncType{pos, params, results}, scope +} + +func (p *parser) parseMethodSpec(scope *ast.Scope) *ast.Field { + if p.trace { + defer un(trace(p, "MethodSpec")) + } + + doc := p.leadComment + var idents []*ast.Ident + var typ ast.Expr + x := p.parseTypeName() + if ident, isIdent := x.(*ast.Ident); isIdent && p.tok == token.LPAREN { + // method + idents = []*ast.Ident{ident} + scope := ast.NewScope(nil) // method scope + params, results := p.parseSignature(scope) + typ = &ast.FuncType{token.NoPos, params, results} + } else { + // embedded interface + typ = x + } + p.expectSemi() // call before accessing p.linecomment + + spec := &ast.Field{doc, idents, typ, nil, p.lineComment} + p.declare(spec, scope, ast.Fun, idents...) + + return spec +} + +func (p *parser) parseInterfaceType() *ast.InterfaceType { + if p.trace { + defer un(trace(p, "InterfaceType")) + } + + pos := p.expect(token.INTERFACE) + lbrace := p.expect(token.LBRACE) + scope := ast.NewScope(nil) // interface scope + var list []*ast.Field + for p.tok == token.IDENT { + list = append(list, p.parseMethodSpec(scope)) + } + rbrace := p.expect(token.RBRACE) + + // TODO(gri): store interface scope in AST + return &ast.InterfaceType{pos, &ast.FieldList{lbrace, list, rbrace}, false} +} + +func (p *parser) parseMapType() *ast.MapType { + if p.trace { + defer un(trace(p, "MapType")) + } + + pos := p.expect(token.MAP) + p.expect(token.LBRACK) + key := p.parseType() + p.expect(token.RBRACK) + value := p.parseType() + + return &ast.MapType{pos, key, value} +} + +func (p *parser) parseChanType() *ast.ChanType { + if p.trace { + defer un(trace(p, "ChanType")) + } + + pos := p.pos + dir := ast.SEND | ast.RECV + if p.tok == token.CHAN { + p.next() + if p.tok == token.ARROW { + p.next() + dir = ast.SEND + } + } else { + p.expect(token.ARROW) + p.expect(token.CHAN) + dir = ast.RECV + } + value := p.parseType() + + return &ast.ChanType{pos, dir, value} +} + +// If the result is an identifier, it is not resolved. +func (p *parser) tryIdentOrType(ellipsisOk bool) ast.Expr { + switch p.tok { + case token.IDENT: + return p.parseTypeName() + case token.LBRACK: + return p.parseArrayType(ellipsisOk) + case token.STRUCT: + return p.parseStructType() + case token.MUL: + return p.parsePointerType() + case token.FUNC: + typ, _ := p.parseFuncType() + return typ + case token.INTERFACE: + return p.parseInterfaceType() + case token.MAP: + return p.parseMapType() + case token.CHAN, token.ARROW: + return p.parseChanType() + case token.LPAREN: + lparen := p.pos + p.next() + typ := p.parseType() + rparen := p.expect(token.RPAREN) + return &ast.ParenExpr{lparen, typ, rparen} + } + + // no type found + return nil +} + +func (p *parser) tryType() ast.Expr { + typ := p.tryIdentOrType(false) + if typ != nil { + p.resolve(typ) + } + return typ +} + +// ---------------------------------------------------------------------------- +// Blocks + +func (p *parser) parseStmtList() (list []ast.Stmt) { + if p.trace { + defer un(trace(p, "StatementList")) + } + + for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF { + list = append(list, p.parseStmt()) + } + + return +} + +func (p *parser) parseBody(scope *ast.Scope) *ast.BlockStmt { + if p.trace { + defer un(trace(p, "Body")) + } + + lbrace := p.expect(token.LBRACE) + p.topScope = scope // open function scope + p.openLabelScope() + list := p.parseStmtList() + p.closeLabelScope() + p.closeScope() + rbrace := p.expect(token.RBRACE) + + return &ast.BlockStmt{lbrace, list, rbrace} +} + +func (p *parser) parseBlockStmt() *ast.BlockStmt { + if p.trace { + defer un(trace(p, "BlockStmt")) + } + + lbrace := p.expect(token.LBRACE) + p.openScope() + list := p.parseStmtList() + p.closeScope() + rbrace := p.expect(token.RBRACE) + + return &ast.BlockStmt{lbrace, list, rbrace} +} + +// ---------------------------------------------------------------------------- +// Expressions + +func (p *parser) parseFuncTypeOrLit() ast.Expr { + if p.trace { + defer un(trace(p, "FuncTypeOrLit")) + } + + typ, scope := p.parseFuncType() + if p.tok != token.LBRACE { + // function type only + return typ + } + + p.exprLev++ + body := p.parseBody(scope) + p.exprLev-- + + return &ast.FuncLit{typ, body} +} + +// parseOperand may return an expression or a raw type (incl. array +// types of the form [...]T. Callers must verify the result. +// If lhs is set and the result is an identifier, it is not resolved. +func (p *parser) parseOperand(lhs bool) ast.Expr { + if p.trace { + defer un(trace(p, "Operand")) + } + + switch p.tok { + case token.IDENT: + x := p.parseIdent() + if !lhs { + p.resolve(x) + } + return x + + case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING: + x := &ast.BasicLit{p.pos, p.tok, p.lit} + p.next() + return x + + case token.LPAREN: + lparen := p.pos + p.next() + p.exprLev++ + x := p.parseRhs() + p.exprLev-- + rparen := p.expect(token.RPAREN) + return &ast.ParenExpr{lparen, x, rparen} + + case token.FUNC: + return p.parseFuncTypeOrLit() + + default: + if typ := p.tryIdentOrType(true); typ != nil { + // could be type for composite literal or conversion + _, isIdent := typ.(*ast.Ident) + assert(!isIdent, "type cannot be identifier") + return typ + } + } + + pos := p.pos + p.errorExpected(pos, "operand") + p.next() // make progress + return &ast.BadExpr{pos, p.pos} +} + +func (p *parser) parseSelector(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "Selector")) + } + + sel := p.parseIdent() + + return &ast.SelectorExpr{x, sel} +} + +func (p *parser) parseTypeAssertion(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "TypeAssertion")) + } + + p.expect(token.LPAREN) + var typ ast.Expr + if p.tok == token.TYPE { + // type switch: typ == nil + p.next() + } else { + typ = p.parseType() + } + p.expect(token.RPAREN) + + return &ast.TypeAssertExpr{x, typ} +} + +func (p *parser) parseIndexOrSlice(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "IndexOrSlice")) + } + + lbrack := p.expect(token.LBRACK) + p.exprLev++ + var low, high ast.Expr + isSlice := false + if p.tok != token.COLON { + low = p.parseRhs() + } + if p.tok == token.COLON { + isSlice = true + p.next() + if p.tok != token.RBRACK { + high = p.parseRhs() + } + } + p.exprLev-- + rbrack := p.expect(token.RBRACK) + + if isSlice { + return &ast.SliceExpr{x, lbrack, low, high, rbrack} + } + return &ast.IndexExpr{x, lbrack, low, rbrack} +} + +func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr { + if p.trace { + defer un(trace(p, "CallOrConversion")) + } + + lparen := p.expect(token.LPAREN) + p.exprLev++ + var list []ast.Expr + var ellipsis token.Pos + for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() { + list = append(list, p.parseRhs()) + if p.tok == token.ELLIPSIS { + ellipsis = p.pos + p.next() + } + if p.tok != token.COMMA { + break + } + p.next() + } + p.exprLev-- + rparen := p.expect(token.RPAREN) + + return &ast.CallExpr{fun, lparen, list, ellipsis, rparen} +} + +func (p *parser) parseElement(keyOk bool) ast.Expr { + if p.trace { + defer un(trace(p, "Element")) + } + + if p.tok == token.LBRACE { + return p.parseLiteralValue(nil) + } + + x := p.parseExpr(keyOk) // don't resolve if map key + if keyOk { + if p.tok == token.COLON { + colon := p.pos + p.next() + return &ast.KeyValueExpr{x, colon, p.parseElement(false)} + } + p.resolve(x) // not a map key + } + + return x +} + +func (p *parser) parseElementList() (list []ast.Expr) { + if p.trace { + defer un(trace(p, "ElementList")) + } + + for p.tok != token.RBRACE && p.tok != token.EOF { + list = append(list, p.parseElement(true)) + if p.tok != token.COMMA { + break + } + p.next() + } + + return +} + +func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "LiteralValue")) + } + + lbrace := p.expect(token.LBRACE) + var elts []ast.Expr + p.exprLev++ + if p.tok != token.RBRACE { + elts = p.parseElementList() + } + p.exprLev-- + rbrace := p.expect(token.RBRACE) + return &ast.CompositeLit{typ, lbrace, elts, rbrace} +} + +// checkExpr checks that x is an expression (and not a type). +func (p *parser) checkExpr(x ast.Expr) ast.Expr { + switch t := unparen(x).(type) { + case *ast.BadExpr: + case *ast.Ident: + case *ast.BasicLit: + case *ast.FuncLit: + case *ast.CompositeLit: + case *ast.ParenExpr: + panic("unreachable") + case *ast.SelectorExpr: + case *ast.IndexExpr: + case *ast.SliceExpr: + case *ast.TypeAssertExpr: + if t.Type == nil { + // the form X.(type) is only allowed in type switch expressions + p.errorExpected(x.Pos(), "expression") + x = &ast.BadExpr{x.Pos(), x.End()} + } + case *ast.CallExpr: + case *ast.StarExpr: + case *ast.UnaryExpr: + if t.Op == token.RANGE { + // the range operator is only allowed at the top of a for statement + p.errorExpected(x.Pos(), "expression") + x = &ast.BadExpr{x.Pos(), x.End()} + } + case *ast.BinaryExpr: + default: + // all other nodes are not proper expressions + p.errorExpected(x.Pos(), "expression") + x = &ast.BadExpr{x.Pos(), x.End()} + } + return x +} + +// isTypeName reports whether x is a (qualified) TypeName. +func isTypeName(x ast.Expr) bool { + switch t := x.(type) { + case *ast.BadExpr: + case *ast.Ident: + case *ast.SelectorExpr: + _, isIdent := t.X.(*ast.Ident) + return isIdent + default: + return false // all other nodes are not type names + } + return true +} + +// isLiteralType reports whether x is a legal composite literal type. +func isLiteralType(x ast.Expr) bool { + switch t := x.(type) { + case *ast.BadExpr: + case *ast.Ident: + case *ast.SelectorExpr: + _, isIdent := t.X.(*ast.Ident) + return isIdent + case *ast.ArrayType: + case *ast.StructType: + case *ast.MapType: + default: + return false // all other nodes are not legal composite literal types + } + return true +} + +// If x is of the form *T, deref returns T, otherwise it returns x. +func deref(x ast.Expr) ast.Expr { + if p, isPtr := x.(*ast.StarExpr); isPtr { + x = p.X + } + return x +} + +// If x is of the form (T), unparen returns unparen(T), otherwise it returns x. +func unparen(x ast.Expr) ast.Expr { + if p, isParen := x.(*ast.ParenExpr); isParen { + x = unparen(p.X) + } + return x +} + +// checkExprOrType checks that x is an expression or a type +// (and not a raw type such as [...]T). +func (p *parser) checkExprOrType(x ast.Expr) ast.Expr { + switch t := unparen(x).(type) { + case *ast.ParenExpr: + panic("unreachable") + case *ast.UnaryExpr: + if t.Op == token.RANGE { + // the range operator is only allowed at the top of a for statement + p.errorExpected(x.Pos(), "expression") + x = &ast.BadExpr{x.Pos(), x.End()} + } + case *ast.ArrayType: + if len, isEllipsis := t.Len.(*ast.Ellipsis); isEllipsis { + p.error(len.Pos(), "expected array length, found '...'") + x = &ast.BadExpr{x.Pos(), x.End()} + } + } + + // all other nodes are expressions or types + return x +} + +// If lhs is set and the result is an identifier, it is not resolved. +func (p *parser) parsePrimaryExpr(lhs bool) ast.Expr { + if p.trace { + defer un(trace(p, "PrimaryExpr")) + } + + x := p.parseOperand(lhs) +L: + for { + switch p.tok { + case token.PERIOD: + p.next() + if lhs { + p.resolve(x) + } + switch p.tok { + case token.IDENT: + x = p.parseSelector(p.checkExpr(x)) + case token.LPAREN: + x = p.parseTypeAssertion(p.checkExpr(x)) + default: + pos := p.pos + p.next() // make progress + p.errorExpected(pos, "selector or type assertion") + x = &ast.BadExpr{pos, p.pos} + } + case token.LBRACK: + if lhs { + p.resolve(x) + } + x = p.parseIndexOrSlice(p.checkExpr(x)) + case token.LPAREN: + if lhs { + p.resolve(x) + } + x = p.parseCallOrConversion(p.checkExprOrType(x)) + case token.LBRACE: + if isLiteralType(x) && (p.exprLev >= 0 || !isTypeName(x)) { + if lhs { + p.resolve(x) + } + x = p.parseLiteralValue(x) + } else { + break L + } + default: + break L + } + lhs = false // no need to try to resolve again + } + + return x +} + +// If lhs is set and the result is an identifier, it is not resolved. +func (p *parser) parseUnaryExpr(lhs bool) ast.Expr { + if p.trace { + defer un(trace(p, "UnaryExpr")) + } + + switch p.tok { + case token.ADD, token.SUB, token.NOT, token.XOR, token.AND, token.RANGE: + pos, op := p.pos, p.tok + p.next() + x := p.parseUnaryExpr(false) + return &ast.UnaryExpr{pos, op, p.checkExpr(x)} + + case token.ARROW: + // channel type or receive expression + pos := p.pos + p.next() + if p.tok == token.CHAN { + p.next() + value := p.parseType() + return &ast.ChanType{pos, ast.RECV, value} + } + + x := p.parseUnaryExpr(false) + return &ast.UnaryExpr{pos, token.ARROW, p.checkExpr(x)} + + case token.MUL: + // pointer type or unary "*" expression + pos := p.pos + p.next() + x := p.parseUnaryExpr(false) + return &ast.StarExpr{pos, p.checkExprOrType(x)} + } + + return p.parsePrimaryExpr(lhs) +} + +// If lhs is set and the result is an identifier, it is not resolved. +func (p *parser) parseBinaryExpr(lhs bool, prec1 int) ast.Expr { + if p.trace { + defer un(trace(p, "BinaryExpr")) + } + + x := p.parseUnaryExpr(lhs) + for prec := p.tok.Precedence(); prec >= prec1; prec-- { + for p.tok.Precedence() == prec { + pos, op := p.pos, p.tok + p.next() + if lhs { + p.resolve(x) + lhs = false + } + y := p.parseBinaryExpr(false, prec+1) + x = &ast.BinaryExpr{p.checkExpr(x), pos, op, p.checkExpr(y)} + } + } + + return x +} + +// If lhs is set and the result is an identifier, it is not resolved. +// TODO(gri): parseExpr may return a type or even a raw type ([..]int) - +// should reject when a type/raw type is obviously not allowed +func (p *parser) parseExpr(lhs bool) ast.Expr { + if p.trace { + defer un(trace(p, "Expression")) + } + + return p.parseBinaryExpr(lhs, token.LowestPrec+1) +} + +func (p *parser) parseRhs() ast.Expr { + return p.parseExpr(false) +} + +// ---------------------------------------------------------------------------- +// Statements + +func (p *parser) parseSimpleStmt(labelOk bool) ast.Stmt { + if p.trace { + defer un(trace(p, "SimpleStmt")) + } + + x := p.parseLhsList() + + switch p.tok { + case + token.DEFINE, token.ASSIGN, token.ADD_ASSIGN, + token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN, + token.REM_ASSIGN, token.AND_ASSIGN, token.OR_ASSIGN, + token.XOR_ASSIGN, token.SHL_ASSIGN, token.SHR_ASSIGN, token.AND_NOT_ASSIGN: + // assignment statement + pos, tok := p.pos, p.tok + p.next() + y := p.parseRhsList() + return &ast.AssignStmt{x, pos, tok, y} + } + + if len(x) > 1 { + p.errorExpected(x[0].Pos(), "1 expression") + // continue with first expression + } + + switch p.tok { + case token.COLON: + // labeled statement + colon := p.pos + p.next() + if label, isIdent := x[0].(*ast.Ident); labelOk && isIdent { + // Go spec: The scope of a label is the body of the function + // in which it is declared and excludes the body of any nested + // function. + stmt := &ast.LabeledStmt{label, colon, p.parseStmt()} + p.declare(stmt, p.labelScope, ast.Lbl, label) + return stmt + } + p.error(x[0].Pos(), "illegal label declaration") + return &ast.BadStmt{x[0].Pos(), colon + 1} + + case token.ARROW: + // send statement + arrow := p.pos + p.next() // consume "<-" + y := p.parseRhs() + return &ast.SendStmt{x[0], arrow, y} + + case token.INC, token.DEC: + // increment or decrement + s := &ast.IncDecStmt{x[0], p.pos, p.tok} + p.next() // consume "++" or "--" + return s + } + + // expression + return &ast.ExprStmt{x[0]} +} + +func (p *parser) parseCallExpr() *ast.CallExpr { + x := p.parseRhs() + if call, isCall := x.(*ast.CallExpr); isCall { + return call + } + p.errorExpected(x.Pos(), "function/method call") + return nil +} + +func (p *parser) parseGoStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "GoStmt")) + } + + pos := p.expect(token.GO) + call := p.parseCallExpr() + p.expectSemi() + if call == nil { + return &ast.BadStmt{pos, pos + 2} // len("go") + } + + return &ast.GoStmt{pos, call} +} + +func (p *parser) parseDeferStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "DeferStmt")) + } + + pos := p.expect(token.DEFER) + call := p.parseCallExpr() + p.expectSemi() + if call == nil { + return &ast.BadStmt{pos, pos + 5} // len("defer") + } + + return &ast.DeferStmt{pos, call} +} + +func (p *parser) parseReturnStmt() *ast.ReturnStmt { + if p.trace { + defer un(trace(p, "ReturnStmt")) + } + + pos := p.pos + p.expect(token.RETURN) + var x []ast.Expr + if p.tok != token.SEMICOLON && p.tok != token.RBRACE { + x = p.parseRhsList() + } + p.expectSemi() + + return &ast.ReturnStmt{pos, x} +} + +func (p *parser) parseBranchStmt(tok token.Token) *ast.BranchStmt { + if p.trace { + defer un(trace(p, "BranchStmt")) + } + + pos := p.expect(tok) + var label *ast.Ident + if tok != token.FALLTHROUGH && p.tok == token.IDENT { + label = p.parseIdent() + // add to list of unresolved targets + n := len(p.targetStack) - 1 + p.targetStack[n] = append(p.targetStack[n], label) + } + p.expectSemi() + + return &ast.BranchStmt{pos, tok, label} +} + +func (p *parser) makeExpr(s ast.Stmt) ast.Expr { + if s == nil { + return nil + } + if es, isExpr := s.(*ast.ExprStmt); isExpr { + return p.checkExpr(es.X) + } + p.error(s.Pos(), "expected condition, found simple statement") + return &ast.BadExpr{s.Pos(), s.End()} +} + +func (p *parser) parseIfStmt() *ast.IfStmt { + if p.trace { + defer un(trace(p, "IfStmt")) + } + + pos := p.expect(token.IF) + p.openScope() + defer p.closeScope() + + var s ast.Stmt + var x ast.Expr + { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok == token.SEMICOLON { + p.next() + x = p.parseRhs() + } else { + s = p.parseSimpleStmt(false) + if p.tok == token.SEMICOLON { + p.next() + x = p.parseRhs() + } else { + x = p.makeExpr(s) + s = nil + } + } + p.exprLev = prevLev + } + + body := p.parseBlockStmt() + var else_ ast.Stmt + if p.tok == token.ELSE { + p.next() + else_ = p.parseStmt() + } else { + p.expectSemi() + } + + return &ast.IfStmt{pos, s, x, body, else_} +} + +func (p *parser) parseTypeList() (list []ast.Expr) { + if p.trace { + defer un(trace(p, "TypeList")) + } + + list = append(list, p.parseType()) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseType()) + } + + return +} + +func (p *parser) parseCaseClause(exprSwitch bool) *ast.CaseClause { + if p.trace { + defer un(trace(p, "CaseClause")) + } + + pos := p.pos + var list []ast.Expr + if p.tok == token.CASE { + p.next() + if exprSwitch { + list = p.parseRhsList() + } else { + list = p.parseTypeList() + } + } else { + p.expect(token.DEFAULT) + } + + colon := p.expect(token.COLON) + p.openScope() + body := p.parseStmtList() + p.closeScope() + + return &ast.CaseClause{pos, list, colon, body} +} + +func isExprSwitch(s ast.Stmt) bool { + if s == nil { + return true + } + if e, ok := s.(*ast.ExprStmt); ok { + if a, ok := e.X.(*ast.TypeAssertExpr); ok { + return a.Type != nil // regular type assertion + } + return true + } + return false +} + +func (p *parser) parseSwitchStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "SwitchStmt")) + } + + pos := p.expect(token.SWITCH) + p.openScope() + defer p.closeScope() + + var s1, s2 ast.Stmt + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + if p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.LBRACE { + s2 = p.parseSimpleStmt(false) + } + } + p.exprLev = prevLev + } + + exprSwitch := isExprSwitch(s2) + lbrace := p.expect(token.LBRACE) + var list []ast.Stmt + for p.tok == token.CASE || p.tok == token.DEFAULT { + list = append(list, p.parseCaseClause(exprSwitch)) + } + rbrace := p.expect(token.RBRACE) + p.expectSemi() + body := &ast.BlockStmt{lbrace, list, rbrace} + + if exprSwitch { + return &ast.SwitchStmt{pos, s1, p.makeExpr(s2), body} + } + // type switch + // TODO(gri): do all the checks! + return &ast.TypeSwitchStmt{pos, s1, s2, body} +} + +func (p *parser) parseCommClause() *ast.CommClause { + if p.trace { + defer un(trace(p, "CommClause")) + } + + p.openScope() + pos := p.pos + var comm ast.Stmt + if p.tok == token.CASE { + p.next() + lhs := p.parseLhsList() + if p.tok == token.ARROW { + // SendStmt + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression + } + arrow := p.pos + p.next() + rhs := p.parseRhs() + comm = &ast.SendStmt{lhs[0], arrow, rhs} + } else { + // RecvStmt + pos := p.pos + tok := p.tok + var rhs ast.Expr + if tok == token.ASSIGN || tok == token.DEFINE { + // RecvStmt with assignment + if len(lhs) > 2 { + p.errorExpected(lhs[0].Pos(), "1 or 2 expressions") + // continue with first two expressions + lhs = lhs[0:2] + } + p.next() + rhs = p.parseRhs() + } else { + // rhs must be single receive operation + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression + } + rhs = lhs[0] + lhs = nil // there is no lhs + } + if x, isUnary := rhs.(*ast.UnaryExpr); !isUnary || x.Op != token.ARROW { + p.errorExpected(rhs.Pos(), "send or receive operation") + rhs = &ast.BadExpr{rhs.Pos(), rhs.End()} + } + if lhs != nil { + comm = &ast.AssignStmt{lhs, pos, tok, []ast.Expr{rhs}} + } else { + comm = &ast.ExprStmt{rhs} + } + } + } else { + p.expect(token.DEFAULT) + } + + colon := p.expect(token.COLON) + body := p.parseStmtList() + p.closeScope() + + return &ast.CommClause{pos, comm, colon, body} +} + +func (p *parser) parseSelectStmt() *ast.SelectStmt { + if p.trace { + defer un(trace(p, "SelectStmt")) + } + + pos := p.expect(token.SELECT) + lbrace := p.expect(token.LBRACE) + var list []ast.Stmt + for p.tok == token.CASE || p.tok == token.DEFAULT { + list = append(list, p.parseCommClause()) + } + rbrace := p.expect(token.RBRACE) + p.expectSemi() + body := &ast.BlockStmt{lbrace, list, rbrace} + + return &ast.SelectStmt{pos, body} +} + +func (p *parser) parseForStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "ForStmt")) + } + + pos := p.expect(token.FOR) + p.openScope() + defer p.closeScope() + + var s1, s2, s3 ast.Stmt + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + if p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.SEMICOLON { + s2 = p.parseSimpleStmt(false) + } + p.expectSemi() + if p.tok != token.LBRACE { + s3 = p.parseSimpleStmt(false) + } + } + p.exprLev = prevLev + } + + body := p.parseBlockStmt() + p.expectSemi() + + if as, isAssign := s2.(*ast.AssignStmt); isAssign { + // possibly a for statement with a range clause; check assignment operator + if as.Tok != token.ASSIGN && as.Tok != token.DEFINE { + p.errorExpected(as.TokPos, "'=' or ':='") + return &ast.BadStmt{pos, body.End()} + } + // check lhs + var key, value ast.Expr + switch len(as.Lhs) { + case 2: + key, value = as.Lhs[0], as.Lhs[1] + case 1: + key = as.Lhs[0] + default: + p.errorExpected(as.Lhs[0].Pos(), "1 or 2 expressions") + return &ast.BadStmt{pos, body.End()} + } + // check rhs + if len(as.Rhs) != 1 { + p.errorExpected(as.Rhs[0].Pos(), "1 expression") + return &ast.BadStmt{pos, body.End()} + } + if rhs, isUnary := as.Rhs[0].(*ast.UnaryExpr); isUnary && rhs.Op == token.RANGE { + // rhs is range expression + // (any short variable declaration was handled by parseSimpleStat above) + return &ast.RangeStmt{pos, key, value, as.TokPos, as.Tok, rhs.X, body} + } + p.errorExpected(s2.Pos(), "range clause") + return &ast.BadStmt{pos, body.End()} + } + + // regular for statement + return &ast.ForStmt{pos, s1, p.makeExpr(s2), s3, body} +} + +func (p *parser) parseStmt() (s ast.Stmt) { + if p.trace { + defer un(trace(p, "Statement")) + } + + switch p.tok { + case token.CONST, token.TYPE, token.VAR: + s = &ast.DeclStmt{p.parseDecl()} + case + // tokens that may start a top-level expression + token.IDENT, token.INT, token.FLOAT, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operand + token.LBRACK, token.STRUCT, // composite type + token.MUL, token.AND, token.ARROW, token.ADD, token.SUB, token.XOR: // unary operators + s = p.parseSimpleStmt(true) + // because of the required look-ahead, labeled statements are + // parsed by parseSimpleStmt - don't expect a semicolon after + // them + if _, isLabeledStmt := s.(*ast.LabeledStmt); !isLabeledStmt { + p.expectSemi() + } + case token.GO: + s = p.parseGoStmt() + case token.DEFER: + s = p.parseDeferStmt() + case token.RETURN: + s = p.parseReturnStmt() + case token.BREAK, token.CONTINUE, token.GOTO, token.FALLTHROUGH: + s = p.parseBranchStmt(p.tok) + case token.LBRACE: + s = p.parseBlockStmt() + p.expectSemi() + case token.IF: + s = p.parseIfStmt() + case token.SWITCH: + s = p.parseSwitchStmt() + case token.SELECT: + s = p.parseSelectStmt() + case token.FOR: + s = p.parseForStmt() + case token.SEMICOLON: + s = &ast.EmptyStmt{p.pos} + p.next() + case token.RBRACE: + // a semicolon may be omitted before a closing "}" + s = &ast.EmptyStmt{p.pos} + default: + // no statement found + pos := p.pos + p.errorExpected(pos, "statement") + p.next() // make progress + s = &ast.BadStmt{pos, p.pos} + } + + return +} + +// ---------------------------------------------------------------------------- +// Declarations + +type parseSpecFunction func(p *parser, doc *ast.CommentGroup, iota int) ast.Spec + +func parseImportSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "ImportSpec")) + } + + var ident *ast.Ident + switch p.tok { + case token.PERIOD: + ident = &ast.Ident{p.pos, ".", nil} + p.next() + case token.IDENT: + ident = p.parseIdent() + } + + var path *ast.BasicLit + if p.tok == token.STRING { + path = &ast.BasicLit{p.pos, p.tok, p.lit} + p.next() + } else { + p.expect(token.STRING) // use expect() error handling + } + p.expectSemi() // call before accessing p.linecomment + + // collect imports + spec := &ast.ImportSpec{doc, ident, path, p.lineComment} + p.imports = append(p.imports, spec) + + return spec +} + +func parseConstSpec(p *parser, doc *ast.CommentGroup, iota int) ast.Spec { + if p.trace { + defer un(trace(p, "ConstSpec")) + } + + idents := p.parseIdentList() + typ := p.tryType() + var values []ast.Expr + if typ != nil || p.tok == token.ASSIGN || iota == 0 { + p.expect(token.ASSIGN) + values = p.parseRhsList() + } + p.expectSemi() // call before accessing p.linecomment + + // Go spec: The scope of a constant or variable identifier declared inside + // a function begins at the end of the ConstSpec or VarSpec and ends at + // the end of the innermost containing block. + // (Global identifiers are resolved in a separate phase after parsing.) + spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment} + p.declare(spec, p.topScope, ast.Con, idents...) + + return spec +} + +func parseTypeSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "TypeSpec")) + } + + ident := p.parseIdent() + + // Go spec: The scope of a type identifier declared inside a function begins + // at the identifier in the TypeSpec and ends at the end of the innermost + // containing block. + // (Global identifiers are resolved in a separate phase after parsing.) + spec := &ast.TypeSpec{doc, ident, nil, nil} + p.declare(spec, p.topScope, ast.Typ, ident) + + spec.Type = p.parseType() + p.expectSemi() // call before accessing p.linecomment + spec.Comment = p.lineComment + + return spec +} + +func parseVarSpec(p *parser, doc *ast.CommentGroup, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "VarSpec")) + } + + idents := p.parseIdentList() + typ := p.tryType() + var values []ast.Expr + if typ == nil || p.tok == token.ASSIGN { + p.expect(token.ASSIGN) + values = p.parseRhsList() + } + p.expectSemi() // call before accessing p.linecomment + + // Go spec: The scope of a constant or variable identifier declared inside + // a function begins at the end of the ConstSpec or VarSpec and ends at + // the end of the innermost containing block. + // (Global identifiers are resolved in a separate phase after parsing.) + spec := &ast.ValueSpec{doc, idents, typ, values, p.lineComment} + p.declare(spec, p.topScope, ast.Var, idents...) + + return spec +} + +func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.GenDecl { + if p.trace { + defer un(trace(p, "GenDecl("+keyword.String()+")")) + } + + doc := p.leadComment + pos := p.expect(keyword) + var lparen, rparen token.Pos + var list []ast.Spec + if p.tok == token.LPAREN { + lparen = p.pos + p.next() + for iota := 0; p.tok != token.RPAREN && p.tok != token.EOF; iota++ { + list = append(list, f(p, p.leadComment, iota)) + } + rparen = p.expect(token.RPAREN) + p.expectSemi() + } else { + list = append(list, f(p, nil, 0)) + } + + return &ast.GenDecl{doc, pos, keyword, lparen, list, rparen} +} + +func (p *parser) parseReceiver(scope *ast.Scope) *ast.FieldList { + if p.trace { + defer un(trace(p, "Receiver")) + } + + pos := p.pos + par := p.parseParameters(scope, false) + + // must have exactly one receiver + if par.NumFields() != 1 { + p.errorExpected(pos, "exactly one receiver") + // TODO determine a better range for BadExpr below + par.List = []*ast.Field{{Type: &ast.BadExpr{pos, pos}}} + return par + } + + // recv type must be of the form ["*"] identifier + recv := par.List[0] + base := deref(recv.Type) + if _, isIdent := base.(*ast.Ident); !isIdent { + p.errorExpected(base.Pos(), "(unqualified) identifier") + par.List = []*ast.Field{{Type: &ast.BadExpr{recv.Pos(), recv.End()}}} + } + + return par +} + +func (p *parser) parseFuncDecl() *ast.FuncDecl { + if p.trace { + defer un(trace(p, "FunctionDecl")) + } + + doc := p.leadComment + pos := p.expect(token.FUNC) + scope := ast.NewScope(p.topScope) // function scope + + var recv *ast.FieldList + if p.tok == token.LPAREN { + recv = p.parseReceiver(scope) + } + + ident := p.parseIdent() + + params, results := p.parseSignature(scope) + + var body *ast.BlockStmt + if p.tok == token.LBRACE { + body = p.parseBody(scope) + } + p.expectSemi() + + decl := &ast.FuncDecl{doc, recv, ident, &ast.FuncType{pos, params, results}, body} + if recv == nil { + // Go spec: The scope of an identifier denoting a constant, type, + // variable, or function (but not method) declared at top level + // (outside any function) is the package block. + // + // init() functions cannot be referred to and there may + // be more than one - don't put them in the pkgScope + if ident.Name != "init" { + p.declare(decl, p.pkgScope, ast.Fun, ident) + } + } + + return decl +} + +func (p *parser) parseDecl() ast.Decl { + if p.trace { + defer un(trace(p, "Declaration")) + } + + var f parseSpecFunction + switch p.tok { + case token.CONST: + f = parseConstSpec + + case token.TYPE: + f = parseTypeSpec + + case token.VAR: + f = parseVarSpec + + case token.FUNC: + return p.parseFuncDecl() + + default: + pos := p.pos + p.errorExpected(pos, "declaration") + p.next() // make progress + decl := &ast.BadDecl{pos, p.pos} + return decl + } + + return p.parseGenDecl(p.tok, f) +} + +func (p *parser) parseDeclList() (list []ast.Decl) { + if p.trace { + defer un(trace(p, "DeclList")) + } + + for p.tok != token.EOF { + list = append(list, p.parseDecl()) + } + + return +} + +// ---------------------------------------------------------------------------- +// Source files + +func (p *parser) parseFile() *ast.File { + if p.trace { + defer un(trace(p, "File")) + } + + // package clause + doc := p.leadComment + pos := p.expect(token.PACKAGE) + // Go spec: The package clause is not a declaration; + // the package name does not appear in any scope. + ident := p.parseIdent() + if ident.Name == "_" { + p.error(p.pos, "invalid package name _") + } + p.expectSemi() + + var decls []ast.Decl + + // Don't bother parsing the rest if we had errors already. + // Likely not a Go source file at all. + + if p.ErrorCount() == 0 && p.mode&PackageClauseOnly == 0 { + // import decls + for p.tok == token.IMPORT { + decls = append(decls, p.parseGenDecl(token.IMPORT, parseImportSpec)) + } + + if p.mode&ImportsOnly == 0 { + // rest of package body + for p.tok != token.EOF { + decls = append(decls, p.parseDecl()) + } + } + } + + assert(p.topScope == p.pkgScope, "imbalanced scopes") + + // resolve global identifiers within the same file + i := 0 + for _, ident := range p.unresolved { + // i <= index for current ident + assert(ident.Obj == unresolved, "object already resolved") + ident.Obj = p.pkgScope.Lookup(ident.Name) // also removes unresolved sentinel + if ident.Obj == nil { + p.unresolved[i] = ident + i++ + } + } + + // TODO(gri): store p.imports in AST + return &ast.File{doc, pos, ident, decls, p.pkgScope, p.imports, p.unresolved[0:i], p.comments} +} diff --git a/src/go/printer/testdata/slow.golden b/src/go/printer/testdata/slow.golden new file mode 100644 index 0000000..43a15cb --- /dev/null +++ b/src/go/printer/testdata/slow.golden @@ -0,0 +1,85 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package deepequal_test + +import ( + "testing" + "google3/spam/archer/frontend/deepequal" +) + +func TestTwoNilValues(t *testing.T) { + if err := deepequal.Check(nil, nil); err != nil { + t.Errorf("expected nil, saw %v", err) + } +} + +type Foo struct { + bar *Bar + bang *Bar +} + +type Bar struct { + baz *Baz + foo []*Foo +} + +type Baz struct { + entries map[int]interface{} + whatever string +} + +func newFoo() *Foo { + return &Foo{bar: &Bar{baz: &Baz{ + entries: map[int]interface{}{ + 42: &Foo{}, + 21: &Bar{}, + 11: &Baz{whatever: "it's just a test"}}}}, + bang: &Bar{foo: []*Foo{ + &Foo{bar: &Bar{baz: &Baz{ + entries: map[int]interface{}{ + 43: &Foo{}, + 22: &Bar{}, + 13: &Baz{whatever: "this is nuts"}}}}, + bang: &Bar{foo: []*Foo{ + &Foo{bar: &Bar{baz: &Baz{ + entries: map[int]interface{}{ + 61: &Foo{}, + 71: &Bar{}, + 11: &Baz{whatever: "no, it's Go"}}}}, + bang: &Bar{foo: []*Foo{ + &Foo{bar: &Bar{baz: &Baz{ + entries: map[int]interface{}{ + 0: &Foo{}, + -2: &Bar{}, + -11: &Baz{whatever: "we need to go deeper"}}}}, + bang: &Bar{foo: []*Foo{ + &Foo{bar: &Bar{baz: &Baz{ + entries: map[int]interface{}{ + -2: &Foo{}, + -5: &Bar{}, + -7: &Baz{whatever: "are you serious?"}}}}, + bang: &Bar{foo: []*Foo{}}}, + &Foo{bar: &Bar{baz: &Baz{ + entries: map[int]interface{}{ + -100: &Foo{}, + 50: &Bar{}, + 20: &Baz{whatever: "na, not really ..."}}}}, + bang: &Bar{foo: []*Foo{}}}}}}}}}, + &Foo{bar: &Bar{baz: &Baz{ + entries: map[int]interface{}{ + 2: &Foo{}, + 1: &Bar{}, + -1: &Baz{whatever: "... it's just a test."}}}}, + bang: &Bar{foo: []*Foo{}}}}}}}}} +} + +func TestElaborate(t *testing.T) { + a := newFoo() + b := newFoo() + + if err := deepequal.Check(a, b); err != nil { + t.Errorf("expected nil, saw %v", err) + } +} diff --git a/src/go/printer/testdata/slow.input b/src/go/printer/testdata/slow.input new file mode 100644 index 0000000..0e5a23d --- /dev/null +++ b/src/go/printer/testdata/slow.input @@ -0,0 +1,85 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package deepequal_test + +import ( + "testing" + "google3/spam/archer/frontend/deepequal" +) + +func TestTwoNilValues(t *testing.T) { + if err := deepequal.Check(nil, nil); err != nil { + t.Errorf("expected nil, saw %v", err) + } +} + +type Foo struct { + bar *Bar + bang *Bar +} + +type Bar struct { + baz *Baz + foo []*Foo +} + +type Baz struct { + entries map[int]interface{} + whatever string +} + +func newFoo() (*Foo) { +return &Foo{bar: &Bar{ baz: &Baz{ +entries: map[int]interface{}{ +42: &Foo{}, +21: &Bar{}, +11: &Baz{ whatever: "it's just a test" }}}}, + bang: &Bar{foo: []*Foo{ +&Foo{bar: &Bar{ baz: &Baz{ +entries: map[int]interface{}{ +43: &Foo{}, +22: &Bar{}, +13: &Baz{ whatever: "this is nuts" }}}}, + bang: &Bar{foo: []*Foo{ +&Foo{bar: &Bar{ baz: &Baz{ +entries: map[int]interface{}{ +61: &Foo{}, +71: &Bar{}, +11: &Baz{ whatever: "no, it's Go" }}}}, + bang: &Bar{foo: []*Foo{ +&Foo{bar: &Bar{ baz: &Baz{ +entries: map[int]interface{}{ +0: &Foo{}, +-2: &Bar{}, +-11: &Baz{ whatever: "we need to go deeper" }}}}, + bang: &Bar{foo: []*Foo{ +&Foo{bar: &Bar{ baz: &Baz{ +entries: map[int]interface{}{ +-2: &Foo{}, +-5: &Bar{}, +-7: &Baz{ whatever: "are you serious?" }}}}, + bang: &Bar{foo: []*Foo{}}}, +&Foo{bar: &Bar{ baz: &Baz{ +entries: map[int]interface{}{ +-100: &Foo{}, +50: &Bar{}, +20: &Baz{ whatever: "na, not really ..." }}}}, + bang: &Bar{foo: []*Foo{}}}}}}}}}, +&Foo{bar: &Bar{ baz: &Baz{ +entries: map[int]interface{}{ +2: &Foo{}, +1: &Bar{}, +-1: &Baz{ whatever: "... it's just a test." }}}}, + bang: &Bar{foo: []*Foo{}}}}}}}}} +} + +func TestElaborate(t *testing.T) { + a := newFoo() + b := newFoo() + + if err := deepequal.Check(a, b); err != nil { + t.Errorf("expected nil, saw %v", err) + } +} diff --git a/src/go/printer/testdata/statements.golden b/src/go/printer/testdata/statements.golden new file mode 100644 index 0000000..4b13460 --- /dev/null +++ b/src/go/printer/testdata/statements.golden @@ -0,0 +1,644 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package statements + +var expr bool + +func use(x interface{}) {} + +// Formatting of multi-line return statements. +func _f() { + return + return x, y, z + return T{} + return T{1, 2, 3}, + x, y, z + return T{1, 2, 3}, + x, y, + z + return T{1, + 2, + 3} + return T{1, + 2, + 3, + } + return T{ + 1, + 2, + 3} + return T{ + 1, + 2, + 3, + } + return T{ + 1, + T{1, 2, 3}, + 3, + } + return T{ + 1, + T{1, + 2, 3}, + 3, + } + return T{ + 1, + T{1, + 2, + 3}, + 3, + } + return T{ + 1, + 2, + }, nil + return T{ + 1, + 2, + }, + T{ + x: 3, + y: 4, + }, nil + return T{ + 1, + 2, + }, + nil + return T{ + 1, + 2, + }, + T{ + x: 3, + y: 4, + }, + nil + return x + y + + z + return func() {} + return func() { + _ = 0 + }, T{ + 1, 2, + } + return func() { + _ = 0 + } + return func() T { + return T{ + 1, 2, + } + } +} + +// Formatting of multi-line returns: test cases from issue 1207. +func F() (*T, os.Error) { + return &T{ + X: 1, + Y: 2, + }, + nil +} + +func G() (*T, *T, os.Error) { + return &T{ + X: 1, + Y: 2, + }, + &T{ + X: 3, + Y: 4, + }, + nil +} + +func _() interface{} { + return &fileStat{ + name: basename(file.name), + size: mkSize(d.FileSizeHigh, d.FileSizeLow), + modTime: mkModTime(d.LastWriteTime), + mode: mkMode(d.FileAttributes), + sys: mkSysFromFI(&d), + }, nil +} + +// Formatting of if-statement headers. +func _() { + if true { + } + if true { + } // no semicolon printed + if expr { + } + if expr { + } // no semicolon printed + if expr { + } // no parens printed + if expr { + } // no semicolon and parens printed + if x := expr; true { + use(x) + } + if x := expr; expr { + use(x) + } +} + +// Formatting of switch-statement headers. +func _() { + switch { + } + switch { + } // no semicolon printed + switch expr { + } + switch expr { + } // no semicolon printed + switch expr { + } // no parens printed + switch expr { + } // no semicolon and parens printed + switch x := expr; { + default: + use( + x) + } + switch x := expr; expr { + default: + use(x) + } +} + +// Formatting of switch statement bodies. +func _() { + switch { + } + + switch x := 0; x { + case 1: + use(x) + use(x) // followed by an empty line + + case 2: // followed by an empty line + + use(x) // followed by an empty line + + case 3: // no empty lines + use(x) + use(x) + } + + switch x { + case 0: + use(x) + case 1: // this comment should have no effect on the previous or next line + use(x) + } + + switch x := 0; x { + case 1: + x = 0 + // this comment should be indented + case 2: + x = 0 + // this comment should not be indented, it is aligned with the next case + case 3: + x = 0 + /* indented comment + aligned + aligned + */ + // bla + /* and more */ + case 4: + x = 0 + /* not indented comment + aligned + aligned + */ + // bla + /* and more */ + case 5: + } +} + +// Formatting of selected select statements. +func _() { + select {} + select { /* this comment should not be tab-aligned because the closing } is on the same line */ + } + select { /* this comment should be tab-aligned */ + } + select { // this comment should be tab-aligned + } + select { + case <-c: + } +} + +// Formatting of for-statement headers for single-line for-loops. +func _() { + for { + } + for expr { + } + for expr { + } // no parens printed + for { + } // no semicolons printed + for x := expr; ; { + use(x) + } + for expr { + } // no semicolons printed + for expr { + } // no semicolons and parens printed + for ; ; expr = false { + } + for x := expr; expr; { + use(x) + } + for x := expr; ; expr = false { + use(x) + } + for ; expr; expr = false { + } + for x := expr; expr; expr = false { + use(x) + } + for x := range []int{} { + use(x) + } + for x := range []int{} { + use(x) + } // no parens printed +} + +// Formatting of for-statement headers for multi-line for-loops. +func _() { + for { + } + for expr { + } + for expr { + } // no parens printed + for { + } // no semicolons printed + for x := expr; ; { + use(x) + } + for expr { + } // no semicolons printed + for expr { + } // no semicolons and parens printed + for ; ; expr = false { + } + for x := expr; expr; { + use(x) + } + for x := expr; ; expr = false { + use(x) + } + for ; expr; expr = false { + } + for x := expr; expr; expr = false { + use(x) + } + for range []int{} { + println("foo") + } + for x := range []int{} { + use(x) + } + for x := range []int{} { + use(x) + } // no parens printed +} + +// Formatting of selected short single- and multi-line statements. +func _() { + if cond { + } + if cond { + } // multiple lines + if cond { + } else { + } // else clause always requires multiple lines + + for { + } + for i := 0; i < len(a); 1++ { + } + for i := 0; i < len(a); 1++ { + a[i] = i + } + for i := 0; i < len(a); 1++ { + a[i] = i + } // multiple lines + + for range a { + } + for _ = range a { + } + for _, _ = range a { + } + for i := range a { + } + for i := range a { + a[i] = i + } + for i := range a { + a[i] = i + } // multiple lines + + go func() { + for { + a <- <-b + } + }() + defer func() { + if x := recover(); x != nil { + err = fmt.Sprintf("error: %s", x.msg) + } + }() +} + +// Don't remove mandatory parentheses around composite literals in control clauses. +func _() { + // strip parentheses - no composite literals or composite literals don't start with a type name + if x { + } + if x { + } + if []T{} { + } + if []T{} { + } + if []T{} { + } + + for x { + } + for x { + } + for []T{} { + } + for []T{} { + } + for []T{} { + } + + switch x { + } + switch x { + } + switch []T{} { + } + switch []T{} { + } + + for _ = range []T{T{42}} { + } + + // leave parentheses - composite literals start with a type name + if (T{}) { + } + if (T{}) { + } + if (T{}) { + } + + for (T{}) { + } + for (T{}) { + } + for (T{}) { + } + + switch (T{}) { + } + switch (T{}) { + } + + for _ = range (T1{T{42}}) { + } + + if x == (T{42}[0]) { + } + if (x == T{42}[0]) { + } + if x == (T{42}[0]) { + } + if x == (T{42}[0]) { + } + if x == (T{42}[0]) { + } + if x == a+b*(T{42}[0]) { + } + if (x == a+b*T{42}[0]) { + } + if x == a+b*(T{42}[0]) { + } + if x == a+(b*(T{42}[0])) { + } + if x == a+b*(T{42}[0]) { + } + if (a + b*(T{42}[0])) == x { + } + if (a + b*(T{42}[0])) == x { + } + + if struct{ x bool }{false}.x { + } + if (struct{ x bool }{false}.x) == false { + } + if struct{ x bool }{false}.x == false { + } +} + +// Extra empty lines inside functions. Do respect source code line +// breaks between statement boundaries but print at most one empty +// line at a time. +func _() { + + const _ = 0 + + const _ = 1 + type _ int + type _ float + + var _ = 0 + var x = 1 + + // Each use(x) call below should have at most one empty line before and after. + // Known bug: The first use call may have more than one empty line before + // (see go/printer/nodes.go, func linebreak). + + use(x) + + if x < x { + + use(x) + + } else { + + use(x) + + } +} + +// Formatting around labels. +func _() { +L: +} + +func _() { + // this comment should be indented +L: // no semicolon needed +} + +func _() { + switch 0 { + case 0: + L0: + ; // semicolon required + case 1: + L1: + ; // semicolon required + default: + L2: // no semicolon needed + } +} + +func _() { + f() +L1: + f() +L2: + ; +L3: +} + +func _() { + // this comment should be indented +L: +} + +func _() { +L: + _ = 0 +} + +func _() { + // this comment should be indented +L: + _ = 0 +} + +func _() { + for { + L1: + _ = 0 + L2: + _ = 0 + } +} + +func _() { + // this comment should be indented + for { + L1: + _ = 0 + L2: + _ = 0 + } +} + +func _() { + if true { + _ = 0 + } + _ = 0 // the indentation here should not be affected by the long label name +AnOverlongLabel: + _ = 0 + + if true { + _ = 0 + } + _ = 0 + +L: + _ = 0 +} + +func _() { + for { + goto L + } +L: + + MoreCode() +} + +func _() { + for { + goto L + } +L: // A comment on the same line as the label, followed by a single empty line. + // Known bug: There may be more than one empty line before MoreCode() + // (see go/printer/nodes.go, func linebreak). + + MoreCode() +} + +func _() { + for { + goto L + } +L: + + // There should be a single empty line before this comment. + MoreCode() +} + +func _() { + for { + goto AVeryLongLabelThatShouldNotAffectFormatting + } +AVeryLongLabelThatShouldNotAffectFormatting: + // There should be a single empty line after this comment. + + // There should be a single empty line before this comment. + MoreCode() +} + +// Formatting of empty statements. +func _() { + +} + +func _() { +} + +func _() { +} + +func _() { + f() +} + +func _() { +L: + ; +} + +func _() { +L: + ; + f() +} diff --git a/src/go/printer/testdata/statements.input b/src/go/printer/testdata/statements.input new file mode 100644 index 0000000..cade157 --- /dev/null +++ b/src/go/printer/testdata/statements.input @@ -0,0 +1,555 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package statements + +var expr bool + +func use(x interface{}) {} + +// Formatting of multi-line return statements. +func _f() { + return + return x, y, z + return T{} + return T{1, 2, 3}, + x, y, z + return T{1, 2, 3}, + x, y, + z + return T{1, + 2, + 3} + return T{1, + 2, + 3, + } + return T{ + 1, + 2, + 3} + return T{ + 1, + 2, + 3, + } + return T{ + 1, + T{1, 2, 3}, + 3, + } + return T{ + 1, + T{1, + 2, 3}, + 3, + } + return T{ + 1, + T{1, + 2, + 3}, + 3, + } + return T{ + 1, + 2, + }, nil + return T{ + 1, + 2, + }, + T{ + x: 3, + y: 4, + }, nil + return T{ + 1, + 2, + }, + nil + return T{ + 1, + 2, + }, + T{ + x: 3, + y: 4, + }, + nil + return x + y + + z + return func() {} + return func() { + _ = 0 + }, T{ + 1, 2, + } + return func() { + _ = 0 + } + return func() T { + return T { + 1, 2, + } + } +} + +// Formatting of multi-line returns: test cases from issue 1207. +func F() (*T, os.Error) { + return &T{ + X: 1, + Y: 2, + }, + nil +} + +func G() (*T, *T, os.Error) { + return &T{ + X: 1, + Y: 2, + }, + &T{ + X: 3, + Y: 4, + }, + nil +} + +func _() interface{} { + return &fileStat{ + name: basename(file.name), + size: mkSize(d.FileSizeHigh, d.FileSizeLow), + modTime: mkModTime(d.LastWriteTime), + mode: mkMode(d.FileAttributes), + sys: mkSysFromFI(&d), + }, nil +} + +// Formatting of if-statement headers. +func _() { + if true {} + if; true {} // no semicolon printed + if expr{} + if;expr{} // no semicolon printed + if (expr){} // no parens printed + if;((expr)){} // no semicolon and parens printed + if x:=expr;true{ + use(x)} + if x:=expr; expr {use(x)} +} + + +// Formatting of switch-statement headers. +func _() { + switch {} + switch;{} // no semicolon printed + switch expr {} + switch;expr{} // no semicolon printed + switch (expr) {} // no parens printed + switch;((expr)){} // no semicolon and parens printed + switch x := expr; { default:use( +x) + } + switch x := expr; expr {default:use(x)} +} + + +// Formatting of switch statement bodies. +func _() { + switch { + } + + switch x := 0; x { + case 1: + use(x) + use(x) // followed by an empty line + + case 2: // followed by an empty line + + use(x) // followed by an empty line + + case 3: // no empty lines + use(x) + use(x) + } + + switch x { + case 0: + use(x) + case 1: // this comment should have no effect on the previous or next line + use(x) + } + + switch x := 0; x { + case 1: + x = 0 + // this comment should be indented + case 2: + x = 0 + // this comment should not be indented, it is aligned with the next case + case 3: + x = 0 + /* indented comment + aligned + aligned + */ + // bla + /* and more */ + case 4: + x = 0 + /* not indented comment + aligned + aligned + */ + // bla + /* and more */ + case 5: + } +} + + +// Formatting of selected select statements. +func _() { + select { + } + select { /* this comment should not be tab-aligned because the closing } is on the same line */ } + select { /* this comment should be tab-aligned */ + } + select { // this comment should be tab-aligned + } + select { case <-c: } +} + + +// Formatting of for-statement headers for single-line for-loops. +func _() { + for{} + for expr {} + for (expr) {} // no parens printed + for;;{} // no semicolons printed + for x :=expr;; {use( x)} + for; expr;{} // no semicolons printed + for; ((expr));{} // no semicolons and parens printed + for; ; expr = false {} + for x :=expr; expr; {use(x)} + for x := expr;; expr=false {use(x)} + for;expr;expr =false {} + for x := expr;expr;expr = false { use(x) } + for x := range []int{} { use(x) } + for x := range (([]int{})) { use(x) } // no parens printed +} + + +// Formatting of for-statement headers for multi-line for-loops. +func _() { + for{ + } + for expr { + } + for (expr) { + } // no parens printed + for;;{ + } // no semicolons printed + for x :=expr;; {use( x) + } + for; expr;{ + } // no semicolons printed + for; ((expr));{ + } // no semicolons and parens printed + for; ; expr = false { + } + for x :=expr; expr; {use(x) + } + for x := expr;; expr=false {use(x) + } + for;expr;expr =false { + } + for x := expr;expr;expr = false { + use(x) + } + for range []int{} { + println("foo")} + for x := range []int{} { + use(x) } + for x := range (([]int{})) { + use(x) } // no parens printed +} + + +// Formatting of selected short single- and multi-line statements. +func _() { + if cond {} + if cond { + } // multiple lines + if cond {} else {} // else clause always requires multiple lines + + for {} + for i := 0; i < len(a); 1++ {} + for i := 0; i < len(a); 1++ { a[i] = i } + for i := 0; i < len(a); 1++ { a[i] = i + } // multiple lines + + for range a{} + for _ = range a{} + for _, _ = range a{} + for i := range a {} + for i := range a { a[i] = i } + for i := range a { a[i] = i + } // multiple lines + + go func() { for { a <- <-b } }() + defer func() { if x := recover(); x != nil { err = fmt.Sprintf("error: %s", x.msg) } }() +} + + +// Don't remove mandatory parentheses around composite literals in control clauses. +func _() { + // strip parentheses - no composite literals or composite literals don't start with a type name + if (x) {} + if (((x))) {} + if ([]T{}) {} + if (([]T{})) {} + if ; (((([]T{})))) {} + + for (x) {} + for (((x))) {} + for ([]T{}) {} + for (([]T{})) {} + for ; (((([]T{})))) ; {} + + switch (x) {} + switch (((x))) {} + switch ([]T{}) {} + switch ; (((([]T{})))) {} + + for _ = range ((([]T{T{42}}))) {} + + // leave parentheses - composite literals start with a type name + if (T{}) {} + if ((T{})) {} + if ; ((((T{})))) {} + + for (T{}) {} + for ((T{})) {} + for ; ((((T{})))) ; {} + + switch (T{}) {} + switch ; ((((T{})))) {} + + for _ = range (((T1{T{42}}))) {} + + if x == (T{42}[0]) {} + if (x == T{42}[0]) {} + if (x == (T{42}[0])) {} + if (x == (((T{42}[0])))) {} + if (((x == (T{42}[0])))) {} + if x == a + b*(T{42}[0]) {} + if (x == a + b*T{42}[0]) {} + if (x == a + b*(T{42}[0])) {} + if (x == a + ((b * (T{42}[0])))) {} + if (((x == a + b * (T{42}[0])))) {} + if (((a + b * (T{42}[0])) == x)) {} + if (((a + b * (T{42}[0])))) == x {} + + if (struct{x bool}{false}.x) {} + if (struct{x bool}{false}.x) == false {} + if (struct{x bool}{false}.x == false) {} +} + + +// Extra empty lines inside functions. Do respect source code line +// breaks between statement boundaries but print at most one empty +// line at a time. +func _() { + + const _ = 0 + + const _ = 1 + type _ int + type _ float + + var _ = 0 + var x = 1 + + // Each use(x) call below should have at most one empty line before and after. + // Known bug: The first use call may have more than one empty line before + // (see go/printer/nodes.go, func linebreak). + + + + use(x) + + if x < x { + + use(x) + + } else { + + use(x) + + } +} + + +// Formatting around labels. +func _() { + L: +} + + +func _() { + // this comment should be indented + L: ; // no semicolon needed +} + + +func _() { + switch 0 { + case 0: + L0: ; // semicolon required + case 1: + L1: ; // semicolon required + default: + L2: ; // no semicolon needed + } +} + + +func _() { + f() +L1: + f() +L2: + ; +L3: +} + + +func _() { + // this comment should be indented + L: +} + + +func _() { + L: _ = 0 +} + + +func _() { + // this comment should be indented + L: _ = 0 +} + + +func _() { + for { + L1: _ = 0 + L2: + _ = 0 + } +} + + +func _() { + // this comment should be indented + for { + L1: _ = 0 + L2: + _ = 0 + } +} + + +func _() { + if true { + _ = 0 + } + _ = 0 // the indentation here should not be affected by the long label name +AnOverlongLabel: + _ = 0 + + if true { + _ = 0 + } + _ = 0 + +L: _ = 0 +} + + +func _() { + for { + goto L + } +L: + + MoreCode() +} + + +func _() { + for { + goto L + } +L: // A comment on the same line as the label, followed by a single empty line. + // Known bug: There may be more than one empty line before MoreCode() + // (see go/printer/nodes.go, func linebreak). + + + + + MoreCode() +} + + +func _() { + for { + goto L + } +L: + + + + + // There should be a single empty line before this comment. + MoreCode() +} + + +func _() { + for { + goto AVeryLongLabelThatShouldNotAffectFormatting + } +AVeryLongLabelThatShouldNotAffectFormatting: + // There should be a single empty line after this comment. + + // There should be a single empty line before this comment. + MoreCode() +} + + +// Formatting of empty statements. +func _() { + ;;;;;;;;;;;;;;;;;;;;;;;;; +} + +func _() {;;;;;;;;;;;;;;;;;;;;;;;;; +} + +func _() {;;;;;;;;;;;;;;;;;;;;;;;;;} + +func _() { +f();;;;;;;;;;;;;;;;;;;;;;;;; +} + +func _() { +L:;;;;;;;;;;;; +} + +func _() { +L:;;;;;;;;;;;; + f() +} diff --git a/src/go/scanner/errors.go b/src/go/scanner/errors.go new file mode 100644 index 0000000..3e9c365 --- /dev/null +++ b/src/go/scanner/errors.go @@ -0,0 +1,120 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scanner + +import ( + "fmt" + "go/token" + "io" + "sort" +) + +// In an ErrorList, an error is represented by an *Error. +// The position Pos, if valid, points to the beginning of +// the offending token, and the error condition is described +// by Msg. +type Error struct { + Pos token.Position + Msg string +} + +// Error implements the error interface. +func (e Error) Error() string { + if e.Pos.Filename != "" || e.Pos.IsValid() { + // don't print "" + // TODO(gri) reconsider the semantics of Position.IsValid + return e.Pos.String() + ": " + e.Msg + } + return e.Msg +} + +// ErrorList is a list of *Errors. +// The zero value for an ErrorList is an empty ErrorList ready to use. +type ErrorList []*Error + +// Add adds an Error with given position and error message to an ErrorList. +func (p *ErrorList) Add(pos token.Position, msg string) { + *p = append(*p, &Error{pos, msg}) +} + +// Reset resets an ErrorList to no errors. +func (p *ErrorList) Reset() { *p = (*p)[0:0] } + +// ErrorList implements the sort Interface. +func (p ErrorList) Len() int { return len(p) } +func (p ErrorList) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +func (p ErrorList) Less(i, j int) bool { + e := &p[i].Pos + f := &p[j].Pos + // Note that it is not sufficient to simply compare file offsets because + // the offsets do not reflect modified line information (through //line + // comments). + if e.Filename != f.Filename { + return e.Filename < f.Filename + } + if e.Line != f.Line { + return e.Line < f.Line + } + if e.Column != f.Column { + return e.Column < f.Column + } + return p[i].Msg < p[j].Msg +} + +// Sort sorts an ErrorList. *Error entries are sorted by position, +// other errors are sorted by error message, and before any *Error +// entry. +func (p ErrorList) Sort() { + sort.Sort(p) +} + +// RemoveMultiples sorts an ErrorList and removes all but the first error per line. +func (p *ErrorList) RemoveMultiples() { + sort.Sort(p) + var last token.Position // initial last.Line is != any legal error line + i := 0 + for _, e := range *p { + if e.Pos.Filename != last.Filename || e.Pos.Line != last.Line { + last = e.Pos + (*p)[i] = e + i++ + } + } + *p = (*p)[0:i] +} + +// An ErrorList implements the error interface. +func (p ErrorList) Error() string { + switch len(p) { + case 0: + return "no errors" + case 1: + return p[0].Error() + } + return fmt.Sprintf("%s (and %d more errors)", p[0], len(p)-1) +} + +// Err returns an error equivalent to this error list. +// If the list is empty, Err returns nil. +func (p ErrorList) Err() error { + if len(p) == 0 { + return nil + } + return p +} + +// PrintError is a utility function that prints a list of errors to w, +// one error per line, if the err parameter is an ErrorList. Otherwise +// it prints the err string. +func PrintError(w io.Writer, err error) { + if list, ok := err.(ErrorList); ok { + for _, e := range list { + fmt.Fprintf(w, "%s\n", e) + } + } else if err != nil { + fmt.Fprintf(w, "%s\n", err) + } +} diff --git a/src/go/scanner/example_test.go b/src/go/scanner/example_test.go new file mode 100644 index 0000000..7493585 --- /dev/null +++ b/src/go/scanner/example_test.go @@ -0,0 +1,46 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scanner_test + +import ( + "fmt" + "go/scanner" + "go/token" +) + +func ExampleScanner_Scan() { + // src is the input that we want to tokenize. + src := []byte("cos(x) + 1i*sin(x) // Euler") + + // Initialize the scanner. + var s scanner.Scanner + fset := token.NewFileSet() // positions are relative to fset + file := fset.AddFile("", fset.Base(), len(src)) // register input "file" + s.Init(file, src, nil /* no error handler */, scanner.ScanComments) + + // Repeated calls to Scan yield the token sequence found in the input. + for { + pos, tok, lit := s.Scan() + if tok == token.EOF { + break + } + fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit) + } + + // output: + // 1:1 IDENT "cos" + // 1:4 ( "" + // 1:5 IDENT "x" + // 1:6 ) "" + // 1:8 + "" + // 1:10 IMAG "1i" + // 1:12 * "" + // 1:13 IDENT "sin" + // 1:16 ( "" + // 1:17 IDENT "x" + // 1:18 ) "" + // 1:20 COMMENT "// Euler" + // 1:28 ; "\n" +} diff --git a/src/go/scanner/scanner.go b/src/go/scanner/scanner.go new file mode 100644 index 0000000..0cd9f59 --- /dev/null +++ b/src/go/scanner/scanner.go @@ -0,0 +1,957 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package scanner implements a scanner for Go source text. +// It takes a []byte as source which can then be tokenized +// through repeated calls to the Scan method. +package scanner + +import ( + "bytes" + "fmt" + "go/token" + "path/filepath" + "strconv" + "unicode" + "unicode/utf8" +) + +// An ErrorHandler may be provided to Scanner.Init. If a syntax error is +// encountered and a handler was installed, the handler is called with a +// position and an error message. The position points to the beginning of +// the offending token. +type ErrorHandler func(pos token.Position, msg string) + +// A Scanner holds the scanner's internal state while processing +// a given text. It can be allocated as part of another data +// structure but must be initialized via Init before use. +type Scanner struct { + // immutable state + file *token.File // source file handle + dir string // directory portion of file.Name() + src []byte // source + err ErrorHandler // error reporting; or nil + mode Mode // scanning mode + + // scanning state + ch rune // current character + offset int // character offset + rdOffset int // reading offset (position after current character) + lineOffset int // current line offset + insertSemi bool // insert a semicolon before next newline + nlPos token.Pos // position of newline in preceding comment + + // public state - ok to modify + ErrorCount int // number of errors encountered +} + +const ( + bom = 0xFEFF // byte order mark, only permitted as very first character + eof = -1 // end of file +) + +// Read the next Unicode char into s.ch. +// s.ch < 0 means end-of-file. +// +// For optimization, there is some overlap between this method and +// s.scanIdentifier. +func (s *Scanner) next() { + if s.rdOffset < len(s.src) { + s.offset = s.rdOffset + if s.ch == '\n' { + s.lineOffset = s.offset + s.file.AddLine(s.offset) + } + r, w := rune(s.src[s.rdOffset]), 1 + switch { + case r == 0: + s.error(s.offset, "illegal character NUL") + case r >= utf8.RuneSelf: + // not ASCII + r, w = utf8.DecodeRune(s.src[s.rdOffset:]) + if r == utf8.RuneError && w == 1 { + s.error(s.offset, "illegal UTF-8 encoding") + } else if r == bom && s.offset > 0 { + s.error(s.offset, "illegal byte order mark") + } + } + s.rdOffset += w + s.ch = r + } else { + s.offset = len(s.src) + if s.ch == '\n' { + s.lineOffset = s.offset + s.file.AddLine(s.offset) + } + s.ch = eof + } +} + +// peek returns the byte following the most recently read character without +// advancing the scanner. If the scanner is at EOF, peek returns 0. +func (s *Scanner) peek() byte { + if s.rdOffset < len(s.src) { + return s.src[s.rdOffset] + } + return 0 +} + +// A mode value is a set of flags (or 0). +// They control scanner behavior. +type Mode uint + +const ( + ScanComments Mode = 1 << iota // return comments as COMMENT tokens + dontInsertSemis // do not automatically insert semicolons - for testing only +) + +// Init prepares the scanner s to tokenize the text src by setting the +// scanner at the beginning of src. The scanner uses the file set file +// for position information and it adds line information for each line. +// It is ok to re-use the same file when re-scanning the same file as +// line information which is already present is ignored. Init causes a +// panic if the file size does not match the src size. +// +// Calls to Scan will invoke the error handler err if they encounter a +// syntax error and err is not nil. Also, for each error encountered, +// the Scanner field ErrorCount is incremented by one. The mode parameter +// determines how comments are handled. +// +// Note that Init may call err if there is an error in the first character +// of the file. +func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) { + // Explicitly initialize all fields since a scanner may be reused. + if file.Size() != len(src) { + panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src))) + } + s.file = file + s.dir, _ = filepath.Split(file.Name()) + s.src = src + s.err = err + s.mode = mode + + s.ch = ' ' + s.offset = 0 + s.rdOffset = 0 + s.lineOffset = 0 + s.insertSemi = false + s.ErrorCount = 0 + + s.next() + if s.ch == bom { + s.next() // ignore BOM at file beginning + } +} + +func (s *Scanner) error(offs int, msg string) { + if s.err != nil { + s.err(s.file.Position(s.file.Pos(offs)), msg) + } + s.ErrorCount++ +} + +func (s *Scanner) errorf(offs int, format string, args ...any) { + s.error(offs, fmt.Sprintf(format, args...)) +} + +// scanComment returns the text of the comment and (if nonzero) +// the offset of the first newline within it, which implies a +// /*...*/ comment. +func (s *Scanner) scanComment() (string, int) { + // initial '/' already consumed; s.ch == '/' || s.ch == '*' + offs := s.offset - 1 // position of initial '/' + next := -1 // position immediately following the comment; < 0 means invalid comment + numCR := 0 + nlOffset := 0 // offset of first newline within /*...*/ comment + + if s.ch == '/' { + //-style comment + // (the final '\n' is not considered part of the comment) + s.next() + for s.ch != '\n' && s.ch >= 0 { + if s.ch == '\r' { + numCR++ + } + s.next() + } + // if we are at '\n', the position following the comment is afterwards + next = s.offset + if s.ch == '\n' { + next++ + } + goto exit + } + + /*-style comment */ + s.next() + for s.ch >= 0 { + ch := s.ch + if ch == '\r' { + numCR++ + } else if ch == '\n' && nlOffset == 0 { + nlOffset = s.offset + } + s.next() + if ch == '*' && s.ch == '/' { + s.next() + next = s.offset + goto exit + } + } + + s.error(offs, "comment not terminated") + +exit: + lit := s.src[offs:s.offset] + + // On Windows, a (//-comment) line may end in "\r\n". + // Remove the final '\r' before analyzing the text for + // line directives (matching the compiler). Remove any + // other '\r' afterwards (matching the pre-existing be- + // havior of the scanner). + if numCR > 0 && len(lit) >= 2 && lit[1] == '/' && lit[len(lit)-1] == '\r' { + lit = lit[:len(lit)-1] + numCR-- + } + + // interpret line directives + // (//line directives must start at the beginning of the current line) + if next >= 0 /* implies valid comment */ && (lit[1] == '*' || offs == s.lineOffset) && bytes.HasPrefix(lit[2:], prefix) { + s.updateLineInfo(next, offs, lit) + } + + if numCR > 0 { + lit = stripCR(lit, lit[1] == '*') + } + + return string(lit), nlOffset +} + +var prefix = []byte("line ") + +// updateLineInfo parses the incoming comment text at offset offs +// as a line directive. If successful, it updates the line info table +// for the position next per the line directive. +func (s *Scanner) updateLineInfo(next, offs int, text []byte) { + // extract comment text + if text[1] == '*' { + text = text[:len(text)-2] // lop off trailing "*/" + } + text = text[7:] // lop off leading "//line " or "/*line " + offs += 7 + + i, n, ok := trailingDigits(text) + if i == 0 { + return // ignore (not a line directive) + } + // i > 0 + + if !ok { + // text has a suffix :xxx but xxx is not a number + s.error(offs+i, "invalid line number: "+string(text[i:])) + return + } + + // Put a cap on the maximum size of line and column numbers. + // 30 bits allows for some additional space before wrapping an int32. + const maxLineCol = 1<<30 - 1 + var line, col int + i2, n2, ok2 := trailingDigits(text[:i-1]) + if ok2 { + //line filename:line:col + i, i2 = i2, i + line, col = n2, n + if col == 0 || col > maxLineCol { + s.error(offs+i2, "invalid column number: "+string(text[i2:])) + return + } + text = text[:i2-1] // lop off ":col" + } else { + //line filename:line + line = n + } + + if line == 0 || line > maxLineCol { + s.error(offs+i, "invalid line number: "+string(text[i:])) + return + } + + // If we have a column (//line filename:line:col form), + // an empty filename means to use the previous filename. + filename := string(text[:i-1]) // lop off ":line", and trim white space + if filename == "" && ok2 { + filename = s.file.Position(s.file.Pos(offs)).Filename + } else if filename != "" { + // Put a relative filename in the current directory. + // This is for compatibility with earlier releases. + // See issue 26671. + filename = filepath.Clean(filename) + if !filepath.IsAbs(filename) { + filename = filepath.Join(s.dir, filename) + } + } + + s.file.AddLineColumnInfo(next, filename, line, col) +} + +func trailingDigits(text []byte) (int, int, bool) { + i := bytes.LastIndexByte(text, ':') // look from right (Windows filenames may contain ':') + if i < 0 { + return 0, 0, false // no ":" + } + // i >= 0 + n, err := strconv.ParseUint(string(text[i+1:]), 10, 0) + return i + 1, int(n), err == nil +} + +func isLetter(ch rune) bool { + return 'a' <= lower(ch) && lower(ch) <= 'z' || ch == '_' || ch >= utf8.RuneSelf && unicode.IsLetter(ch) +} + +func isDigit(ch rune) bool { + return isDecimal(ch) || ch >= utf8.RuneSelf && unicode.IsDigit(ch) +} + +// scanIdentifier reads the string of valid identifier characters at s.offset. +// It must only be called when s.ch is known to be a valid letter. +// +// Be careful when making changes to this function: it is optimized and affects +// scanning performance significantly. +func (s *Scanner) scanIdentifier() string { + offs := s.offset + + // Optimize for the common case of an ASCII identifier. + // + // Ranging over s.src[s.rdOffset:] lets us avoid some bounds checks, and + // avoids conversions to runes. + // + // In case we encounter a non-ASCII character, fall back on the slower path + // of calling into s.next(). + for rdOffset, b := range s.src[s.rdOffset:] { + if 'a' <= b && b <= 'z' || 'A' <= b && b <= 'Z' || b == '_' || '0' <= b && b <= '9' { + // Avoid assigning a rune for the common case of an ascii character. + continue + } + s.rdOffset += rdOffset + if 0 < b && b < utf8.RuneSelf { + // Optimization: we've encountered an ASCII character that's not a letter + // or number. Avoid the call into s.next() and corresponding set up. + // + // Note that s.next() does some line accounting if s.ch is '\n', so this + // shortcut is only possible because we know that the preceding character + // is not '\n'. + s.ch = rune(b) + s.offset = s.rdOffset + s.rdOffset++ + goto exit + } + // We know that the preceding character is valid for an identifier because + // scanIdentifier is only called when s.ch is a letter, so calling s.next() + // at s.rdOffset resets the scanner state. + s.next() + for isLetter(s.ch) || isDigit(s.ch) { + s.next() + } + goto exit + } + s.offset = len(s.src) + s.rdOffset = len(s.src) + s.ch = eof + +exit: + return string(s.src[offs:s.offset]) +} + +func digitVal(ch rune) int { + switch { + case '0' <= ch && ch <= '9': + return int(ch - '0') + case 'a' <= lower(ch) && lower(ch) <= 'f': + return int(lower(ch) - 'a' + 10) + } + return 16 // larger than any legal digit val +} + +func lower(ch rune) rune { return ('a' - 'A') | ch } // returns lower-case ch iff ch is ASCII letter +func isDecimal(ch rune) bool { return '0' <= ch && ch <= '9' } +func isHex(ch rune) bool { return '0' <= ch && ch <= '9' || 'a' <= lower(ch) && lower(ch) <= 'f' } + +// digits accepts the sequence { digit | '_' }. +// If base <= 10, digits accepts any decimal digit but records +// the offset (relative to the source start) of a digit >= base +// in *invalid, if *invalid < 0. +// digits returns a bitset describing whether the sequence contained +// digits (bit 0 is set), or separators '_' (bit 1 is set). +func (s *Scanner) digits(base int, invalid *int) (digsep int) { + if base <= 10 { + max := rune('0' + base) + for isDecimal(s.ch) || s.ch == '_' { + ds := 1 + if s.ch == '_' { + ds = 2 + } else if s.ch >= max && *invalid < 0 { + *invalid = s.offset // record invalid rune offset + } + digsep |= ds + s.next() + } + } else { + for isHex(s.ch) || s.ch == '_' { + ds := 1 + if s.ch == '_' { + ds = 2 + } + digsep |= ds + s.next() + } + } + return +} + +func (s *Scanner) scanNumber() (token.Token, string) { + offs := s.offset + tok := token.ILLEGAL + + base := 10 // number base + prefix := rune(0) // one of 0 (decimal), '0' (0-octal), 'x', 'o', or 'b' + digsep := 0 // bit 0: digit present, bit 1: '_' present + invalid := -1 // index of invalid digit in literal, or < 0 + + // integer part + if s.ch != '.' { + tok = token.INT + if s.ch == '0' { + s.next() + switch lower(s.ch) { + case 'x': + s.next() + base, prefix = 16, 'x' + case 'o': + s.next() + base, prefix = 8, 'o' + case 'b': + s.next() + base, prefix = 2, 'b' + default: + base, prefix = 8, '0' + digsep = 1 // leading 0 + } + } + digsep |= s.digits(base, &invalid) + } + + // fractional part + if s.ch == '.' { + tok = token.FLOAT + if prefix == 'o' || prefix == 'b' { + s.error(s.offset, "invalid radix point in "+litname(prefix)) + } + s.next() + digsep |= s.digits(base, &invalid) + } + + if digsep&1 == 0 { + s.error(s.offset, litname(prefix)+" has no digits") + } + + // exponent + if e := lower(s.ch); e == 'e' || e == 'p' { + switch { + case e == 'e' && prefix != 0 && prefix != '0': + s.errorf(s.offset, "%q exponent requires decimal mantissa", s.ch) + case e == 'p' && prefix != 'x': + s.errorf(s.offset, "%q exponent requires hexadecimal mantissa", s.ch) + } + s.next() + tok = token.FLOAT + if s.ch == '+' || s.ch == '-' { + s.next() + } + ds := s.digits(10, nil) + digsep |= ds + if ds&1 == 0 { + s.error(s.offset, "exponent has no digits") + } + } else if prefix == 'x' && tok == token.FLOAT { + s.error(s.offset, "hexadecimal mantissa requires a 'p' exponent") + } + + // suffix 'i' + if s.ch == 'i' { + tok = token.IMAG + s.next() + } + + lit := string(s.src[offs:s.offset]) + if tok == token.INT && invalid >= 0 { + s.errorf(invalid, "invalid digit %q in %s", lit[invalid-offs], litname(prefix)) + } + if digsep&2 != 0 { + if i := invalidSep(lit); i >= 0 { + s.error(offs+i, "'_' must separate successive digits") + } + } + + return tok, lit +} + +func litname(prefix rune) string { + switch prefix { + case 'x': + return "hexadecimal literal" + case 'o', '0': + return "octal literal" + case 'b': + return "binary literal" + } + return "decimal literal" +} + +// invalidSep returns the index of the first invalid separator in x, or -1. +func invalidSep(x string) int { + x1 := ' ' // prefix char, we only care if it's 'x' + d := '.' // digit, one of '_', '0' (a digit), or '.' (anything else) + i := 0 + + // a prefix counts as a digit + if len(x) >= 2 && x[0] == '0' { + x1 = lower(rune(x[1])) + if x1 == 'x' || x1 == 'o' || x1 == 'b' { + d = '0' + i = 2 + } + } + + // mantissa and exponent + for ; i < len(x); i++ { + p := d // previous digit + d = rune(x[i]) + switch { + case d == '_': + if p != '0' { + return i + } + case isDecimal(d) || x1 == 'x' && isHex(d): + d = '0' + default: + if p == '_' { + return i - 1 + } + d = '.' + } + } + if d == '_' { + return len(x) - 1 + } + + return -1 +} + +// scanEscape parses an escape sequence where rune is the accepted +// escaped quote. In case of a syntax error, it stops at the offending +// character (without consuming it) and returns false. Otherwise +// it returns true. +func (s *Scanner) scanEscape(quote rune) bool { + offs := s.offset + + var n int + var base, max uint32 + switch s.ch { + case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote: + s.next() + return true + case '0', '1', '2', '3', '4', '5', '6', '7': + n, base, max = 3, 8, 255 + case 'x': + s.next() + n, base, max = 2, 16, 255 + case 'u': + s.next() + n, base, max = 4, 16, unicode.MaxRune + case 'U': + s.next() + n, base, max = 8, 16, unicode.MaxRune + default: + msg := "unknown escape sequence" + if s.ch < 0 { + msg = "escape sequence not terminated" + } + s.error(offs, msg) + return false + } + + var x uint32 + for n > 0 { + d := uint32(digitVal(s.ch)) + if d >= base { + msg := fmt.Sprintf("illegal character %#U in escape sequence", s.ch) + if s.ch < 0 { + msg = "escape sequence not terminated" + } + s.error(s.offset, msg) + return false + } + x = x*base + d + s.next() + n-- + } + + if x > max || 0xD800 <= x && x < 0xE000 { + s.error(offs, "escape sequence is invalid Unicode code point") + return false + } + + return true +} + +func (s *Scanner) scanRune() string { + // '\'' opening already consumed + offs := s.offset - 1 + + valid := true + n := 0 + for { + ch := s.ch + if ch == '\n' || ch < 0 { + // only report error if we don't have one already + if valid { + s.error(offs, "rune literal not terminated") + valid = false + } + break + } + s.next() + if ch == '\'' { + break + } + n++ + if ch == '\\' { + if !s.scanEscape('\'') { + valid = false + } + // continue to read to closing quote + } + } + + if valid && n != 1 { + s.error(offs, "illegal rune literal") + } + + return string(s.src[offs:s.offset]) +} + +func (s *Scanner) scanString() string { + // '"' opening already consumed + offs := s.offset - 1 + + for { + ch := s.ch + if ch == '\n' || ch < 0 { + s.error(offs, "string literal not terminated") + break + } + s.next() + if ch == '"' { + break + } + if ch == '\\' { + s.scanEscape('"') + } + } + + return string(s.src[offs:s.offset]) +} + +func stripCR(b []byte, comment bool) []byte { + c := make([]byte, len(b)) + i := 0 + for j, ch := range b { + // In a /*-style comment, don't strip \r from *\r/ (incl. + // sequences of \r from *\r\r...\r/) since the resulting + // */ would terminate the comment too early unless the \r + // is immediately following the opening /* in which case + // it's ok because /*/ is not closed yet (issue #11151). + if ch != '\r' || comment && i > len("/*") && c[i-1] == '*' && j+1 < len(b) && b[j+1] == '/' { + c[i] = ch + i++ + } + } + return c[:i] +} + +func (s *Scanner) scanRawString() string { + // '`' opening already consumed + offs := s.offset - 1 + + hasCR := false + for { + ch := s.ch + if ch < 0 { + s.error(offs, "raw string literal not terminated") + break + } + s.next() + if ch == '`' { + break + } + if ch == '\r' { + hasCR = true + } + } + + lit := s.src[offs:s.offset] + if hasCR { + lit = stripCR(lit, false) + } + + return string(lit) +} + +func (s *Scanner) skipWhitespace() { + for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' && !s.insertSemi || s.ch == '\r' { + s.next() + } +} + +// Helper functions for scanning multi-byte tokens such as >> += >>= . +// Different routines recognize different length tok_i based on matches +// of ch_i. If a token ends in '=', the result is tok1 or tok3 +// respectively. Otherwise, the result is tok0 if there was no other +// matching character, or tok2 if the matching character was ch2. + +func (s *Scanner) switch2(tok0, tok1 token.Token) token.Token { + if s.ch == '=' { + s.next() + return tok1 + } + return tok0 +} + +func (s *Scanner) switch3(tok0, tok1 token.Token, ch2 rune, tok2 token.Token) token.Token { + if s.ch == '=' { + s.next() + return tok1 + } + if s.ch == ch2 { + s.next() + return tok2 + } + return tok0 +} + +func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Token) token.Token { + if s.ch == '=' { + s.next() + return tok1 + } + if s.ch == ch2 { + s.next() + if s.ch == '=' { + s.next() + return tok3 + } + return tok2 + } + return tok0 +} + +// Scan scans the next token and returns the token position, the token, +// and its literal string if applicable. The source end is indicated by +// token.EOF. +// +// If the returned token is a literal (token.IDENT, token.INT, token.FLOAT, +// token.IMAG, token.CHAR, token.STRING) or token.COMMENT, the literal string +// has the corresponding value. +// +// If the returned token is a keyword, the literal string is the keyword. +// +// If the returned token is token.SEMICOLON, the corresponding +// literal string is ";" if the semicolon was present in the source, +// and "\n" if the semicolon was inserted because of a newline or +// at EOF. +// +// If the returned token is token.ILLEGAL, the literal string is the +// offending character. +// +// In all other cases, Scan returns an empty literal string. +// +// For more tolerant parsing, Scan will return a valid token if +// possible even if a syntax error was encountered. Thus, even +// if the resulting token sequence contains no illegal tokens, +// a client may not assume that no error occurred. Instead it +// must check the scanner's ErrorCount or the number of calls +// of the error handler, if there was one installed. +// +// Scan adds line information to the file added to the file +// set with Init. Token positions are relative to that file +// and thus relative to the file set. +func (s *Scanner) Scan() (pos token.Pos, tok token.Token, lit string) { +scanAgain: + if s.nlPos.IsValid() { + // Return artificial ';' token after /*...*/ comment + // containing newline, at position of first newline. + pos, tok, lit = s.nlPos, token.SEMICOLON, "\n" + s.nlPos = token.NoPos + return + } + + s.skipWhitespace() + + // current token start + pos = s.file.Pos(s.offset) + + // determine token value + insertSemi := false + switch ch := s.ch; { + case isLetter(ch): + lit = s.scanIdentifier() + if len(lit) > 1 { + // keywords are longer than one letter - avoid lookup otherwise + tok = token.Lookup(lit) + switch tok { + case token.IDENT, token.BREAK, token.CONTINUE, token.FALLTHROUGH, token.RETURN: + insertSemi = true + } + } else { + insertSemi = true + tok = token.IDENT + } + case isDecimal(ch) || ch == '.' && isDecimal(rune(s.peek())): + insertSemi = true + tok, lit = s.scanNumber() + default: + s.next() // always make progress + switch ch { + case eof: + if s.insertSemi { + s.insertSemi = false // EOF consumed + return pos, token.SEMICOLON, "\n" + } + tok = token.EOF + case '\n': + // we only reach here if s.insertSemi was + // set in the first place and exited early + // from s.skipWhitespace() + s.insertSemi = false // newline consumed + return pos, token.SEMICOLON, "\n" + case '"': + insertSemi = true + tok = token.STRING + lit = s.scanString() + case '\'': + insertSemi = true + tok = token.CHAR + lit = s.scanRune() + case '`': + insertSemi = true + tok = token.STRING + lit = s.scanRawString() + case ':': + tok = s.switch2(token.COLON, token.DEFINE) + case '.': + // fractions starting with a '.' are handled by outer switch + tok = token.PERIOD + if s.ch == '.' && s.peek() == '.' { + s.next() + s.next() // consume last '.' + tok = token.ELLIPSIS + } + case ',': + tok = token.COMMA + case ';': + tok = token.SEMICOLON + lit = ";" + case '(': + tok = token.LPAREN + case ')': + insertSemi = true + tok = token.RPAREN + case '[': + tok = token.LBRACK + case ']': + insertSemi = true + tok = token.RBRACK + case '{': + tok = token.LBRACE + case '}': + insertSemi = true + tok = token.RBRACE + case '+': + tok = s.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC) + if tok == token.INC { + insertSemi = true + } + case '-': + tok = s.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC) + if tok == token.DEC { + insertSemi = true + } + case '*': + tok = s.switch2(token.MUL, token.MUL_ASSIGN) + case '/': + if s.ch == '/' || s.ch == '*' { + // comment + comment, nlOffset := s.scanComment() + if s.insertSemi && nlOffset != 0 { + // For /*...*/ containing \n, return + // COMMENT then artificial SEMICOLON. + s.nlPos = s.file.Pos(nlOffset) + s.insertSemi = false + } else { + insertSemi = s.insertSemi // preserve insertSemi info + } + if s.mode&ScanComments == 0 { + // skip comment + goto scanAgain + } + tok = token.COMMENT + lit = comment + } else { + // division + tok = s.switch2(token.QUO, token.QUO_ASSIGN) + } + case '%': + tok = s.switch2(token.REM, token.REM_ASSIGN) + case '^': + tok = s.switch2(token.XOR, token.XOR_ASSIGN) + case '<': + if s.ch == '-' { + s.next() + tok = token.ARROW + } else { + tok = s.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN) + } + case '>': + tok = s.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN) + case '=': + tok = s.switch2(token.ASSIGN, token.EQL) + case '!': + tok = s.switch2(token.NOT, token.NEQ) + case '&': + if s.ch == '^' { + s.next() + tok = s.switch2(token.AND_NOT, token.AND_NOT_ASSIGN) + } else { + tok = s.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND) + } + case '|': + tok = s.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR) + case '~': + tok = token.TILDE + default: + // next reports unexpected BOMs - don't repeat + if ch != bom { + s.errorf(s.file.Offset(pos), "illegal character %#U", ch) + } + insertSemi = s.insertSemi // preserve insertSemi info + tok = token.ILLEGAL + lit = string(ch) + } + } + if s.mode&dontInsertSemis == 0 { + s.insertSemi = insertSemi + } + + return +} diff --git a/src/go/scanner/scanner_test.go b/src/go/scanner/scanner_test.go new file mode 100644 index 0000000..4f320ee --- /dev/null +++ b/src/go/scanner/scanner_test.go @@ -0,0 +1,1125 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package scanner + +import ( + "go/token" + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +var fset = token.NewFileSet() + +const /* class */ ( + special = iota + literal + operator + keyword +) + +func tokenclass(tok token.Token) int { + switch { + case tok.IsLiteral(): + return literal + case tok.IsOperator(): + return operator + case tok.IsKeyword(): + return keyword + } + return special +} + +type elt struct { + tok token.Token + lit string + class int +} + +var tokens = []elt{ + // Special tokens + {token.COMMENT, "/* a comment */", special}, + {token.COMMENT, "// a comment \n", special}, + {token.COMMENT, "/*\r*/", special}, + {token.COMMENT, "/**\r/*/", special}, // issue 11151 + {token.COMMENT, "/**\r\r/*/", special}, + {token.COMMENT, "//\r\n", special}, + + // Identifiers and basic type literals + {token.IDENT, "foobar", literal}, + {token.IDENT, "aŪ°Ū±Ūø", literal}, + {token.IDENT, "fooą„¬ą„Ŗ", literal}, + {token.IDENT, "barļ¼™ļ¼˜ļ¼—ļ¼–", literal}, + {token.IDENT, "ŝ", literal}, // was bug (issue 4000) + {token.IDENT, "ŝfoo", literal}, // was bug (issue 4000) + {token.INT, "0", literal}, + {token.INT, "1", literal}, + {token.INT, "123456789012345678890", literal}, + {token.INT, "01234567", literal}, + {token.INT, "0xcafebabe", literal}, + {token.FLOAT, "0.", literal}, + {token.FLOAT, ".0", literal}, + {token.FLOAT, "3.14159265", literal}, + {token.FLOAT, "1e0", literal}, + {token.FLOAT, "1e+100", literal}, + {token.FLOAT, "1e-100", literal}, + {token.FLOAT, "2.71828e-1000", literal}, + {token.IMAG, "0i", literal}, + {token.IMAG, "1i", literal}, + {token.IMAG, "012345678901234567889i", literal}, + {token.IMAG, "123456789012345678890i", literal}, + {token.IMAG, "0.i", literal}, + {token.IMAG, ".0i", literal}, + {token.IMAG, "3.14159265i", literal}, + {token.IMAG, "1e0i", literal}, + {token.IMAG, "1e+100i", literal}, + {token.IMAG, "1e-100i", literal}, + {token.IMAG, "2.71828e-1000i", literal}, + {token.CHAR, "'a'", literal}, + {token.CHAR, "'\\000'", literal}, + {token.CHAR, "'\\xFF'", literal}, + {token.CHAR, "'\\uff16'", literal}, + {token.CHAR, "'\\U0000ff16'", literal}, + {token.STRING, "`foobar`", literal}, + {token.STRING, "`" + `foo + bar` + + "`", + literal, + }, + {token.STRING, "`\r`", literal}, + {token.STRING, "`foo\r\nbar`", literal}, + + // Operators and delimiters + {token.ADD, "+", operator}, + {token.SUB, "-", operator}, + {token.MUL, "*", operator}, + {token.QUO, "/", operator}, + {token.REM, "%", operator}, + + {token.AND, "&", operator}, + {token.OR, "|", operator}, + {token.XOR, "^", operator}, + {token.SHL, "<<", operator}, + {token.SHR, ">>", operator}, + {token.AND_NOT, "&^", operator}, + + {token.ADD_ASSIGN, "+=", operator}, + {token.SUB_ASSIGN, "-=", operator}, + {token.MUL_ASSIGN, "*=", operator}, + {token.QUO_ASSIGN, "/=", operator}, + {token.REM_ASSIGN, "%=", operator}, + + {token.AND_ASSIGN, "&=", operator}, + {token.OR_ASSIGN, "|=", operator}, + {token.XOR_ASSIGN, "^=", operator}, + {token.SHL_ASSIGN, "<<=", operator}, + {token.SHR_ASSIGN, ">>=", operator}, + {token.AND_NOT_ASSIGN, "&^=", operator}, + + {token.LAND, "&&", operator}, + {token.LOR, "||", operator}, + {token.ARROW, "<-", operator}, + {token.INC, "++", operator}, + {token.DEC, "--", operator}, + + {token.EQL, "==", operator}, + {token.LSS, "<", operator}, + {token.GTR, ">", operator}, + {token.ASSIGN, "=", operator}, + {token.NOT, "!", operator}, + + {token.NEQ, "!=", operator}, + {token.LEQ, "<=", operator}, + {token.GEQ, ">=", operator}, + {token.DEFINE, ":=", operator}, + {token.ELLIPSIS, "...", operator}, + + {token.LPAREN, "(", operator}, + {token.LBRACK, "[", operator}, + {token.LBRACE, "{", operator}, + {token.COMMA, ",", operator}, + {token.PERIOD, ".", operator}, + + {token.RPAREN, ")", operator}, + {token.RBRACK, "]", operator}, + {token.RBRACE, "}", operator}, + {token.SEMICOLON, ";", operator}, + {token.COLON, ":", operator}, + {token.TILDE, "~", operator}, + + // Keywords + {token.BREAK, "break", keyword}, + {token.CASE, "case", keyword}, + {token.CHAN, "chan", keyword}, + {token.CONST, "const", keyword}, + {token.CONTINUE, "continue", keyword}, + + {token.DEFAULT, "default", keyword}, + {token.DEFER, "defer", keyword}, + {token.ELSE, "else", keyword}, + {token.FALLTHROUGH, "fallthrough", keyword}, + {token.FOR, "for", keyword}, + + {token.FUNC, "func", keyword}, + {token.GO, "go", keyword}, + {token.GOTO, "goto", keyword}, + {token.IF, "if", keyword}, + {token.IMPORT, "import", keyword}, + + {token.INTERFACE, "interface", keyword}, + {token.MAP, "map", keyword}, + {token.PACKAGE, "package", keyword}, + {token.RANGE, "range", keyword}, + {token.RETURN, "return", keyword}, + + {token.SELECT, "select", keyword}, + {token.STRUCT, "struct", keyword}, + {token.SWITCH, "switch", keyword}, + {token.TYPE, "type", keyword}, + {token.VAR, "var", keyword}, +} + +const whitespace = " \t \n\n\n" // to separate tokens + +var source = func() []byte { + var src []byte + for _, t := range tokens { + src = append(src, t.lit...) + src = append(src, whitespace...) + } + return src +}() + +func newlineCount(s string) int { + n := 0 + for i := 0; i < len(s); i++ { + if s[i] == '\n' { + n++ + } + } + return n +} + +func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) { + pos := fset.Position(p) + // Check cleaned filenames so that we don't have to worry about + // different os.PathSeparator values. + if pos.Filename != expected.Filename && filepath.Clean(pos.Filename) != filepath.Clean(expected.Filename) { + t.Errorf("bad filename for %q: got %s, expected %s", lit, pos.Filename, expected.Filename) + } + if pos.Offset != expected.Offset { + t.Errorf("bad position for %q: got %d, expected %d", lit, pos.Offset, expected.Offset) + } + if pos.Line != expected.Line { + t.Errorf("bad line for %q: got %d, expected %d", lit, pos.Line, expected.Line) + } + if pos.Column != expected.Column { + t.Errorf("bad column for %q: got %d, expected %d", lit, pos.Column, expected.Column) + } +} + +// Verify that calling Scan() provides the correct results. +func TestScan(t *testing.T) { + whitespace_linecount := newlineCount(whitespace) + + // error handler + eh := func(_ token.Position, msg string) { + t.Errorf("error handler called (msg = %s)", msg) + } + + // verify scan + var s Scanner + s.Init(fset.AddFile("", fset.Base(), len(source)), source, eh, ScanComments|dontInsertSemis) + + // set up expected position + epos := token.Position{ + Filename: "", + Offset: 0, + Line: 1, + Column: 1, + } + + index := 0 + for { + pos, tok, lit := s.Scan() + + // check position + if tok == token.EOF { + // correction for EOF + epos.Line = newlineCount(string(source)) + epos.Column = 2 + } + checkPos(t, lit, pos, epos) + + // check token + e := elt{token.EOF, "", special} + if index < len(tokens) { + e = tokens[index] + index++ + } + if tok != e.tok { + t.Errorf("bad token for %q: got %s, expected %s", lit, tok, e.tok) + } + + // check token class + if tokenclass(tok) != e.class { + t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class) + } + + // check literal + elit := "" + switch e.tok { + case token.COMMENT: + // no CRs in comments + elit = string(stripCR([]byte(e.lit), e.lit[1] == '*')) + //-style comment literal doesn't contain newline + if elit[1] == '/' { + elit = elit[0 : len(elit)-1] + } + case token.IDENT: + elit = e.lit + case token.SEMICOLON: + elit = ";" + default: + if e.tok.IsLiteral() { + // no CRs in raw string literals + elit = e.lit + if elit[0] == '`' { + elit = string(stripCR([]byte(elit), false)) + } + } else if e.tok.IsKeyword() { + elit = e.lit + } + } + if lit != elit { + t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, elit) + } + + if tok == token.EOF { + break + } + + // update position + epos.Offset += len(e.lit) + len(whitespace) + epos.Line += newlineCount(e.lit) + whitespace_linecount + + } + + if s.ErrorCount != 0 { + t.Errorf("found %d errors", s.ErrorCount) + } +} + +func TestStripCR(t *testing.T) { + for _, test := range []struct{ have, want string }{ + {"//\n", "//\n"}, + {"//\r\n", "//\n"}, + {"//\r\r\r\n", "//\n"}, + {"//\r*\r/\r\n", "//*/\n"}, + {"/**/", "/**/"}, + {"/*\r/*/", "/*/*/"}, + {"/*\r*/", "/**/"}, + {"/**\r/*/", "/**\r/*/"}, + {"/*\r/\r*\r/*/", "/*/*\r/*/"}, + {"/*\r\r\r\r*/", "/**/"}, + } { + got := string(stripCR([]byte(test.have), len(test.have) >= 2 && test.have[1] == '*')) + if got != test.want { + t.Errorf("stripCR(%q) = %q; want %q", test.have, got, test.want) + } + } +} + +func checkSemi(t *testing.T, input, want string, mode Mode) { + if mode&ScanComments == 0 { + want = strings.ReplaceAll(want, "COMMENT ", "") + want = strings.ReplaceAll(want, " COMMENT", "") // if at end + want = strings.ReplaceAll(want, "COMMENT", "") // if sole token + } + + file := fset.AddFile("TestSemis", fset.Base(), len(input)) + var scan Scanner + scan.Init(file, []byte(input), nil, mode) + var tokens []string + for { + pos, tok, lit := scan.Scan() + if tok == token.EOF { + break + } + if tok == token.SEMICOLON && lit != ";" { + // Artifical semicolon: + // assert that position is EOF or that of a newline. + off := file.Offset(pos) + if off != len(input) && input[off] != '\n' { + t.Errorf("scanning <<%s>>, got SEMICOLON at offset %d, want newline or EOF", input, off) + } + } + lit = tok.String() // "\n" => ";" + tokens = append(tokens, lit) + } + if got := strings.Join(tokens, " "); got != want { + t.Errorf("scanning <<%s>>, got [%s], want [%s]", input, got, want) + } +} + +var semicolonTests = [...]struct{ input, want string }{ + {"", ""}, + {"\ufeff;", ";"}, // first BOM is ignored + {";", ";"}, + {"foo\n", "IDENT ;"}, + {"123\n", "INT ;"}, + {"1.2\n", "FLOAT ;"}, + {"'x'\n", "CHAR ;"}, + {`"x"` + "\n", "STRING ;"}, + {"`x`\n", "STRING ;"}, + + {"+\n", "+"}, + {"-\n", "-"}, + {"*\n", "*"}, + {"/\n", "/"}, + {"%\n", "%"}, + + {"&\n", "&"}, + {"|\n", "|"}, + {"^\n", "^"}, + {"<<\n", "<<"}, + {">>\n", ">>"}, + {"&^\n", "&^"}, + + {"+=\n", "+="}, + {"-=\n", "-="}, + {"*=\n", "*="}, + {"/=\n", "/="}, + {"%=\n", "%="}, + + {"&=\n", "&="}, + {"|=\n", "|="}, + {"^=\n", "^="}, + {"<<=\n", "<<="}, + {">>=\n", ">>="}, + {"&^=\n", "&^="}, + + {"&&\n", "&&"}, + {"||\n", "||"}, + {"<-\n", "<-"}, + {"++\n", "++ ;"}, + {"--\n", "-- ;"}, + + {"==\n", "=="}, + {"<\n", "<"}, + {">\n", ">"}, + {"=\n", "="}, + {"!\n", "!"}, + + {"!=\n", "!="}, + {"<=\n", "<="}, + {">=\n", ">="}, + {":=\n", ":="}, + {"...\n", "..."}, + + {"(\n", "("}, + {"[\n", "["}, + {"{\n", "{"}, + {",\n", ","}, + {".\n", "."}, + + {")\n", ") ;"}, + {"]\n", "] ;"}, + {"}\n", "} ;"}, + {";\n", ";"}, + {":\n", ":"}, + + {"break\n", "break ;"}, + {"case\n", "case"}, + {"chan\n", "chan"}, + {"const\n", "const"}, + {"continue\n", "continue ;"}, + + {"default\n", "default"}, + {"defer\n", "defer"}, + {"else\n", "else"}, + {"fallthrough\n", "fallthrough ;"}, + {"for\n", "for"}, + + {"func\n", "func"}, + {"go\n", "go"}, + {"goto\n", "goto"}, + {"if\n", "if"}, + {"import\n", "import"}, + + {"interface\n", "interface"}, + {"map\n", "map"}, + {"package\n", "package"}, + {"range\n", "range"}, + {"return\n", "return ;"}, + + {"select\n", "select"}, + {"struct\n", "struct"}, + {"switch\n", "switch"}, + {"type\n", "type"}, + {"var\n", "var"}, + + {"foo//comment\n", "IDENT COMMENT ;"}, + {"foo//comment", "IDENT COMMENT ;"}, + {"foo/*comment*/\n", "IDENT COMMENT ;"}, + {"foo/*\n*/", "IDENT COMMENT ;"}, + {"foo/*comment*/ \n", "IDENT COMMENT ;"}, + {"foo/*\n*/ ", "IDENT COMMENT ;"}, + + {"foo // comment\n", "IDENT COMMENT ;"}, + {"foo // comment", "IDENT COMMENT ;"}, + {"foo /*comment*/\n", "IDENT COMMENT ;"}, + {"foo /*\n*/", "IDENT COMMENT ;"}, + {"foo /* */ /* \n */ bar/**/\n", "IDENT COMMENT COMMENT ; IDENT COMMENT ;"}, + {"foo /*0*/ /*1*/ /*2*/\n", "IDENT COMMENT COMMENT COMMENT ;"}, + + {"foo /*comment*/ \n", "IDENT COMMENT ;"}, + {"foo /*0*/ /*1*/ /*2*/ \n", "IDENT COMMENT COMMENT COMMENT ;"}, + {"foo /**/ /*-------------*/ /*----\n*/bar /* \n*/baa\n", "IDENT COMMENT COMMENT COMMENT ; IDENT COMMENT ; IDENT ;"}, + {"foo /* an EOF terminates a line */", "IDENT COMMENT ;"}, + {"foo /* an EOF terminates a line */ /*", "IDENT COMMENT COMMENT ;"}, + {"foo /* an EOF terminates a line */ //", "IDENT COMMENT COMMENT ;"}, + + {"package main\n\nfunc main() {\n\tif {\n\t\treturn /* */ }\n}\n", "package IDENT ; func IDENT ( ) { if { return COMMENT } ; } ;"}, + {"package main", "package IDENT ;"}, +} + +func TestSemicolons(t *testing.T) { + for _, test := range semicolonTests { + input, want := test.input, test.want + checkSemi(t, input, want, 0) + checkSemi(t, input, want, ScanComments) + + // if the input ended in newlines, the input must tokenize the + // same with or without those newlines + for i := len(input) - 1; i >= 0 && input[i] == '\n'; i-- { + checkSemi(t, input[0:i], want, 0) + checkSemi(t, input[0:i], want, ScanComments) + } + } +} + +type segment struct { + srcline string // a line of source text + filename string // filename for current token; error message for invalid line directives + line, column int // line and column for current token; error position for invalid line directives +} + +var segments = []segment{ + // exactly one token per line since the test consumes one token per segment + {" line1", "TestLineDirectives", 1, 3}, + {"\nline2", "TestLineDirectives", 2, 1}, + {"\nline3 //line File1.go:100", "TestLineDirectives", 3, 1}, // bad line comment, ignored + {"\nline4", "TestLineDirectives", 4, 1}, + {"\n//line File1.go:100\n line100", "File1.go", 100, 0}, + {"\n//line \t :42\n line1", " \t ", 42, 0}, + {"\n//line File2.go:200\n line200", "File2.go", 200, 0}, + {"\n//line foo\t:42\n line42", "foo\t", 42, 0}, + {"\n //line foo:42\n line43", "foo\t", 44, 0}, // bad line comment, ignored (use existing, prior filename) + {"\n//line foo 42\n line44", "foo\t", 46, 0}, // bad line comment, ignored (use existing, prior filename) + {"\n//line /bar:42\n line45", "/bar", 42, 0}, + {"\n//line ./foo:42\n line46", "foo", 42, 0}, + {"\n//line a/b/c/File1.go:100\n line100", "a/b/c/File1.go", 100, 0}, + {"\n//line c:\\bar:42\n line200", "c:\\bar", 42, 0}, + {"\n//line c:\\dir\\File1.go:100\n line201", "c:\\dir\\File1.go", 100, 0}, + + // tests for new line directive syntax + {"\n//line :100\na1", "", 100, 0}, // missing filename means empty filename + {"\n//line bar:100\nb1", "bar", 100, 0}, + {"\n//line :100:10\nc1", "bar", 100, 10}, // missing filename means current filename + {"\n//line foo:100:10\nd1", "foo", 100, 10}, + + {"\n/*line :100*/a2", "", 100, 0}, // missing filename means empty filename + {"\n/*line bar:100*/b2", "bar", 100, 0}, + {"\n/*line :100:10*/c2", "bar", 100, 10}, // missing filename means current filename + {"\n/*line foo:100:10*/d2", "foo", 100, 10}, + {"\n/*line foo:100:10*/ e2", "foo", 100, 14}, // line-directive relative column + {"\n/*line foo:100:10*/\n\nf2", "foo", 102, 1}, // absolute column since on new line +} + +var dirsegments = []segment{ + // exactly one token per line since the test consumes one token per segment + {" line1", "TestLineDir/TestLineDirectives", 1, 3}, + {"\n//line File1.go:100\n line100", "TestLineDir/File1.go", 100, 0}, +} + +var dirUnixSegments = []segment{ + {"\n//line /bar:42\n line42", "/bar", 42, 0}, +} + +var dirWindowsSegments = []segment{ + {"\n//line c:\\bar:42\n line42", "c:\\bar", 42, 0}, +} + +// Verify that line directives are interpreted correctly. +func TestLineDirectives(t *testing.T) { + testSegments(t, segments, "TestLineDirectives") + testSegments(t, dirsegments, "TestLineDir/TestLineDirectives") + if runtime.GOOS == "windows" { + testSegments(t, dirWindowsSegments, "TestLineDir/TestLineDirectives") + } else { + testSegments(t, dirUnixSegments, "TestLineDir/TestLineDirectives") + } +} + +func testSegments(t *testing.T, segments []segment, filename string) { + var src string + for _, e := range segments { + src += e.srcline + } + + // verify scan + var S Scanner + file := fset.AddFile(filename, fset.Base(), len(src)) + S.Init(file, []byte(src), func(pos token.Position, msg string) { t.Error(Error{pos, msg}) }, dontInsertSemis) + for _, s := range segments { + p, _, lit := S.Scan() + pos := file.Position(p) + checkPos(t, lit, p, token.Position{ + Filename: s.filename, + Offset: pos.Offset, + Line: s.line, + Column: s.column, + }) + } + + if S.ErrorCount != 0 { + t.Errorf("got %d errors", S.ErrorCount) + } +} + +// The filename is used for the error message in these test cases. +// The first line directive is valid and used to control the expected error line. +var invalidSegments = []segment{ + {"\n//line :1:1\n//line foo:42 extra text\ndummy", "invalid line number: 42 extra text", 1, 12}, + {"\n//line :2:1\n//line foobar:\ndummy", "invalid line number: ", 2, 15}, + {"\n//line :5:1\n//line :0\ndummy", "invalid line number: 0", 5, 9}, + {"\n//line :10:1\n//line :1:0\ndummy", "invalid column number: 0", 10, 11}, + {"\n//line :1:1\n//line :foo:0\ndummy", "invalid line number: 0", 1, 13}, // foo is considered part of the filename +} + +// Verify that invalid line directives get the correct error message. +func TestInvalidLineDirectives(t *testing.T) { + // make source + var src string + for _, e := range invalidSegments { + src += e.srcline + } + + // verify scan + var S Scanner + var s segment // current segment + file := fset.AddFile(filepath.Join("dir", "TestInvalidLineDirectives"), fset.Base(), len(src)) + S.Init(file, []byte(src), func(pos token.Position, msg string) { + if msg != s.filename { + t.Errorf("got error %q; want %q", msg, s.filename) + } + if pos.Line != s.line || pos.Column != s.column { + t.Errorf("got position %d:%d; want %d:%d", pos.Line, pos.Column, s.line, s.column) + } + }, dontInsertSemis) + for _, s = range invalidSegments { + S.Scan() + } + + if S.ErrorCount != len(invalidSegments) { + t.Errorf("got %d errors; want %d", S.ErrorCount, len(invalidSegments)) + } +} + +// Verify that initializing the same scanner more than once works correctly. +func TestInit(t *testing.T) { + var s Scanner + + // 1st init + src1 := "if true { }" + f1 := fset.AddFile("src1", fset.Base(), len(src1)) + s.Init(f1, []byte(src1), nil, dontInsertSemis) + if f1.Size() != len(src1) { + t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1)) + } + s.Scan() // if + s.Scan() // true + _, tok, _ := s.Scan() // { + if tok != token.LBRACE { + t.Errorf("bad token: got %s, expected %s", tok, token.LBRACE) + } + + // 2nd init + src2 := "go true { ]" + f2 := fset.AddFile("src2", fset.Base(), len(src2)) + s.Init(f2, []byte(src2), nil, dontInsertSemis) + if f2.Size() != len(src2) { + t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2)) + } + _, tok, _ = s.Scan() // go + if tok != token.GO { + t.Errorf("bad token: got %s, expected %s", tok, token.GO) + } + + if s.ErrorCount != 0 { + t.Errorf("found %d errors", s.ErrorCount) + } +} + +func TestStdErrorHandler(t *testing.T) { + const src = "@\n" + // illegal character, cause an error + "@ @\n" + // two errors on the same line + "//line File2:20\n" + + "@\n" + // different file, but same line + "//line File2:1\n" + + "@ @\n" + // same file, decreasing line number + "//line File1:1\n" + + "@ @ @" // original file, line 1 again + + var list ErrorList + eh := func(pos token.Position, msg string) { list.Add(pos, msg) } + + var s Scanner + s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), eh, dontInsertSemis) + for { + if _, tok, _ := s.Scan(); tok == token.EOF { + break + } + } + + if len(list) != s.ErrorCount { + t.Errorf("found %d errors, expected %d", len(list), s.ErrorCount) + } + + if len(list) != 9 { + t.Errorf("found %d raw errors, expected 9", len(list)) + PrintError(os.Stderr, list) + } + + list.Sort() + if len(list) != 9 { + t.Errorf("found %d sorted errors, expected 9", len(list)) + PrintError(os.Stderr, list) + } + + list.RemoveMultiples() + if len(list) != 4 { + t.Errorf("found %d one-per-line errors, expected 4", len(list)) + PrintError(os.Stderr, list) + } +} + +type errorCollector struct { + cnt int // number of errors encountered + msg string // last error message encountered + pos token.Position // last error position encountered +} + +func checkError(t *testing.T, src string, tok token.Token, pos int, lit, err string) { + var s Scanner + var h errorCollector + eh := func(pos token.Position, msg string) { + h.cnt++ + h.msg = msg + h.pos = pos + } + s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), eh, ScanComments|dontInsertSemis) + _, tok0, lit0 := s.Scan() + if tok0 != tok { + t.Errorf("%q: got %s, expected %s", src, tok0, tok) + } + if tok0 != token.ILLEGAL && lit0 != lit { + t.Errorf("%q: got literal %q, expected %q", src, lit0, lit) + } + cnt := 0 + if err != "" { + cnt = 1 + } + if h.cnt != cnt { + t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt) + } + if h.msg != err { + t.Errorf("%q: got msg %q, expected %q", src, h.msg, err) + } + if h.pos.Offset != pos { + t.Errorf("%q: got offset %d, expected %d", src, h.pos.Offset, pos) + } +} + +var errors = []struct { + src string + tok token.Token + pos int + lit string + err string +}{ + {"\a", token.ILLEGAL, 0, "", "illegal character U+0007"}, + {`#`, token.ILLEGAL, 0, "", "illegal character U+0023 '#'"}, + {`ā€¦`, token.ILLEGAL, 0, "", "illegal character U+2026 'ā€¦'"}, + {"..", token.PERIOD, 0, "", ""}, // two periods, not invalid token (issue #28112) + {`' '`, token.CHAR, 0, `' '`, ""}, + {`''`, token.CHAR, 0, `''`, "illegal rune literal"}, + {`'12'`, token.CHAR, 0, `'12'`, "illegal rune literal"}, + {`'123'`, token.CHAR, 0, `'123'`, "illegal rune literal"}, + {`'\0'`, token.CHAR, 3, `'\0'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\07'`, token.CHAR, 4, `'\07'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\8'`, token.CHAR, 2, `'\8'`, "unknown escape sequence"}, + {`'\08'`, token.CHAR, 3, `'\08'`, "illegal character U+0038 '8' in escape sequence"}, + {`'\x'`, token.CHAR, 3, `'\x'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\x0'`, token.CHAR, 4, `'\x0'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\x0g'`, token.CHAR, 4, `'\x0g'`, "illegal character U+0067 'g' in escape sequence"}, + {`'\u'`, token.CHAR, 3, `'\u'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\u0'`, token.CHAR, 4, `'\u0'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\u00'`, token.CHAR, 5, `'\u00'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\u000'`, token.CHAR, 6, `'\u000'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\u000`, token.CHAR, 6, `'\u000`, "escape sequence not terminated"}, + {`'\u0000'`, token.CHAR, 0, `'\u0000'`, ""}, + {`'\U'`, token.CHAR, 3, `'\U'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\U0'`, token.CHAR, 4, `'\U0'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\U00'`, token.CHAR, 5, `'\U00'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\U000'`, token.CHAR, 6, `'\U000'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\U0000'`, token.CHAR, 7, `'\U0000'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\U00000'`, token.CHAR, 8, `'\U00000'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\U000000'`, token.CHAR, 9, `'\U000000'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\U0000000'`, token.CHAR, 10, `'\U0000000'`, "illegal character U+0027 ''' in escape sequence"}, + {`'\U0000000`, token.CHAR, 10, `'\U0000000`, "escape sequence not terminated"}, + {`'\U00000000'`, token.CHAR, 0, `'\U00000000'`, ""}, + {`'\Uffffffff'`, token.CHAR, 2, `'\Uffffffff'`, "escape sequence is invalid Unicode code point"}, + {`'`, token.CHAR, 0, `'`, "rune literal not terminated"}, + {`'\`, token.CHAR, 2, `'\`, "escape sequence not terminated"}, + {"'\n", token.CHAR, 0, "'", "rune literal not terminated"}, + {"'\n ", token.CHAR, 0, "'", "rune literal not terminated"}, + {`""`, token.STRING, 0, `""`, ""}, + {`"abc`, token.STRING, 0, `"abc`, "string literal not terminated"}, + {"\"abc\n", token.STRING, 0, `"abc`, "string literal not terminated"}, + {"\"abc\n ", token.STRING, 0, `"abc`, "string literal not terminated"}, + {"``", token.STRING, 0, "``", ""}, + {"`", token.STRING, 0, "`", "raw string literal not terminated"}, + {"/**/", token.COMMENT, 0, "/**/", ""}, + {"/*", token.COMMENT, 0, "/*", "comment not terminated"}, + {"077", token.INT, 0, "077", ""}, + {"078.", token.FLOAT, 0, "078.", ""}, + {"07801234567.", token.FLOAT, 0, "07801234567.", ""}, + {"078e0", token.FLOAT, 0, "078e0", ""}, + {"0E", token.FLOAT, 2, "0E", "exponent has no digits"}, // issue 17621 + {"078", token.INT, 2, "078", "invalid digit '8' in octal literal"}, + {"07090000008", token.INT, 3, "07090000008", "invalid digit '9' in octal literal"}, + {"0x", token.INT, 2, "0x", "hexadecimal literal has no digits"}, + {"\"abc\x00def\"", token.STRING, 4, "\"abc\x00def\"", "illegal character NUL"}, + {"\"abc\x80def\"", token.STRING, 4, "\"abc\x80def\"", "illegal UTF-8 encoding"}, + {"\ufeff\ufeff", token.ILLEGAL, 3, "\ufeff\ufeff", "illegal byte order mark"}, // only first BOM is ignored + {"//\ufeff", token.COMMENT, 2, "//\ufeff", "illegal byte order mark"}, // only first BOM is ignored + {"'\ufeff" + `'`, token.CHAR, 1, "'\ufeff" + `'`, "illegal byte order mark"}, // only first BOM is ignored + {`"` + "abc\ufeffdef" + `"`, token.STRING, 4, `"` + "abc\ufeffdef" + `"`, "illegal byte order mark"}, // only first BOM is ignored + {"abc\x00def", token.IDENT, 3, "abc", "illegal character NUL"}, + {"abc\x00", token.IDENT, 3, "abc", "illegal character NUL"}, +} + +func TestScanErrors(t *testing.T) { + for _, e := range errors { + checkError(t, e.src, e.tok, e.pos, e.lit, e.err) + } +} + +// Verify that no comments show up as literal values when skipping comments. +func TestIssue10213(t *testing.T) { + const src = ` + var ( + A = 1 // foo + ) + + var ( + B = 2 + // foo + ) + + var C = 3 // foo + + var D = 4 + // foo + + func anycode() { + // foo + } + ` + var s Scanner + s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), nil, 0) + for { + pos, tok, lit := s.Scan() + class := tokenclass(tok) + if lit != "" && class != keyword && class != literal && tok != token.SEMICOLON { + t.Errorf("%s: tok = %s, lit = %q", fset.Position(pos), tok, lit) + } + if tok <= token.EOF { + break + } + } +} + +func TestIssue28112(t *testing.T) { + const src = "... .. 0.. .." // make sure to have stand-alone ".." immediately before EOF to test EOF behavior + tokens := []token.Token{token.ELLIPSIS, token.PERIOD, token.PERIOD, token.FLOAT, token.PERIOD, token.PERIOD, token.PERIOD, token.EOF} + var s Scanner + s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), nil, 0) + for _, want := range tokens { + pos, got, lit := s.Scan() + if got != want { + t.Errorf("%s: got %s, want %s", fset.Position(pos), got, want) + } + // literals expect to have a (non-empty) literal string and we don't care about other tokens for this test + if tokenclass(got) == literal && lit == "" { + t.Errorf("%s: for %s got empty literal string", fset.Position(pos), got) + } + } +} + +func BenchmarkScan(b *testing.B) { + b.StopTimer() + fset := token.NewFileSet() + file := fset.AddFile("", fset.Base(), len(source)) + var s Scanner + b.StartTimer() + for i := 0; i < b.N; i++ { + s.Init(file, source, nil, ScanComments) + for { + _, tok, _ := s.Scan() + if tok == token.EOF { + break + } + } + } +} + +func BenchmarkScanFiles(b *testing.B) { + // Scan a few arbitrary large files, and one small one, to provide some + // variety in benchmarks. + for _, p := range []string{ + "go/types/expr.go", + "go/parser/parser.go", + "net/http/server.go", + "go/scanner/errors.go", + } { + b.Run(p, func(b *testing.B) { + b.StopTimer() + filename := filepath.Join("..", "..", filepath.FromSlash(p)) + src, err := os.ReadFile(filename) + if err != nil { + b.Fatal(err) + } + fset := token.NewFileSet() + file := fset.AddFile(filename, fset.Base(), len(src)) + b.SetBytes(int64(len(src))) + var s Scanner + b.StartTimer() + for i := 0; i < b.N; i++ { + s.Init(file, src, nil, ScanComments) + for { + _, tok, _ := s.Scan() + if tok == token.EOF { + break + } + } + } + }) + } +} + +func TestNumbers(t *testing.T) { + for _, test := range []struct { + tok token.Token + src, tokens, err string + }{ + // binaries + {token.INT, "0b0", "0b0", ""}, + {token.INT, "0b1010", "0b1010", ""}, + {token.INT, "0B1110", "0B1110", ""}, + + {token.INT, "0b", "0b", "binary literal has no digits"}, + {token.INT, "0b0190", "0b0190", "invalid digit '9' in binary literal"}, + {token.INT, "0b01a0", "0b01 a0", ""}, // only accept 0-9 + + {token.FLOAT, "0b.", "0b.", "invalid radix point in binary literal"}, + {token.FLOAT, "0b.1", "0b.1", "invalid radix point in binary literal"}, + {token.FLOAT, "0b1.0", "0b1.0", "invalid radix point in binary literal"}, + {token.FLOAT, "0b1e10", "0b1e10", "'e' exponent requires decimal mantissa"}, + {token.FLOAT, "0b1P-1", "0b1P-1", "'P' exponent requires hexadecimal mantissa"}, + + {token.IMAG, "0b10i", "0b10i", ""}, + {token.IMAG, "0b10.0i", "0b10.0i", "invalid radix point in binary literal"}, + + // octals + {token.INT, "0o0", "0o0", ""}, + {token.INT, "0o1234", "0o1234", ""}, + {token.INT, "0O1234", "0O1234", ""}, + + {token.INT, "0o", "0o", "octal literal has no digits"}, + {token.INT, "0o8123", "0o8123", "invalid digit '8' in octal literal"}, + {token.INT, "0o1293", "0o1293", "invalid digit '9' in octal literal"}, + {token.INT, "0o12a3", "0o12 a3", ""}, // only accept 0-9 + + {token.FLOAT, "0o.", "0o.", "invalid radix point in octal literal"}, + {token.FLOAT, "0o.2", "0o.2", "invalid radix point in octal literal"}, + {token.FLOAT, "0o1.2", "0o1.2", "invalid radix point in octal literal"}, + {token.FLOAT, "0o1E+2", "0o1E+2", "'E' exponent requires decimal mantissa"}, + {token.FLOAT, "0o1p10", "0o1p10", "'p' exponent requires hexadecimal mantissa"}, + + {token.IMAG, "0o10i", "0o10i", ""}, + {token.IMAG, "0o10e0i", "0o10e0i", "'e' exponent requires decimal mantissa"}, + + // 0-octals + {token.INT, "0", "0", ""}, + {token.INT, "0123", "0123", ""}, + + {token.INT, "08123", "08123", "invalid digit '8' in octal literal"}, + {token.INT, "01293", "01293", "invalid digit '9' in octal literal"}, + {token.INT, "0F.", "0 F .", ""}, // only accept 0-9 + {token.INT, "0123F.", "0123 F .", ""}, + {token.INT, "0123456x", "0123456 x", ""}, + + // decimals + {token.INT, "1", "1", ""}, + {token.INT, "1234", "1234", ""}, + + {token.INT, "1f", "1 f", ""}, // only accept 0-9 + + {token.IMAG, "0i", "0i", ""}, + {token.IMAG, "0678i", "0678i", ""}, + + // decimal floats + {token.FLOAT, "0.", "0.", ""}, + {token.FLOAT, "123.", "123.", ""}, + {token.FLOAT, "0123.", "0123.", ""}, + + {token.FLOAT, ".0", ".0", ""}, + {token.FLOAT, ".123", ".123", ""}, + {token.FLOAT, ".0123", ".0123", ""}, + + {token.FLOAT, "0.0", "0.0", ""}, + {token.FLOAT, "123.123", "123.123", ""}, + {token.FLOAT, "0123.0123", "0123.0123", ""}, + + {token.FLOAT, "0e0", "0e0", ""}, + {token.FLOAT, "123e+0", "123e+0", ""}, + {token.FLOAT, "0123E-1", "0123E-1", ""}, + + {token.FLOAT, "0.e+1", "0.e+1", ""}, + {token.FLOAT, "123.E-10", "123.E-10", ""}, + {token.FLOAT, "0123.e123", "0123.e123", ""}, + + {token.FLOAT, ".0e-1", ".0e-1", ""}, + {token.FLOAT, ".123E+10", ".123E+10", ""}, + {token.FLOAT, ".0123E123", ".0123E123", ""}, + + {token.FLOAT, "0.0e1", "0.0e1", ""}, + {token.FLOAT, "123.123E-10", "123.123E-10", ""}, + {token.FLOAT, "0123.0123e+456", "0123.0123e+456", ""}, + + {token.FLOAT, "0e", "0e", "exponent has no digits"}, + {token.FLOAT, "0E+", "0E+", "exponent has no digits"}, + {token.FLOAT, "1e+f", "1e+ f", "exponent has no digits"}, + {token.FLOAT, "0p0", "0p0", "'p' exponent requires hexadecimal mantissa"}, + {token.FLOAT, "1.0P-1", "1.0P-1", "'P' exponent requires hexadecimal mantissa"}, + + {token.IMAG, "0.i", "0.i", ""}, + {token.IMAG, ".123i", ".123i", ""}, + {token.IMAG, "123.123i", "123.123i", ""}, + {token.IMAG, "123e+0i", "123e+0i", ""}, + {token.IMAG, "123.E-10i", "123.E-10i", ""}, + {token.IMAG, ".123E+10i", ".123E+10i", ""}, + + // hexadecimals + {token.INT, "0x0", "0x0", ""}, + {token.INT, "0x1234", "0x1234", ""}, + {token.INT, "0xcafef00d", "0xcafef00d", ""}, + {token.INT, "0XCAFEF00D", "0XCAFEF00D", ""}, + + {token.INT, "0x", "0x", "hexadecimal literal has no digits"}, + {token.INT, "0x1g", "0x1 g", ""}, + + {token.IMAG, "0xf00i", "0xf00i", ""}, + + // hexadecimal floats + {token.FLOAT, "0x0p0", "0x0p0", ""}, + {token.FLOAT, "0x12efp-123", "0x12efp-123", ""}, + {token.FLOAT, "0xABCD.p+0", "0xABCD.p+0", ""}, + {token.FLOAT, "0x.0189P-0", "0x.0189P-0", ""}, + {token.FLOAT, "0x1.ffffp+1023", "0x1.ffffp+1023", ""}, + + {token.FLOAT, "0x.", "0x.", "hexadecimal literal has no digits"}, + {token.FLOAT, "0x0.", "0x0.", "hexadecimal mantissa requires a 'p' exponent"}, + {token.FLOAT, "0x.0", "0x.0", "hexadecimal mantissa requires a 'p' exponent"}, + {token.FLOAT, "0x1.1", "0x1.1", "hexadecimal mantissa requires a 'p' exponent"}, + {token.FLOAT, "0x1.1e0", "0x1.1e0", "hexadecimal mantissa requires a 'p' exponent"}, + {token.FLOAT, "0x1.2gp1a", "0x1.2 gp1a", "hexadecimal mantissa requires a 'p' exponent"}, + {token.FLOAT, "0x0p", "0x0p", "exponent has no digits"}, + {token.FLOAT, "0xeP-", "0xeP-", "exponent has no digits"}, + {token.FLOAT, "0x1234PAB", "0x1234P AB", "exponent has no digits"}, + {token.FLOAT, "0x1.2p1a", "0x1.2p1 a", ""}, + + {token.IMAG, "0xf00.bap+12i", "0xf00.bap+12i", ""}, + + // separators + {token.INT, "0b_1000_0001", "0b_1000_0001", ""}, + {token.INT, "0o_600", "0o_600", ""}, + {token.INT, "0_466", "0_466", ""}, + {token.INT, "1_000", "1_000", ""}, + {token.FLOAT, "1_000.000_1", "1_000.000_1", ""}, + {token.IMAG, "10e+1_2_3i", "10e+1_2_3i", ""}, + {token.INT, "0x_f00d", "0x_f00d", ""}, + {token.FLOAT, "0x_f00d.0p1_2", "0x_f00d.0p1_2", ""}, + + {token.INT, "0b__1000", "0b__1000", "'_' must separate successive digits"}, + {token.INT, "0o60___0", "0o60___0", "'_' must separate successive digits"}, + {token.INT, "0466_", "0466_", "'_' must separate successive digits"}, + {token.FLOAT, "1_.", "1_.", "'_' must separate successive digits"}, + {token.FLOAT, "0._1", "0._1", "'_' must separate successive digits"}, + {token.FLOAT, "2.7_e0", "2.7_e0", "'_' must separate successive digits"}, + {token.IMAG, "10e+12_i", "10e+12_i", "'_' must separate successive digits"}, + {token.INT, "0x___0", "0x___0", "'_' must separate successive digits"}, + {token.FLOAT, "0x1.0_p0", "0x1.0_p0", "'_' must separate successive digits"}, + } { + var s Scanner + var err string + s.Init(fset.AddFile("", fset.Base(), len(test.src)), []byte(test.src), func(_ token.Position, msg string) { + if err == "" { + err = msg + } + }, 0) + for i, want := range strings.Split(test.tokens, " ") { + err = "" + _, tok, lit := s.Scan() + + // compute lit where for tokens where lit is not defined + switch tok { + case token.PERIOD: + lit = "." + case token.ADD: + lit = "+" + case token.SUB: + lit = "-" + } + + if i == 0 { + if tok != test.tok { + t.Errorf("%q: got token %s; want %s", test.src, tok, test.tok) + } + if err != test.err { + t.Errorf("%q: got error %q; want %q", test.src, err, test.err) + } + } + + if lit != want { + t.Errorf("%q: got literal %q (%s); want %s", test.src, lit, tok, want) + } + } + + // make sure we read all + _, tok, _ := s.Scan() + if tok == token.SEMICOLON { + _, tok, _ = s.Scan() + } + if tok != token.EOF { + t.Errorf("%q: got %s; want EOF", test.src, tok) + } + } +} diff --git a/src/go/token/example_test.go b/src/go/token/example_test.go new file mode 100644 index 0000000..0011703 --- /dev/null +++ b/src/go/token/example_test.go @@ -0,0 +1,77 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package token_test + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" +) + +func Example_retrievePositionInfo() { + fset := token.NewFileSet() + + const src = `package main + +import "fmt" + +import "go/token" + +//line :1:5 +type p = token.Pos + +const bad = token.NoPos + +//line fake.go:42:11 +func ok(pos p) bool { + return pos != bad +} + +/*line :7:9*/func main() { + fmt.Println(ok(bad) == bad.IsValid()) +} +` + + f, err := parser.ParseFile(fset, "main.go", src, 0) + if err != nil { + fmt.Println(err) + return + } + + // Print the location and kind of each declaration in f. + for _, decl := range f.Decls { + // Get the filename, line, and column back via the file set. + // We get both the relative and absolute position. + // The relative position is relative to the last line directive. + // The absolute position is the exact position in the source. + pos := decl.Pos() + relPosition := fset.Position(pos) + absPosition := fset.PositionFor(pos, false) + + // Either a FuncDecl or GenDecl, since we exit on error. + kind := "func" + if gen, ok := decl.(*ast.GenDecl); ok { + kind = gen.Tok.String() + } + + // If the relative and absolute positions differ, show both. + fmtPosition := relPosition.String() + if relPosition != absPosition { + fmtPosition += "[" + absPosition.String() + "]" + } + + fmt.Printf("%s: %s\n", fmtPosition, kind) + } + + //Output: + // + // main.go:3:1: import + // main.go:5:1: import + // main.go:1:5[main.go:8:1]: type + // main.go:3:1[main.go:10:1]: const + // fake.go:42:11[main.go:13:1]: func + // fake.go:7:9[main.go:17:14]: func +} diff --git a/src/go/token/position.go b/src/go/token/position.go new file mode 100644 index 0000000..cbc2ddb --- /dev/null +++ b/src/go/token/position.go @@ -0,0 +1,556 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package token + +import ( + "fmt" + "sort" + "sync" + "sync/atomic" +) + +// ----------------------------------------------------------------------------- +// Positions + +// Position describes an arbitrary source position +// including the file, line, and column location. +// A Position is valid if the line number is > 0. +type Position struct { + Filename string // filename, if any + Offset int // offset, starting at 0 + Line int // line number, starting at 1 + Column int // column number, starting at 1 (byte count) +} + +// IsValid reports whether the position is valid. +func (pos *Position) IsValid() bool { return pos.Line > 0 } + +// String returns a string in one of several forms: +// +// file:line:column valid position with file name +// file:line valid position with file name but no column (column == 0) +// line:column valid position without file name +// line valid position without file name and no column (column == 0) +// file invalid position with file name +// - invalid position without file name +func (pos Position) String() string { + s := pos.Filename + if pos.IsValid() { + if s != "" { + s += ":" + } + s += fmt.Sprintf("%d", pos.Line) + if pos.Column != 0 { + s += fmt.Sprintf(":%d", pos.Column) + } + } + if s == "" { + s = "-" + } + return s +} + +// Pos is a compact encoding of a source position within a file set. +// It can be converted into a Position for a more convenient, but much +// larger, representation. +// +// The Pos value for a given file is a number in the range [base, base+size], +// where base and size are specified when a file is added to the file set. +// The difference between a Pos value and the corresponding file base +// corresponds to the byte offset of that position (represented by the Pos value) +// from the beginning of the file. Thus, the file base offset is the Pos value +// representing the first byte in the file. +// +// To create the Pos value for a specific source offset (measured in bytes), +// first add the respective file to the current file set using FileSet.AddFile +// and then call File.Pos(offset) for that file. Given a Pos value p +// for a specific file set fset, the corresponding Position value is +// obtained by calling fset.Position(p). +// +// Pos values can be compared directly with the usual comparison operators: +// If two Pos values p and q are in the same file, comparing p and q is +// equivalent to comparing the respective source file offsets. If p and q +// are in different files, p < q is true if the file implied by p was added +// to the respective file set before the file implied by q. +type Pos int + +// The zero value for Pos is NoPos; there is no file and line information +// associated with it, and NoPos.IsValid() is false. NoPos is always +// smaller than any other Pos value. The corresponding Position value +// for NoPos is the zero value for Position. +const NoPos Pos = 0 + +// IsValid reports whether the position is valid. +func (p Pos) IsValid() bool { + return p != NoPos +} + +// ----------------------------------------------------------------------------- +// File + +// A File is a handle for a file belonging to a FileSet. +// A File has a name, size, and line offset table. +type File struct { + name string // file name as provided to AddFile + base int // Pos value range for this file is [base...base+size] + size int // file size as provided to AddFile + + // lines and infos are protected by mutex + mutex sync.Mutex + lines []int // lines contains the offset of the first character for each line (the first entry is always 0) + infos []lineInfo +} + +// Name returns the file name of file f as registered with AddFile. +func (f *File) Name() string { + return f.name +} + +// Base returns the base offset of file f as registered with AddFile. +func (f *File) Base() int { + return f.base +} + +// Size returns the size of file f as registered with AddFile. +func (f *File) Size() int { + return f.size +} + +// LineCount returns the number of lines in file f. +func (f *File) LineCount() int { + f.mutex.Lock() + n := len(f.lines) + f.mutex.Unlock() + return n +} + +// AddLine adds the line offset for a new line. +// The line offset must be larger than the offset for the previous line +// and smaller than the file size; otherwise the line offset is ignored. +func (f *File) AddLine(offset int) { + f.mutex.Lock() + if i := len(f.lines); (i == 0 || f.lines[i-1] < offset) && offset < f.size { + f.lines = append(f.lines, offset) + } + f.mutex.Unlock() +} + +// MergeLine merges a line with the following line. It is akin to replacing +// the newline character at the end of the line with a space (to not change the +// remaining offsets). To obtain the line number, consult e.g. Position.Line. +// MergeLine will panic if given an invalid line number. +func (f *File) MergeLine(line int) { + if line < 1 { + panic(fmt.Sprintf("invalid line number %d (should be >= 1)", line)) + } + f.mutex.Lock() + defer f.mutex.Unlock() + if line >= len(f.lines) { + panic(fmt.Sprintf("invalid line number %d (should be < %d)", line, len(f.lines))) + } + // To merge the line numbered with the line numbered , + // we need to remove the entry in lines corresponding to the line + // numbered . The entry in lines corresponding to the line + // numbered is located at index , since indices in lines + // are 0-based and line numbers are 1-based. + copy(f.lines[line:], f.lines[line+1:]) + f.lines = f.lines[:len(f.lines)-1] +} + +// SetLines sets the line offsets for a file and reports whether it succeeded. +// The line offsets are the offsets of the first character of each line; +// for instance for the content "ab\nc\n" the line offsets are {0, 3}. +// An empty file has an empty line offset table. +// Each line offset must be larger than the offset for the previous line +// and smaller than the file size; otherwise SetLines fails and returns +// false. +// Callers must not mutate the provided slice after SetLines returns. +func (f *File) SetLines(lines []int) bool { + // verify validity of lines table + size := f.size + for i, offset := range lines { + if i > 0 && offset <= lines[i-1] || size <= offset { + return false + } + } + + // set lines table + f.mutex.Lock() + f.lines = lines + f.mutex.Unlock() + return true +} + +// SetLinesForContent sets the line offsets for the given file content. +// It ignores position-altering //line comments. +func (f *File) SetLinesForContent(content []byte) { + var lines []int + line := 0 + for offset, b := range content { + if line >= 0 { + lines = append(lines, line) + } + line = -1 + if b == '\n' { + line = offset + 1 + } + } + + // set lines table + f.mutex.Lock() + f.lines = lines + f.mutex.Unlock() +} + +// LineStart returns the Pos value of the start of the specified line. +// It ignores any alternative positions set using AddLineColumnInfo. +// LineStart panics if the 1-based line number is invalid. +func (f *File) LineStart(line int) Pos { + if line < 1 { + panic(fmt.Sprintf("invalid line number %d (should be >= 1)", line)) + } + f.mutex.Lock() + defer f.mutex.Unlock() + if line > len(f.lines) { + panic(fmt.Sprintf("invalid line number %d (should be < %d)", line, len(f.lines))) + } + return Pos(f.base + f.lines[line-1]) +} + +// A lineInfo object describes alternative file, line, and column +// number information (such as provided via a //line directive) +// for a given file offset. +type lineInfo struct { + // fields are exported to make them accessible to gob + Offset int + Filename string + Line, Column int +} + +// AddLineInfo is like AddLineColumnInfo with a column = 1 argument. +// It is here for backward-compatibility for code prior to Go 1.11. +func (f *File) AddLineInfo(offset int, filename string, line int) { + f.AddLineColumnInfo(offset, filename, line, 1) +} + +// AddLineColumnInfo adds alternative file, line, and column number +// information for a given file offset. The offset must be larger +// than the offset for the previously added alternative line info +// and smaller than the file size; otherwise the information is +// ignored. +// +// AddLineColumnInfo is typically used to register alternative position +// information for line directives such as //line filename:line:column. +func (f *File) AddLineColumnInfo(offset int, filename string, line, column int) { + f.mutex.Lock() + if i := len(f.infos); (i == 0 || f.infos[i-1].Offset < offset) && offset < f.size { + f.infos = append(f.infos, lineInfo{offset, filename, line, column}) + } + f.mutex.Unlock() +} + +// Pos returns the Pos value for the given file offset; +// the offset must be <= f.Size(). +// f.Pos(f.Offset(p)) == p. +func (f *File) Pos(offset int) Pos { + if offset > f.size { + panic(fmt.Sprintf("invalid file offset %d (should be <= %d)", offset, f.size)) + } + return Pos(f.base + offset) +} + +// Offset returns the offset for the given file position p; +// p must be a valid Pos value in that file. +// f.Offset(f.Pos(offset)) == offset. +func (f *File) Offset(p Pos) int { + if int(p) < f.base || int(p) > f.base+f.size { + panic(fmt.Sprintf("invalid Pos value %d (should be in [%d, %d])", p, f.base, f.base+f.size)) + } + return int(p) - f.base +} + +// Line returns the line number for the given file position p; +// p must be a Pos value in that file or NoPos. +func (f *File) Line(p Pos) int { + return f.Position(p).Line +} + +func searchLineInfos(a []lineInfo, x int) int { + return sort.Search(len(a), func(i int) bool { return a[i].Offset > x }) - 1 +} + +// unpack returns the filename and line and column number for a file offset. +// If adjusted is set, unpack will return the filename and line information +// possibly adjusted by //line comments; otherwise those comments are ignored. +func (f *File) unpack(offset int, adjusted bool) (filename string, line, column int) { + f.mutex.Lock() + filename = f.name + if i := searchInts(f.lines, offset); i >= 0 { + line, column = i+1, offset-f.lines[i]+1 + } + if adjusted && len(f.infos) > 0 { + // few files have extra line infos + if i := searchLineInfos(f.infos, offset); i >= 0 { + alt := &f.infos[i] + filename = alt.Filename + if i := searchInts(f.lines, alt.Offset); i >= 0 { + // i+1 is the line at which the alternative position was recorded + d := line - (i + 1) // line distance from alternative position base + line = alt.Line + d + if alt.Column == 0 { + // alternative column is unknown => relative column is unknown + // (the current specification for line directives requires + // this to apply until the next PosBase/line directive, + // not just until the new newline) + column = 0 + } else if d == 0 { + // the alternative position base is on the current line + // => column is relative to alternative column + column = alt.Column + (offset - alt.Offset) + } + } + } + } + // TODO(mvdan): move Unlock back under Lock with a defer statement once + // https://go.dev/issue/38471 is fixed to remove the performance penalty. + f.mutex.Unlock() + return +} + +func (f *File) position(p Pos, adjusted bool) (pos Position) { + offset := int(p) - f.base + pos.Offset = offset + pos.Filename, pos.Line, pos.Column = f.unpack(offset, adjusted) + return +} + +// PositionFor returns the Position value for the given file position p. +// If adjusted is set, the position may be adjusted by position-altering +// //line comments; otherwise those comments are ignored. +// p must be a Pos value in f or NoPos. +func (f *File) PositionFor(p Pos, adjusted bool) (pos Position) { + if p != NoPos { + if int(p) < f.base || int(p) > f.base+f.size { + panic(fmt.Sprintf("invalid Pos value %d (should be in [%d, %d])", p, f.base, f.base+f.size)) + } + pos = f.position(p, adjusted) + } + return +} + +// Position returns the Position value for the given file position p. +// Calling f.Position(p) is equivalent to calling f.PositionFor(p, true). +func (f *File) Position(p Pos) (pos Position) { + return f.PositionFor(p, true) +} + +// ----------------------------------------------------------------------------- +// FileSet + +// A FileSet represents a set of source files. +// Methods of file sets are synchronized; multiple goroutines +// may invoke them concurrently. +// +// The byte offsets for each file in a file set are mapped into +// distinct (integer) intervals, one interval [base, base+size] +// per file. Base represents the first byte in the file, and size +// is the corresponding file size. A Pos value is a value in such +// an interval. By determining the interval a Pos value belongs +// to, the file, its file base, and thus the byte offset (position) +// the Pos value is representing can be computed. +// +// When adding a new file, a file base must be provided. That can +// be any integer value that is past the end of any interval of any +// file already in the file set. For convenience, FileSet.Base provides +// such a value, which is simply the end of the Pos interval of the most +// recently added file, plus one. Unless there is a need to extend an +// interval later, using the FileSet.Base should be used as argument +// for FileSet.AddFile. +// +// A File may be removed from a FileSet when it is no longer needed. +// This may reduce memory usage in a long-running application. +type FileSet struct { + mutex sync.RWMutex // protects the file set + base int // base offset for the next file + files []*File // list of files in the order added to the set + last atomic.Pointer[File] // cache of last file looked up +} + +// NewFileSet creates a new file set. +func NewFileSet() *FileSet { + return &FileSet{ + base: 1, // 0 == NoPos + } +} + +// Base returns the minimum base offset that must be provided to +// AddFile when adding the next file. +func (s *FileSet) Base() int { + s.mutex.RLock() + b := s.base + s.mutex.RUnlock() + return b + +} + +// AddFile adds a new file with a given filename, base offset, and file size +// to the file set s and returns the file. Multiple files may have the same +// name. The base offset must not be smaller than the FileSet's Base(), and +// size must not be negative. As a special case, if a negative base is provided, +// the current value of the FileSet's Base() is used instead. +// +// Adding the file will set the file set's Base() value to base + size + 1 +// as the minimum base value for the next file. The following relationship +// exists between a Pos value p for a given file offset offs: +// +// int(p) = base + offs +// +// with offs in the range [0, size] and thus p in the range [base, base+size]. +// For convenience, File.Pos may be used to create file-specific position +// values from a file offset. +func (s *FileSet) AddFile(filename string, base, size int) *File { + // Allocate f outside the critical section. + f := &File{name: filename, size: size, lines: []int{0}} + + s.mutex.Lock() + defer s.mutex.Unlock() + if base < 0 { + base = s.base + } + if base < s.base { + panic(fmt.Sprintf("invalid base %d (should be >= %d)", base, s.base)) + } + f.base = base + if size < 0 { + panic(fmt.Sprintf("invalid size %d (should be >= 0)", size)) + } + // base >= s.base && size >= 0 + base += size + 1 // +1 because EOF also has a position + if base < 0 { + panic("token.Pos offset overflow (> 2G of source code in file set)") + } + // add the file to the file set + s.base = base + s.files = append(s.files, f) + s.last.Store(f) + return f +} + +// RemoveFile removes a file from the FileSet so that subsequent +// queries for its Pos interval yield a negative result. +// This reduces the memory usage of a long-lived FileSet that +// encounters an unbounded stream of files. +// +// Removing a file that does not belong to the set has no effect. +func (s *FileSet) RemoveFile(file *File) { + s.last.CompareAndSwap(file, nil) // clear last file cache + + s.mutex.Lock() + defer s.mutex.Unlock() + + if i := searchFiles(s.files, file.base); i >= 0 && s.files[i] == file { + last := &s.files[len(s.files)-1] + s.files = append(s.files[:i], s.files[i+1:]...) + *last = nil // don't prolong lifetime when popping last element + } +} + +// Iterate calls f for the files in the file set in the order they were added +// until f returns false. +func (s *FileSet) Iterate(f func(*File) bool) { + for i := 0; ; i++ { + var file *File + s.mutex.RLock() + if i < len(s.files) { + file = s.files[i] + } + s.mutex.RUnlock() + if file == nil || !f(file) { + break + } + } +} + +func searchFiles(a []*File, x int) int { + return sort.Search(len(a), func(i int) bool { return a[i].base > x }) - 1 +} + +func (s *FileSet) file(p Pos) *File { + // common case: p is in last file. + if f := s.last.Load(); f != nil && f.base <= int(p) && int(p) <= f.base+f.size { + return f + } + + s.mutex.RLock() + defer s.mutex.RUnlock() + + // p is not in last file - search all files + if i := searchFiles(s.files, int(p)); i >= 0 { + f := s.files[i] + // f.base <= int(p) by definition of searchFiles + if int(p) <= f.base+f.size { + // Update cache of last file. A race is ok, + // but an exclusive lock causes heavy contention. + s.last.Store(f) + return f + } + } + return nil +} + +// File returns the file that contains the position p. +// If no such file is found (for instance for p == NoPos), +// the result is nil. +func (s *FileSet) File(p Pos) (f *File) { + if p != NoPos { + f = s.file(p) + } + return +} + +// PositionFor converts a Pos p in the fileset into a Position value. +// If adjusted is set, the position may be adjusted by position-altering +// //line comments; otherwise those comments are ignored. +// p must be a Pos value in s or NoPos. +func (s *FileSet) PositionFor(p Pos, adjusted bool) (pos Position) { + if p != NoPos { + if f := s.file(p); f != nil { + return f.position(p, adjusted) + } + } + return +} + +// Position converts a Pos p in the fileset into a Position value. +// Calling s.Position(p) is equivalent to calling s.PositionFor(p, true). +func (s *FileSet) Position(p Pos) (pos Position) { + return s.PositionFor(p, true) +} + +// ----------------------------------------------------------------------------- +// Helper functions + +func searchInts(a []int, x int) int { + // This function body is a manually inlined version of: + // + // return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1 + // + // With better compiler optimizations, this may not be needed in the + // future, but at the moment this change improves the go/printer + // benchmark performance by ~30%. This has a direct impact on the + // speed of gofmt and thus seems worthwhile (2011-04-29). + // TODO(gri): Remove this when compilers have caught up. + i, j := 0, len(a) + for i < j { + h := int(uint(i+j) >> 1) // avoid overflow when computing h + // i ā‰¤ h < j + if a[h] <= x { + i = h + 1 + } else { + j = h + } + } + return i - 1 +} diff --git a/src/go/token/position_bench_test.go b/src/go/token/position_bench_test.go new file mode 100644 index 0000000..41be728 --- /dev/null +++ b/src/go/token/position_bench_test.go @@ -0,0 +1,24 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package token + +import ( + "testing" +) + +func BenchmarkSearchInts(b *testing.B) { + data := make([]int, 10000) + for i := 0; i < 10000; i++ { + data[i] = i + } + const x = 8 + if r := searchInts(data, x); r != x { + b.Errorf("got index = %d; want %d", r, x) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + searchInts(data, x) + } +} diff --git a/src/go/token/position_test.go b/src/go/token/position_test.go new file mode 100644 index 0000000..65cb242 --- /dev/null +++ b/src/go/token/position_test.go @@ -0,0 +1,477 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package token + +import ( + "fmt" + "math/rand" + "reflect" + "sync" + "testing" +) + +func checkPos(t *testing.T, msg string, got, want Position) { + if got.Filename != want.Filename { + t.Errorf("%s: got filename = %q; want %q", msg, got.Filename, want.Filename) + } + if got.Offset != want.Offset { + t.Errorf("%s: got offset = %d; want %d", msg, got.Offset, want.Offset) + } + if got.Line != want.Line { + t.Errorf("%s: got line = %d; want %d", msg, got.Line, want.Line) + } + if got.Column != want.Column { + t.Errorf("%s: got column = %d; want %d", msg, got.Column, want.Column) + } +} + +func TestNoPos(t *testing.T) { + if NoPos.IsValid() { + t.Errorf("NoPos should not be valid") + } + var fset *FileSet + checkPos(t, "nil NoPos", fset.Position(NoPos), Position{}) + fset = NewFileSet() + checkPos(t, "fset NoPos", fset.Position(NoPos), Position{}) +} + +var tests = []struct { + filename string + source []byte // may be nil + size int + lines []int +}{ + {"a", []byte{}, 0, []int{}}, + {"b", []byte("01234"), 5, []int{0}}, + {"c", []byte("\n\n\n\n\n\n\n\n\n"), 9, []int{0, 1, 2, 3, 4, 5, 6, 7, 8}}, + {"d", nil, 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}}, + {"e", nil, 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}}, + {"f", []byte("package p\n\nimport \"fmt\""), 23, []int{0, 10, 11}}, + {"g", []byte("package p\n\nimport \"fmt\"\n"), 24, []int{0, 10, 11}}, + {"h", []byte("package p\n\nimport \"fmt\"\n "), 25, []int{0, 10, 11, 24}}, +} + +func linecol(lines []int, offs int) (int, int) { + prevLineOffs := 0 + for line, lineOffs := range lines { + if offs < lineOffs { + return line, offs - prevLineOffs + 1 + } + prevLineOffs = lineOffs + } + return len(lines), offs - prevLineOffs + 1 +} + +func verifyPositions(t *testing.T, fset *FileSet, f *File, lines []int) { + for offs := 0; offs < f.Size(); offs++ { + p := f.Pos(offs) + offs2 := f.Offset(p) + if offs2 != offs { + t.Errorf("%s, Offset: got offset %d; want %d", f.Name(), offs2, offs) + } + line, col := linecol(lines, offs) + msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p) + checkPos(t, msg, f.Position(f.Pos(offs)), Position{f.Name(), offs, line, col}) + checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col}) + } +} + +func makeTestSource(size int, lines []int) []byte { + src := make([]byte, size) + for _, offs := range lines { + if offs > 0 { + src[offs-1] = '\n' + } + } + return src +} + +func TestPositions(t *testing.T) { + const delta = 7 // a non-zero base offset increment + fset := NewFileSet() + for _, test := range tests { + // verify consistency of test case + if test.source != nil && len(test.source) != test.size { + t.Errorf("%s: inconsistent test case: got file size %d; want %d", test.filename, len(test.source), test.size) + } + + // add file and verify name and size + f := fset.AddFile(test.filename, fset.Base()+delta, test.size) + if f.Name() != test.filename { + t.Errorf("got filename %q; want %q", f.Name(), test.filename) + } + if f.Size() != test.size { + t.Errorf("%s: got file size %d; want %d", f.Name(), f.Size(), test.size) + } + if fset.File(f.Pos(0)) != f { + t.Errorf("%s: f.Pos(0) was not found in f", f.Name()) + } + + // add lines individually and verify all positions + for i, offset := range test.lines { + f.AddLine(offset) + if f.LineCount() != i+1 { + t.Errorf("%s, AddLine: got line count %d; want %d", f.Name(), f.LineCount(), i+1) + } + // adding the same offset again should be ignored + f.AddLine(offset) + if f.LineCount() != i+1 { + t.Errorf("%s, AddLine: got unchanged line count %d; want %d", f.Name(), f.LineCount(), i+1) + } + verifyPositions(t, fset, f, test.lines[0:i+1]) + } + + // add lines with SetLines and verify all positions + if ok := f.SetLines(test.lines); !ok { + t.Errorf("%s: SetLines failed", f.Name()) + } + if f.LineCount() != len(test.lines) { + t.Errorf("%s, SetLines: got line count %d; want %d", f.Name(), f.LineCount(), len(test.lines)) + } + verifyPositions(t, fset, f, test.lines) + + // add lines with SetLinesForContent and verify all positions + src := test.source + if src == nil { + // no test source available - create one from scratch + src = makeTestSource(test.size, test.lines) + } + f.SetLinesForContent(src) + if f.LineCount() != len(test.lines) { + t.Errorf("%s, SetLinesForContent: got line count %d; want %d", f.Name(), f.LineCount(), len(test.lines)) + } + verifyPositions(t, fset, f, test.lines) + } +} + +func TestLineInfo(t *testing.T) { + fset := NewFileSet() + f := fset.AddFile("foo", fset.Base(), 500) + lines := []int{0, 42, 77, 100, 210, 220, 277, 300, 333, 401} + // add lines individually and provide alternative line information + for _, offs := range lines { + f.AddLine(offs) + f.AddLineInfo(offs, "bar", 42) + } + // verify positions for all offsets + for offs := 0; offs <= f.Size(); offs++ { + p := f.Pos(offs) + _, col := linecol(lines, offs) + msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p) + checkPos(t, msg, f.Position(f.Pos(offs)), Position{"bar", offs, 42, col}) + checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col}) + } +} + +func TestFiles(t *testing.T) { + fset := NewFileSet() + for i, test := range tests { + base := fset.Base() + if i%2 == 1 { + // Setting a negative base is equivalent to + // fset.Base(), so test some of each. + base = -1 + } + fset.AddFile(test.filename, base, test.size) + j := 0 + fset.Iterate(func(f *File) bool { + if f.Name() != tests[j].filename { + t.Errorf("got filename = %s; want %s", f.Name(), tests[j].filename) + } + j++ + return true + }) + if j != i+1 { + t.Errorf("got %d files; want %d", j, i+1) + } + } +} + +// FileSet.File should return nil if Pos is past the end of the FileSet. +func TestFileSetPastEnd(t *testing.T) { + fset := NewFileSet() + for _, test := range tests { + fset.AddFile(test.filename, fset.Base(), test.size) + } + if f := fset.File(Pos(fset.Base())); f != nil { + t.Errorf("got %v, want nil", f) + } +} + +func TestFileSetCacheUnlikely(t *testing.T) { + fset := NewFileSet() + offsets := make(map[string]int) + for _, test := range tests { + offsets[test.filename] = fset.Base() + fset.AddFile(test.filename, fset.Base(), test.size) + } + for file, pos := range offsets { + f := fset.File(Pos(pos)) + if f.Name() != file { + t.Errorf("got %q at position %d, want %q", f.Name(), pos, file) + } + } +} + +// issue 4345. Test that concurrent use of FileSet.Pos does not trigger a +// race in the FileSet position cache. +func TestFileSetRace(t *testing.T) { + fset := NewFileSet() + for i := 0; i < 100; i++ { + fset.AddFile(fmt.Sprintf("file-%d", i), fset.Base(), 1031) + } + max := int32(fset.Base()) + var stop sync.WaitGroup + r := rand.New(rand.NewSource(7)) + for i := 0; i < 2; i++ { + r := rand.New(rand.NewSource(r.Int63())) + stop.Add(1) + go func() { + for i := 0; i < 1000; i++ { + fset.Position(Pos(r.Int31n(max))) + } + stop.Done() + }() + } + stop.Wait() +} + +// issue 16548. Test that concurrent use of File.AddLine and FileSet.PositionFor +// does not trigger a race in the FileSet position cache. +func TestFileSetRace2(t *testing.T) { + const N = 1e3 + var ( + fset = NewFileSet() + file = fset.AddFile("", -1, N) + ch = make(chan int, 2) + ) + + go func() { + for i := 0; i < N; i++ { + file.AddLine(i) + } + ch <- 1 + }() + + go func() { + pos := file.Pos(0) + for i := 0; i < N; i++ { + fset.PositionFor(pos, false) + } + ch <- 1 + }() + + <-ch + <-ch +} + +func TestPositionFor(t *testing.T) { + src := []byte(` +foo +b +ar +//line :100 +foobar +//line bar:3 +done +`) + + const filename = "foo" + fset := NewFileSet() + f := fset.AddFile(filename, fset.Base(), len(src)) + f.SetLinesForContent(src) + + // verify position info + for i, offs := range f.lines { + got1 := f.PositionFor(f.Pos(offs), false) + got2 := f.PositionFor(f.Pos(offs), true) + got3 := f.Position(f.Pos(offs)) + want := Position{filename, offs, i + 1, 1} + checkPos(t, "1. PositionFor unadjusted", got1, want) + checkPos(t, "1. PositionFor adjusted", got2, want) + checkPos(t, "1. Position", got3, want) + } + + // manually add //line info on lines l1, l2 + const l1, l2 = 5, 7 + f.AddLineInfo(f.lines[l1-1], "", 100) + f.AddLineInfo(f.lines[l2-1], "bar", 3) + + // unadjusted position info must remain unchanged + for i, offs := range f.lines { + got1 := f.PositionFor(f.Pos(offs), false) + want := Position{filename, offs, i + 1, 1} + checkPos(t, "2. PositionFor unadjusted", got1, want) + } + + // adjusted position info should have changed + for i, offs := range f.lines { + got2 := f.PositionFor(f.Pos(offs), true) + got3 := f.Position(f.Pos(offs)) + want := Position{filename, offs, i + 1, 1} + // manually compute wanted filename and line + line := want.Line + if i+1 >= l1 { + want.Filename = "" + want.Line = line - l1 + 100 + } + if i+1 >= l2 { + want.Filename = "bar" + want.Line = line - l2 + 3 + } + checkPos(t, "3. PositionFor adjusted", got2, want) + checkPos(t, "3. Position", got3, want) + } +} + +func TestLineStart(t *testing.T) { + const src = "one\ntwo\nthree\n" + fset := NewFileSet() + f := fset.AddFile("input", -1, len(src)) + f.SetLinesForContent([]byte(src)) + + for line := 1; line <= 3; line++ { + pos := f.LineStart(line) + position := fset.Position(pos) + if position.Line != line || position.Column != 1 { + t.Errorf("LineStart(%d) returned wrong pos %d: %s", line, pos, position) + } + } +} + +func TestRemoveFile(t *testing.T) { + contentA := []byte("this\nis\nfileA") + contentB := []byte("this\nis\nfileB") + fset := NewFileSet() + a := fset.AddFile("fileA", -1, len(contentA)) + a.SetLinesForContent(contentA) + b := fset.AddFile("fileB", -1, len(contentB)) + b.SetLinesForContent(contentB) + + checkPos := func(pos Pos, want string) { + if got := fset.Position(pos).String(); got != want { + t.Errorf("Position(%d) = %s, want %s", pos, got, want) + } + } + checkNumFiles := func(want int) { + got := 0 + fset.Iterate(func(*File) bool { got++; return true }) + if got != want { + t.Errorf("Iterate called %d times, want %d", got, want) + } + } + + apos3 := a.Pos(3) + bpos3 := b.Pos(3) + checkPos(apos3, "fileA:1:4") + checkPos(bpos3, "fileB:1:4") + checkNumFiles(2) + + // After removal, queries on fileA fail. + fset.RemoveFile(a) + checkPos(apos3, "-") + checkPos(bpos3, "fileB:1:4") + checkNumFiles(1) + + // idempotent / no effect + fset.RemoveFile(a) + checkPos(apos3, "-") + checkPos(bpos3, "fileB:1:4") + checkNumFiles(1) +} + +func TestFileAddLineColumnInfo(t *testing.T) { + const ( + filename = "test.go" + filesize = 100 + ) + + tests := []struct { + name string + infos []lineInfo + want []lineInfo + }{ + { + name: "normal", + infos: []lineInfo{ + {Offset: 10, Filename: filename, Line: 2, Column: 1}, + {Offset: 50, Filename: filename, Line: 3, Column: 1}, + {Offset: 80, Filename: filename, Line: 4, Column: 2}, + }, + want: []lineInfo{ + {Offset: 10, Filename: filename, Line: 2, Column: 1}, + {Offset: 50, Filename: filename, Line: 3, Column: 1}, + {Offset: 80, Filename: filename, Line: 4, Column: 2}, + }, + }, + { + name: "offset1 == file size", + infos: []lineInfo{ + {Offset: filesize, Filename: filename, Line: 2, Column: 1}, + }, + want: nil, + }, + { + name: "offset1 > file size", + infos: []lineInfo{ + {Offset: filesize + 1, Filename: filename, Line: 2, Column: 1}, + }, + want: nil, + }, + { + name: "offset2 == file size", + infos: []lineInfo{ + {Offset: 10, Filename: filename, Line: 2, Column: 1}, + {Offset: filesize, Filename: filename, Line: 3, Column: 1}, + }, + want: []lineInfo{ + {Offset: 10, Filename: filename, Line: 2, Column: 1}, + }, + }, + { + name: "offset2 > file size", + infos: []lineInfo{ + {Offset: 10, Filename: filename, Line: 2, Column: 1}, + {Offset: filesize + 1, Filename: filename, Line: 3, Column: 1}, + }, + want: []lineInfo{ + {Offset: 10, Filename: filename, Line: 2, Column: 1}, + }, + }, + { + name: "offset2 == offset1", + infos: []lineInfo{ + {Offset: 10, Filename: filename, Line: 2, Column: 1}, + {Offset: 10, Filename: filename, Line: 3, Column: 1}, + }, + want: []lineInfo{ + {Offset: 10, Filename: filename, Line: 2, Column: 1}, + }, + }, + { + name: "offset2 < offset1", + infos: []lineInfo{ + {Offset: 10, Filename: filename, Line: 2, Column: 1}, + {Offset: 9, Filename: filename, Line: 3, Column: 1}, + }, + want: []lineInfo{ + {Offset: 10, Filename: filename, Line: 2, Column: 1}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fs := NewFileSet() + f := fs.AddFile(filename, -1, filesize) + for _, info := range test.infos { + f.AddLineColumnInfo(info.Offset, info.Filename, info.Line, info.Column) + } + if !reflect.DeepEqual(f.infos, test.want) { + t.Errorf("\ngot %+v, \nwant %+v", f.infos, test.want) + } + }) + } +} diff --git a/src/go/token/serialize.go b/src/go/token/serialize.go new file mode 100644 index 0000000..04a48d9 --- /dev/null +++ b/src/go/token/serialize.go @@ -0,0 +1,70 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package token + +type serializedFile struct { + // fields correspond 1:1 to fields with same (lower-case) name in File + Name string + Base int + Size int + Lines []int + Infos []lineInfo +} + +type serializedFileSet struct { + Base int + Files []serializedFile +} + +// Read calls decode to deserialize a file set into s; s must not be nil. +func (s *FileSet) Read(decode func(any) error) error { + var ss serializedFileSet + if err := decode(&ss); err != nil { + return err + } + + s.mutex.Lock() + s.base = ss.Base + files := make([]*File, len(ss.Files)) + for i := 0; i < len(ss.Files); i++ { + f := &ss.Files[i] + files[i] = &File{ + name: f.Name, + base: f.Base, + size: f.Size, + lines: f.Lines, + infos: f.Infos, + } + } + s.files = files + s.last.Store(nil) + s.mutex.Unlock() + + return nil +} + +// Write calls encode to serialize the file set s. +func (s *FileSet) Write(encode func(any) error) error { + var ss serializedFileSet + + s.mutex.Lock() + ss.Base = s.base + files := make([]serializedFile, len(s.files)) + for i, f := range s.files { + f.mutex.Lock() + files[i] = serializedFile{ + Name: f.name, + Base: f.base, + Size: f.size, + Lines: append([]int(nil), f.lines...), + Infos: append([]lineInfo(nil), f.infos...), + } + f.mutex.Unlock() + } + ss.Files = files + s.mutex.Unlock() + + return encode(ss) +} diff --git a/src/go/token/serialize_test.go b/src/go/token/serialize_test.go new file mode 100644 index 0000000..8d97995 --- /dev/null +++ b/src/go/token/serialize_test.go @@ -0,0 +1,105 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package token + +import ( + "bytes" + "encoding/gob" + "fmt" + "testing" +) + +// equal returns nil if p and q describe the same file set; +// otherwise it returns an error describing the discrepancy. +func equal(p, q *FileSet) error { + if p == q { + // avoid deadlock if p == q + return nil + } + + // not strictly needed for the test + p.mutex.Lock() + q.mutex.Lock() + defer q.mutex.Unlock() + defer p.mutex.Unlock() + + if p.base != q.base { + return fmt.Errorf("different bases: %d != %d", p.base, q.base) + } + + if len(p.files) != len(q.files) { + return fmt.Errorf("different number of files: %d != %d", len(p.files), len(q.files)) + } + + for i, f := range p.files { + g := q.files[i] + if f.name != g.name { + return fmt.Errorf("different filenames: %q != %q", f.name, g.name) + } + if f.base != g.base { + return fmt.Errorf("different base for %q: %d != %d", f.name, f.base, g.base) + } + if f.size != g.size { + return fmt.Errorf("different size for %q: %d != %d", f.name, f.size, g.size) + } + for j, l := range f.lines { + m := g.lines[j] + if l != m { + return fmt.Errorf("different offsets for %q", f.name) + } + } + for j, l := range f.infos { + m := g.infos[j] + if l.Offset != m.Offset || l.Filename != m.Filename || l.Line != m.Line { + return fmt.Errorf("different infos for %q", f.name) + } + } + } + + // we don't care about .last - it's just a cache + return nil +} + +func checkSerialize(t *testing.T, p *FileSet) { + var buf bytes.Buffer + encode := func(x any) error { + return gob.NewEncoder(&buf).Encode(x) + } + if err := p.Write(encode); err != nil { + t.Errorf("writing fileset failed: %s", err) + return + } + q := NewFileSet() + decode := func(x any) error { + return gob.NewDecoder(&buf).Decode(x) + } + if err := q.Read(decode); err != nil { + t.Errorf("reading fileset failed: %s", err) + return + } + if err := equal(p, q); err != nil { + t.Errorf("filesets not identical: %s", err) + } +} + +func TestSerialization(t *testing.T) { + p := NewFileSet() + checkSerialize(t, p) + // add some files + for i := 0; i < 10; i++ { + f := p.AddFile(fmt.Sprintf("file%d", i), p.Base()+i, i*100) + checkSerialize(t, p) + // add some lines and alternative file infos + line := 1000 + for offs := 0; offs < f.Size(); offs += 40 + i { + f.AddLine(offs) + if offs%7 == 0 { + f.AddLineInfo(offs, fmt.Sprintf("file%d", offs), line) + line += 33 + } + } + checkSerialize(t, p) + } +} diff --git a/src/go/token/token.go b/src/go/token/token.go new file mode 100644 index 0000000..3ae10d8 --- /dev/null +++ b/src/go/token/token.go @@ -0,0 +1,341 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package token defines constants representing the lexical tokens of the Go +// programming language and basic operations on tokens (printing, predicates). +package token + +import ( + "strconv" + "unicode" + "unicode/utf8" +) + +// Token is the set of lexical tokens of the Go programming language. +type Token int + +// The list of tokens. +const ( + // Special tokens + ILLEGAL Token = iota + EOF + COMMENT + + literal_beg + // Identifiers and basic type literals + // (these tokens stand for classes of literals) + IDENT // main + INT // 12345 + FLOAT // 123.45 + IMAG // 123.45i + CHAR // 'a' + STRING // "abc" + literal_end + + operator_beg + // Operators and delimiters + ADD // + + SUB // - + MUL // * + QUO // / + REM // % + + AND // & + OR // | + XOR // ^ + SHL // << + SHR // >> + AND_NOT // &^ + + ADD_ASSIGN // += + SUB_ASSIGN // -= + MUL_ASSIGN // *= + QUO_ASSIGN // /= + REM_ASSIGN // %= + + AND_ASSIGN // &= + OR_ASSIGN // |= + XOR_ASSIGN // ^= + SHL_ASSIGN // <<= + SHR_ASSIGN // >>= + AND_NOT_ASSIGN // &^= + + LAND // && + LOR // || + ARROW // <- + INC // ++ + DEC // -- + + EQL // == + LSS // < + GTR // > + ASSIGN // = + NOT // ! + + NEQ // != + LEQ // <= + GEQ // >= + DEFINE // := + ELLIPSIS // ... + + LPAREN // ( + LBRACK // [ + LBRACE // { + COMMA // , + PERIOD // . + + RPAREN // ) + RBRACK // ] + RBRACE // } + SEMICOLON // ; + COLON // : + operator_end + + keyword_beg + // Keywords + BREAK + CASE + CHAN + CONST + CONTINUE + + DEFAULT + DEFER + ELSE + FALLTHROUGH + FOR + + FUNC + GO + GOTO + IF + IMPORT + + INTERFACE + MAP + PACKAGE + RANGE + RETURN + + SELECT + STRUCT + SWITCH + TYPE + VAR + keyword_end + + additional_beg + // additional tokens, handled in an ad-hoc manner + TILDE + additional_end +) + +var tokens = [...]string{ + ILLEGAL: "ILLEGAL", + + EOF: "EOF", + COMMENT: "COMMENT", + + IDENT: "IDENT", + INT: "INT", + FLOAT: "FLOAT", + IMAG: "IMAG", + CHAR: "CHAR", + STRING: "STRING", + + ADD: "+", + SUB: "-", + MUL: "*", + QUO: "/", + REM: "%", + + AND: "&", + OR: "|", + XOR: "^", + SHL: "<<", + SHR: ">>", + AND_NOT: "&^", + + ADD_ASSIGN: "+=", + SUB_ASSIGN: "-=", + MUL_ASSIGN: "*=", + QUO_ASSIGN: "/=", + REM_ASSIGN: "%=", + + AND_ASSIGN: "&=", + OR_ASSIGN: "|=", + XOR_ASSIGN: "^=", + SHL_ASSIGN: "<<=", + SHR_ASSIGN: ">>=", + AND_NOT_ASSIGN: "&^=", + + LAND: "&&", + LOR: "||", + ARROW: "<-", + INC: "++", + DEC: "--", + + EQL: "==", + LSS: "<", + GTR: ">", + ASSIGN: "=", + NOT: "!", + + NEQ: "!=", + LEQ: "<=", + GEQ: ">=", + DEFINE: ":=", + ELLIPSIS: "...", + + LPAREN: "(", + LBRACK: "[", + LBRACE: "{", + COMMA: ",", + PERIOD: ".", + + RPAREN: ")", + RBRACK: "]", + RBRACE: "}", + SEMICOLON: ";", + COLON: ":", + + BREAK: "break", + CASE: "case", + CHAN: "chan", + CONST: "const", + CONTINUE: "continue", + + DEFAULT: "default", + DEFER: "defer", + ELSE: "else", + FALLTHROUGH: "fallthrough", + FOR: "for", + + FUNC: "func", + GO: "go", + GOTO: "goto", + IF: "if", + IMPORT: "import", + + INTERFACE: "interface", + MAP: "map", + PACKAGE: "package", + RANGE: "range", + RETURN: "return", + + SELECT: "select", + STRUCT: "struct", + SWITCH: "switch", + TYPE: "type", + VAR: "var", + + TILDE: "~", +} + +// String returns the string corresponding to the token tok. +// For operators, delimiters, and keywords the string is the actual +// token character sequence (e.g., for the token ADD, the string is +// "+"). For all other tokens the string corresponds to the token +// constant name (e.g. for the token IDENT, the string is "IDENT"). +func (tok Token) String() string { + s := "" + if 0 <= tok && tok < Token(len(tokens)) { + s = tokens[tok] + } + if s == "" { + s = "token(" + strconv.Itoa(int(tok)) + ")" + } + return s +} + +// A set of constants for precedence-based expression parsing. +// Non-operators have lowest precedence, followed by operators +// starting with precedence 1 up to unary operators. The highest +// precedence serves as "catch-all" precedence for selector, +// indexing, and other operator and delimiter tokens. +const ( + LowestPrec = 0 // non-operators + UnaryPrec = 6 + HighestPrec = 7 +) + +// Precedence returns the operator precedence of the binary +// operator op. If op is not a binary operator, the result +// is LowestPrecedence. +func (op Token) Precedence() int { + switch op { + case LOR: + return 1 + case LAND: + return 2 + case EQL, NEQ, LSS, LEQ, GTR, GEQ: + return 3 + case ADD, SUB, OR, XOR: + return 4 + case MUL, QUO, REM, SHL, SHR, AND, AND_NOT: + return 5 + } + return LowestPrec +} + +var keywords map[string]Token + +func init() { + keywords = make(map[string]Token, keyword_end-(keyword_beg+1)) + for i := keyword_beg + 1; i < keyword_end; i++ { + keywords[tokens[i]] = i + } +} + +// Lookup maps an identifier to its keyword token or IDENT (if not a keyword). +func Lookup(ident string) Token { + if tok, is_keyword := keywords[ident]; is_keyword { + return tok + } + return IDENT +} + +// Predicates + +// IsLiteral returns true for tokens corresponding to identifiers +// and basic type literals; it returns false otherwise. +func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end } + +// IsOperator returns true for tokens corresponding to operators and +// delimiters; it returns false otherwise. +func (tok Token) IsOperator() bool { + return (operator_beg < tok && tok < operator_end) || tok == TILDE +} + +// IsKeyword returns true for tokens corresponding to keywords; +// it returns false otherwise. +func (tok Token) IsKeyword() bool { return keyword_beg < tok && tok < keyword_end } + +// IsExported reports whether name starts with an upper-case letter. +func IsExported(name string) bool { + ch, _ := utf8.DecodeRuneInString(name) + return unicode.IsUpper(ch) +} + +// IsKeyword reports whether name is a Go keyword, such as "func" or "return". +func IsKeyword(name string) bool { + // TODO: opt: use a perfect hash function instead of a global map. + _, ok := keywords[name] + return ok +} + +// IsIdentifier reports whether name is a Go identifier, that is, a non-empty +// string made up of letters, digits, and underscores, where the first character +// is not a digit. Keywords are not identifiers. +func IsIdentifier(name string) bool { + if name == "" || IsKeyword(name) { + return false + } + for i, c := range name { + if !unicode.IsLetter(c) && c != '_' && (i == 0 || !unicode.IsDigit(c)) { + return false + } + } + return true +} diff --git a/src/go/token/token_test.go b/src/go/token/token_test.go new file mode 100644 index 0000000..eff38cc --- /dev/null +++ b/src/go/token/token_test.go @@ -0,0 +1,33 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package token + +import "testing" + +func TestIsIdentifier(t *testing.T) { + tests := []struct { + name string + in string + want bool + }{ + {"Empty", "", false}, + {"Space", " ", false}, + {"SpaceSuffix", "foo ", false}, + {"Number", "123", false}, + {"Keyword", "func", false}, + + {"LettersASCII", "foo", true}, + {"MixedASCII", "_bar123", true}, + {"UppercaseKeyword", "Func", true}, + {"LettersUnicode", "fĆ³Ć¶", true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := IsIdentifier(test.in); got != test.want { + t.Fatalf("IsIdentifier(%q) = %t, want %v", test.in, got, test.want) + } + }) + } +} diff --git a/src/go/types/api.go b/src/go/types/api.go new file mode 100644 index 0000000..eda41b3 --- /dev/null +++ b/src/go/types/api.go @@ -0,0 +1,492 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package types declares the data types and implements +// the algorithms for type-checking of Go packages. Use +// Config.Check to invoke the type checker for a package. +// Alternatively, create a new type checker with NewChecker +// and invoke it incrementally by calling Checker.Files. +// +// Type-checking consists of several interdependent phases: +// +// Name resolution maps each identifier (ast.Ident) in the program to the +// language object (Object) it denotes. +// Use Info.{Defs,Uses,Implicits} for the results of name resolution. +// +// Constant folding computes the exact constant value (constant.Value) +// for every expression (ast.Expr) that is a compile-time constant. +// Use Info.Types[expr].Value for the results of constant folding. +// +// Type inference computes the type (Type) of every expression (ast.Expr) +// and checks for compliance with the language specification. +// Use Info.Types[expr].Type for the results of type inference. +// +// For a tutorial, see https://golang.org/s/types-tutorial. +package types + +import ( + "bytes" + "fmt" + "go/ast" + "go/constant" + "go/token" + . "internal/types/errors" +) + +// An Error describes a type-checking error; it implements the error interface. +// A "soft" error is an error that still permits a valid interpretation of a +// package (such as "unused variable"); "hard" errors may lead to unpredictable +// behavior if ignored. +type Error struct { + Fset *token.FileSet // file set for interpretation of Pos + Pos token.Pos // error position + Msg string // error message + Soft bool // if set, error is "soft" + + // go116code is a future API, unexported as the set of error codes is large + // and likely to change significantly during experimentation. Tools wishing + // to preview this feature may read go116code using reflection (see + // errorcodes_test.go), but beware that there is no guarantee of future + // compatibility. + go116code Code + go116start token.Pos + go116end token.Pos +} + +// Error returns an error string formatted as follows: +// filename:line:column: message +func (err Error) Error() string { + return fmt.Sprintf("%s: %s", err.Fset.Position(err.Pos), err.Msg) +} + +// An ArgumentError holds an error associated with an argument index. +type ArgumentError struct { + Index int + Err error +} + +func (e *ArgumentError) Error() string { return e.Err.Error() } +func (e *ArgumentError) Unwrap() error { return e.Err } + +// An Importer resolves import paths to Packages. +// +// CAUTION: This interface does not support the import of locally +// vendored packages. See https://golang.org/s/go15vendor. +// If possible, external implementations should implement ImporterFrom. +type Importer interface { + // Import returns the imported package for the given import path. + // The semantics is like for ImporterFrom.ImportFrom except that + // dir and mode are ignored (since they are not present). + Import(path string) (*Package, error) +} + +// ImportMode is reserved for future use. +type ImportMode int + +// An ImporterFrom resolves import paths to packages; it +// supports vendoring per https://golang.org/s/go15vendor. +// Use go/importer to obtain an ImporterFrom implementation. +type ImporterFrom interface { + // Importer is present for backward-compatibility. Calling + // Import(path) is the same as calling ImportFrom(path, "", 0); + // i.e., locally vendored packages may not be found. + // The types package does not call Import if an ImporterFrom + // is present. + Importer + + // ImportFrom returns the imported package for the given import + // path when imported by a package file located in dir. + // If the import failed, besides returning an error, ImportFrom + // is encouraged to cache and return a package anyway, if one + // was created. This will reduce package inconsistencies and + // follow-on type checker errors due to the missing package. + // The mode value must be 0; it is reserved for future use. + // Two calls to ImportFrom with the same path and dir must + // return the same package. + ImportFrom(path, dir string, mode ImportMode) (*Package, error) +} + +// A Config specifies the configuration for type checking. +// The zero value for Config is a ready-to-use default configuration. +type Config struct { + // Context is the context used for resolving global identifiers. If nil, the + // type checker will initialize this field with a newly created context. + Context *Context + + // GoVersion describes the accepted Go language version. The string + // must follow the format "go%d.%d" (e.g. "go1.12") or it must be + // empty; an empty string indicates the latest language version. + // If the format is invalid, invoking the type checker will cause a + // panic. + GoVersion string + + // If IgnoreFuncBodies is set, function bodies are not + // type-checked. + IgnoreFuncBodies bool + + // If FakeImportC is set, `import "C"` (for packages requiring Cgo) + // declares an empty "C" package and errors are omitted for qualified + // identifiers referring to package C (which won't find an object). + // This feature is intended for the standard library cmd/api tool. + // + // Caution: Effects may be unpredictable due to follow-on errors. + // Do not use casually! + FakeImportC bool + + // If go115UsesCgo is set, the type checker expects the + // _cgo_gotypes.go file generated by running cmd/cgo to be + // provided as a package source file. Qualified identifiers + // referring to package C will be resolved to cgo-provided + // declarations within _cgo_gotypes.go. + // + // It is an error to set both FakeImportC and go115UsesCgo. + go115UsesCgo bool + + // If Error != nil, it is called with each error found + // during type checking; err has dynamic type Error. + // Secondary errors (for instance, to enumerate all types + // involved in an invalid recursive type declaration) have + // error strings that start with a '\t' character. + // If Error == nil, type-checking stops with the first + // error found. + Error func(err error) + + // An importer is used to import packages referred to from + // import declarations. + // If the installed importer implements ImporterFrom, the type + // checker calls ImportFrom instead of Import. + // The type checker reports an error if an importer is needed + // but none was installed. + Importer Importer + + // If Sizes != nil, it provides the sizing functions for package unsafe. + // Otherwise SizesFor("gc", "amd64") is used instead. + Sizes Sizes + + // If DisableUnusedImportCheck is set, packages are not checked + // for unused imports. + DisableUnusedImportCheck bool + + // If oldComparableSemantics is set, ordinary (non-type parameter) + // interfaces do not satisfy the comparable constraint. + // TODO(gri) remove this flag for Go 1.21 + oldComparableSemantics bool +} + +func srcimporter_setUsesCgo(conf *Config) { + conf.go115UsesCgo = true +} + +// Info holds result type information for a type-checked package. +// Only the information for which a map is provided is collected. +// If the package has type errors, the collected information may +// be incomplete. +type Info struct { + // Types maps expressions to their types, and for constant + // expressions, also their values. Invalid expressions are + // omitted. + // + // For (possibly parenthesized) identifiers denoting built-in + // functions, the recorded signatures are call-site specific: + // if the call result is not a constant, the recorded type is + // an argument-specific signature. Otherwise, the recorded type + // is invalid. + // + // The Types map does not record the type of every identifier, + // only those that appear where an arbitrary expression is + // permitted. For instance, the identifier f in a selector + // expression x.f is found only in the Selections map, the + // identifier z in a variable declaration 'var z int' is found + // only in the Defs map, and identifiers denoting packages in + // qualified identifiers are collected in the Uses map. + Types map[ast.Expr]TypeAndValue + + // Instances maps identifiers denoting generic types or functions to their + // type arguments and instantiated type. + // + // For example, Instances will map the identifier for 'T' in the type + // instantiation T[int, string] to the type arguments [int, string] and + // resulting instantiated *Named type. Given a generic function + // func F[A any](A), Instances will map the identifier for 'F' in the call + // expression F(int(1)) to the inferred type arguments [int], and resulting + // instantiated *Signature. + // + // Invariant: Instantiating Uses[id].Type() with Instances[id].TypeArgs + // results in an equivalent of Instances[id].Type. + Instances map[*ast.Ident]Instance + + // Defs maps identifiers to the objects they define (including + // package names, dots "." of dot-imports, and blank "_" identifiers). + // For identifiers that do not denote objects (e.g., the package name + // in package clauses, or symbolic variables t in t := x.(type) of + // type switch headers), the corresponding objects are nil. + // + // For an embedded field, Defs returns the field *Var it defines. + // + // Invariant: Defs[id] == nil || Defs[id].Pos() == id.Pos() + Defs map[*ast.Ident]Object + + // Uses maps identifiers to the objects they denote. + // + // For an embedded field, Uses returns the *TypeName it denotes. + // + // Invariant: Uses[id].Pos() != id.Pos() + Uses map[*ast.Ident]Object + + // Implicits maps nodes to their implicitly declared objects, if any. + // The following node and object types may appear: + // + // node declared object + // + // *ast.ImportSpec *PkgName for imports without renames + // *ast.CaseClause type-specific *Var for each type switch case clause (incl. default) + // *ast.Field anonymous parameter *Var (incl. unnamed results) + // + Implicits map[ast.Node]Object + + // Selections maps selector expressions (excluding qualified identifiers) + // to their corresponding selections. + Selections map[*ast.SelectorExpr]*Selection + + // Scopes maps ast.Nodes to the scopes they define. Package scopes are not + // associated with a specific node but with all files belonging to a package. + // Thus, the package scope can be found in the type-checked Package object. + // Scopes nest, with the Universe scope being the outermost scope, enclosing + // the package scope, which contains (one or more) files scopes, which enclose + // function scopes which in turn enclose statement and function literal scopes. + // Note that even though package-level functions are declared in the package + // scope, the function scopes are embedded in the file scope of the file + // containing the function declaration. + // + // The following node types may appear in Scopes: + // + // *ast.File + // *ast.FuncType + // *ast.TypeSpec + // *ast.BlockStmt + // *ast.IfStmt + // *ast.SwitchStmt + // *ast.TypeSwitchStmt + // *ast.CaseClause + // *ast.CommClause + // *ast.ForStmt + // *ast.RangeStmt + // + Scopes map[ast.Node]*Scope + + // InitOrder is the list of package-level initializers in the order in which + // they must be executed. Initializers referring to variables related by an + // initialization dependency appear in topological order, the others appear + // in source order. Variables without an initialization expression do not + // appear in this list. + InitOrder []*Initializer +} + +// TypeOf returns the type of expression e, or nil if not found. +// Precondition: the Types, Uses and Defs maps are populated. +func (info *Info) TypeOf(e ast.Expr) Type { + if t, ok := info.Types[e]; ok { + return t.Type + } + if id, _ := e.(*ast.Ident); id != nil { + if obj := info.ObjectOf(id); obj != nil { + return obj.Type() + } + } + return nil +} + +// ObjectOf returns the object denoted by the specified id, +// or nil if not found. +// +// If id is an embedded struct field, ObjectOf returns the field (*Var) +// it defines, not the type (*TypeName) it uses. +// +// Precondition: the Uses and Defs maps are populated. +func (info *Info) ObjectOf(id *ast.Ident) Object { + if obj := info.Defs[id]; obj != nil { + return obj + } + return info.Uses[id] +} + +// TypeAndValue reports the type and value (for constants) +// of the corresponding expression. +type TypeAndValue struct { + mode operandMode + Type Type + Value constant.Value +} + +// IsVoid reports whether the corresponding expression +// is a function call without results. +func (tv TypeAndValue) IsVoid() bool { + return tv.mode == novalue +} + +// IsType reports whether the corresponding expression specifies a type. +func (tv TypeAndValue) IsType() bool { + return tv.mode == typexpr +} + +// IsBuiltin reports whether the corresponding expression denotes +// a (possibly parenthesized) built-in function. +func (tv TypeAndValue) IsBuiltin() bool { + return tv.mode == builtin +} + +// IsValue reports whether the corresponding expression is a value. +// Builtins are not considered values. Constant values have a non- +// nil Value. +func (tv TypeAndValue) IsValue() bool { + switch tv.mode { + case constant_, variable, mapindex, value, commaok, commaerr: + return true + } + return false +} + +// IsNil reports whether the corresponding expression denotes the +// predeclared value nil. +func (tv TypeAndValue) IsNil() bool { + return tv.mode == value && tv.Type == Typ[UntypedNil] +} + +// Addressable reports whether the corresponding expression +// is addressable (https://golang.org/ref/spec#Address_operators). +func (tv TypeAndValue) Addressable() bool { + return tv.mode == variable +} + +// Assignable reports whether the corresponding expression +// is assignable to (provided a value of the right type). +func (tv TypeAndValue) Assignable() bool { + return tv.mode == variable || tv.mode == mapindex +} + +// HasOk reports whether the corresponding expression may be +// used on the rhs of a comma-ok assignment. +func (tv TypeAndValue) HasOk() bool { + return tv.mode == commaok || tv.mode == mapindex +} + +// Instance reports the type arguments and instantiated type for type and +// function instantiations. For type instantiations, Type will be of dynamic +// type *Named. For function instantiations, Type will be of dynamic type +// *Signature. +type Instance struct { + TypeArgs *TypeList + Type Type +} + +// An Initializer describes a package-level variable, or a list of variables in case +// of a multi-valued initialization expression, and the corresponding initialization +// expression. +type Initializer struct { + Lhs []*Var // var Lhs = Rhs + Rhs ast.Expr +} + +func (init *Initializer) String() string { + var buf bytes.Buffer + for i, lhs := range init.Lhs { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(lhs.Name()) + } + buf.WriteString(" = ") + WriteExpr(&buf, init.Rhs) + return buf.String() +} + +// Check type-checks a package and returns the resulting package object and +// the first error if any. Additionally, if info != nil, Check populates each +// of the non-nil maps in the Info struct. +// +// The package is marked as complete if no errors occurred, otherwise it is +// incomplete. See Config.Error for controlling behavior in the presence of +// errors. +// +// The package is specified by a list of *ast.Files and corresponding +// file set, and the package path the package is identified with. +// The clean path must not be empty or dot ("."). +func (conf *Config) Check(path string, fset *token.FileSet, files []*ast.File, info *Info) (*Package, error) { + pkg := NewPackage(path, "") + return pkg, NewChecker(conf, fset, pkg, info).Files(files) +} + +// AssertableTo reports whether a value of type V can be asserted to have type T. +// +// The behavior of AssertableTo is unspecified in three cases: +// - if T is Typ[Invalid] +// - if V is a generalized interface; i.e., an interface that may only be used +// as a type constraint in Go code +// - if T is an uninstantiated generic type +func AssertableTo(V *Interface, T Type) bool { + // Checker.newAssertableTo suppresses errors for invalid types, so we need special + // handling here. + if T.Underlying() == Typ[Invalid] { + return false + } + return (*Checker)(nil).newAssertableTo(V, T) +} + +// AssignableTo reports whether a value of type V is assignable to a variable +// of type T. +// +// The behavior of AssignableTo is unspecified if V or T is Typ[Invalid] or an +// uninstantiated generic type. +func AssignableTo(V, T Type) bool { + x := operand{mode: value, typ: V} + ok, _ := x.assignableTo(nil, T, nil) // check not needed for non-constant x + return ok +} + +// ConvertibleTo reports whether a value of type V is convertible to a value of +// type T. +// +// The behavior of ConvertibleTo is unspecified if V or T is Typ[Invalid] or an +// uninstantiated generic type. +func ConvertibleTo(V, T Type) bool { + x := operand{mode: value, typ: V} + return x.convertibleTo(nil, T, nil) // check not needed for non-constant x +} + +// Implements reports whether type V implements interface T. +// +// The behavior of Implements is unspecified if V is Typ[Invalid] or an uninstantiated +// generic type. +func Implements(V Type, T *Interface) bool { + if T.Empty() { + // All types (even Typ[Invalid]) implement the empty interface. + return true + } + // Checker.implements suppresses errors for invalid types, so we need special + // handling here. + if V.Underlying() == Typ[Invalid] { + return false + } + return (*Checker)(nil).implements(V, T, false, nil) +} + +// Satisfies reports whether type V satisfies the constraint T. +// +// The behavior of Satisfies is unspecified if V is Typ[Invalid] or an uninstantiated +// generic type. +func Satisfies(V Type, T *Interface) bool { + return (*Checker)(nil).implements(V, T, true, nil) +} + +// Identical reports whether x and y are identical types. +// Receivers of Signature types are ignored. +func Identical(x, y Type) bool { + return identical(x, y, true, nil) +} + +// IdenticalIgnoreTags reports whether x and y are identical types if tags are ignored. +// Receivers of Signature types are ignored. +func IdenticalIgnoreTags(x, y Type) bool { + return identical(x, y, false, nil) +} diff --git a/src/go/types/api_test.go b/src/go/types/api_test.go new file mode 100644 index 0000000..2b72607 --- /dev/null +++ b/src/go/types/api_test.go @@ -0,0 +1,2596 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "errors" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "internal/testenv" + "reflect" + "regexp" + "sort" + "strings" + "testing" + + . "go/types" +) + +func parse(fset *token.FileSet, filename, src string) (*ast.File, error) { + return parser.ParseFile(fset, filename, src, 0) +} + +func mustParse(fset *token.FileSet, filename, src string) *ast.File { + f, err := parse(fset, filename, src) + if err != nil { + panic(err) // so we don't need to pass *testing.T + } + return f +} + +func typecheck(path, src string, info *Info) (*Package, error) { + fset := token.NewFileSet() + f, err := parse(fset, path, src) + if f == nil { // ignore errors unless f is nil + return nil, err + } + conf := Config{ + Error: func(err error) {}, // collect all errors + Importer: importer.Default(), + } + return conf.Check(f.Name.Name, fset, []*ast.File{f}, info) +} + +func mustTypecheck(path, src string, info *Info) *Package { + pkg, err := typecheck(path, src, info) + if err != nil { + panic(err) // so we don't need to pass *testing.T + } + return pkg +} + +func TestValuesInfo(t *testing.T) { + var tests = []struct { + src string + expr string // constant expression + typ string // constant type + val string // constant value + }{ + {`package a0; const _ = false`, `false`, `untyped bool`, `false`}, + {`package a1; const _ = 0`, `0`, `untyped int`, `0`}, + {`package a2; const _ = 'A'`, `'A'`, `untyped rune`, `65`}, + {`package a3; const _ = 0.`, `0.`, `untyped float`, `0`}, + {`package a4; const _ = 0i`, `0i`, `untyped complex`, `(0 + 0i)`}, + {`package a5; const _ = "foo"`, `"foo"`, `untyped string`, `"foo"`}, + + {`package b0; var _ = false`, `false`, `bool`, `false`}, + {`package b1; var _ = 0`, `0`, `int`, `0`}, + {`package b2; var _ = 'A'`, `'A'`, `rune`, `65`}, + {`package b3; var _ = 0.`, `0.`, `float64`, `0`}, + {`package b4; var _ = 0i`, `0i`, `complex128`, `(0 + 0i)`}, + {`package b5; var _ = "foo"`, `"foo"`, `string`, `"foo"`}, + + {`package c0a; var _ = bool(false)`, `false`, `bool`, `false`}, + {`package c0b; var _ = bool(false)`, `bool(false)`, `bool`, `false`}, + {`package c0c; type T bool; var _ = T(false)`, `T(false)`, `c0c.T`, `false`}, + + {`package c1a; var _ = int(0)`, `0`, `int`, `0`}, + {`package c1b; var _ = int(0)`, `int(0)`, `int`, `0`}, + {`package c1c; type T int; var _ = T(0)`, `T(0)`, `c1c.T`, `0`}, + + {`package c2a; var _ = rune('A')`, `'A'`, `rune`, `65`}, + {`package c2b; var _ = rune('A')`, `rune('A')`, `rune`, `65`}, + {`package c2c; type T rune; var _ = T('A')`, `T('A')`, `c2c.T`, `65`}, + + {`package c3a; var _ = float32(0.)`, `0.`, `float32`, `0`}, + {`package c3b; var _ = float32(0.)`, `float32(0.)`, `float32`, `0`}, + {`package c3c; type T float32; var _ = T(0.)`, `T(0.)`, `c3c.T`, `0`}, + + {`package c4a; var _ = complex64(0i)`, `0i`, `complex64`, `(0 + 0i)`}, + {`package c4b; var _ = complex64(0i)`, `complex64(0i)`, `complex64`, `(0 + 0i)`}, + {`package c4c; type T complex64; var _ = T(0i)`, `T(0i)`, `c4c.T`, `(0 + 0i)`}, + + {`package c5a; var _ = string("foo")`, `"foo"`, `string`, `"foo"`}, + {`package c5b; var _ = string("foo")`, `string("foo")`, `string`, `"foo"`}, + {`package c5c; type T string; var _ = T("foo")`, `T("foo")`, `c5c.T`, `"foo"`}, + {`package c5d; var _ = string(65)`, `65`, `untyped int`, `65`}, + {`package c5e; var _ = string('A')`, `'A'`, `untyped rune`, `65`}, + {`package c5f; type T string; var _ = T('A')`, `'A'`, `untyped rune`, `65`}, + + {`package d0; var _ = []byte("foo")`, `"foo"`, `string`, `"foo"`}, + {`package d1; var _ = []byte(string("foo"))`, `"foo"`, `string`, `"foo"`}, + {`package d2; var _ = []byte(string("foo"))`, `string("foo")`, `string`, `"foo"`}, + {`package d3; type T []byte; var _ = T("foo")`, `"foo"`, `string`, `"foo"`}, + + {`package e0; const _ = float32( 1e-200)`, `float32(1e-200)`, `float32`, `0`}, + {`package e1; const _ = float32(-1e-200)`, `float32(-1e-200)`, `float32`, `0`}, + {`package e2; const _ = float64( 1e-2000)`, `float64(1e-2000)`, `float64`, `0`}, + {`package e3; const _ = float64(-1e-2000)`, `float64(-1e-2000)`, `float64`, `0`}, + {`package e4; const _ = complex64( 1e-200)`, `complex64(1e-200)`, `complex64`, `(0 + 0i)`}, + {`package e5; const _ = complex64(-1e-200)`, `complex64(-1e-200)`, `complex64`, `(0 + 0i)`}, + {`package e6; const _ = complex128( 1e-2000)`, `complex128(1e-2000)`, `complex128`, `(0 + 0i)`}, + {`package e7; const _ = complex128(-1e-2000)`, `complex128(-1e-2000)`, `complex128`, `(0 + 0i)`}, + + {`package f0 ; var _ float32 = 1e-200`, `1e-200`, `float32`, `0`}, + {`package f1 ; var _ float32 = -1e-200`, `-1e-200`, `float32`, `0`}, + {`package f2a; var _ float64 = 1e-2000`, `1e-2000`, `float64`, `0`}, + {`package f3a; var _ float64 = -1e-2000`, `-1e-2000`, `float64`, `0`}, + {`package f2b; var _ = 1e-2000`, `1e-2000`, `float64`, `0`}, + {`package f3b; var _ = -1e-2000`, `-1e-2000`, `float64`, `0`}, + {`package f4 ; var _ complex64 = 1e-200 `, `1e-200`, `complex64`, `(0 + 0i)`}, + {`package f5 ; var _ complex64 = -1e-200 `, `-1e-200`, `complex64`, `(0 + 0i)`}, + {`package f6a; var _ complex128 = 1e-2000i`, `1e-2000i`, `complex128`, `(0 + 0i)`}, + {`package f7a; var _ complex128 = -1e-2000i`, `-1e-2000i`, `complex128`, `(0 + 0i)`}, + {`package f6b; var _ = 1e-2000i`, `1e-2000i`, `complex128`, `(0 + 0i)`}, + {`package f7b; var _ = -1e-2000i`, `-1e-2000i`, `complex128`, `(0 + 0i)`}, + + {`package g0; const (a = len([iota]int{}); b; c); const _ = c`, `c`, `int`, `2`}, // issue #22341 + {`package g1; var(j int32; s int; n = 1.0< 1 { + t.Errorf("package %s: %d Implicits entries found", name, len(info.Implicits)) + continue + } + + // extract Implicits entry, if any + var got string + for n, obj := range info.Implicits { + switch x := n.(type) { + case *ast.ImportSpec: + got = "importSpec" + case *ast.CaseClause: + got = "caseClause" + case *ast.Field: + got = "field" + default: + t.Fatalf("package %s: unexpected %T", name, x) + } + got += ": " + obj.String() + } + + // verify entry + if got != test.want { + t.Errorf("package %s: got %q; want %q", name, got, test.want) + } + } +} + +func predString(tv TypeAndValue) string { + var buf strings.Builder + pred := func(b bool, s string) { + if b { + if buf.Len() > 0 { + buf.WriteString(", ") + } + buf.WriteString(s) + } + } + + pred(tv.IsVoid(), "void") + pred(tv.IsType(), "type") + pred(tv.IsBuiltin(), "builtin") + pred(tv.IsValue() && tv.Value != nil, "const") + pred(tv.IsValue() && tv.Value == nil, "value") + pred(tv.IsNil(), "nil") + pred(tv.Addressable(), "addressable") + pred(tv.Assignable(), "assignable") + pred(tv.HasOk(), "hasOk") + + if buf.Len() == 0 { + return "invalid" + } + return buf.String() +} + +func TestPredicatesInfo(t *testing.T) { + testenv.MustHaveGoBuild(t) + + var tests = []struct { + src string + expr string + pred string + }{ + // void + {`package n0; func f() { f() }`, `f()`, `void`}, + + // types + {`package t0; type _ int`, `int`, `type`}, + {`package t1; type _ []int`, `[]int`, `type`}, + {`package t2; type _ func()`, `func()`, `type`}, + {`package t3; type _ func(int)`, `int`, `type`}, + {`package t3; type _ func(...int)`, `...int`, `type`}, + + // built-ins + {`package b0; var _ = len("")`, `len`, `builtin`}, + {`package b1; var _ = (len)("")`, `(len)`, `builtin`}, + + // constants + {`package c0; var _ = 42`, `42`, `const`}, + {`package c1; var _ = "foo" + "bar"`, `"foo" + "bar"`, `const`}, + {`package c2; const (i = 1i; _ = i)`, `i`, `const`}, + + // values + {`package v0; var (a, b int; _ = a + b)`, `a + b`, `value`}, + {`package v1; var _ = &[]int{1}`, `[]int{ā€¦}`, `value`}, + {`package v2; var _ = func(){}`, `(func() literal)`, `value`}, + {`package v4; func f() { _ = f }`, `f`, `value`}, + {`package v3; var _ *int = nil`, `nil`, `value, nil`}, + {`package v3; var _ *int = (nil)`, `(nil)`, `value, nil`}, + + // addressable (and thus assignable) operands + {`package a0; var (x int; _ = x)`, `x`, `value, addressable, assignable`}, + {`package a1; var (p *int; _ = *p)`, `*p`, `value, addressable, assignable`}, + {`package a2; var (s []int; _ = s[0])`, `s[0]`, `value, addressable, assignable`}, + {`package a3; var (s struct{f int}; _ = s.f)`, `s.f`, `value, addressable, assignable`}, + {`package a4; var (a [10]int; _ = a[0])`, `a[0]`, `value, addressable, assignable`}, + {`package a5; func _(x int) { _ = x }`, `x`, `value, addressable, assignable`}, + {`package a6; func _()(x int) { _ = x; return }`, `x`, `value, addressable, assignable`}, + {`package a7; type T int; func (x T) _() { _ = x }`, `x`, `value, addressable, assignable`}, + // composite literals are not addressable + + // assignable but not addressable values + {`package s0; var (m map[int]int; _ = m[0])`, `m[0]`, `value, assignable, hasOk`}, + {`package s1; var (m map[int]int; _, _ = m[0])`, `m[0]`, `value, assignable, hasOk`}, + + // hasOk expressions + {`package k0; var (ch chan int; _ = <-ch)`, `<-ch`, `value, hasOk`}, + {`package k1; var (ch chan int; _, _ = <-ch)`, `<-ch`, `value, hasOk`}, + + // missing entries + // - package names are collected in the Uses map + // - identifiers being declared are collected in the Defs map + {`package m0; import "os"; func _() { _ = os.Stdout }`, `os`, ``}, + {`package m1; import p "os"; func _() { _ = p.Stdout }`, `p`, ``}, + {`package m2; const c = 0`, `c`, ``}, + {`package m3; type T int`, `T`, ``}, + {`package m4; var v int`, `v`, ``}, + {`package m5; func f() {}`, `f`, ``}, + {`package m6; func _(x int) {}`, `x`, ``}, + {`package m6; func _()(x int) { return }`, `x`, ``}, + {`package m6; type T int; func (x T) _() {}`, `x`, ``}, + } + + for _, test := range tests { + info := Info{Types: make(map[ast.Expr]TypeAndValue)} + name := mustTypecheck("PredicatesInfo", test.src, &info).Name() + + // look for expression predicates + got := "" + for e, tv := range info.Types { + //println(name, ExprString(e)) + if ExprString(e) == test.expr { + got = predString(tv) + break + } + } + + if got != test.pred { + t.Errorf("package %s: got %s; want %s", name, got, test.pred) + } + } +} + +func TestScopesInfo(t *testing.T) { + testenv.MustHaveGoBuild(t) + + var tests = []struct { + src string + scopes []string // list of scope descriptors of the form kind:varlist + }{ + {`package p0`, []string{ + "file:", + }}, + {`package p1; import ( "fmt"; m "math"; _ "os" ); var ( _ = fmt.Println; _ = m.Pi )`, []string{ + "file:fmt m", + }}, + {`package p2; func _() {}`, []string{ + "file:", "func:", + }}, + {`package p3; func _(x, y int) {}`, []string{ + "file:", "func:x y", + }}, + {`package p4; func _(x, y int) { x, z := 1, 2; _ = z }`, []string{ + "file:", "func:x y z", // redeclaration of x + }}, + {`package p5; func _(x, y int) (u, _ int) { return }`, []string{ + "file:", "func:u x y", + }}, + {`package p6; func _() { { var x int; _ = x } }`, []string{ + "file:", "func:", "block:x", + }}, + {`package p7; func _() { if true {} }`, []string{ + "file:", "func:", "if:", "block:", + }}, + {`package p8; func _() { if x := 0; x < 0 { y := x; _ = y } }`, []string{ + "file:", "func:", "if:x", "block:y", + }}, + {`package p9; func _() { switch x := 0; x {} }`, []string{ + "file:", "func:", "switch:x", + }}, + {`package p10; func _() { switch x := 0; x { case 1: y := x; _ = y; default: }}`, []string{ + "file:", "func:", "switch:x", "case:y", "case:", + }}, + {`package p11; func _(t interface{}) { switch t.(type) {} }`, []string{ + "file:", "func:t", "type switch:", + }}, + {`package p12; func _(t interface{}) { switch t := t; t.(type) {} }`, []string{ + "file:", "func:t", "type switch:t", + }}, + {`package p13; func _(t interface{}) { switch x := t.(type) { case int: _ = x } }`, []string{ + "file:", "func:t", "type switch:", "case:x", // x implicitly declared + }}, + {`package p14; func _() { select{} }`, []string{ + "file:", "func:", + }}, + {`package p15; func _(c chan int) { select{ case <-c: } }`, []string{ + "file:", "func:c", "comm:", + }}, + {`package p16; func _(c chan int) { select{ case i := <-c: x := i; _ = x} }`, []string{ + "file:", "func:c", "comm:i x", + }}, + {`package p17; func _() { for{} }`, []string{ + "file:", "func:", "for:", "block:", + }}, + {`package p18; func _(n int) { for i := 0; i < n; i++ { _ = i } }`, []string{ + "file:", "func:n", "for:i", "block:", + }}, + {`package p19; func _(a []int) { for i := range a { _ = i} }`, []string{ + "file:", "func:a", "range:i", "block:", + }}, + {`package p20; var s int; func _(a []int) { for i, x := range a { s += x; _ = i } }`, []string{ + "file:", "func:a", "range:i x", "block:", + }}, + } + + for _, test := range tests { + info := Info{Scopes: make(map[ast.Node]*Scope)} + name := mustTypecheck("ScopesInfo", test.src, &info).Name() + + // number of scopes must match + if len(info.Scopes) != len(test.scopes) { + t.Errorf("package %s: got %d scopes; want %d", name, len(info.Scopes), len(test.scopes)) + } + + // scope descriptions must match + for node, scope := range info.Scopes { + kind := "" + switch node.(type) { + case *ast.File: + kind = "file" + case *ast.FuncType: + kind = "func" + case *ast.BlockStmt: + kind = "block" + case *ast.IfStmt: + kind = "if" + case *ast.SwitchStmt: + kind = "switch" + case *ast.TypeSwitchStmt: + kind = "type switch" + case *ast.CaseClause: + kind = "case" + case *ast.CommClause: + kind = "comm" + case *ast.ForStmt: + kind = "for" + case *ast.RangeStmt: + kind = "range" + } + + // look for matching scope description + desc := kind + ":" + strings.Join(scope.Names(), " ") + found := false + for _, d := range test.scopes { + if desc == d { + found = true + break + } + } + if !found { + t.Errorf("package %s: no matching scope found for %s", name, desc) + } + } + } +} + +func TestInitOrderInfo(t *testing.T) { + var tests = []struct { + src string + inits []string + }{ + {`package p0; var (x = 1; y = x)`, []string{ + "x = 1", "y = x", + }}, + {`package p1; var (a = 1; b = 2; c = 3)`, []string{ + "a = 1", "b = 2", "c = 3", + }}, + {`package p2; var (a, b, c = 1, 2, 3)`, []string{ + "a = 1", "b = 2", "c = 3", + }}, + {`package p3; var _ = f(); func f() int { return 1 }`, []string{ + "_ = f()", // blank var + }}, + {`package p4; var (a = 0; x = y; y = z; z = 0)`, []string{ + "a = 0", "z = 0", "y = z", "x = y", + }}, + {`package p5; var (a, _ = m[0]; m map[int]string)`, []string{ + "a, _ = m[0]", // blank var + }}, + {`package p6; var a, b = f(); func f() (_, _ int) { return z, z }; var z = 0`, []string{ + "z = 0", "a, b = f()", + }}, + {`package p7; var (a = func() int { return b }(); b = 1)`, []string{ + "b = 1", "a = (func() int literal)()", + }}, + {`package p8; var (a, b = func() (_, _ int) { return c, c }(); c = 1)`, []string{ + "c = 1", "a, b = (func() (_, _ int) literal)()", + }}, + {`package p9; type T struct{}; func (T) m() int { _ = y; return 0 }; var x, y = T.m, 1`, []string{ + "y = 1", "x = T.m", + }}, + {`package p10; var (d = c + b; a = 0; b = 0; c = 0)`, []string{ + "a = 0", "b = 0", "c = 0", "d = c + b", + }}, + {`package p11; var (a = e + c; b = d + c; c = 0; d = 0; e = 0)`, []string{ + "c = 0", "d = 0", "b = d + c", "e = 0", "a = e + c", + }}, + // emit an initializer for n:1 initializations only once (not for each node + // on the lhs which may appear in different order in the dependency graph) + {`package p12; var (a = x; b = 0; x, y = m[0]; m map[int]int)`, []string{ + "b = 0", "x, y = m[0]", "a = x", + }}, + // test case from spec section on package initialization + {`package p12 + + var ( + a = c + b + b = f() + c = f() + d = 3 + ) + + func f() int { + d++ + return d + }`, []string{ + "d = 3", "b = f()", "c = f()", "a = c + b", + }}, + // test case for issue 7131 + {`package main + + var counter int + func next() int { counter++; return counter } + + var _ = makeOrder() + func makeOrder() []int { return []int{f, b, d, e, c, a} } + + var a = next() + var b, c = next(), next() + var d, e, f = next(), next(), next() + `, []string{ + "a = next()", "b = next()", "c = next()", "d = next()", "e = next()", "f = next()", "_ = makeOrder()", + }}, + // test case for issue 10709 + {`package p13 + + var ( + v = t.m() + t = makeT(0) + ) + + type T struct{} + + func (T) m() int { return 0 } + + func makeT(n int) T { + if n > 0 { + return makeT(n-1) + } + return T{} + }`, []string{ + "t = makeT(0)", "v = t.m()", + }}, + // test case for issue 10709: same as test before, but variable decls swapped + {`package p14 + + var ( + t = makeT(0) + v = t.m() + ) + + type T struct{} + + func (T) m() int { return 0 } + + func makeT(n int) T { + if n > 0 { + return makeT(n-1) + } + return T{} + }`, []string{ + "t = makeT(0)", "v = t.m()", + }}, + // another candidate possibly causing problems with issue 10709 + {`package p15 + + var y1 = f1() + + func f1() int { return g1() } + func g1() int { f1(); return x1 } + + var x1 = 0 + + var y2 = f2() + + func f2() int { return g2() } + func g2() int { return x2 } + + var x2 = 0`, []string{ + "x1 = 0", "y1 = f1()", "x2 = 0", "y2 = f2()", + }}, + } + + for _, test := range tests { + info := Info{} + name := mustTypecheck("InitOrderInfo", test.src, &info).Name() + + // number of initializers must match + if len(info.InitOrder) != len(test.inits) { + t.Errorf("package %s: got %d initializers; want %d", name, len(info.InitOrder), len(test.inits)) + continue + } + + // initializers must match + for i, want := range test.inits { + got := info.InitOrder[i].String() + if got != want { + t.Errorf("package %s, init %d: got %s; want %s", name, i, got, want) + continue + } + } + } +} + +func TestMultiFileInitOrder(t *testing.T) { + fset := token.NewFileSet() + fileA := mustParse(fset, "", `package main; var a = 1`) + fileB := mustParse(fset, "", `package main; var b = 2`) + + // The initialization order must not depend on the parse + // order of the files, only on the presentation order to + // the type-checker. + for _, test := range []struct { + files []*ast.File + want string + }{ + {[]*ast.File{fileA, fileB}, "[a = 1 b = 2]"}, + {[]*ast.File{fileB, fileA}, "[b = 2 a = 1]"}, + } { + var info Info + if _, err := new(Config).Check("main", fset, test.files, &info); err != nil { + t.Fatal(err) + } + if got := fmt.Sprint(info.InitOrder); got != test.want { + t.Fatalf("got %s; want %s", got, test.want) + } + } +} + +func TestFiles(t *testing.T) { + var sources = []string{ + "package p; type T struct{}; func (T) m1() {}", + "package p; func (T) m2() {}; var x interface{ m1(); m2() } = T{}", + "package p; func (T) m3() {}; var y interface{ m1(); m2(); m3() } = T{}", + "package p", + } + + var conf Config + fset := token.NewFileSet() + pkg := NewPackage("p", "p") + var info Info + check := NewChecker(&conf, fset, pkg, &info) + + for i, src := range sources { + filename := fmt.Sprintf("sources%d", i) + f := mustParse(fset, filename, src) + if err := check.Files([]*ast.File{f}); err != nil { + t.Error(err) + } + } + + // check InitOrder is [x y] + var vars []string + for _, init := range info.InitOrder { + for _, v := range init.Lhs { + vars = append(vars, v.Name()) + } + } + if got, want := fmt.Sprint(vars), "[x y]"; got != want { + t.Errorf("InitOrder == %s, want %s", got, want) + } +} + +type testImporter map[string]*Package + +func (m testImporter) Import(path string) (*Package, error) { + if pkg := m[path]; pkg != nil { + return pkg, nil + } + return nil, fmt.Errorf("package %q not found", path) +} + +func TestSelection(t *testing.T) { + selections := make(map[*ast.SelectorExpr]*Selection) + + fset := token.NewFileSet() + imports := make(testImporter) + conf := Config{Importer: imports} + makePkg := func(path, src string) { + f := mustParse(fset, path+".go", src) + pkg, err := conf.Check(path, fset, []*ast.File{f}, &Info{Selections: selections}) + if err != nil { + t.Fatal(err) + } + imports[path] = pkg + } + + const libSrc = ` +package lib +type T float64 +const C T = 3 +var V T +func F() {} +func (T) M() {} +` + const mainSrc = ` +package main +import "lib" + +type A struct { + *B + C +} + +type B struct { + b int +} + +func (B) f(int) + +type C struct { + c int +} + +type G[P any] struct { + p P +} + +func (G[P]) m(P) {} + +var Inst G[int] + +func (C) g() +func (*C) h() + +func main() { + // qualified identifiers + var _ lib.T + _ = lib.C + _ = lib.F + _ = lib.V + _ = lib.T.M + + // fields + _ = A{}.B + _ = new(A).B + + _ = A{}.C + _ = new(A).C + + _ = A{}.b + _ = new(A).b + + _ = A{}.c + _ = new(A).c + + _ = Inst.p + _ = G[string]{}.p + + // methods + _ = A{}.f + _ = new(A).f + _ = A{}.g + _ = new(A).g + _ = new(A).h + + _ = B{}.f + _ = new(B).f + + _ = C{}.g + _ = new(C).g + _ = new(C).h + _ = Inst.m + + // method expressions + _ = A.f + _ = (*A).f + _ = B.f + _ = (*B).f + _ = G[string].m +}` + + wantOut := map[string][2]string{ + "lib.T.M": {"method expr (lib.T) M(lib.T)", ".[0]"}, + + "A{}.B": {"field (main.A) B *main.B", ".[0]"}, + "new(A).B": {"field (*main.A) B *main.B", "->[0]"}, + "A{}.C": {"field (main.A) C main.C", ".[1]"}, + "new(A).C": {"field (*main.A) C main.C", "->[1]"}, + "A{}.b": {"field (main.A) b int", "->[0 0]"}, + "new(A).b": {"field (*main.A) b int", "->[0 0]"}, + "A{}.c": {"field (main.A) c int", ".[1 0]"}, + "new(A).c": {"field (*main.A) c int", "->[1 0]"}, + "Inst.p": {"field (main.G[int]) p int", ".[0]"}, + + "A{}.f": {"method (main.A) f(int)", "->[0 0]"}, + "new(A).f": {"method (*main.A) f(int)", "->[0 0]"}, + "A{}.g": {"method (main.A) g()", ".[1 0]"}, + "new(A).g": {"method (*main.A) g()", "->[1 0]"}, + "new(A).h": {"method (*main.A) h()", "->[1 1]"}, // TODO(gri) should this report .[1 1] ? + "B{}.f": {"method (main.B) f(int)", ".[0]"}, + "new(B).f": {"method (*main.B) f(int)", "->[0]"}, + "C{}.g": {"method (main.C) g()", ".[0]"}, + "new(C).g": {"method (*main.C) g()", "->[0]"}, + "new(C).h": {"method (*main.C) h()", "->[1]"}, // TODO(gri) should this report .[1] ? + "Inst.m": {"method (main.G[int]) m(int)", ".[0]"}, + + "A.f": {"method expr (main.A) f(main.A, int)", "->[0 0]"}, + "(*A).f": {"method expr (*main.A) f(*main.A, int)", "->[0 0]"}, + "B.f": {"method expr (main.B) f(main.B, int)", ".[0]"}, + "(*B).f": {"method expr (*main.B) f(*main.B, int)", "->[0]"}, + "G[string].m": {"method expr (main.G[string]) m(main.G[string], string)", ".[0]"}, + "G[string]{}.p": {"field (main.G[string]) p string", ".[0]"}, + } + + makePkg("lib", libSrc) + makePkg("main", mainSrc) + + for e, sel := range selections { + _ = sel.String() // assertion: must not panic + + start := fset.Position(e.Pos()).Offset + end := fset.Position(e.End()).Offset + syntax := mainSrc[start:end] // (all SelectorExprs are in main, not lib) + + direct := "." + if sel.Indirect() { + direct = "->" + } + got := [2]string{ + sel.String(), + fmt.Sprintf("%s%v", direct, sel.Index()), + } + want := wantOut[syntax] + if want != got { + t.Errorf("%s: got %q; want %q", syntax, got, want) + } + delete(wantOut, syntax) + + // We must explicitly assert properties of the + // Signature's receiver since it doesn't participate + // in Identical() or String(). + sig, _ := sel.Type().(*Signature) + if sel.Kind() == MethodVal { + got := sig.Recv().Type() + want := sel.Recv() + if !Identical(got, want) { + t.Errorf("%s: Recv() = %s, want %s", syntax, got, want) + } + } else if sig != nil && sig.Recv() != nil { + t.Errorf("%s: signature has receiver %s", sig, sig.Recv().Type()) + } + } + // Assert that all wantOut entries were used exactly once. + for syntax := range wantOut { + t.Errorf("no ast.Selection found with syntax %q", syntax) + } +} + +func TestIssue8518(t *testing.T) { + fset := token.NewFileSet() + imports := make(testImporter) + conf := Config{ + Error: func(err error) { t.Log(err) }, // don't exit after first error + Importer: imports, + } + makePkg := func(path, src string) { + f := mustParse(fset, path, src) + pkg, _ := conf.Check(path, fset, []*ast.File{f}, nil) // errors logged via conf.Error + imports[path] = pkg + } + + const libSrc = ` +package a +import "missing" +const C1 = foo +const C2 = missing.C +` + + const mainSrc = ` +package main +import "a" +var _ = a.C1 +var _ = a.C2 +` + + makePkg("a", libSrc) + makePkg("main", mainSrc) // don't crash when type-checking this package +} + +func TestLookupFieldOrMethodOnNil(t *testing.T) { + // LookupFieldOrMethod on a nil type is expected to produce a run-time panic. + defer func() { + const want = "LookupFieldOrMethod on nil type" + p := recover() + if s, ok := p.(string); !ok || s != want { + t.Fatalf("got %v, want %s", p, want) + } + }() + LookupFieldOrMethod(nil, false, nil, "") +} + +func TestLookupFieldOrMethod(t *testing.T) { + // Test cases assume a lookup of the form a.f or x.f, where a stands for an + // addressable value, and x for a non-addressable value (even though a variable + // for ease of test case writing). + // + // Should be kept in sync with TestMethodSet. + var tests = []struct { + src string + found bool + index []int + indirect bool + }{ + // field lookups + {"var x T; type T struct{}", false, nil, false}, + {"var x T; type T struct{ f int }", true, []int{0}, false}, + {"var x T; type T struct{ a, b, f, c int }", true, []int{2}, false}, + + // field lookups on a generic type + {"var x T[int]; type T[P any] struct{}", false, nil, false}, + {"var x T[int]; type T[P any] struct{ f P }", true, []int{0}, false}, + {"var x T[int]; type T[P any] struct{ a, b, f, c P }", true, []int{2}, false}, + + // method lookups + {"var a T; type T struct{}; func (T) f() {}", true, []int{0}, false}, + {"var a *T; type T struct{}; func (T) f() {}", true, []int{0}, true}, + {"var a T; type T struct{}; func (*T) f() {}", true, []int{0}, false}, + {"var a *T; type T struct{}; func (*T) f() {}", true, []int{0}, true}, // TODO(gri) should this report indirect = false? + + // method lookups on a generic type + {"var a T[int]; type T[P any] struct{}; func (T[P]) f() {}", true, []int{0}, false}, + {"var a *T[int]; type T[P any] struct{}; func (T[P]) f() {}", true, []int{0}, true}, + {"var a T[int]; type T[P any] struct{}; func (*T[P]) f() {}", true, []int{0}, false}, + {"var a *T[int]; type T[P any] struct{}; func (*T[P]) f() {}", true, []int{0}, true}, // TODO(gri) should this report indirect = false? + + // collisions + {"type ( E1 struct{ f int }; E2 struct{ f int }; x struct{ E1; *E2 })", false, []int{1, 0}, false}, + {"type ( E1 struct{ f int }; E2 struct{}; x struct{ E1; *E2 }); func (E2) f() {}", false, []int{1, 0}, false}, + + // collisions on a generic type + {"type ( E1[P any] struct{ f P }; E2[P any] struct{ f P }; x struct{ E1[int]; *E2[int] })", false, []int{1, 0}, false}, + {"type ( E1[P any] struct{ f P }; E2[P any] struct{}; x struct{ E1[int]; *E2[int] }); func (E2[P]) f() {}", false, []int{1, 0}, false}, + + // outside methodset + // (*T).f method exists, but value of type T is not addressable + {"var x T; type T struct{}; func (*T) f() {}", false, nil, true}, + + // outside method set of a generic type + {"var x T[int]; type T[P any] struct{}; func (*T[P]) f() {}", false, nil, true}, + + // recursive generic types; see golang/go#52715 + {"var a T[int]; type ( T[P any] struct { *N[P] }; N[P any] struct { *T[P] } ); func (N[P]) f() {}", true, []int{0, 0}, true}, + {"var a T[int]; type ( T[P any] struct { *N[P] }; N[P any] struct { *T[P] } ); func (T[P]) f() {}", true, []int{0}, false}, + } + + for _, test := range tests { + pkg := mustTypecheck("test", "package p;"+test.src, nil) + + obj := pkg.Scope().Lookup("a") + if obj == nil { + if obj = pkg.Scope().Lookup("x"); obj == nil { + t.Errorf("%s: incorrect test case - no object a or x", test.src) + continue + } + } + + f, index, indirect := LookupFieldOrMethod(obj.Type(), obj.Name() == "a", pkg, "f") + if (f != nil) != test.found { + if f == nil { + t.Errorf("%s: got no object; want one", test.src) + } else { + t.Errorf("%s: got object = %v; want none", test.src, f) + } + } + if !sameSlice(index, test.index) { + t.Errorf("%s: got index = %v; want %v", test.src, index, test.index) + } + if indirect != test.indirect { + t.Errorf("%s: got indirect = %v; want %v", test.src, indirect, test.indirect) + } + } +} + +// Test for golang/go#52715 +func TestLookupFieldOrMethod_RecursiveGeneric(t *testing.T) { + const src = ` +package pkg + +type Tree[T any] struct { + *Node[T] +} + +func (*Tree[R]) N(r R) R { return r } + +type Node[T any] struct { + *Tree[T] +} + +type Instance = *Tree[int] +` + + fset := token.NewFileSet() + f := mustParse(fset, "foo.go", src) + pkg := NewPackage("pkg", f.Name.Name) + if err := NewChecker(nil, fset, pkg, nil).Files([]*ast.File{f}); err != nil { + panic(err) + } + + T := pkg.Scope().Lookup("Instance").Type() + _, _, _ = LookupFieldOrMethod(T, false, pkg, "M") // verify that LookupFieldOrMethod terminates +} + +func sameSlice(a, b []int) bool { + if len(a) != len(b) { + return false + } + for i, x := range a { + if x != b[i] { + return false + } + } + return true +} + +// TestScopeLookupParent ensures that (*Scope).LookupParent returns +// the correct result at various positions with the source. +func TestScopeLookupParent(t *testing.T) { + fset := token.NewFileSet() + imports := make(testImporter) + conf := Config{Importer: imports} + var info Info + makePkg := func(path string, files ...*ast.File) { + var err error + imports[path], err = conf.Check(path, fset, files, &info) + if err != nil { + t.Fatal(err) + } + } + + makePkg("lib", mustParse(fset, "", "package lib; var X int")) + // Each /*name=kind:line*/ comment makes the test look up the + // name at that point and checks that it resolves to a decl of + // the specified kind and line number. "undef" means undefined. + mainSrc := ` +/*lib=pkgname:5*/ /*X=var:1*/ /*Pi=const:8*/ /*T=typename:9*/ /*Y=var:10*/ /*F=func:12*/ +package main + +import "lib" +import . "lib" + +const Pi = 3.1415 +type T struct{} +var Y, _ = lib.X, X + +func F(){ + const pi, e = 3.1415, /*pi=undef*/ 2.71828 /*pi=const:13*/ /*e=const:13*/ + type /*t=undef*/ t /*t=typename:14*/ *t + print(Y) /*Y=var:10*/ + x, Y := Y, /*x=undef*/ /*Y=var:10*/ Pi /*x=var:16*/ /*Y=var:16*/ ; _ = x; _ = Y + var F = /*F=func:12*/ F /*F=var:17*/ ; _ = F + + var a []int + for i, x := range a /*i=undef*/ /*x=var:16*/ { _ = i; _ = x } + + var i interface{} + switch y := i.(type) { /*y=undef*/ + case /*y=undef*/ int /*y=var:23*/ : + case float32, /*y=undef*/ float64 /*y=var:23*/ : + default /*y=var:23*/: + println(y) + } + /*y=undef*/ + + switch int := i.(type) { + case /*int=typename:0*/ int /*int=var:31*/ : + println(int) + default /*int=var:31*/ : + } +} +/*main=undef*/ +` + + info.Uses = make(map[*ast.Ident]Object) + f := mustParse(fset, "", mainSrc) + makePkg("main", f) + mainScope := imports["main"].Scope() + rx := regexp.MustCompile(`^/\*(\w*)=([\w:]*)\*/$`) + for _, group := range f.Comments { + for _, comment := range group.List { + // Parse the assertion in the comment. + m := rx.FindStringSubmatch(comment.Text) + if m == nil { + t.Errorf("%s: bad comment: %s", + fset.Position(comment.Pos()), comment.Text) + continue + } + name, want := m[1], m[2] + + // Look up the name in the innermost enclosing scope. + inner := mainScope.Innermost(comment.Pos()) + if inner == nil { + t.Errorf("%s: at %s: can't find innermost scope", + fset.Position(comment.Pos()), comment.Text) + continue + } + got := "undef" + if _, obj := inner.LookupParent(name, comment.Pos()); obj != nil { + kind := strings.ToLower(strings.TrimPrefix(reflect.TypeOf(obj).String(), "*types.")) + got = fmt.Sprintf("%s:%d", kind, fset.Position(obj.Pos()).Line) + } + if got != want { + t.Errorf("%s: at %s: %s resolved to %s, want %s", + fset.Position(comment.Pos()), comment.Text, name, got, want) + } + } + } + + // Check that for each referring identifier, + // a lookup of its name on the innermost + // enclosing scope returns the correct object. + + for id, wantObj := range info.Uses { + inner := mainScope.Innermost(id.Pos()) + if inner == nil { + t.Errorf("%s: can't find innermost scope enclosing %q", + fset.Position(id.Pos()), id.Name) + continue + } + + // Exclude selectors and qualified identifiers---lexical + // refs only. (Ideally, we'd see if the AST parent is a + // SelectorExpr, but that requires PathEnclosingInterval + // from golang.org/x/tools/go/ast/astutil.) + if id.Name == "X" { + continue + } + + _, gotObj := inner.LookupParent(id.Name, id.Pos()) + if gotObj != wantObj { + t.Errorf("%s: got %v, want %v", + fset.Position(id.Pos()), gotObj, wantObj) + continue + } + } +} + +// newDefined creates a new defined type named T with the given underlying type. +// Helper function for use with TestIncompleteInterfaces only. +func newDefined(underlying Type) *Named { + tname := NewTypeName(token.NoPos, nil, "T", nil) + return NewNamed(tname, underlying, nil) +} + +func TestConvertibleTo(t *testing.T) { + for _, test := range []struct { + v, t Type + want bool + }{ + {Typ[Int], Typ[Int], true}, + {Typ[Int], Typ[Float32], true}, + {Typ[Int], Typ[String], true}, + {newDefined(Typ[Int]), Typ[Int], true}, + {newDefined(new(Struct)), new(Struct), true}, + {newDefined(Typ[Int]), new(Struct), false}, + {Typ[UntypedInt], Typ[Int], true}, + {NewSlice(Typ[Int]), NewArray(Typ[Int], 10), true}, + {NewSlice(Typ[Int]), NewArray(Typ[Uint], 10), false}, + {NewSlice(Typ[Int]), NewPointer(NewArray(Typ[Int], 10)), true}, + {NewSlice(Typ[Int]), NewPointer(NewArray(Typ[Uint], 10)), false}, + // Untyped string values are not permitted by the spec, so the behavior below is undefined. + {Typ[UntypedString], Typ[String], true}, + } { + if got := ConvertibleTo(test.v, test.t); got != test.want { + t.Errorf("ConvertibleTo(%v, %v) = %t, want %t", test.v, test.t, got, test.want) + } + } +} + +func TestAssignableTo(t *testing.T) { + for _, test := range []struct { + v, t Type + want bool + }{ + {Typ[Int], Typ[Int], true}, + {Typ[Int], Typ[Float32], false}, + {newDefined(Typ[Int]), Typ[Int], false}, + {newDefined(new(Struct)), new(Struct), true}, + {Typ[UntypedBool], Typ[Bool], true}, + {Typ[UntypedString], Typ[Bool], false}, + // Neither untyped string nor untyped numeric assignments arise during + // normal type checking, so the below behavior is technically undefined by + // the spec. + {Typ[UntypedString], Typ[String], true}, + {Typ[UntypedInt], Typ[Int], true}, + } { + if got := AssignableTo(test.v, test.t); got != test.want { + t.Errorf("AssignableTo(%v, %v) = %t, want %t", test.v, test.t, got, test.want) + } + } +} + +func TestIdentical(t *testing.T) { + // For each test, we compare the types of objects X and Y in the source. + tests := []struct { + src string + want bool + }{ + // Basic types. + {"var X int; var Y int", true}, + {"var X int; var Y string", false}, + + // TODO: add more tests for complex types. + + // Named types. + {"type X int; type Y int", false}, + + // Aliases. + {"type X = int; type Y = int", true}, + + // Functions. + {`func X(int) string { return "" }; func Y(int) string { return "" }`, true}, + {`func X() string { return "" }; func Y(int) string { return "" }`, false}, + {`func X(int) string { return "" }; func Y(int) {}`, false}, + + // Generic functions. Type parameters should be considered identical modulo + // renaming. See also issue #49722. + {`func X[P ~int](){}; func Y[Q ~int]() {}`, true}, + {`func X[P1 any, P2 ~*P1](){}; func Y[Q1 any, Q2 ~*Q1]() {}`, true}, + {`func X[P1 any, P2 ~[]P1](){}; func Y[Q1 any, Q2 ~*Q1]() {}`, false}, + {`func X[P ~int](P){}; func Y[Q ~int](Q) {}`, true}, + {`func X[P ~string](P){}; func Y[Q ~int](Q) {}`, false}, + {`func X[P ~int]([]P){}; func Y[Q ~int]([]Q) {}`, true}, + } + + for _, test := range tests { + pkg := mustTypecheck("test", "package p;"+test.src, nil) + X := pkg.Scope().Lookup("X") + Y := pkg.Scope().Lookup("Y") + if X == nil || Y == nil { + t.Fatal("test must declare both X and Y") + } + if got := Identical(X.Type(), Y.Type()); got != test.want { + t.Errorf("Identical(%s, %s) = %t, want %t", X.Type(), Y.Type(), got, test.want) + } + } +} + +func TestIdentical_issue15173(t *testing.T) { + // Identical should allow nil arguments and be symmetric. + for _, test := range []struct { + x, y Type + want bool + }{ + {Typ[Int], Typ[Int], true}, + {Typ[Int], nil, false}, + {nil, Typ[Int], false}, + {nil, nil, true}, + } { + if got := Identical(test.x, test.y); got != test.want { + t.Errorf("Identical(%v, %v) = %t", test.x, test.y, got) + } + } +} + +func TestIdenticalUnions(t *testing.T) { + tname := NewTypeName(token.NoPos, nil, "myInt", nil) + myInt := NewNamed(tname, Typ[Int], nil) + tmap := map[string]*Term{ + "int": NewTerm(false, Typ[Int]), + "~int": NewTerm(true, Typ[Int]), + "string": NewTerm(false, Typ[String]), + "~string": NewTerm(true, Typ[String]), + "myInt": NewTerm(false, myInt), + } + makeUnion := func(s string) *Union { + parts := strings.Split(s, "|") + var terms []*Term + for _, p := range parts { + term := tmap[p] + if term == nil { + t.Fatalf("missing term %q", p) + } + terms = append(terms, term) + } + return NewUnion(terms) + } + for _, test := range []struct { + x, y string + want bool + }{ + // These tests are just sanity checks. The tests for type sets and + // interfaces provide much more test coverage. + {"int|~int", "~int", true}, + {"myInt|~int", "~int", true}, + {"int|string", "string|int", true}, + {"int|int|string", "string|int", true}, + {"myInt|string", "int|string", false}, + } { + x := makeUnion(test.x) + y := makeUnion(test.y) + if got := Identical(x, y); got != test.want { + t.Errorf("Identical(%v, %v) = %t", test.x, test.y, got) + } + } +} + +func TestIssue61737(t *testing.T) { + // This test verifies that it is possible to construct invalid interfaces + // containing duplicate methods using the go/types API. + // + // It must be possible for importers to construct such invalid interfaces. + // Previously, this panicked. + + sig1 := NewSignatureType(nil, nil, nil, NewTuple(NewParam(token.NoPos, nil, "", Typ[Int])), nil, false) + sig2 := NewSignatureType(nil, nil, nil, NewTuple(NewParam(token.NoPos, nil, "", Typ[String])), nil, false) + + methods := []*Func{ + NewFunc(token.NoPos, nil, "M", sig1), + NewFunc(token.NoPos, nil, "M", sig2), + } + + embeddedMethods := []*Func{ + NewFunc(token.NoPos, nil, "M", sig2), + } + embedded := NewInterfaceType(embeddedMethods, nil) + iface := NewInterfaceType(methods, []Type{embedded}) + iface.Complete() +} + +func TestIssue15305(t *testing.T) { + const src = "package p; func f() int16; var _ = f(undef)" + fset := token.NewFileSet() + f := mustParse(fset, "issue15305.go", src) + conf := Config{ + Error: func(err error) {}, // allow errors + } + info := &Info{ + Types: make(map[ast.Expr]TypeAndValue), + } + conf.Check("p", fset, []*ast.File{f}, info) // ignore result + for e, tv := range info.Types { + if _, ok := e.(*ast.CallExpr); ok { + if tv.Type != Typ[Int16] { + t.Errorf("CallExpr has type %v, want int16", tv.Type) + } + return + } + } + t.Errorf("CallExpr has no type") +} + +// TestCompositeLitTypes verifies that Info.Types registers the correct +// types for composite literal expressions and composite literal type +// expressions. +func TestCompositeLitTypes(t *testing.T) { + for _, test := range []struct { + lit, typ string + }{ + {`[16]byte{}`, `[16]byte`}, + {`[...]byte{}`, `[0]byte`}, // test for issue #14092 + {`[...]int{1, 2, 3}`, `[3]int`}, // test for issue #14092 + {`[...]int{90: 0, 98: 1, 2}`, `[100]int`}, // test for issue #14092 + {`[]int{}`, `[]int`}, + {`map[string]bool{"foo": true}`, `map[string]bool`}, + {`struct{}{}`, `struct{}`}, + {`struct{x, y int; z complex128}{}`, `struct{x int; y int; z complex128}`}, + } { + fset := token.NewFileSet() + f := mustParse(fset, test.lit, "package p; var _ = "+test.lit) + types := make(map[ast.Expr]TypeAndValue) + if _, err := new(Config).Check("p", fset, []*ast.File{f}, &Info{Types: types}); err != nil { + t.Fatalf("%s: %v", test.lit, err) + } + + cmptype := func(x ast.Expr, want string) { + tv, ok := types[x] + if !ok { + t.Errorf("%s: no Types entry found", test.lit) + return + } + if tv.Type == nil { + t.Errorf("%s: type is nil", test.lit) + return + } + if got := tv.Type.String(); got != want { + t.Errorf("%s: got %v, want %s", test.lit, got, want) + } + } + + // test type of composite literal expression + rhs := f.Decls[0].(*ast.GenDecl).Specs[0].(*ast.ValueSpec).Values[0] + cmptype(rhs, test.typ) + + // test type of composite literal type expression + cmptype(rhs.(*ast.CompositeLit).Type, test.typ) + } +} + +// TestObjectParents verifies that objects have parent scopes or not +// as specified by the Object interface. +func TestObjectParents(t *testing.T) { + const src = ` +package p + +const C = 0 + +type T1 struct { + a, b int + T2 +} + +type T2 interface { + im1() + im2() +} + +func (T1) m1() {} +func (*T1) m2() {} + +func f(x int) { y := x; print(y) } +` + + fset := token.NewFileSet() + f := mustParse(fset, "src", src) + + info := &Info{ + Defs: make(map[*ast.Ident]Object), + } + if _, err := new(Config).Check("p", fset, []*ast.File{f}, info); err != nil { + t.Fatal(err) + } + + for ident, obj := range info.Defs { + if obj == nil { + // only package names and implicit vars have a nil object + // (in this test we only need to handle the package name) + if ident.Name != "p" { + t.Errorf("%v has nil object", ident) + } + continue + } + + // struct fields, type-associated and interface methods + // have no parent scope + wantParent := true + switch obj := obj.(type) { + case *Var: + if obj.IsField() { + wantParent = false + } + case *Func: + if obj.Type().(*Signature).Recv() != nil { // method + wantParent = false + } + } + + gotParent := obj.Parent() != nil + switch { + case gotParent && !wantParent: + t.Errorf("%v: want no parent, got %s", ident, obj.Parent()) + case !gotParent && wantParent: + t.Errorf("%v: no parent found", ident) + } + } +} + +// TestFailedImport tests that we don't get follow-on errors +// elsewhere in a package due to failing to import a package. +func TestFailedImport(t *testing.T) { + testenv.MustHaveGoBuild(t) + + const src = ` +package p + +import foo "go/types/thisdirectorymustnotexistotherwisethistestmayfail/foo" // should only see an error here + +const c = foo.C +type T = foo.T +var v T = c +func f(x T) T { return foo.F(x) } +` + fset := token.NewFileSet() + f := mustParse(fset, "src", src) + files := []*ast.File{f} + + // type-check using all possible importers + for _, compiler := range []string{"gc", "gccgo", "source"} { + errcount := 0 + conf := Config{ + Error: func(err error) { + // we should only see the import error + if errcount > 0 || !strings.Contains(err.Error(), "could not import") { + t.Errorf("for %s importer, got unexpected error: %v", compiler, err) + } + errcount++ + }, + Importer: importer.For(compiler, nil), + } + + info := &Info{ + Uses: make(map[*ast.Ident]Object), + } + pkg, _ := conf.Check("p", fset, files, info) + if pkg == nil { + t.Errorf("for %s importer, type-checking failed to return a package", compiler) + continue + } + + imports := pkg.Imports() + if len(imports) != 1 { + t.Errorf("for %s importer, got %d imports, want 1", compiler, len(imports)) + continue + } + imp := imports[0] + if imp.Name() != "foo" { + t.Errorf(`for %s importer, got %q, want "foo"`, compiler, imp.Name()) + continue + } + + // verify that all uses of foo refer to the imported package foo (imp) + for ident, obj := range info.Uses { + if ident.Name == "foo" { + if obj, ok := obj.(*PkgName); ok { + if obj.Imported() != imp { + t.Errorf("%s resolved to %v; want %v", ident, obj.Imported(), imp) + } + } else { + t.Errorf("%s resolved to %v; want package name", ident, obj) + } + } + } + } +} + +func TestInstantiate(t *testing.T) { + // eventually we like more tests but this is a start + const src = "package p; type T[P any] *T[P]" + pkg := mustTypecheck(".", src, nil) + + // type T should have one type parameter + T := pkg.Scope().Lookup("T").Type().(*Named) + if n := T.TypeParams().Len(); n != 1 { + t.Fatalf("expected 1 type parameter; found %d", n) + } + + // instantiation should succeed (no endless recursion) + // even with a nil *Checker + res, err := Instantiate(nil, T, []Type{Typ[Int]}, false) + if err != nil { + t.Fatal(err) + } + + // instantiated type should point to itself + if p := res.Underlying().(*Pointer).Elem(); p != res { + t.Fatalf("unexpected result type: %s points to %s", res, p) + } +} + +func TestInstantiateErrors(t *testing.T) { + tests := []struct { + src string // by convention, T must be the type being instantiated + targs []Type + wantAt int // -1 indicates no error + }{ + {"type T[P interface{~string}] int", []Type{Typ[Int]}, 0}, + {"type T[P1 interface{int}, P2 interface{~string}] int", []Type{Typ[Int], Typ[Int]}, 1}, + {"type T[P1 any, P2 interface{~[]P1}] int", []Type{Typ[Int], NewSlice(Typ[String])}, 1}, + {"type T[P1 interface{~[]P2}, P2 any] int", []Type{NewSlice(Typ[String]), Typ[Int]}, 0}, + } + + for _, test := range tests { + src := "package p; " + test.src + pkg := mustTypecheck(".", src, nil) + + T := pkg.Scope().Lookup("T").Type().(*Named) + + _, err := Instantiate(nil, T, test.targs, true) + if err == nil { + t.Fatalf("Instantiate(%v, %v) returned nil error, want non-nil", T, test.targs) + } + + var argErr *ArgumentError + if !errors.As(err, &argErr) { + t.Fatalf("Instantiate(%v, %v): error is not an *ArgumentError", T, test.targs) + } + + if argErr.Index != test.wantAt { + t.Errorf("Instantiate(%v, %v): error at index %d, want index %d", T, test.targs, argErr.Index, test.wantAt) + } + } +} + +func TestArgumentErrorUnwrapping(t *testing.T) { + var err error = &ArgumentError{ + Index: 1, + Err: Error{Msg: "test"}, + } + var e Error + if !errors.As(err, &e) { + t.Fatalf("error %v does not wrap types.Error", err) + } + if e.Msg != "test" { + t.Errorf("e.Msg = %q, want %q", e.Msg, "test") + } +} + +func TestInstanceIdentity(t *testing.T) { + imports := make(testImporter) + conf := Config{Importer: imports} + makePkg := func(src string) { + fset := token.NewFileSet() + f := mustParse(fset, "", src) + name := f.Name.Name + pkg, err := conf.Check(name, fset, []*ast.File{f}, nil) + if err != nil { + t.Fatal(err) + } + imports[name] = pkg + } + makePkg(`package lib; type T[P any] struct{}`) + makePkg(`package a; import "lib"; var A lib.T[int]`) + makePkg(`package b; import "lib"; var B lib.T[int]`) + a := imports["a"].Scope().Lookup("A") + b := imports["b"].Scope().Lookup("B") + if !Identical(a.Type(), b.Type()) { + t.Errorf("mismatching types: a.A: %s, b.B: %s", a.Type(), b.Type()) + } +} + +// TestInstantiatedObjects verifies properties of instantiated objects. +func TestInstantiatedObjects(t *testing.T) { + const src = ` +package p + +type T[P any] struct { + field P +} + +func (recv *T[Q]) concreteMethod(mParam Q) (mResult Q) { return } + +type FT[P any] func(ftParam P) (ftResult P) + +func F[P any](fParam P) (fResult P){ return } + +type I[P any] interface { + interfaceMethod(P) +} + +type R[P any] T[P] + +func (R[P]) m() {} // having a method triggers expansion of R + +var ( + t T[int] + ft FT[int] + f = F[int] + i I[int] +) + +func fn() { + var r R[int] + _ = r +} +` + info := &Info{ + Defs: make(map[*ast.Ident]Object), + } + fset := token.NewFileSet() + f := mustParse(fset, "p.go", src) + conf := Config{} + pkg, err := conf.Check(f.Name.Name, fset, []*ast.File{f}, info) + if err != nil { + t.Fatal(err) + } + + lookup := func(name string) Type { return pkg.Scope().Lookup(name).Type() } + fnScope := pkg.Scope().Lookup("fn").(*Func).Scope() + + tests := []struct { + name string + obj Object + }{ + // Struct fields + {"field", lookup("t").Underlying().(*Struct).Field(0)}, + {"field", fnScope.Lookup("r").Type().Underlying().(*Struct).Field(0)}, + + // Methods and method fields + {"concreteMethod", lookup("t").(*Named).Method(0)}, + {"recv", lookup("t").(*Named).Method(0).Type().(*Signature).Recv()}, + {"mParam", lookup("t").(*Named).Method(0).Type().(*Signature).Params().At(0)}, + {"mResult", lookup("t").(*Named).Method(0).Type().(*Signature).Results().At(0)}, + + // Interface methods + {"interfaceMethod", lookup("i").Underlying().(*Interface).Method(0)}, + + // Function type fields + {"ftParam", lookup("ft").Underlying().(*Signature).Params().At(0)}, + {"ftResult", lookup("ft").Underlying().(*Signature).Results().At(0)}, + + // Function fields + {"fParam", lookup("f").(*Signature).Params().At(0)}, + {"fResult", lookup("f").(*Signature).Results().At(0)}, + } + + // Collect all identifiers by name. + idents := make(map[string][]*ast.Ident) + ast.Inspect(f, func(n ast.Node) bool { + if id, ok := n.(*ast.Ident); ok { + idents[id.Name] = append(idents[id.Name], id) + } + return true + }) + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + if got := len(idents[test.name]); got != 1 { + t.Fatalf("found %d identifiers named %s, want 1", got, test.name) + } + ident := idents[test.name][0] + def := info.Defs[ident] + if def == test.obj { + t.Fatalf("info.Defs[%s] contains the test object", test.name) + } + if orig := originObject(test.obj); def != orig { + t.Errorf("info.Defs[%s] does not match obj.Origin()", test.name) + } + if def.Pkg() != test.obj.Pkg() { + t.Errorf("Pkg() = %v, want %v", def.Pkg(), test.obj.Pkg()) + } + if def.Name() != test.obj.Name() { + t.Errorf("Name() = %v, want %v", def.Name(), test.obj.Name()) + } + if def.Pos() != test.obj.Pos() { + t.Errorf("Pos() = %v, want %v", def.Pos(), test.obj.Pos()) + } + if def.Parent() != test.obj.Parent() { + t.Fatalf("Parent() = %v, want %v", def.Parent(), test.obj.Parent()) + } + if def.Exported() != test.obj.Exported() { + t.Fatalf("Exported() = %v, want %v", def.Exported(), test.obj.Exported()) + } + if def.Id() != test.obj.Id() { + t.Fatalf("Id() = %v, want %v", def.Id(), test.obj.Id()) + } + // String and Type are expected to differ. + }) + } +} + +func originObject(obj Object) Object { + switch obj := obj.(type) { + case *Var: + return obj.Origin() + case *Func: + return obj.Origin() + } + return obj +} + +func TestImplements(t *testing.T) { + const src = ` +package p + +type EmptyIface interface{} + +type I interface { + m() +} + +type C interface { + m() + ~int +} + +type Integer interface{ + int8 | int16 | int32 | int64 +} + +type EmptyTypeSet interface{ + Integer + ~string +} + +type N1 int +func (N1) m() {} + +type N2 int +func (*N2) m() {} + +type N3 int +func (N3) m(int) {} + +type N4 string +func (N4) m() + +type Bad Bad // invalid type +` + + fset := token.NewFileSet() + f := mustParse(fset, "p.go", src) + conf := Config{Error: func(error) {}} + pkg, _ := conf.Check(f.Name.Name, fset, []*ast.File{f}, nil) + + lookup := func(tname string) Type { return pkg.Scope().Lookup(tname).Type() } + var ( + EmptyIface = lookup("EmptyIface").Underlying().(*Interface) + I = lookup("I").(*Named) + II = I.Underlying().(*Interface) + C = lookup("C").(*Named) + CI = C.Underlying().(*Interface) + Integer = lookup("Integer").Underlying().(*Interface) + EmptyTypeSet = lookup("EmptyTypeSet").Underlying().(*Interface) + N1 = lookup("N1") + N1p = NewPointer(N1) + N2 = lookup("N2") + N2p = NewPointer(N2) + N3 = lookup("N3") + N4 = lookup("N4") + Bad = lookup("Bad") + ) + + tests := []struct { + V Type + T *Interface + want bool + }{ + {I, II, true}, + {I, CI, false}, + {C, II, true}, + {C, CI, true}, + {Typ[Int8], Integer, true}, + {Typ[Int64], Integer, true}, + {Typ[String], Integer, false}, + {EmptyTypeSet, II, true}, + {EmptyTypeSet, EmptyTypeSet, true}, + {Typ[Int], EmptyTypeSet, false}, + {N1, II, true}, + {N1, CI, true}, + {N1p, II, true}, + {N1p, CI, false}, + {N2, II, false}, + {N2, CI, false}, + {N2p, II, true}, + {N2p, CI, false}, + {N3, II, false}, + {N3, CI, false}, + {N4, II, true}, + {N4, CI, false}, + {Bad, II, false}, + {Bad, CI, false}, + {Bad, EmptyIface, true}, + } + + for _, test := range tests { + if got := Implements(test.V, test.T); got != test.want { + t.Errorf("Implements(%s, %s) = %t, want %t", test.V, test.T, got, test.want) + } + + // The type assertion x.(T) is valid if T is an interface or if T implements the type of x. + // The assertion is never valid if T is a bad type. + V := test.T + T := test.V + want := false + if _, ok := T.Underlying().(*Interface); (ok || Implements(T, V)) && T != Bad { + want = true + } + if got := AssertableTo(V, T); got != want { + t.Errorf("AssertableTo(%s, %s) = %t, want %t", V, T, got, want) + } + } +} + +func TestMissingMethodAlternative(t *testing.T) { + const src = ` +package p +type T interface { + m() +} + +type V0 struct{} +func (V0) m() {} + +type V1 struct{} + +type V2 struct{} +func (V2) m() int + +type V3 struct{} +func (*V3) m() + +type V4 struct{} +func (V4) M() +` + + pkg := mustTypecheck("p.go", src, nil) + + T := pkg.Scope().Lookup("T").Type().Underlying().(*Interface) + lookup := func(name string) (*Func, bool) { + return MissingMethod(pkg.Scope().Lookup(name).Type(), T, true) + } + + // V0 has method m with correct signature. Should not report wrongType. + method, wrongType := lookup("V0") + if method != nil || wrongType { + t.Fatalf("V0: got method = %v, wrongType = %v", method, wrongType) + } + + checkMissingMethod := func(tname string, reportWrongType bool) { + method, wrongType := lookup(tname) + if method == nil || method.Name() != "m" || wrongType != reportWrongType { + t.Fatalf("%s: got method = %v, wrongType = %v", tname, method, wrongType) + } + } + + // V1 has no method m. Should not report wrongType. + checkMissingMethod("V1", false) + + // V2 has method m with wrong signature type (ignoring receiver). Should report wrongType. + checkMissingMethod("V2", true) + + // V3 has no method m but it exists on *V3. Should report wrongType. + checkMissingMethod("V3", true) + + // V4 has no method m but has M. Should not report wrongType. + checkMissingMethod("V4", false) +} diff --git a/src/go/types/array.go b/src/go/types/array.go new file mode 100644 index 0000000..5b28474 --- /dev/null +++ b/src/go/types/array.go @@ -0,0 +1,25 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +// An Array represents an array type. +type Array struct { + len int64 + elem Type +} + +// NewArray returns a new array type for the given element type and length. +// A negative length indicates an unknown length. +func NewArray(elem Type, len int64) *Array { return &Array{len: len, elem: elem} } + +// Len returns the length of array a. +// A negative result indicates an unknown length. +func (a *Array) Len() int64 { return a.len } + +// Elem returns element type of array a. +func (a *Array) Elem() Type { return a.elem } + +func (t *Array) Underlying() Type { return t } +func (t *Array) String() string { return TypeString(t, nil) } diff --git a/src/go/types/assignments.go b/src/go/types/assignments.go new file mode 100644 index 0000000..4d5acb1 --- /dev/null +++ b/src/go/types/assignments.go @@ -0,0 +1,475 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements initialization and assignment checks. + +package types + +import ( + "fmt" + "go/ast" + "go/token" + . "internal/types/errors" + "strings" +) + +// assignment reports whether x can be assigned to a variable of type T, +// if necessary by attempting to convert untyped values to the appropriate +// type. context describes the context in which the assignment takes place. +// Use T == nil to indicate assignment to an untyped blank identifier. +// x.mode is set to invalid if the assignment failed. +func (check *Checker) assignment(x *operand, T Type, context string) { + check.singleValue(x) + + switch x.mode { + case invalid: + return // error reported before + case constant_, variable, mapindex, value, commaok, commaerr: + // ok + default: + // we may get here because of other problems (issue #39634, crash 12) + // TODO(gri) do we need a new "generic" error code here? + check.errorf(x, IncompatibleAssign, "cannot assign %s to %s in %s", x, T, context) + return + } + + if isUntyped(x.typ) { + target := T + // spec: "If an untyped constant is assigned to a variable of interface + // type or the blank identifier, the constant is first converted to type + // bool, rune, int, float64, complex128 or string respectively, depending + // on whether the value is a boolean, rune, integer, floating-point, + // complex, or string constant." + if T == nil || isNonTypeParamInterface(T) { + if T == nil && x.typ == Typ[UntypedNil] { + check.errorf(x, UntypedNilUse, "use of untyped nil in %s", context) + x.mode = invalid + return + } + target = Default(x.typ) + } + newType, val, code := check.implicitTypeAndValue(x, target) + if code != 0 { + msg := check.sprintf("cannot use %s as %s value in %s", x, target, context) + switch code { + case TruncatedFloat: + msg += " (truncated)" + case NumericOverflow: + msg += " (overflows)" + default: + code = IncompatibleAssign + } + check.error(x, code, msg) + x.mode = invalid + return + } + if val != nil { + x.val = val + check.updateExprVal(x.expr, val) + } + if newType != x.typ { + x.typ = newType + check.updateExprType(x.expr, newType, false) + } + } + + // A generic (non-instantiated) function value cannot be assigned to a variable. + if sig, _ := under(x.typ).(*Signature); sig != nil && sig.TypeParams().Len() > 0 { + check.errorf(x, WrongTypeArgCount, "cannot use generic function %s without instantiation in %s", x, context) + } + + // spec: "If a left-hand side is the blank identifier, any typed or + // non-constant value except for the predeclared identifier nil may + // be assigned to it." + if T == nil { + return + } + + cause := "" + if ok, code := x.assignableTo(check, T, &cause); !ok { + if cause != "" { + check.errorf(x, code, "cannot use %s as %s value in %s: %s", x, T, context, cause) + } else { + check.errorf(x, code, "cannot use %s as %s value in %s", x, T, context) + } + x.mode = invalid + } +} + +func (check *Checker) initConst(lhs *Const, x *operand) { + if x.mode == invalid || x.typ == Typ[Invalid] || lhs.typ == Typ[Invalid] { + if lhs.typ == nil { + lhs.typ = Typ[Invalid] + } + return + } + + // rhs must be a constant + if x.mode != constant_ { + check.errorf(x, InvalidConstInit, "%s is not constant", x) + if lhs.typ == nil { + lhs.typ = Typ[Invalid] + } + return + } + assert(isConstType(x.typ)) + + // If the lhs doesn't have a type yet, use the type of x. + if lhs.typ == nil { + lhs.typ = x.typ + } + + check.assignment(x, lhs.typ, "constant declaration") + if x.mode == invalid { + return + } + + lhs.val = x.val +} + +func (check *Checker) initVar(lhs *Var, x *operand, context string) Type { + if x.mode == invalid || x.typ == Typ[Invalid] || lhs.typ == Typ[Invalid] { + if lhs.typ == nil { + lhs.typ = Typ[Invalid] + } + return nil + } + + // If the lhs doesn't have a type yet, use the type of x. + if lhs.typ == nil { + typ := x.typ + if isUntyped(typ) { + // convert untyped types to default types + if typ == Typ[UntypedNil] { + check.errorf(x, UntypedNilUse, "use of untyped nil in %s", context) + lhs.typ = Typ[Invalid] + return nil + } + typ = Default(typ) + } + lhs.typ = typ + } + + check.assignment(x, lhs.typ, context) + if x.mode == invalid { + return nil + } + + return x.typ +} + +func (check *Checker) assignVar(lhs ast.Expr, x *operand) Type { + if x.mode == invalid || x.typ == Typ[Invalid] { + check.useLHS(lhs) + return nil + } + + // Determine if the lhs is a (possibly parenthesized) identifier. + ident, _ := unparen(lhs).(*ast.Ident) + + // Don't evaluate lhs if it is the blank identifier. + if ident != nil && ident.Name == "_" { + check.recordDef(ident, nil) + check.assignment(x, nil, "assignment to _ identifier") + if x.mode == invalid { + return nil + } + return x.typ + } + + // If the lhs is an identifier denoting a variable v, this assignment + // is not a 'use' of v. Remember current value of v.used and restore + // after evaluating the lhs via check.expr. + var v *Var + var v_used bool + if ident != nil { + if obj := check.lookup(ident.Name); obj != nil { + // It's ok to mark non-local variables, but ignore variables + // from other packages to avoid potential race conditions with + // dot-imported variables. + if w, _ := obj.(*Var); w != nil && w.pkg == check.pkg { + v = w + v_used = v.used + } + } + } + + var z operand + check.expr(&z, lhs) + if v != nil { + v.used = v_used // restore v.used + } + + if z.mode == invalid || z.typ == Typ[Invalid] { + return nil + } + + // spec: "Each left-hand side operand must be addressable, a map index + // expression, or the blank identifier. Operands may be parenthesized." + switch z.mode { + case invalid: + return nil + case variable, mapindex: + // ok + default: + if sel, ok := z.expr.(*ast.SelectorExpr); ok { + var op operand + check.expr(&op, sel.X) + if op.mode == mapindex { + check.errorf(&z, UnaddressableFieldAssign, "cannot assign to struct field %s in map", ExprString(z.expr)) + return nil + } + } + check.errorf(&z, UnassignableOperand, "cannot assign to %s", &z) + return nil + } + + check.assignment(x, z.typ, "assignment") + if x.mode == invalid { + return nil + } + + return x.typ +} + +// operandTypes returns the list of types for the given operands. +func operandTypes(list []*operand) (res []Type) { + for _, x := range list { + res = append(res, x.typ) + } + return res +} + +// varTypes returns the list of types for the given variables. +func varTypes(list []*Var) (res []Type) { + for _, x := range list { + res = append(res, x.typ) + } + return res +} + +// typesSummary returns a string of the form "(t1, t2, ...)" where the +// ti's are user-friendly string representations for the given types. +// If variadic is set and the last type is a slice, its string is of +// the form "...E" where E is the slice's element type. +func (check *Checker) typesSummary(list []Type, variadic bool) string { + var res []string + for i, t := range list { + var s string + switch { + case t == nil: + fallthrough // should not happen but be cautious + case t == Typ[Invalid]: + s = "" + case isUntyped(t): + if isNumeric(t) { + // Do not imply a specific type requirement: + // "have number, want float64" is better than + // "have untyped int, want float64" or + // "have int, want float64". + s = "number" + } else { + // If we don't have a number, omit the "untyped" qualifier + // for compactness. + s = strings.Replace(t.(*Basic).name, "untyped ", "", -1) + } + case variadic && i == len(list)-1: + s = check.sprintf("...%s", t.(*Slice).elem) + } + if s == "" { + s = check.sprintf("%s", t) + } + res = append(res, s) + } + return "(" + strings.Join(res, ", ") + ")" +} + +func measure(x int, unit string) string { + if x != 1 { + unit += "s" + } + return fmt.Sprintf("%d %s", x, unit) +} + +func (check *Checker) assignError(rhs []ast.Expr, nvars, nvals int) { + vars := measure(nvars, "variable") + vals := measure(nvals, "value") + rhs0 := rhs[0] + + if len(rhs) == 1 { + if call, _ := unparen(rhs0).(*ast.CallExpr); call != nil { + check.errorf(rhs0, WrongAssignCount, "assignment mismatch: %s but %s returns %s", vars, call.Fun, vals) + return + } + } + check.errorf(rhs0, WrongAssignCount, "assignment mismatch: %s but %s", vars, vals) +} + +// If returnStmt != nil, initVars is called to type-check the assignment +// of return expressions, and returnStmt is the return statement. +func (check *Checker) initVars(lhs []*Var, origRHS []ast.Expr, returnStmt ast.Stmt) { + rhs, commaOk := check.exprList(origRHS, len(lhs) == 2 && returnStmt == nil) + + if len(lhs) != len(rhs) { + // invalidate lhs + for _, obj := range lhs { + obj.used = true // avoid declared and not used errors + if obj.typ == nil { + obj.typ = Typ[Invalid] + } + } + // don't report an error if we already reported one + for _, x := range rhs { + if x.mode == invalid { + return + } + } + if returnStmt != nil { + var at positioner = returnStmt + qualifier := "not enough" + if len(rhs) > len(lhs) { + at = rhs[len(lhs)].expr // report at first extra value + qualifier = "too many" + } else if len(rhs) > 0 { + at = rhs[len(rhs)-1].expr // report at last value + } + err := newErrorf(at, WrongResultCount, "%s return values", qualifier) + err.errorf(token.NoPos, "have %s", check.typesSummary(operandTypes(rhs), false)) + err.errorf(token.NoPos, "want %s", check.typesSummary(varTypes(lhs), false)) + check.report(err) + return + } + check.assignError(origRHS, len(lhs), len(rhs)) + return + } + + context := "assignment" + if returnStmt != nil { + context = "return statement" + } + + if commaOk { + var a [2]Type + for i := range a { + a[i] = check.initVar(lhs[i], rhs[i], context) + } + check.recordCommaOkTypes(origRHS[0], a) + return + } + + for i, lhs := range lhs { + check.initVar(lhs, rhs[i], context) + } +} + +func (check *Checker) assignVars(lhs, origRHS []ast.Expr) { + rhs, commaOk := check.exprList(origRHS, len(lhs) == 2) + + if len(lhs) != len(rhs) { + check.useLHS(lhs...) + // don't report an error if we already reported one + for _, x := range rhs { + if x.mode == invalid { + return + } + } + check.assignError(origRHS, len(lhs), len(rhs)) + return + } + + if commaOk { + var a [2]Type + for i := range a { + a[i] = check.assignVar(lhs[i], rhs[i]) + } + check.recordCommaOkTypes(origRHS[0], a) + return + } + + for i, lhs := range lhs { + check.assignVar(lhs, rhs[i]) + } +} + +func (check *Checker) shortVarDecl(pos positioner, lhs, rhs []ast.Expr) { + top := len(check.delayed) + scope := check.scope + + // collect lhs variables + seen := make(map[string]bool, len(lhs)) + lhsVars := make([]*Var, len(lhs)) + newVars := make([]*Var, 0, len(lhs)) + hasErr := false + for i, lhs := range lhs { + ident, _ := lhs.(*ast.Ident) + if ident == nil { + check.useLHS(lhs) + // TODO(rFindley) this is redundant with a parser error. Consider omitting? + check.errorf(lhs, BadDecl, "non-name %s on left side of :=", lhs) + hasErr = true + continue + } + + name := ident.Name + if name != "_" { + if seen[name] { + check.errorf(lhs, RepeatedDecl, "%s repeated on left side of :=", lhs) + hasErr = true + continue + } + seen[name] = true + } + + // Use the correct obj if the ident is redeclared. The + // variable's scope starts after the declaration; so we + // must use Scope.Lookup here and call Scope.Insert + // (via check.declare) later. + if alt := scope.Lookup(name); alt != nil { + check.recordUse(ident, alt) + // redeclared object must be a variable + if obj, _ := alt.(*Var); obj != nil { + lhsVars[i] = obj + } else { + check.errorf(lhs, UnassignableOperand, "cannot assign to %s", lhs) + hasErr = true + } + continue + } + + // declare new variable + obj := NewVar(ident.Pos(), check.pkg, name, nil) + lhsVars[i] = obj + if name != "_" { + newVars = append(newVars, obj) + } + check.recordDef(ident, obj) + } + + // create dummy variables where the lhs is invalid + for i, obj := range lhsVars { + if obj == nil { + lhsVars[i] = NewVar(lhs[i].Pos(), check.pkg, "_", nil) + } + } + + check.initVars(lhsVars, rhs, nil) + + // process function literals in rhs expressions before scope changes + check.processDelayed(top) + + if len(newVars) == 0 && !hasErr { + check.softErrorf(pos, NoNewVar, "no new variables on left side of :=") + return + } + + // declare new variables + // spec: "The scope of a constant or variable identifier declared inside + // a function begins at the end of the ConstSpec or VarSpec (ShortVarDecl + // for short variable declarations) and ends at the end of the innermost + // containing block." + scopePos := rhs[len(rhs)-1].End() + for _, obj := range newVars { + check.declare(scope, nil, obj, scopePos) // id = nil: recordDef already called + } +} diff --git a/src/go/types/basic.go b/src/go/types/basic.go new file mode 100644 index 0000000..215923f --- /dev/null +++ b/src/go/types/basic.go @@ -0,0 +1,82 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +// BasicKind describes the kind of basic type. +type BasicKind int + +const ( + Invalid BasicKind = iota // type is invalid + + // predeclared types + Bool + Int + Int8 + Int16 + Int32 + Int64 + Uint + Uint8 + Uint16 + Uint32 + Uint64 + Uintptr + Float32 + Float64 + Complex64 + Complex128 + String + UnsafePointer + + // types for untyped values + UntypedBool + UntypedInt + UntypedRune + UntypedFloat + UntypedComplex + UntypedString + UntypedNil + + // aliases + Byte = Uint8 + Rune = Int32 +) + +// BasicInfo is a set of flags describing properties of a basic type. +type BasicInfo int + +// Properties of basic types. +const ( + IsBoolean BasicInfo = 1 << iota + IsInteger + IsUnsigned + IsFloat + IsComplex + IsString + IsUntyped + + IsOrdered = IsInteger | IsFloat | IsString + IsNumeric = IsInteger | IsFloat | IsComplex + IsConstType = IsBoolean | IsNumeric | IsString +) + +// A Basic represents a basic type. +type Basic struct { + kind BasicKind + info BasicInfo + name string +} + +// Kind returns the kind of basic type b. +func (b *Basic) Kind() BasicKind { return b.kind } + +// Info returns information about properties of basic type b. +func (b *Basic) Info() BasicInfo { return b.info } + +// Name returns the name of basic type b. +func (b *Basic) Name() string { return b.name } + +func (t *Basic) Underlying() Type { return t } +func (t *Basic) String() string { return TypeString(t, nil) } diff --git a/src/go/types/builtins.go b/src/go/types/builtins.go new file mode 100644 index 0000000..d3bca60 --- /dev/null +++ b/src/go/types/builtins.go @@ -0,0 +1,1017 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements typechecking of builtin function calls. + +package types + +import ( + "go/ast" + "go/constant" + "go/token" + . "internal/types/errors" +) + +// builtin type-checks a call to the built-in specified by id and +// reports whether the call is valid, with *x holding the result; +// but x.expr is not set. If the call is invalid, the result is +// false, and *x is undefined. +func (check *Checker) builtin(x *operand, call *ast.CallExpr, id builtinId) (_ bool) { + // append is the only built-in that permits the use of ... for the last argument + bin := predeclaredFuncs[id] + if call.Ellipsis.IsValid() && id != _Append { + check.errorf(atPos(call.Ellipsis), + InvalidDotDotDot, + invalidOp+"invalid use of ... with built-in %s", bin.name) + check.use(call.Args...) + return + } + + // For len(x) and cap(x) we need to know if x contains any function calls or + // receive operations. Save/restore current setting and set hasCallOrRecv to + // false for the evaluation of x so that we can check it afterwards. + // Note: We must do this _before_ calling exprList because exprList evaluates + // all arguments. + if id == _Len || id == _Cap { + defer func(b bool) { + check.hasCallOrRecv = b + }(check.hasCallOrRecv) + check.hasCallOrRecv = false + } + + // determine actual arguments + var arg func(*operand, int) // TODO(gri) remove use of arg getter in favor of using xlist directly + nargs := len(call.Args) + switch id { + default: + // make argument getter + xlist, _ := check.exprList(call.Args, false) + arg = func(x *operand, i int) { *x = *xlist[i] } + nargs = len(xlist) + // evaluate first argument, if present + if nargs > 0 { + arg(x, 0) + if x.mode == invalid { + return + } + } + case _Make, _New, _Offsetof, _Trace: + // arguments require special handling + } + + // check argument count + { + msg := "" + if nargs < bin.nargs { + msg = "not enough" + } else if !bin.variadic && nargs > bin.nargs { + msg = "too many" + } + if msg != "" { + check.errorf(inNode(call, call.Rparen), WrongArgCount, invalidOp+"%s arguments for %s (expected %d, found %d)", msg, call, bin.nargs, nargs) + return + } + } + + switch id { + case _Append: + // append(s S, x ...T) S, where T is the element type of S + // spec: "The variadic function append appends zero or more values x to s of type + // S, which must be a slice type, and returns the resulting slice, also of type S. + // The values x are passed to a parameter of type ...T where T is the element type + // of S and the respective parameter passing rules apply." + S := x.typ + var T Type + if s, _ := coreType(S).(*Slice); s != nil { + T = s.elem + } else { + var cause string + switch { + case x.isNil(): + cause = "have untyped nil" + case isTypeParam(S): + if u := coreType(S); u != nil { + cause = check.sprintf("%s has core type %s", x, u) + } else { + cause = check.sprintf("%s has no core type", x) + } + default: + cause = check.sprintf("have %s", x) + } + // don't use Checker.invalidArg here as it would repeat "argument" in the error message + check.errorf(x, InvalidAppend, "first argument to append must be a slice; %s", cause) + return + } + + // remember arguments that have been evaluated already + alist := []operand{*x} + + // spec: "As a special case, append also accepts a first argument assignable + // to type []byte with a second argument of string type followed by ... . + // This form appends the bytes of the string. + if nargs == 2 && call.Ellipsis.IsValid() { + if ok, _ := x.assignableTo(check, NewSlice(universeByte), nil); ok { + arg(x, 1) + if x.mode == invalid { + return + } + if t := coreString(x.typ); t != nil && isString(t) { + if check.Types != nil { + sig := makeSig(S, S, x.typ) + sig.variadic = true + check.recordBuiltinType(call.Fun, sig) + } + x.mode = value + x.typ = S + break + } + alist = append(alist, *x) + // fallthrough + } + } + + // check general case by creating custom signature + sig := makeSig(S, S, NewSlice(T)) // []T required for variadic signature + sig.variadic = true + var xlist []*operand + // convert []operand to []*operand + for i := range alist { + xlist = append(xlist, &alist[i]) + } + for i := len(alist); i < nargs; i++ { + var x operand + arg(&x, i) + xlist = append(xlist, &x) + } + check.arguments(call, sig, nil, xlist, nil) // discard result (we know the result type) + // ok to continue even if check.arguments reported errors + + x.mode = value + x.typ = S + if check.Types != nil { + check.recordBuiltinType(call.Fun, sig) + } + + case _Cap, _Len: + // cap(x) + // len(x) + mode := invalid + var val constant.Value + switch t := arrayPtrDeref(under(x.typ)).(type) { + case *Basic: + if isString(t) && id == _Len { + if x.mode == constant_ { + mode = constant_ + val = constant.MakeInt64(int64(len(constant.StringVal(x.val)))) + } else { + mode = value + } + } + + case *Array: + mode = value + // spec: "The expressions len(s) and cap(s) are constants + // if the type of s is an array or pointer to an array and + // the expression s does not contain channel receives or + // function calls; in this case s is not evaluated." + if !check.hasCallOrRecv { + mode = constant_ + if t.len >= 0 { + val = constant.MakeInt64(t.len) + } else { + val = constant.MakeUnknown() + } + } + + case *Slice, *Chan: + mode = value + + case *Map: + if id == _Len { + mode = value + } + + case *Interface: + if !isTypeParam(x.typ) { + break + } + if t.typeSet().underIs(func(t Type) bool { + switch t := arrayPtrDeref(t).(type) { + case *Basic: + if isString(t) && id == _Len { + return true + } + case *Array, *Slice, *Chan: + return true + case *Map: + if id == _Len { + return true + } + } + return false + }) { + mode = value + } + } + + if mode == invalid && under(x.typ) != Typ[Invalid] { + code := InvalidCap + if id == _Len { + code = InvalidLen + } + check.errorf(x, code, invalidArg+"%s for %s", x, bin.name) + return + } + + // record the signature before changing x.typ + if check.Types != nil && mode != constant_ { + check.recordBuiltinType(call.Fun, makeSig(Typ[Int], x.typ)) + } + + x.mode = mode + x.typ = Typ[Int] + x.val = val + + case _Clear: + // clear(m) + if !check.allowVersion(check.pkg, 1, 21) { + check.error(call.Fun, UnsupportedFeature, "clear requires go1.21 or later") + return + } + + if !underIs(x.typ, func(u Type) bool { + switch u := u.(type) { + case *Map, *Slice: + return true + case *Pointer: + if _, ok := under(u.base).(*Array); ok { + return true + } + } + check.errorf(x, InvalidClear, invalidArg+"cannot clear %s: argument must be (or constrained by) map, slice, or array pointer", x) + return false + }) { + return + } + + x.mode = novalue + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(nil, x.typ)) + } + + case _Close: + // close(c) + if !underIs(x.typ, func(u Type) bool { + uch, _ := u.(*Chan) + if uch == nil { + check.errorf(x, InvalidClose, invalidOp+"cannot close non-channel %s", x) + return false + } + if uch.dir == RecvOnly { + check.errorf(x, InvalidClose, invalidOp+"cannot close receive-only channel %s", x) + return false + } + return true + }) { + return + } + x.mode = novalue + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(nil, x.typ)) + } + + case _Complex: + // complex(x, y floatT) complexT + var y operand + arg(&y, 1) + if y.mode == invalid { + return + } + + // convert or check untyped arguments + d := 0 + if isUntyped(x.typ) { + d |= 1 + } + if isUntyped(y.typ) { + d |= 2 + } + switch d { + case 0: + // x and y are typed => nothing to do + case 1: + // only x is untyped => convert to type of y + check.convertUntyped(x, y.typ) + case 2: + // only y is untyped => convert to type of x + check.convertUntyped(&y, x.typ) + case 3: + // x and y are untyped => + // 1) if both are constants, convert them to untyped + // floating-point numbers if possible, + // 2) if one of them is not constant (possible because + // it contains a shift that is yet untyped), convert + // both of them to float64 since they must have the + // same type to succeed (this will result in an error + // because shifts of floats are not permitted) + if x.mode == constant_ && y.mode == constant_ { + toFloat := func(x *operand) { + if isNumeric(x.typ) && constant.Sign(constant.Imag(x.val)) == 0 { + x.typ = Typ[UntypedFloat] + } + } + toFloat(x) + toFloat(&y) + } else { + check.convertUntyped(x, Typ[Float64]) + check.convertUntyped(&y, Typ[Float64]) + // x and y should be invalid now, but be conservative + // and check below + } + } + if x.mode == invalid || y.mode == invalid { + return + } + + // both argument types must be identical + if !Identical(x.typ, y.typ) { + check.errorf(x, InvalidComplex, invalidArg+"mismatched types %s and %s", x.typ, y.typ) + return + } + + // the argument types must be of floating-point type + // (applyTypeFunc never calls f with a type parameter) + f := func(typ Type) Type { + assert(!isTypeParam(typ)) + if t, _ := under(typ).(*Basic); t != nil { + switch t.kind { + case Float32: + return Typ[Complex64] + case Float64: + return Typ[Complex128] + case UntypedFloat: + return Typ[UntypedComplex] + } + } + return nil + } + resTyp := check.applyTypeFunc(f, x, id) + if resTyp == nil { + check.errorf(x, InvalidComplex, invalidArg+"arguments have type %s, expected floating-point", x.typ) + return + } + + // if both arguments are constants, the result is a constant + if x.mode == constant_ && y.mode == constant_ { + x.val = constant.BinaryOp(constant.ToFloat(x.val), token.ADD, constant.MakeImag(constant.ToFloat(y.val))) + } else { + x.mode = value + } + + if check.Types != nil && x.mode != constant_ { + check.recordBuiltinType(call.Fun, makeSig(resTyp, x.typ, x.typ)) + } + + x.typ = resTyp + + case _Copy: + // copy(x, y []T) int + dst, _ := coreType(x.typ).(*Slice) + + var y operand + arg(&y, 1) + if y.mode == invalid { + return + } + src0 := coreString(y.typ) + if src0 != nil && isString(src0) { + src0 = NewSlice(universeByte) + } + src, _ := src0.(*Slice) + + if dst == nil || src == nil { + check.errorf(x, InvalidCopy, invalidArg+"copy expects slice arguments; found %s and %s", x, &y) + return + } + + if !Identical(dst.elem, src.elem) { + check.errorf(x, InvalidCopy, "arguments to copy %s and %s have different element types %s and %s", x, &y, dst.elem, src.elem) + return + } + + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(Typ[Int], x.typ, y.typ)) + } + x.mode = value + x.typ = Typ[Int] + + case _Delete: + // delete(map_, key) + // map_ must be a map type or a type parameter describing map types. + // The key cannot be a type parameter for now. + map_ := x.typ + var key Type + if !underIs(map_, func(u Type) bool { + map_, _ := u.(*Map) + if map_ == nil { + check.errorf(x, InvalidDelete, invalidArg+"%s is not a map", x) + return false + } + if key != nil && !Identical(map_.key, key) { + check.errorf(x, InvalidDelete, invalidArg+"maps of %s must have identical key types", x) + return false + } + key = map_.key + return true + }) { + return + } + + arg(x, 1) // k + if x.mode == invalid { + return + } + + check.assignment(x, key, "argument to delete") + if x.mode == invalid { + return + } + + x.mode = novalue + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(nil, map_, key)) + } + + case _Imag, _Real: + // imag(complexT) floatT + // real(complexT) floatT + + // convert or check untyped argument + if isUntyped(x.typ) { + if x.mode == constant_ { + // an untyped constant number can always be considered + // as a complex constant + if isNumeric(x.typ) { + x.typ = Typ[UntypedComplex] + } + } else { + // an untyped non-constant argument may appear if + // it contains a (yet untyped non-constant) shift + // expression: convert it to complex128 which will + // result in an error (shift of complex value) + check.convertUntyped(x, Typ[Complex128]) + // x should be invalid now, but be conservative and check + if x.mode == invalid { + return + } + } + } + + // the argument must be of complex type + // (applyTypeFunc never calls f with a type parameter) + f := func(typ Type) Type { + assert(!isTypeParam(typ)) + if t, _ := under(typ).(*Basic); t != nil { + switch t.kind { + case Complex64: + return Typ[Float32] + case Complex128: + return Typ[Float64] + case UntypedComplex: + return Typ[UntypedFloat] + } + } + return nil + } + resTyp := check.applyTypeFunc(f, x, id) + if resTyp == nil { + code := InvalidImag + if id == _Real { + code = InvalidReal + } + check.errorf(x, code, invalidArg+"argument has type %s, expected complex type", x.typ) + return + } + + // if the argument is a constant, the result is a constant + if x.mode == constant_ { + if id == _Real { + x.val = constant.Real(x.val) + } else { + x.val = constant.Imag(x.val) + } + } else { + x.mode = value + } + + if check.Types != nil && x.mode != constant_ { + check.recordBuiltinType(call.Fun, makeSig(resTyp, x.typ)) + } + + x.typ = resTyp + + case _Make: + // make(T, n) + // make(T, n, m) + // (no argument evaluated yet) + arg0 := call.Args[0] + T := check.varType(arg0) + if T == Typ[Invalid] { + return + } + + var min int // minimum number of arguments + switch coreType(T).(type) { + case *Slice: + min = 2 + case *Map, *Chan: + min = 1 + case nil: + check.errorf(arg0, InvalidMake, "cannot make %s: no core type", arg0) + return + default: + check.errorf(arg0, InvalidMake, invalidArg+"cannot make %s; type must be slice, map, or channel", arg0) + return + } + if nargs < min || min+1 < nargs { + check.errorf(call, WrongArgCount, invalidOp+"%v expects %d or %d arguments; found %d", call, min, min+1, nargs) + return + } + + types := []Type{T} + var sizes []int64 // constant integer arguments, if any + for _, arg := range call.Args[1:] { + typ, size := check.index(arg, -1) // ok to continue with typ == Typ[Invalid] + types = append(types, typ) + if size >= 0 { + sizes = append(sizes, size) + } + } + if len(sizes) == 2 && sizes[0] > sizes[1] { + check.error(call.Args[1], SwappedMakeArgs, invalidArg+"length and capacity swapped") + // safe to continue + } + x.mode = value + x.typ = T + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(x.typ, types...)) + } + + case _New: + // new(T) + // (no argument evaluated yet) + T := check.varType(call.Args[0]) + if T == Typ[Invalid] { + return + } + + x.mode = value + x.typ = &Pointer{base: T} + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(x.typ, T)) + } + + case _Panic: + // panic(x) + // record panic call if inside a function with result parameters + // (for use in Checker.isTerminating) + if check.sig != nil && check.sig.results.Len() > 0 { + // function has result parameters + p := check.isPanic + if p == nil { + // allocate lazily + p = make(map[*ast.CallExpr]bool) + check.isPanic = p + } + p[call] = true + } + + check.assignment(x, &emptyInterface, "argument to panic") + if x.mode == invalid { + return + } + + x.mode = novalue + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(nil, &emptyInterface)) + } + + case _Print, _Println: + // print(x, y, ...) + // println(x, y, ...) + var params []Type + if nargs > 0 { + params = make([]Type, nargs) + for i := 0; i < nargs; i++ { + if i > 0 { + arg(x, i) // first argument already evaluated + } + check.assignment(x, nil, "argument to "+predeclaredFuncs[id].name) + if x.mode == invalid { + // TODO(gri) "use" all arguments? + return + } + params[i] = x.typ + } + } + + x.mode = novalue + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(nil, params...)) + } + + case _Recover: + // recover() interface{} + x.mode = value + x.typ = &emptyInterface + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(x.typ)) + } + + case _Add: + // unsafe.Add(ptr unsafe.Pointer, len IntegerType) unsafe.Pointer + if !check.allowVersion(check.pkg, 1, 17) { + check.error(call.Fun, UnsupportedFeature, "unsafe.Add requires go1.17 or later") + return + } + + check.assignment(x, Typ[UnsafePointer], "argument to unsafe.Add") + if x.mode == invalid { + return + } + + var y operand + arg(&y, 1) + if !check.isValidIndex(&y, InvalidUnsafeAdd, "length", true) { + return + } + + x.mode = value + x.typ = Typ[UnsafePointer] + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(x.typ, x.typ, y.typ)) + } + + case _Alignof: + // unsafe.Alignof(x T) uintptr + check.assignment(x, nil, "argument to unsafe.Alignof") + if x.mode == invalid { + return + } + + if hasVarSize(x.typ, nil) { + x.mode = value + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(Typ[Uintptr], x.typ)) + } + } else { + x.mode = constant_ + x.val = constant.MakeInt64(check.conf.alignof(x.typ)) + // result is constant - no need to record signature + } + x.typ = Typ[Uintptr] + + case _Offsetof: + // unsafe.Offsetof(x T) uintptr, where x must be a selector + // (no argument evaluated yet) + arg0 := call.Args[0] + selx, _ := unparen(arg0).(*ast.SelectorExpr) + if selx == nil { + check.errorf(arg0, BadOffsetofSyntax, invalidArg+"%s is not a selector expression", arg0) + check.use(arg0) + return + } + + check.expr(x, selx.X) + if x.mode == invalid { + return + } + + base := derefStructPtr(x.typ) + sel := selx.Sel.Name + obj, index, indirect := LookupFieldOrMethod(base, false, check.pkg, sel) + switch obj.(type) { + case nil: + check.errorf(x, MissingFieldOrMethod, invalidArg+"%s has no single field %s", base, sel) + return + case *Func: + // TODO(gri) Using derefStructPtr may result in methods being found + // that don't actually exist. An error either way, but the error + // message is confusing. See: https://play.golang.org/p/al75v23kUy , + // but go/types reports: "invalid argument: x.m is a method value". + check.errorf(arg0, InvalidOffsetof, invalidArg+"%s is a method value", arg0) + return + } + if indirect { + check.errorf(x, InvalidOffsetof, invalidArg+"field %s is embedded via a pointer in %s", sel, base) + return + } + + // TODO(gri) Should we pass x.typ instead of base (and have indirect report if derefStructPtr indirected)? + check.recordSelection(selx, FieldVal, base, obj, index, false) + + // record the selector expression (was bug - issue #47895) + { + mode := value + if x.mode == variable || indirect { + mode = variable + } + check.record(&operand{mode, selx, obj.Type(), nil, 0}) + } + + // The field offset is considered a variable even if the field is declared before + // the part of the struct which is variable-sized. This makes both the rules + // simpler and also permits (or at least doesn't prevent) a compiler from re- + // arranging struct fields if it wanted to. + if hasVarSize(base, nil) { + x.mode = value + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(Typ[Uintptr], obj.Type())) + } + } else { + x.mode = constant_ + x.val = constant.MakeInt64(check.conf.offsetof(base, index)) + // result is constant - no need to record signature + } + x.typ = Typ[Uintptr] + + case _Sizeof: + // unsafe.Sizeof(x T) uintptr + check.assignment(x, nil, "argument to unsafe.Sizeof") + if x.mode == invalid { + return + } + + if hasVarSize(x.typ, nil) { + x.mode = value + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(Typ[Uintptr], x.typ)) + } + } else { + x.mode = constant_ + x.val = constant.MakeInt64(check.conf.sizeof(x.typ)) + // result is constant - no need to record signature + } + x.typ = Typ[Uintptr] + + case _Slice: + // unsafe.Slice(ptr *T, len IntegerType) []T + if !check.allowVersion(check.pkg, 1, 17) { + check.error(call.Fun, UnsupportedFeature, "unsafe.Slice requires go1.17 or later") + return + } + + ptr, _ := under(x.typ).(*Pointer) // TODO(gri) should this be coreType rather than under? + if ptr == nil { + check.errorf(x, InvalidUnsafeSlice, invalidArg+"%s is not a pointer", x) + return + } + + var y operand + arg(&y, 1) + if !check.isValidIndex(&y, InvalidUnsafeSlice, "length", false) { + return + } + + x.mode = value + x.typ = NewSlice(ptr.base) + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(x.typ, ptr, y.typ)) + } + + case _SliceData: + // unsafe.SliceData(slice []T) *T + if !check.allowVersion(check.pkg, 1, 20) { + check.error(call.Fun, UnsupportedFeature, "unsafe.SliceData requires go1.20 or later") + return + } + + slice, _ := under(x.typ).(*Slice) // TODO(gri) should this be coreType rather than under? + if slice == nil { + check.errorf(x, InvalidUnsafeSliceData, invalidArg+"%s is not a slice", x) + return + } + + x.mode = value + x.typ = NewPointer(slice.elem) + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(x.typ, slice)) + } + + case _String: + // unsafe.String(ptr *byte, len IntegerType) string + if !check.allowVersion(check.pkg, 1, 20) { + check.error(call.Fun, UnsupportedFeature, "unsafe.String requires go1.20 or later") + return + } + + check.assignment(x, NewPointer(universeByte), "argument to unsafe.String") + if x.mode == invalid { + return + } + + var y operand + arg(&y, 1) + if !check.isValidIndex(&y, InvalidUnsafeString, "length", false) { + return + } + + x.mode = value + x.typ = Typ[String] + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(x.typ, NewPointer(universeByte), y.typ)) + } + + case _StringData: + // unsafe.StringData(str string) *byte + if !check.allowVersion(check.pkg, 1, 20) { + check.error(call.Fun, UnsupportedFeature, "unsafe.StringData requires go1.20 or later") + return + } + + check.assignment(x, Typ[String], "argument to unsafe.StringData") + if x.mode == invalid { + return + } + + x.mode = value + x.typ = NewPointer(universeByte) + if check.Types != nil { + check.recordBuiltinType(call.Fun, makeSig(x.typ, Typ[String])) + } + + case _Assert: + // assert(pred) causes a typechecker error if pred is false. + // The result of assert is the value of pred if there is no error. + // Note: assert is only available in self-test mode. + if x.mode != constant_ || !isBoolean(x.typ) { + check.errorf(x, Test, invalidArg+"%s is not a boolean constant", x) + return + } + if x.val.Kind() != constant.Bool { + check.errorf(x, Test, "internal error: value of %s should be a boolean constant", x) + return + } + if !constant.BoolVal(x.val) { + check.errorf(call, Test, "%v failed", call) + // compile-time assertion failure - safe to continue + } + // result is constant - no need to record signature + + case _Trace: + // trace(x, y, z, ...) dumps the positions, expressions, and + // values of its arguments. The result of trace is the value + // of the first argument. + // Note: trace is only available in self-test mode. + // (no argument evaluated yet) + if nargs == 0 { + check.dump("%v: trace() without arguments", call.Pos()) + x.mode = novalue + break + } + var t operand + x1 := x + for _, arg := range call.Args { + check.rawExpr(x1, arg, nil, false) // permit trace for types, e.g.: new(trace(T)) + check.dump("%v: %s", x1.Pos(), x1) + x1 = &t // use incoming x only for first argument + } + // trace is only available in test mode - no need to record signature + + default: + unreachable() + } + + return true +} + +// hasVarSize reports if the size of type t is variable due to type parameters +// or if the type is infinitely-sized due to a cycle for which the type has not +// yet been checked. +func hasVarSize(t Type, seen map[*Named]bool) (varSized bool) { + // Cycles are only possible through *Named types. + // The seen map is used to detect cycles and track + // the results of previously seen types. + if named, _ := t.(*Named); named != nil { + if v, ok := seen[named]; ok { + return v + } + if seen == nil { + seen = make(map[*Named]bool) + } + seen[named] = true // possibly cyclic until proven otherwise + defer func() { + seen[named] = varSized // record final determination for named + }() + } + + switch u := under(t).(type) { + case *Array: + return hasVarSize(u.elem, seen) + case *Struct: + for _, f := range u.fields { + if hasVarSize(f.typ, seen) { + return true + } + } + case *Interface: + return isTypeParam(t) + case *Named, *Union: + unreachable() + } + return false +} + +// applyTypeFunc applies f to x. If x is a type parameter, +// the result is a type parameter constrained by an new +// interface bound. The type bounds for that interface +// are computed by applying f to each of the type bounds +// of x. If any of these applications of f return nil, +// applyTypeFunc returns nil. +// If x is not a type parameter, the result is f(x). +func (check *Checker) applyTypeFunc(f func(Type) Type, x *operand, id builtinId) Type { + if tp, _ := x.typ.(*TypeParam); tp != nil { + // Test if t satisfies the requirements for the argument + // type and collect possible result types at the same time. + var terms []*Term + if !tp.is(func(t *term) bool { + if t == nil { + return false + } + if r := f(t.typ); r != nil { + terms = append(terms, NewTerm(t.tilde, r)) + return true + } + return false + }) { + return nil + } + + // We can type-check this fine but we're introducing a synthetic + // type parameter for the result. It's not clear what the API + // implications are here. Report an error for 1.18 (see #50912), + // but continue type-checking. + var code Code + switch id { + case _Real: + code = InvalidReal + case _Imag: + code = InvalidImag + case _Complex: + code = InvalidComplex + default: + unreachable() + } + check.softErrorf(x, code, "%s not supported as argument to %s for go1.18 (see issue #50937)", x, predeclaredFuncs[id].name) + + // Construct a suitable new type parameter for the result type. + // The type parameter is placed in the current package so export/import + // works as expected. + tpar := NewTypeName(token.NoPos, check.pkg, tp.obj.name, nil) + ptyp := check.newTypeParam(tpar, NewInterfaceType(nil, []Type{NewUnion(terms)})) // assigns type to tpar as a side-effect + ptyp.index = tp.index + + return ptyp + } + + return f(x.typ) +} + +// makeSig makes a signature for the given argument and result types. +// Default types are used for untyped arguments, and res may be nil. +func makeSig(res Type, args ...Type) *Signature { + list := make([]*Var, len(args)) + for i, param := range args { + list[i] = NewVar(token.NoPos, nil, "", Default(param)) + } + params := NewTuple(list...) + var result *Tuple + if res != nil { + assert(!isUntyped(res)) + result = NewTuple(NewVar(token.NoPos, nil, "", res)) + } + return &Signature{params: params, results: result} +} + +// arrayPtrDeref returns A if typ is of the form *A and A is an array; +// otherwise it returns typ. +func arrayPtrDeref(typ Type) Type { + if p, ok := typ.(*Pointer); ok { + if a, _ := under(p.base).(*Array); a != nil { + return a + } + } + return typ +} + +// unparen returns e with any enclosing parentheses stripped. +func unparen(e ast.Expr) ast.Expr { + for { + p, ok := e.(*ast.ParenExpr) + if !ok { + return e + } + e = p.X + } +} diff --git a/src/go/types/builtins_test.go b/src/go/types/builtins_test.go new file mode 100644 index 0000000..fb71c48 --- /dev/null +++ b/src/go/types/builtins_test.go @@ -0,0 +1,254 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "fmt" + "go/ast" + "go/importer" + "go/parser" + "testing" + + . "go/types" +) + +var builtinCalls = []struct { + name, src, sig string +}{ + {"append", `var s []int; _ = append(s)`, `func([]int, ...int) []int`}, + {"append", `var s []int; _ = append(s, 0)`, `func([]int, ...int) []int`}, + {"append", `var s []int; _ = (append)(s, 0)`, `func([]int, ...int) []int`}, + {"append", `var s []byte; _ = ((append))(s, 0)`, `func([]byte, ...byte) []byte`}, + {"append", `var s []byte; _ = append(s, "foo"...)`, `func([]byte, string...) []byte`}, + {"append", `type T []byte; var s T; var str string; _ = append(s, str...)`, `func(p.T, string...) p.T`}, + {"append", `type T []byte; type U string; var s T; var str U; _ = append(s, str...)`, `func(p.T, p.U...) p.T`}, + + {"cap", `var s [10]int; _ = cap(s)`, `invalid type`}, // constant + {"cap", `var s [10]int; _ = cap(&s)`, `invalid type`}, // constant + {"cap", `var s []int64; _ = cap(s)`, `func([]int64) int`}, + {"cap", `var c chan<-bool; _ = cap(c)`, `func(chan<- bool) int`}, + {"cap", `type S []byte; var s S; _ = cap(s)`, `func(p.S) int`}, + {"cap", `var s P; _ = cap(s)`, `func(P) int`}, + + {"len", `_ = len("foo")`, `invalid type`}, // constant + {"len", `var s string; _ = len(s)`, `func(string) int`}, + {"len", `var s [10]int; _ = len(s)`, `invalid type`}, // constant + {"len", `var s [10]int; _ = len(&s)`, `invalid type`}, // constant + {"len", `var s []int64; _ = len(s)`, `func([]int64) int`}, + {"len", `var c chan<-bool; _ = len(c)`, `func(chan<- bool) int`}, + {"len", `var m map[string]float32; _ = len(m)`, `func(map[string]float32) int`}, + {"len", `type S []byte; var s S; _ = len(s)`, `func(p.S) int`}, + {"len", `var s P; _ = len(s)`, `func(P) int`}, + + {"clear", `var m map[float64]int; clear(m)`, `func(map[float64]int)`}, + {"clear", `var s []byte; clear(s)`, `func([]byte)`}, + {"clear", `var p *[10]int; clear(p)`, `func(*[10]int)`}, + {"clear", `var s P; clear(s)`, `func(P)`}, + + {"close", `var c chan int; close(c)`, `func(chan int)`}, + {"close", `var c chan<- chan string; close(c)`, `func(chan<- chan string)`}, + + {"complex", `_ = complex(1, 0)`, `invalid type`}, // constant + {"complex", `var re float32; _ = complex(re, 1.0)`, `func(float32, float32) complex64`}, + {"complex", `var im float64; _ = complex(1, im)`, `func(float64, float64) complex128`}, + {"complex", `type F32 float32; var re, im F32; _ = complex(re, im)`, `func(p.F32, p.F32) complex64`}, + {"complex", `type F64 float64; var re, im F64; _ = complex(re, im)`, `func(p.F64, p.F64) complex128`}, + + {"copy", `var src, dst []byte; copy(dst, src)`, `func([]byte, []byte) int`}, + {"copy", `type T [][]int; var src, dst T; _ = copy(dst, src)`, `func(p.T, p.T) int`}, + {"copy", `var src string; var dst []byte; copy(dst, src)`, `func([]byte, string) int`}, + {"copy", `type T string; type U []byte; var src T; var dst U; copy(dst, src)`, `func(p.U, p.T) int`}, + {"copy", `var dst []byte; copy(dst, "hello")`, `func([]byte, string) int`}, + + {"delete", `var m map[string]bool; delete(m, "foo")`, `func(map[string]bool, string)`}, + {"delete", `type (K string; V int); var m map[K]V; delete(m, "foo")`, `func(map[p.K]p.V, p.K)`}, + + {"imag", `_ = imag(1i)`, `invalid type`}, // constant + {"imag", `var c complex64; _ = imag(c)`, `func(complex64) float32`}, + {"imag", `var c complex128; _ = imag(c)`, `func(complex128) float64`}, + {"imag", `type C64 complex64; var c C64; _ = imag(c)`, `func(p.C64) float32`}, + {"imag", `type C128 complex128; var c C128; _ = imag(c)`, `func(p.C128) float64`}, + + {"real", `_ = real(1i)`, `invalid type`}, // constant + {"real", `var c complex64; _ = real(c)`, `func(complex64) float32`}, + {"real", `var c complex128; _ = real(c)`, `func(complex128) float64`}, + {"real", `type C64 complex64; var c C64; _ = real(c)`, `func(p.C64) float32`}, + {"real", `type C128 complex128; var c C128; _ = real(c)`, `func(p.C128) float64`}, + + {"make", `_ = make([]int, 10)`, `func([]int, int) []int`}, + {"make", `type T []byte; _ = make(T, 10, 20)`, `func(p.T, int, int) p.T`}, + + // issue #37349 + {"make", ` _ = make([]int, 0 )`, `func([]int, int) []int`}, + {"make", `var l int; _ = make([]int, l )`, `func([]int, int) []int`}, + {"make", ` _ = make([]int, 0, 0)`, `func([]int, int, int) []int`}, + {"make", `var l int; _ = make([]int, l, 0)`, `func([]int, int, int) []int`}, + {"make", `var c int; _ = make([]int, 0, c)`, `func([]int, int, int) []int`}, + {"make", `var l, c int; _ = make([]int, l, c)`, `func([]int, int, int) []int`}, + + // issue #37393 + {"make", ` _ = make([]int , 0 )`, `func([]int, int) []int`}, + {"make", `var l byte ; _ = make([]int8 , l )`, `func([]int8, byte) []int8`}, + {"make", ` _ = make([]int16 , 0, 0)`, `func([]int16, int, int) []int16`}, + {"make", `var l int16; _ = make([]string , l, 0)`, `func([]string, int16, int) []string`}, + {"make", `var c int32; _ = make([]float64 , 0, c)`, `func([]float64, int, int32) []float64`}, + {"make", `var l, c uint ; _ = make([]complex128, l, c)`, `func([]complex128, uint, uint) []complex128`}, + + // issue #45667 + {"make", `const l uint = 1; _ = make([]int, l)`, `func([]int, uint) []int`}, + + {"new", `_ = new(int)`, `func(int) *int`}, + {"new", `type T struct{}; _ = new(T)`, `func(p.T) *p.T`}, + + {"panic", `panic(0)`, `func(interface{})`}, + {"panic", `panic("foo")`, `func(interface{})`}, + + {"print", `print()`, `func()`}, + {"print", `print(0)`, `func(int)`}, + {"print", `print(1, 2.0, "foo", true)`, `func(int, float64, string, bool)`}, + + {"println", `println()`, `func()`}, + {"println", `println(0)`, `func(int)`}, + {"println", `println(1, 2.0, "foo", true)`, `func(int, float64, string, bool)`}, + + {"recover", `recover()`, `func() interface{}`}, + {"recover", `_ = recover()`, `func() interface{}`}, + + {"Add", `var p unsafe.Pointer; _ = unsafe.Add(p, -1.0)`, `func(unsafe.Pointer, int) unsafe.Pointer`}, + {"Add", `var p unsafe.Pointer; var n uintptr; _ = unsafe.Add(p, n)`, `func(unsafe.Pointer, uintptr) unsafe.Pointer`}, + {"Add", `_ = unsafe.Add(nil, 0)`, `func(unsafe.Pointer, int) unsafe.Pointer`}, + + {"Alignof", `_ = unsafe.Alignof(0)`, `invalid type`}, // constant + {"Alignof", `var x struct{}; _ = unsafe.Alignof(x)`, `invalid type`}, // constant + {"Alignof", `var x P; _ = unsafe.Alignof(x)`, `func(P) uintptr`}, + + {"Offsetof", `var x struct{f bool}; _ = unsafe.Offsetof(x.f)`, `invalid type`}, // constant + {"Offsetof", `var x struct{_ int; f bool}; _ = unsafe.Offsetof((&x).f)`, `invalid type`}, // constant + {"Offsetof", `var x struct{_ int; f P}; _ = unsafe.Offsetof((&x).f)`, `func(P) uintptr`}, + + {"Sizeof", `_ = unsafe.Sizeof(0)`, `invalid type`}, // constant + {"Sizeof", `var x struct{}; _ = unsafe.Sizeof(x)`, `invalid type`}, // constant + {"Sizeof", `var x P; _ = unsafe.Sizeof(x)`, `func(P) uintptr`}, + + {"Slice", `var p *int; _ = unsafe.Slice(p, 1)`, `func(*int, int) []int`}, + {"Slice", `var p *byte; var n uintptr; _ = unsafe.Slice(p, n)`, `func(*byte, uintptr) []byte`}, + {"Slice", `type B *byte; var b B; _ = unsafe.Slice(b, 0)`, `func(*byte, int) []byte`}, + + {"SliceData", "var s []int; _ = unsafe.SliceData(s)", `func([]int) *int`}, + {"SliceData", "type S []int; var s S; _ = unsafe.SliceData(s)", `func([]int) *int`}, + + {"String", `var p *byte; _ = unsafe.String(p, 1)`, `func(*byte, int) string`}, + {"String", `type B *byte; var b B; _ = unsafe.String(b, 0)`, `func(*byte, int) string`}, + + {"StringData", `var s string; _ = unsafe.StringData(s)`, `func(string) *byte`}, + {"StringData", `_ = unsafe.StringData("foo")`, `func(string) *byte`}, + + {"assert", `assert(true)`, `invalid type`}, // constant + {"assert", `type B bool; const pred B = 1 < 2; assert(pred)`, `invalid type`}, // constant + + // no tests for trace since it produces output as a side-effect +} + +func TestBuiltinSignatures(t *testing.T) { + DefPredeclaredTestFuncs() + + seen := map[string]bool{"trace": true} // no test for trace built-in; add it manually + for _, call := range builtinCalls { + testBuiltinSignature(t, call.name, call.src, call.sig) + seen[call.name] = true + } + + // make sure we didn't miss one + for _, name := range Universe.Names() { + if _, ok := Universe.Lookup(name).(*Builtin); ok && !seen[name] { + t.Errorf("missing test for %s", name) + } + } + for _, name := range Unsafe.Scope().Names() { + if _, ok := Unsafe.Scope().Lookup(name).(*Builtin); ok && !seen[name] { + t.Errorf("missing test for unsafe.%s", name) + } + } +} + +// parseGenericSrc in types2 is not necessary. We can just parse in testBuiltinSignature below. + +func testBuiltinSignature(t *testing.T, name, src0, want string) { + src := fmt.Sprintf(`package p; import "unsafe"; type _ unsafe.Pointer /* use unsafe */; func _[P ~[]byte]() { %s }`, src0) + f, err := parser.ParseFile(fset, "", src, 0) + if err != nil { + t.Errorf("%s: %s", src0, err) + return + } + + conf := Config{Importer: importer.Default()} + uses := make(map[*ast.Ident]Object) + types := make(map[ast.Expr]TypeAndValue) + _, err = conf.Check(f.Name.Name, fset, []*ast.File{f}, &Info{Uses: uses, Types: types}) + if err != nil { + t.Errorf("%s: %s", src0, err) + return + } + + // find called function + n := 0 + var fun ast.Expr + for x := range types { + if call, _ := x.(*ast.CallExpr); call != nil { + fun = call.Fun + n++ + } + } + if n != 1 { + t.Errorf("%s: got %d CallExprs; want 1", src0, n) + return + } + + // check recorded types for fun and descendents (may be parenthesized) + for { + // the recorded type for the built-in must match the wanted signature + typ := types[fun].Type + if typ == nil { + t.Errorf("%s: no type recorded for %s", src0, ExprString(fun)) + return + } + if got := typ.String(); got != want { + t.Errorf("%s: got type %s; want %s", src0, got, want) + return + } + + // called function must be a (possibly parenthesized, qualified) + // identifier denoting the expected built-in + switch p := fun.(type) { + case *ast.Ident: + obj := uses[p] + if obj == nil { + t.Errorf("%s: no object found for %s", src0, p) + return + } + bin, _ := obj.(*Builtin) + if bin == nil { + t.Errorf("%s: %s does not denote a built-in", src0, p) + return + } + if bin.Name() != name { + t.Errorf("%s: got built-in %s; want %s", src0, bin.Name(), name) + return + } + return // we're done + + case *ast.ParenExpr: + fun = p.X // unpack + + case *ast.SelectorExpr: + // built-in from package unsafe - ignore details + return // we're done + + default: + t.Errorf("%s: invalid function call", src0) + return + } + } +} diff --git a/src/go/types/call.go b/src/go/types/call.go new file mode 100644 index 0000000..db603b5 --- /dev/null +++ b/src/go/types/call.go @@ -0,0 +1,817 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements typechecking of call and selector expressions. + +package types + +import ( + "go/ast" + "go/internal/typeparams" + "go/token" + . "internal/types/errors" + "strings" + "unicode" +) + +// funcInst type-checks a function instantiation inst and returns the result in x. +// The operand x must be the evaluation of inst.X and its type must be a signature. +func (check *Checker) funcInst(x *operand, ix *typeparams.IndexExpr) { + if !check.allowVersion(check.pkg, 1, 18) { + check.softErrorf(inNode(ix.Orig, ix.Lbrack), UnsupportedFeature, "function instantiation requires go1.18 or later") + } + + targs := check.typeList(ix.Indices) + if targs == nil { + x.mode = invalid + x.expr = ix.Orig + return + } + assert(len(targs) == len(ix.Indices)) + + // check number of type arguments (got) vs number of type parameters (want) + sig := x.typ.(*Signature) + got, want := len(targs), sig.TypeParams().Len() + if got > want { + check.errorf(ix.Indices[got-1], WrongTypeArgCount, "got %d type arguments but want %d", got, want) + x.mode = invalid + x.expr = ix.Orig + return + } + + if got < want { + targs = check.infer(ix.Orig, sig.TypeParams().list(), targs, nil, nil) + if targs == nil { + // error was already reported + x.mode = invalid + x.expr = ix.Orig + return + } + got = len(targs) + } + assert(got == want) + + // instantiate function signature + sig = check.instantiateSignature(x.Pos(), sig, targs, ix.Indices) + assert(sig.TypeParams().Len() == 0) // signature is not generic anymore + check.recordInstance(ix.Orig, targs, sig) + x.typ = sig + x.mode = value + x.expr = ix.Orig +} + +func (check *Checker) instantiateSignature(pos token.Pos, typ *Signature, targs []Type, xlist []ast.Expr) (res *Signature) { + assert(check != nil) + assert(len(targs) == typ.TypeParams().Len()) + + if trace { + check.trace(pos, "-- instantiating signature %s with %s", typ, targs) + check.indent++ + defer func() { + check.indent-- + check.trace(pos, "=> %s (under = %s)", res, res.Underlying()) + }() + } + + inst := check.instance(pos, typ, targs, nil, check.context()).(*Signature) + assert(len(xlist) <= len(targs)) + + // verify instantiation lazily (was issue #50450) + check.later(func() { + tparams := typ.TypeParams().list() + if i, err := check.verify(pos, tparams, targs, check.context()); err != nil { + // best position for error reporting + pos := pos + if i < len(xlist) { + pos = xlist[i].Pos() + } + check.softErrorf(atPos(pos), InvalidTypeArg, "%s", err) + } else { + check.mono.recordInstance(check.pkg, pos, tparams, targs, xlist) + } + }).describef(atPos(pos), "verify instantiation") + + return inst +} + +func (check *Checker) callExpr(x *operand, call *ast.CallExpr) exprKind { + ix := typeparams.UnpackIndexExpr(call.Fun) + if ix != nil { + if check.indexExpr(x, ix) { + // Delay function instantiation to argument checking, + // where we combine type and value arguments for type + // inference. + assert(x.mode == value) + } else { + ix = nil + } + x.expr = call.Fun + check.record(x) + } else { + check.exprOrType(x, call.Fun, true) + } + // x.typ may be generic + + switch x.mode { + case invalid: + check.use(call.Args...) + x.expr = call + return statement + + case typexpr: + // conversion + check.nonGeneric(x) + if x.mode == invalid { + return conversion + } + T := x.typ + x.mode = invalid + switch n := len(call.Args); n { + case 0: + check.errorf(inNode(call, call.Rparen), WrongArgCount, "missing argument in conversion to %s", T) + case 1: + check.expr(x, call.Args[0]) + if x.mode != invalid { + if call.Ellipsis.IsValid() { + check.errorf(call.Args[0], BadDotDotDotSyntax, "invalid use of ... in conversion to %s", T) + break + } + if t, _ := under(T).(*Interface); t != nil && !isTypeParam(T) { + if !t.IsMethodSet() { + check.errorf(call, MisplacedConstraintIface, "cannot use interface %s in conversion (contains specific type constraints or is comparable)", T) + break + } + } + check.conversion(x, T) + } + default: + check.use(call.Args...) + check.errorf(call.Args[n-1], WrongArgCount, "too many arguments in conversion to %s", T) + } + x.expr = call + return conversion + + case builtin: + // no need to check for non-genericity here + id := x.id + if !check.builtin(x, call, id) { + x.mode = invalid + } + x.expr = call + // a non-constant result implies a function call + if x.mode != invalid && x.mode != constant_ { + check.hasCallOrRecv = true + } + return predeclaredFuncs[id].kind + } + + // ordinary function/method call + // signature may be generic + cgocall := x.mode == cgofunc + + // a type parameter may be "called" if all types have the same signature + sig, _ := coreType(x.typ).(*Signature) + if sig == nil { + check.errorf(x, InvalidCall, invalidOp+"cannot call non-function %s", x) + x.mode = invalid + x.expr = call + return statement + } + + // Capture wasGeneric before sig is potentially instantiated below. + wasGeneric := sig.TypeParams().Len() > 0 + + // evaluate type arguments, if any + var xlist []ast.Expr + var targs []Type + if ix != nil { + xlist = ix.Indices + targs = check.typeList(xlist) + if targs == nil { + check.use(call.Args...) + x.mode = invalid + x.expr = call + return statement + } + assert(len(targs) == len(xlist)) + + // check number of type arguments (got) vs number of type parameters (want) + got, want := len(targs), sig.TypeParams().Len() + if got > want { + check.errorf(xlist[want], WrongTypeArgCount, "got %d type arguments but want %d", got, want) + check.use(call.Args...) + x.mode = invalid + x.expr = call + return statement + } + + // If sig is generic and all type arguments are provided, preempt function + // argument type inference by explicitly instantiating the signature. This + // ensures that we record accurate type information for sig, even if there + // is an error checking its arguments (for example, if an incorrect number + // of arguments is supplied). + if got == want && want > 0 { + if !check.allowVersion(check.pkg, 1, 18) { + check.softErrorf(inNode(call.Fun, ix.Lbrack), UnsupportedFeature, "function instantiation requires go1.18 or later") + } + + sig = check.instantiateSignature(ix.Pos(), sig, targs, xlist) + assert(sig.TypeParams().Len() == 0) // signature is not generic anymore + check.recordInstance(ix.Orig, targs, sig) + + // targs have been consumed; proceed with checking arguments of the + // non-generic signature. + targs = nil + xlist = nil + } + } + + // evaluate arguments + args, _ := check.exprList(call.Args, false) + sig = check.arguments(call, sig, targs, args, xlist) + + if wasGeneric && sig.TypeParams().Len() == 0 { + // Update the recorded type of call.Fun to its instantiated type. + check.recordTypeAndValue(call.Fun, value, sig, nil) + } + + // determine result + switch sig.results.Len() { + case 0: + x.mode = novalue + case 1: + if cgocall { + x.mode = commaerr + } else { + x.mode = value + } + x.typ = sig.results.vars[0].typ // unpack tuple + default: + x.mode = value + x.typ = sig.results + } + x.expr = call + check.hasCallOrRecv = true + + // if type inference failed, a parametrized result must be invalidated + // (operands cannot have a parametrized type) + if x.mode == value && sig.TypeParams().Len() > 0 && isParameterized(sig.TypeParams().list(), x.typ) { + x.mode = invalid + } + + return statement +} + +func (check *Checker) exprList(elist []ast.Expr, allowCommaOk bool) (xlist []*operand, commaOk bool) { + switch len(elist) { + case 0: + // nothing to do + + case 1: + // single (possibly comma-ok) value, or function returning multiple values + e := elist[0] + var x operand + check.multiExpr(&x, e) + if t, ok := x.typ.(*Tuple); ok && x.mode != invalid { + // multiple values + xlist = make([]*operand, t.Len()) + for i, v := range t.vars { + xlist[i] = &operand{mode: value, expr: e, typ: v.typ} + } + break + } + + // exactly one (possibly invalid or comma-ok) value + xlist = []*operand{&x} + if allowCommaOk && (x.mode == mapindex || x.mode == commaok || x.mode == commaerr) { + x2 := &operand{mode: value, expr: e, typ: Typ[UntypedBool]} + if x.mode == commaerr { + x2.typ = universeError + } + xlist = append(xlist, x2) + commaOk = true + } + + default: + // multiple (possibly invalid) values + xlist = make([]*operand, len(elist)) + for i, e := range elist { + var x operand + check.expr(&x, e) + xlist[i] = &x + } + } + + return +} + +// xlist is the list of type argument expressions supplied in the source code. +func (check *Checker) arguments(call *ast.CallExpr, sig *Signature, targs []Type, args []*operand, xlist []ast.Expr) (rsig *Signature) { + rsig = sig + + // TODO(gri) try to eliminate this extra verification loop + for _, a := range args { + switch a.mode { + case typexpr: + check.errorf(a, NotAnExpr, "%s used as value", a) + return + case invalid: + return + } + } + + // Function call argument/parameter count requirements + // + // | standard call | dotdotdot call | + // --------------+------------------+----------------+ + // standard func | nargs == npars | invalid | + // --------------+------------------+----------------+ + // variadic func | nargs >= npars-1 | nargs == npars | + // --------------+------------------+----------------+ + + nargs := len(args) + npars := sig.params.Len() + ddd := call.Ellipsis.IsValid() + + // set up parameters + sigParams := sig.params // adjusted for variadic functions (may be nil for empty parameter lists!) + adjusted := false // indicates if sigParams is different from t.params + if sig.variadic { + if ddd { + // variadic_func(a, b, c...) + if len(call.Args) == 1 && nargs > 1 { + // f()... is not permitted if f() is multi-valued + check.errorf(inNode(call, call.Ellipsis), InvalidDotDotDot, "cannot use ... with %d-valued %s", nargs, call.Args[0]) + return + } + } else { + // variadic_func(a, b, c) + if nargs >= npars-1 { + // Create custom parameters for arguments: keep + // the first npars-1 parameters and add one for + // each argument mapping to the ... parameter. + vars := make([]*Var, npars-1) // npars > 0 for variadic functions + copy(vars, sig.params.vars) + last := sig.params.vars[npars-1] + typ := last.typ.(*Slice).elem + for len(vars) < nargs { + vars = append(vars, NewParam(last.pos, last.pkg, last.name, typ)) + } + sigParams = NewTuple(vars...) // possibly nil! + adjusted = true + npars = nargs + } else { + // nargs < npars-1 + npars-- // for correct error message below + } + } + } else { + if ddd { + // standard_func(a, b, c...) + check.errorf(inNode(call, call.Ellipsis), NonVariadicDotDotDot, "cannot use ... in call to non-variadic %s", call.Fun) + return + } + // standard_func(a, b, c) + } + + // check argument count + if nargs != npars { + var at positioner = call + qualifier := "not enough" + if nargs > npars { + at = args[npars].expr // report at first extra argument + qualifier = "too many" + } else { + at = atPos(call.Rparen) // report at closing ) + } + // take care of empty parameter lists represented by nil tuples + var params []*Var + if sig.params != nil { + params = sig.params.vars + } + err := newErrorf(at, WrongArgCount, "%s arguments in call to %s", qualifier, call.Fun) + err.errorf(token.NoPos, "have %s", check.typesSummary(operandTypes(args), false)) + err.errorf(token.NoPos, "want %s", check.typesSummary(varTypes(params), sig.variadic)) + check.report(err) + return + } + + // infer type arguments and instantiate signature if necessary + if sig.TypeParams().Len() > 0 { + if !check.allowVersion(check.pkg, 1, 18) { + switch call.Fun.(type) { + case *ast.IndexExpr, *ast.IndexListExpr: + ix := typeparams.UnpackIndexExpr(call.Fun) + check.softErrorf(inNode(call.Fun, ix.Lbrack), UnsupportedFeature, "function instantiation requires go1.18 or later") + default: + check.softErrorf(inNode(call, call.Lparen), UnsupportedFeature, "implicit function instantiation requires go1.18 or later") + } + } + targs := check.infer(call, sig.TypeParams().list(), targs, sigParams, args) + if targs == nil { + return // error already reported + } + + // compute result signature + rsig = check.instantiateSignature(call.Pos(), sig, targs, xlist) + assert(rsig.TypeParams().Len() == 0) // signature is not generic anymore + check.recordInstance(call.Fun, targs, rsig) + + // Optimization: Only if the parameter list was adjusted do we + // need to compute it from the adjusted list; otherwise we can + // simply use the result signature's parameter list. + if adjusted { + sigParams = check.subst(call.Pos(), sigParams, makeSubstMap(sig.TypeParams().list(), targs), nil, check.context()).(*Tuple) + } else { + sigParams = rsig.params + } + } + + // check arguments + if len(args) > 0 { + context := check.sprintf("argument to %s", call.Fun) + for i, a := range args { + check.assignment(a, sigParams.vars[i].typ, context) + } + } + + return +} + +var cgoPrefixes = [...]string{ + "_Ciconst_", + "_Cfconst_", + "_Csconst_", + "_Ctype_", + "_Cvar_", // actually a pointer to the var + "_Cfpvar_fp_", + "_Cfunc_", + "_Cmacro_", // function to evaluate the expanded expression +} + +func (check *Checker) selector(x *operand, e *ast.SelectorExpr, def *Named, wantType bool) { + // these must be declared before the "goto Error" statements + var ( + obj Object + index []int + indirect bool + ) + + sel := e.Sel.Name + // If the identifier refers to a package, handle everything here + // so we don't need a "package" mode for operands: package names + // can only appear in qualified identifiers which are mapped to + // selector expressions. + if ident, ok := e.X.(*ast.Ident); ok { + obj := check.lookup(ident.Name) + if pname, _ := obj.(*PkgName); pname != nil { + assert(pname.pkg == check.pkg) + check.recordUse(ident, pname) + pname.used = true + pkg := pname.imported + + var exp Object + funcMode := value + if pkg.cgo { + // cgo special cases C.malloc: it's + // rewritten to _CMalloc and does not + // support two-result calls. + if sel == "malloc" { + sel = "_CMalloc" + } else { + funcMode = cgofunc + } + for _, prefix := range cgoPrefixes { + // cgo objects are part of the current package (in file + // _cgo_gotypes.go). Use regular lookup. + _, exp = check.scope.LookupParent(prefix+sel, check.pos) + if exp != nil { + break + } + } + if exp == nil { + check.errorf(e.Sel, UndeclaredImportedName, "undefined: %s", ast.Expr(e)) // cast to ast.Expr to silence vet + goto Error + } + check.objDecl(exp, nil) + } else { + exp = pkg.scope.Lookup(sel) + if exp == nil { + if !pkg.fake { + check.errorf(e.Sel, UndeclaredImportedName, "undefined: %s", ast.Expr(e)) + } + goto Error + } + if !exp.Exported() { + check.errorf(e.Sel, UnexportedName, "%s not exported by package %s", sel, pkg.name) + // ok to continue + } + } + check.recordUse(e.Sel, exp) + + // Simplified version of the code for *ast.Idents: + // - imported objects are always fully initialized + switch exp := exp.(type) { + case *Const: + assert(exp.Val() != nil) + x.mode = constant_ + x.typ = exp.typ + x.val = exp.val + case *TypeName: + x.mode = typexpr + x.typ = exp.typ + case *Var: + x.mode = variable + x.typ = exp.typ + if pkg.cgo && strings.HasPrefix(exp.name, "_Cvar_") { + x.typ = x.typ.(*Pointer).base + } + case *Func: + x.mode = funcMode + x.typ = exp.typ + if pkg.cgo && strings.HasPrefix(exp.name, "_Cmacro_") { + x.mode = value + x.typ = x.typ.(*Signature).results.vars[0].typ + } + case *Builtin: + x.mode = builtin + x.typ = exp.typ + x.id = exp.id + default: + check.dump("%v: unexpected object %v", e.Sel.Pos(), exp) + unreachable() + } + x.expr = e + return + } + } + + check.exprOrType(x, e.X, false) + switch x.mode { + case typexpr: + // don't crash for "type T T.x" (was issue #51509) + if def != nil && x.typ == def { + check.cycleError([]Object{def.obj}) + goto Error + } + case builtin: + // types2 uses the position of '.' for the error + check.errorf(e.Sel, UncalledBuiltin, "cannot select on %s", x) + goto Error + case invalid: + goto Error + } + + // Avoid crashing when checking an invalid selector in a method declaration + // (i.e., where def is not set): + // + // type S[T any] struct{} + // type V = S[any] + // func (fs *S[T]) M(x V.M) {} + // + // All codepaths below return a non-type expression. If we get here while + // expecting a type expression, it is an error. + // + // See issue #57522 for more details. + // + // TODO(rfindley): We should do better by refusing to check selectors in all cases where + // x.typ is incomplete. + if wantType { + check.errorf(e.Sel, NotAType, "%s is not a type", ast.Expr(e)) + goto Error + } + + obj, index, indirect = LookupFieldOrMethod(x.typ, x.mode == variable, check.pkg, sel) + if obj == nil { + // Don't report another error if the underlying type was invalid (issue #49541). + if under(x.typ) == Typ[Invalid] { + goto Error + } + + if index != nil { + // TODO(gri) should provide actual type where the conflict happens + check.errorf(e.Sel, AmbiguousSelector, "ambiguous selector %s.%s", x.expr, sel) + goto Error + } + + if indirect { + if x.mode == typexpr { + check.errorf(e.Sel, InvalidMethodExpr, "invalid method expression %s.%s (needs pointer receiver (*%s).%s)", x.typ, sel, x.typ, sel) + } else { + check.errorf(e.Sel, InvalidMethodExpr, "cannot call pointer method %s on %s", sel, x.typ) + } + goto Error + } + + var why string + if isInterfacePtr(x.typ) { + why = check.interfacePtrError(x.typ) + } else { + why = check.sprintf("type %s has no field or method %s", x.typ, sel) + // Check if capitalization of sel matters and provide better error message in that case. + // TODO(gri) This code only looks at the first character but LookupFieldOrMethod should + // have an (internal) mechanism for case-insensitive lookup that we should use + // instead (see types2). + if len(sel) > 0 { + var changeCase string + if r := rune(sel[0]); unicode.IsUpper(r) { + changeCase = string(unicode.ToLower(r)) + sel[1:] + } else { + changeCase = string(unicode.ToUpper(r)) + sel[1:] + } + if obj, _, _ = LookupFieldOrMethod(x.typ, x.mode == variable, check.pkg, changeCase); obj != nil { + why += ", but does have " + changeCase + } + } + } + check.errorf(e.Sel, MissingFieldOrMethod, "%s.%s undefined (%s)", x.expr, sel, why) + goto Error + } + + // methods may not have a fully set up signature yet + if m, _ := obj.(*Func); m != nil { + check.objDecl(m, nil) + } + + if x.mode == typexpr { + // method expression + m, _ := obj.(*Func) + if m == nil { + // TODO(gri) should check if capitalization of sel matters and provide better error message in that case + check.errorf(e.Sel, MissingFieldOrMethod, "%s.%s undefined (type %s has no method %s)", x.expr, sel, x.typ, sel) + goto Error + } + + check.recordSelection(e, MethodExpr, x.typ, m, index, indirect) + + sig := m.typ.(*Signature) + if sig.recv == nil { + check.error(e, InvalidDeclCycle, "illegal cycle in method declaration") + goto Error + } + + // the receiver type becomes the type of the first function + // argument of the method expression's function type + var params []*Var + if sig.params != nil { + params = sig.params.vars + } + // Be consistent about named/unnamed parameters. This is not needed + // for type-checking, but the newly constructed signature may appear + // in an error message and then have mixed named/unnamed parameters. + // (An alternative would be to not print parameter names in errors, + // but it's useful to see them; this is cheap and method expressions + // are rare.) + name := "" + if len(params) > 0 && params[0].name != "" { + // name needed + name = sig.recv.name + if name == "" { + name = "_" + } + } + params = append([]*Var{NewVar(sig.recv.pos, sig.recv.pkg, name, x.typ)}, params...) + x.mode = value + x.typ = &Signature{ + tparams: sig.tparams, + params: NewTuple(params...), + results: sig.results, + variadic: sig.variadic, + } + + check.addDeclDep(m) + + } else { + // regular selector + switch obj := obj.(type) { + case *Var: + check.recordSelection(e, FieldVal, x.typ, obj, index, indirect) + if x.mode == variable || indirect { + x.mode = variable + } else { + x.mode = value + } + x.typ = obj.typ + + case *Func: + // TODO(gri) If we needed to take into account the receiver's + // addressability, should we report the type &(x.typ) instead? + check.recordSelection(e, MethodVal, x.typ, obj, index, indirect) + + // TODO(gri) The verification pass below is disabled for now because + // method sets don't match method lookup in some cases. + // For instance, if we made a copy above when creating a + // custom method for a parameterized received type, the + // method set method doesn't match (no copy there). There + /// may be other situations. + disabled := true + if !disabled && debug { + // Verify that LookupFieldOrMethod and MethodSet.Lookup agree. + // TODO(gri) This only works because we call LookupFieldOrMethod + // _before_ calling NewMethodSet: LookupFieldOrMethod completes + // any incomplete interfaces so they are available to NewMethodSet + // (which assumes that interfaces have been completed already). + typ := x.typ + if x.mode == variable { + // If typ is not an (unnamed) pointer or an interface, + // use *typ instead, because the method set of *typ + // includes the methods of typ. + // Variables are addressable, so we can always take their + // address. + if _, ok := typ.(*Pointer); !ok && !IsInterface(typ) { + typ = &Pointer{base: typ} + } + } + // If we created a synthetic pointer type above, we will throw + // away the method set computed here after use. + // TODO(gri) Method set computation should probably always compute + // both, the value and the pointer receiver method set and represent + // them in a single structure. + // TODO(gri) Consider also using a method set cache for the lifetime + // of checker once we rely on MethodSet lookup instead of individual + // lookup. + mset := NewMethodSet(typ) + if m := mset.Lookup(check.pkg, sel); m == nil || m.obj != obj { + check.dump("%v: (%s).%v -> %s", e.Pos(), typ, obj.name, m) + check.dump("%s\n", mset) + // Caution: MethodSets are supposed to be used externally + // only (after all interface types were completed). It's + // now possible that we get here incorrectly. Not urgent + // to fix since we only run this code in debug mode. + // TODO(gri) fix this eventually. + panic("method sets and lookup don't agree") + } + } + + x.mode = value + + // remove receiver + sig := *obj.typ.(*Signature) + sig.recv = nil + x.typ = &sig + + check.addDeclDep(obj) + + default: + unreachable() + } + } + + // everything went well + x.expr = e + return + +Error: + x.mode = invalid + x.expr = e +} + +// use type-checks each argument. +// Useful to make sure expressions are evaluated +// (and variables are "used") in the presence of other errors. +// The arguments may be nil. +func (check *Checker) use(arg ...ast.Expr) { + var x operand + for _, e := range arg { + // The nil check below is necessary since certain AST fields + // may legally be nil (e.g., the ast.SliceExpr.High field). + if e != nil { + check.rawExpr(&x, e, nil, false) + } + } +} + +// useLHS is like use, but doesn't "use" top-level identifiers. +// It should be called instead of use if the arguments are +// expressions on the lhs of an assignment. +// The arguments must not be nil. +func (check *Checker) useLHS(arg ...ast.Expr) { + var x operand + for _, e := range arg { + // If the lhs is an identifier denoting a variable v, this assignment + // is not a 'use' of v. Remember current value of v.used and restore + // after evaluating the lhs via check.rawExpr. + var v *Var + var v_used bool + if ident, _ := unparen(e).(*ast.Ident); ident != nil { + // never type-check the blank name on the lhs + if ident.Name == "_" { + continue + } + if _, obj := check.scope.LookupParent(ident.Name, token.NoPos); obj != nil { + // It's ok to mark non-local variables, but ignore variables + // from other packages to avoid potential race conditions with + // dot-imported variables. + if w, _ := obj.(*Var); w != nil && w.pkg == check.pkg { + v = w + v_used = v.used + } + } + } + check.rawExpr(&x, e, nil, false) + if v != nil { + v.used = v_used // restore v.used + } + } +} diff --git a/src/go/types/chan.go b/src/go/types/chan.go new file mode 100644 index 0000000..1f7b72b --- /dev/null +++ b/src/go/types/chan.go @@ -0,0 +1,35 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +// A Chan represents a channel type. +type Chan struct { + dir ChanDir + elem Type +} + +// A ChanDir value indicates a channel direction. +type ChanDir int + +// The direction of a channel is indicated by one of these constants. +const ( + SendRecv ChanDir = iota + SendOnly + RecvOnly +) + +// NewChan returns a new channel type for the given direction and element type. +func NewChan(dir ChanDir, elem Type) *Chan { + return &Chan{dir: dir, elem: elem} +} + +// Dir returns the direction of channel c. +func (c *Chan) Dir() ChanDir { return c.dir } + +// Elem returns the element type of channel c. +func (c *Chan) Elem() Type { return c.elem } + +func (t *Chan) Underlying() Type { return t } +func (t *Chan) String() string { return TypeString(t, nil) } diff --git a/src/go/types/check.go b/src/go/types/check.go new file mode 100644 index 0000000..50d8afe --- /dev/null +++ b/src/go/types/check.go @@ -0,0 +1,578 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements the Check function, which drives type-checking. + +package types + +import ( + "errors" + "fmt" + "go/ast" + "go/constant" + "go/token" + . "internal/types/errors" +) + +// debugging/development support +const ( + debug = false // leave on during development + trace = false // turn on for detailed type resolution traces +) + +// exprInfo stores information about an untyped expression. +type exprInfo struct { + isLhs bool // expression is lhs operand of a shift with delayed type-check + mode operandMode + typ *Basic + val constant.Value // constant value; or nil (if not a constant) +} + +// An environment represents the environment within which an object is +// type-checked. +type environment struct { + decl *declInfo // package-level declaration whose init expression/function body is checked + scope *Scope // top-most scope for lookups + pos token.Pos // if valid, identifiers are looked up as if at position pos (used by Eval) + iota constant.Value // value of iota in a constant declaration; nil otherwise + errpos positioner // if set, identifier position of a constant with inherited initializer + inTParamList bool // set if inside a type parameter list + sig *Signature // function signature if inside a function; nil otherwise + isPanic map[*ast.CallExpr]bool // set of panic call expressions (used for termination check) + hasLabel bool // set if a function makes use of labels (only ~1% of functions); unused outside functions + hasCallOrRecv bool // set if an expression contains a function call or channel receive operation +} + +// lookup looks up name in the current environment and returns the matching object, or nil. +func (env *environment) lookup(name string) Object { + _, obj := env.scope.LookupParent(name, env.pos) + return obj +} + +// An importKey identifies an imported package by import path and source directory +// (directory containing the file containing the import). In practice, the directory +// may always be the same, or may not matter. Given an (import path, directory), an +// importer must always return the same package (but given two different import paths, +// an importer may still return the same package by mapping them to the same package +// paths). +type importKey struct { + path, dir string +} + +// A dotImportKey describes a dot-imported object in the given scope. +type dotImportKey struct { + scope *Scope + name string +} + +// An action describes a (delayed) action. +type action struct { + f func() // action to be executed + desc *actionDesc // action description; may be nil, requires debug to be set +} + +// If debug is set, describef sets a printf-formatted description for action a. +// Otherwise, it is a no-op. +func (a *action) describef(pos positioner, format string, args ...any) { + if debug { + a.desc = &actionDesc{pos, format, args} + } +} + +// An actionDesc provides information on an action. +// For debugging only. +type actionDesc struct { + pos positioner + format string + args []any +} + +// A Checker maintains the state of the type checker. +// It must be created with NewChecker. +type Checker struct { + // package information + // (initialized by NewChecker, valid for the life-time of checker) + conf *Config + ctxt *Context // context for de-duplicating instances + fset *token.FileSet + pkg *Package + *Info + version version // accepted language version + nextID uint64 // unique Id for type parameters (first valid Id is 1) + objMap map[Object]*declInfo // maps package-level objects and (non-interface) methods to declaration info + impMap map[importKey]*Package // maps (import path, source directory) to (complete or fake) package + valids instanceLookup // valid *Named (incl. instantiated) types per the validType check + + // pkgPathMap maps package names to the set of distinct import paths we've + // seen for that name, anywhere in the import graph. It is used for + // disambiguating package names in error messages. + // + // pkgPathMap is allocated lazily, so that we don't pay the price of building + // it on the happy path. seenPkgMap tracks the packages that we've already + // walked. + pkgPathMap map[string]map[string]bool + seenPkgMap map[*Package]bool + + // information collected during type-checking of a set of package files + // (initialized by Files, valid only for the duration of check.Files; + // maps and lists are allocated on demand) + files []*ast.File // package files + imports []*PkgName // list of imported packages + dotImportMap map[dotImportKey]*PkgName // maps dot-imported objects to the package they were dot-imported through + recvTParamMap map[*ast.Ident]*TypeParam // maps blank receiver type parameters to their type + brokenAliases map[*TypeName]bool // set of aliases with broken (not yet determined) types + unionTypeSets map[*Union]*_TypeSet // computed type sets for union types + mono monoGraph // graph for detecting non-monomorphizable instantiation loops + + firstErr error // first error encountered + methods map[*TypeName][]*Func // maps package scope type names to associated non-blank (non-interface) methods + untyped map[ast.Expr]exprInfo // map of expressions without final type + delayed []action // stack of delayed action segments; segments are processed in FIFO order + objPath []Object // path of object dependencies during type inference (for cycle reporting) + cleaners []cleaner // list of types that may need a final cleanup at the end of type-checking + + // environment within which the current object is type-checked (valid only + // for the duration of type-checking a specific object) + environment + + // debugging + indent int // indentation for tracing +} + +// addDeclDep adds the dependency edge (check.decl -> to) if check.decl exists +func (check *Checker) addDeclDep(to Object) { + from := check.decl + if from == nil { + return // not in a package-level init expression + } + if _, found := check.objMap[to]; !found { + return // to is not a package-level object + } + from.addDep(to) +} + +// brokenAlias records that alias doesn't have a determined type yet. +// It also sets alias.typ to Typ[Invalid]. +func (check *Checker) brokenAlias(alias *TypeName) { + if check.brokenAliases == nil { + check.brokenAliases = make(map[*TypeName]bool) + } + check.brokenAliases[alias] = true + alias.typ = Typ[Invalid] +} + +// validAlias records that alias has the valid type typ (possibly Typ[Invalid]). +func (check *Checker) validAlias(alias *TypeName, typ Type) { + delete(check.brokenAliases, alias) + alias.typ = typ +} + +// isBrokenAlias reports whether alias doesn't have a determined type yet. +func (check *Checker) isBrokenAlias(alias *TypeName) bool { + return alias.typ == Typ[Invalid] && check.brokenAliases[alias] +} + +func (check *Checker) rememberUntyped(e ast.Expr, lhs bool, mode operandMode, typ *Basic, val constant.Value) { + m := check.untyped + if m == nil { + m = make(map[ast.Expr]exprInfo) + check.untyped = m + } + m[e] = exprInfo{lhs, mode, typ, val} +} + +// later pushes f on to the stack of actions that will be processed later; +// either at the end of the current statement, or in case of a local constant +// or variable declaration, before the constant or variable is in scope +// (so that f still sees the scope before any new declarations). +// later returns the pushed action so one can provide a description +// via action.describef for debugging, if desired. +func (check *Checker) later(f func()) *action { + i := len(check.delayed) + check.delayed = append(check.delayed, action{f: f}) + return &check.delayed[i] +} + +// push pushes obj onto the object path and returns its index in the path. +func (check *Checker) push(obj Object) int { + check.objPath = append(check.objPath, obj) + return len(check.objPath) - 1 +} + +// pop pops and returns the topmost object from the object path. +func (check *Checker) pop() Object { + i := len(check.objPath) - 1 + obj := check.objPath[i] + check.objPath[i] = nil + check.objPath = check.objPath[:i] + return obj +} + +type cleaner interface { + cleanup() +} + +// needsCleanup records objects/types that implement the cleanup method +// which will be called at the end of type-checking. +func (check *Checker) needsCleanup(c cleaner) { + check.cleaners = append(check.cleaners, c) +} + +// NewChecker returns a new Checker instance for a given package. +// Package files may be added incrementally via checker.Files. +func NewChecker(conf *Config, fset *token.FileSet, pkg *Package, info *Info) *Checker { + // make sure we have a configuration + if conf == nil { + conf = new(Config) + } + + // make sure we have an info struct + if info == nil { + info = new(Info) + } + + version, err := parseGoVersion(conf.GoVersion) + if err != nil { + panic(fmt.Sprintf("invalid Go version %q (%v)", conf.GoVersion, err)) + } + + return &Checker{ + conf: conf, + ctxt: conf.Context, + fset: fset, + pkg: pkg, + Info: info, + version: version, + objMap: make(map[Object]*declInfo), + impMap: make(map[importKey]*Package), + } +} + +// initFiles initializes the files-specific portion of checker. +// The provided files must all belong to the same package. +func (check *Checker) initFiles(files []*ast.File) { + // start with a clean slate (check.Files may be called multiple times) + check.files = nil + check.imports = nil + check.dotImportMap = nil + + check.firstErr = nil + check.methods = nil + check.untyped = nil + check.delayed = nil + check.objPath = nil + check.cleaners = nil + + // determine package name and collect valid files + pkg := check.pkg + for _, file := range files { + switch name := file.Name.Name; pkg.name { + case "": + if name != "_" { + pkg.name = name + } else { + check.error(file.Name, BlankPkgName, "invalid package name _") + } + fallthrough + + case name: + check.files = append(check.files, file) + + default: + check.errorf(atPos(file.Package), MismatchedPkgName, "package %s; expected %s", name, pkg.name) + // ignore this file + } + } +} + +// A bailout panic is used for early termination. +type bailout struct{} + +func (check *Checker) handleBailout(err *error) { + switch p := recover().(type) { + case nil, bailout: + // normal return or early exit + *err = check.firstErr + default: + // re-panic + panic(p) + } +} + +// Files checks the provided files as part of the checker's package. +func (check *Checker) Files(files []*ast.File) error { return check.checkFiles(files) } + +var errBadCgo = errors.New("cannot use FakeImportC and go115UsesCgo together") + +func (check *Checker) checkFiles(files []*ast.File) (err error) { + if check.conf.FakeImportC && check.conf.go115UsesCgo { + return errBadCgo + } + + defer check.handleBailout(&err) + + print := func(msg string) { + if trace { + fmt.Println() + fmt.Println(msg) + } + } + + print("== initFiles ==") + check.initFiles(files) + + print("== collectObjects ==") + check.collectObjects() + + print("== packageObjects ==") + check.packageObjects() + + print("== processDelayed ==") + check.processDelayed(0) // incl. all functions + + print("== cleanup ==") + check.cleanup() + + print("== initOrder ==") + check.initOrder() + + if !check.conf.DisableUnusedImportCheck { + print("== unusedImports ==") + check.unusedImports() + } + + print("== recordUntyped ==") + check.recordUntyped() + + if check.firstErr == nil { + // TODO(mdempsky): Ensure monomorph is safe when errors exist. + check.monomorph() + } + + check.pkg.complete = true + + // no longer needed - release memory + check.imports = nil + check.dotImportMap = nil + check.pkgPathMap = nil + check.seenPkgMap = nil + check.recvTParamMap = nil + check.brokenAliases = nil + check.unionTypeSets = nil + check.ctxt = nil + + // TODO(rFindley) There's more memory we should release at this point. + + return +} + +// processDelayed processes all delayed actions pushed after top. +func (check *Checker) processDelayed(top int) { + // If each delayed action pushes a new action, the + // stack will continue to grow during this loop. + // However, it is only processing functions (which + // are processed in a delayed fashion) that may + // add more actions (such as nested functions), so + // this is a sufficiently bounded process. + for i := top; i < len(check.delayed); i++ { + a := &check.delayed[i] + if trace { + if a.desc != nil { + check.trace(a.desc.pos.Pos(), "-- "+a.desc.format, a.desc.args...) + } else { + check.trace(token.NoPos, "-- delayed %p", a.f) + } + } + a.f() // may append to check.delayed + if trace { + fmt.Println() + } + } + assert(top <= len(check.delayed)) // stack must not have shrunk + check.delayed = check.delayed[:top] +} + +// cleanup runs cleanup for all collected cleaners. +func (check *Checker) cleanup() { + // Don't use a range clause since Named.cleanup may add more cleaners. + for i := 0; i < len(check.cleaners); i++ { + check.cleaners[i].cleanup() + } + check.cleaners = nil +} + +func (check *Checker) record(x *operand) { + // convert x into a user-friendly set of values + // TODO(gri) this code can be simplified + var typ Type + var val constant.Value + switch x.mode { + case invalid: + typ = Typ[Invalid] + case novalue: + typ = (*Tuple)(nil) + case constant_: + typ = x.typ + val = x.val + default: + typ = x.typ + } + assert(x.expr != nil && typ != nil) + + if isUntyped(typ) { + // delay type and value recording until we know the type + // or until the end of type checking + check.rememberUntyped(x.expr, false, x.mode, typ.(*Basic), val) + } else { + check.recordTypeAndValue(x.expr, x.mode, typ, val) + } +} + +func (check *Checker) recordUntyped() { + if !debug && check.Types == nil { + return // nothing to do + } + + for x, info := range check.untyped { + if debug && isTyped(info.typ) { + check.dump("%v: %s (type %s) is typed", x.Pos(), x, info.typ) + unreachable() + } + check.recordTypeAndValue(x, info.mode, info.typ, info.val) + } +} + +func (check *Checker) recordTypeAndValue(x ast.Expr, mode operandMode, typ Type, val constant.Value) { + assert(x != nil) + assert(typ != nil) + if mode == invalid { + return // omit + } + if mode == constant_ { + assert(val != nil) + // We check allBasic(typ, IsConstType) here as constant expressions may be + // recorded as type parameters. + assert(typ == Typ[Invalid] || allBasic(typ, IsConstType)) + } + if m := check.Types; m != nil { + m[x] = TypeAndValue{mode, typ, val} + } +} + +func (check *Checker) recordBuiltinType(f ast.Expr, sig *Signature) { + // f must be a (possibly parenthesized, possibly qualified) + // identifier denoting a built-in (including unsafe's non-constant + // functions Add and Slice): record the signature for f and possible + // children. + for { + check.recordTypeAndValue(f, builtin, sig, nil) + switch p := f.(type) { + case *ast.Ident, *ast.SelectorExpr: + return // we're done + case *ast.ParenExpr: + f = p.X + default: + unreachable() + } + } +} + +func (check *Checker) recordCommaOkTypes(x ast.Expr, a [2]Type) { + assert(x != nil) + if a[0] == nil || a[1] == nil { + return + } + assert(isTyped(a[0]) && isTyped(a[1]) && (isBoolean(a[1]) || a[1] == universeError)) + if m := check.Types; m != nil { + for { + tv := m[x] + assert(tv.Type != nil) // should have been recorded already + pos := x.Pos() + tv.Type = NewTuple( + NewVar(pos, check.pkg, "", a[0]), + NewVar(pos, check.pkg, "", a[1]), + ) + m[x] = tv + // if x is a parenthesized expression (p.X), update p.X + p, _ := x.(*ast.ParenExpr) + if p == nil { + break + } + x = p.X + } + } +} + +// recordInstance records instantiation information into check.Info, if the +// Instances map is non-nil. The given expr must be an ident, selector, or +// index (list) expr with ident or selector operand. +// +// TODO(rfindley): the expr parameter is fragile. See if we can access the +// instantiated identifier in some other way. +func (check *Checker) recordInstance(expr ast.Expr, targs []Type, typ Type) { + ident := instantiatedIdent(expr) + assert(ident != nil) + assert(typ != nil) + if m := check.Instances; m != nil { + m[ident] = Instance{newTypeList(targs), typ} + } +} + +func instantiatedIdent(expr ast.Expr) *ast.Ident { + var selOrIdent ast.Expr + switch e := expr.(type) { + case *ast.IndexExpr: + selOrIdent = e.X + case *ast.IndexListExpr: + selOrIdent = e.X + case *ast.SelectorExpr, *ast.Ident: + selOrIdent = e + } + switch x := selOrIdent.(type) { + case *ast.Ident: + return x + case *ast.SelectorExpr: + return x.Sel + } + panic("instantiated ident not found") +} + +func (check *Checker) recordDef(id *ast.Ident, obj Object) { + assert(id != nil) + if m := check.Defs; m != nil { + m[id] = obj + } +} + +func (check *Checker) recordUse(id *ast.Ident, obj Object) { + assert(id != nil) + assert(obj != nil) + if m := check.Uses; m != nil { + m[id] = obj + } +} + +func (check *Checker) recordImplicit(node ast.Node, obj Object) { + assert(node != nil) + assert(obj != nil) + if m := check.Implicits; m != nil { + m[node] = obj + } +} + +func (check *Checker) recordSelection(x *ast.SelectorExpr, kind SelectionKind, recv Type, obj Object, index []int, indirect bool) { + assert(obj != nil && (recv == nil || len(index) > 0)) + check.recordUse(x.Sel, obj) + if m := check.Selections; m != nil { + m[x] = &Selection{kind, recv, obj, index, indirect} + } +} + +func (check *Checker) recordScope(node ast.Node, scope *Scope) { + assert(node != nil) + assert(scope != nil) + if m := check.Scopes; m != nil { + m[node] = scope + } +} diff --git a/src/go/types/check_test.go b/src/go/types/check_test.go new file mode 100644 index 0000000..215a836 --- /dev/null +++ b/src/go/types/check_test.go @@ -0,0 +1,425 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements a typechecker test harness. The packages specified +// in tests are typechecked. Error messages reported by the typechecker are +// compared against the error messages expected in the test files. +// +// Expected errors are indicated in the test files by putting a comment +// of the form /* ERROR "rx" */ immediately following an offending token. +// The harness will verify that an error matching the regular expression +// rx is reported at that source position. Consecutive comments may be +// used to indicate multiple errors for the same token position. +// +// For instance, the following test file indicates that a "not declared" +// error should be reported for the undeclared variable x: +// +// package p +// func f() { +// _ = x /* ERROR "not declared" */ + 1 +// } + +package types_test + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/scanner" + "go/token" + "internal/testenv" + "os" + "path/filepath" + "reflect" + "regexp" + "strings" + "testing" + + . "go/types" +) + +var ( + haltOnError = flag.Bool("halt", false, "halt on error") + verifyErrors = flag.Bool("verify", false, "verify errors (rather than list them) in TestManual") +) + +var fset = token.NewFileSet() + +// Positioned errors are of the form filename:line:column: message . +var posMsgRx = regexp.MustCompile(`^(.*:\d+:\d+): *(?s)(.*)`) + +// splitError splits an error's error message into a position string +// and the actual error message. If there's no position information, +// pos is the empty string, and msg is the entire error message. +func splitError(err error) (pos, msg string) { + msg = err.Error() + if m := posMsgRx.FindStringSubmatch(msg); len(m) == 3 { + pos = m[1] + msg = m[2] + } + return +} + +func parseFiles(t *testing.T, filenames []string, srcs [][]byte, mode parser.Mode) ([]*ast.File, []error) { + var files []*ast.File + var errlist []error + for i, filename := range filenames { + file, err := parser.ParseFile(fset, filename, srcs[i], mode) + if file == nil { + t.Fatalf("%s: %s", filename, err) + } + files = append(files, file) + if err != nil { + if list, _ := err.(scanner.ErrorList); len(list) > 0 { + for _, err := range list { + errlist = append(errlist, err) + } + } else { + errlist = append(errlist, err) + } + } + } + return files, errlist +} + +// ERROR comments must start with text `ERROR "rx"` or `ERROR rx` where +// rx is a regular expression that matches the expected error message. +// Space around "rx" or rx is ignored. +var errRx = regexp.MustCompile(`^ *ERROR *"?([^"]*)"?`) + +// errMap collects the regular expressions of ERROR comments found +// in files and returns them as a map of error positions to error messages. +// +// srcs must be a slice of the same length as files, containing the original +// source for the parsed AST. +func errMap(t *testing.T, files []*ast.File, srcs [][]byte) map[string][]string { + // map of position strings to lists of error message patterns + errmap := make(map[string][]string) + + for i, file := range files { + tok := fset.File(file.Package) + src := srcs[i] + var s scanner.Scanner + s.Init(tok, src, nil, scanner.ScanComments) + var prev token.Pos // position of last non-comment, non-semicolon token + + scanFile: + for { + pos, tok, lit := s.Scan() + switch tok { + case token.EOF: + break scanFile + case token.COMMENT: + if lit[1] == '*' { + lit = lit[:len(lit)-2] // strip trailing */ + } + if s := errRx.FindStringSubmatch(lit[2:]); len(s) == 2 { + p := fset.Position(prev).String() + errmap[p] = append(errmap[p], strings.TrimSpace(s[1])) + } + case token.SEMICOLON: + // ignore automatically inserted semicolon + if lit == "\n" { + continue scanFile + } + fallthrough + default: + prev = pos + } + } + } + + return errmap +} + +func eliminate(t *testing.T, errmap map[string][]string, errlist []error) { + for _, err := range errlist { + pos, gotMsg := splitError(err) + list := errmap[pos] + index := -1 // list index of matching message, if any + // we expect one of the messages in list to match the error at pos + for i, wantRx := range list { + rx, err := regexp.Compile(wantRx) + if err != nil { + t.Errorf("%s: %v", pos, err) + continue + } + if rx.MatchString(gotMsg) { + index = i + break + } + } + if index >= 0 { + // eliminate from list + if n := len(list) - 1; n > 0 { + // not the last entry - swap in last element and shorten list by 1 + list[index] = list[n] + errmap[pos] = list[:n] + } else { + // last entry - remove list from map + delete(errmap, pos) + } + } else { + t.Errorf("%s: no error expected: %q", pos, gotMsg) + } + } +} + +// parseFlags parses flags from the first line of the given source +// (from src if present, or by reading from the file) if the line +// starts with "//" (line comment) followed by "-" (possibly with +// spaces between). Otherwise the line is ignored. +func parseFlags(filename string, src []byte, flags *flag.FlagSet) error { + // If there is no src, read from the file. + const maxLen = 256 + if len(src) == 0 { + f, err := os.Open(filename) + if err != nil { + return err + } + + var buf [maxLen]byte + n, err := f.Read(buf[:]) + if err != nil { + return err + } + src = buf[:n] + } + + // we must have a line comment that starts with a "-" + const prefix = "//" + if !bytes.HasPrefix(src, []byte(prefix)) { + return nil // first line is not a line comment + } + src = src[len(prefix):] + if i := bytes.Index(src, []byte("-")); i < 0 || len(bytes.TrimSpace(src[:i])) != 0 { + return nil // comment doesn't start with a "-" + } + end := bytes.Index(src, []byte("\n")) + if end < 0 || end > maxLen { + return fmt.Errorf("flags comment line too long") + } + + return flags.Parse(strings.Fields(string(src[:end]))) +} + +func testFiles(t *testing.T, sizes Sizes, filenames []string, srcs [][]byte, manual bool, imp Importer) { + if len(filenames) == 0 { + t.Fatal("no source files") + } + + var conf Config + conf.Sizes = sizes + flags := flag.NewFlagSet("", flag.PanicOnError) + flags.StringVar(&conf.GoVersion, "lang", "", "") + flags.BoolVar(&conf.FakeImportC, "fakeImportC", false, "") + flags.BoolVar(addrOldComparableSemantics(&conf), "oldComparableSemantics", false, "") + if err := parseFlags(filenames[0], srcs[0], flags); err != nil { + t.Fatal(err) + } + + files, errlist := parseFiles(t, filenames, srcs, parser.AllErrors) + + pkgName := "" + if len(files) > 0 { + pkgName = files[0].Name.Name + } + + listErrors := manual && !*verifyErrors + if listErrors && len(errlist) > 0 { + t.Errorf("--- %s:", pkgName) + for _, err := range errlist { + t.Error(err) + } + } + + // typecheck and collect typechecker errors + if imp == nil { + imp = importer.Default() + } + conf.Importer = imp + conf.Error = func(err error) { + if *haltOnError { + defer panic(err) + } + if listErrors { + t.Error(err) + return + } + // Ignore secondary error messages starting with "\t"; + // they are clarifying messages for a primary error. + if !strings.Contains(err.Error(), ": \t") { + errlist = append(errlist, err) + } + } + conf.Check(pkgName, fset, files, nil) + + if listErrors { + return + } + + for _, err := range errlist { + err, ok := err.(Error) + if !ok { + continue + } + code := readCode(err) + if code == 0 { + t.Errorf("missing error code: %v", err) + } + } + + // match and eliminate errors; + // we are expecting the following errors + errmap := errMap(t, files, srcs) + eliminate(t, errmap, errlist) + + // there should be no expected errors left + if len(errmap) > 0 { + t.Errorf("--- %s: %d source positions with expected (but not reported) errors:", pkgName, len(errmap)) + for pos, list := range errmap { + for _, rx := range list { + t.Errorf("%s: %q", pos, rx) + } + } + } +} + +func readCode(err Error) int { + v := reflect.ValueOf(err) + return int(v.FieldByName("go116code").Int()) +} + +// addrOldComparableSemantics(conf) returns &conf.oldComparableSemantics (unexported field). +func addrOldComparableSemantics(conf *Config) *bool { + v := reflect.Indirect(reflect.ValueOf(conf)) + return (*bool)(v.FieldByName("oldComparableSemantics").Addr().UnsafePointer()) +} + +// TestManual is for manual testing of a package - either provided +// as a list of filenames belonging to the package, or a directory +// name containing the package files - after the test arguments +// (and a separating "--"). For instance, to test the package made +// of the files foo.go and bar.go, use: +// +// go test -run Manual -- foo.go bar.go +// +// If no source arguments are provided, the file testdata/manual.go +// is used instead. +// Provide the -verify flag to verify errors against ERROR comments +// in the input files rather than having a list of errors reported. +// The accepted Go language version can be controlled with the -lang +// flag. +func TestManual(t *testing.T) { + testenv.MustHaveGoBuild(t) + + filenames := flag.Args() + if len(filenames) == 0 { + filenames = []string{filepath.FromSlash("testdata/manual.go")} + } + + info, err := os.Stat(filenames[0]) + if err != nil { + t.Fatalf("TestManual: %v", err) + } + + DefPredeclaredTestFuncs() + if info.IsDir() { + if len(filenames) > 1 { + t.Fatal("TestManual: must have only one directory argument") + } + testDir(t, filenames[0], true) + } else { + testPkg(t, filenames, true) + } +} + +func TestLongConstants(t *testing.T) { + format := "package longconst\n\nconst _ = %s /* ERROR constant overflow */ \nconst _ = %s // ERROR excessively long constant" + src := fmt.Sprintf(format, strings.Repeat("1", 9999), strings.Repeat("1", 10001)) + testFiles(t, nil, []string{"longconst.go"}, [][]byte{[]byte(src)}, false, nil) +} + +// TestIndexRepresentability tests that constant index operands must +// be representable as int even if they already have a type that can +// represent larger values. +func TestIndexRepresentability(t *testing.T) { + const src = "package index\n\nvar s []byte\nvar _ = s[int64 /* ERROR \"int64\\(1\\) << 40 \\(.*\\) overflows int\" */ (1) << 40]" + testFiles(t, &StdSizes{4, 4}, []string{"index.go"}, [][]byte{[]byte(src)}, false, nil) +} + +func TestIssue47243_TypedRHS(t *testing.T) { + // The RHS of the shift expression below overflows uint on 32bit platforms, + // but this is OK as it is explicitly typed. + const src = "package issue47243\n\nvar a uint64; var _ = a << uint64(4294967296)" // uint64(1<<32) + testFiles(t, &StdSizes{4, 4}, []string{"p.go"}, [][]byte{[]byte(src)}, false, nil) +} + +func TestCheck(t *testing.T) { + DefPredeclaredTestFuncs() + testDirFiles(t, "../../internal/types/testdata/check", false) +} +func TestSpec(t *testing.T) { testDirFiles(t, "../../internal/types/testdata/spec", false) } +func TestExamples(t *testing.T) { testDirFiles(t, "../../internal/types/testdata/examples", false) } +func TestFixedbugs(t *testing.T) { testDirFiles(t, "../../internal/types/testdata/fixedbugs", false) } +func TestLocal(t *testing.T) { testDirFiles(t, "testdata/local", false) } + +func testDirFiles(t *testing.T, dir string, manual bool) { + testenv.MustHaveGoBuild(t) + dir = filepath.FromSlash(dir) + + fis, err := os.ReadDir(dir) + if err != nil { + t.Error(err) + return + } + + for _, fi := range fis { + path := filepath.Join(dir, fi.Name()) + + // If fi is a directory, its files make up a single package. + if fi.IsDir() { + testDir(t, path, manual) + } else { + t.Run(filepath.Base(path), func(t *testing.T) { + testPkg(t, []string{path}, manual) + }) + } + } +} + +func testDir(t *testing.T, dir string, manual bool) { + testenv.MustHaveGoBuild(t) + + fis, err := os.ReadDir(dir) + if err != nil { + t.Error(err) + return + } + + var filenames []string + for _, fi := range fis { + filenames = append(filenames, filepath.Join(dir, fi.Name())) + } + + t.Run(filepath.Base(dir), func(t *testing.T) { + testPkg(t, filenames, manual) + }) +} + +// TODO(rFindley) reconcile the different test setup in go/types with types2. +func testPkg(t *testing.T, filenames []string, manual bool) { + srcs := make([][]byte, len(filenames)) + for i, filename := range filenames { + src, err := os.ReadFile(filename) + if err != nil { + t.Fatalf("could not read %s: %v", filename, err) + } + srcs[i] = src + } + testFiles(t, nil, filenames, srcs, manual, nil) +} diff --git a/src/go/types/context.go b/src/go/types/context.go new file mode 100644 index 0000000..15756b0 --- /dev/null +++ b/src/go/types/context.go @@ -0,0 +1,144 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "sync" +) + +// This file contains a definition of the type-checking context; an opaque type +// that may be supplied by users during instantiation. +// +// Contexts serve two purposes: +// - reduce the duplication of identical instances +// - short-circuit instantiation cycles +// +// For the latter purpose, we must always have a context during instantiation, +// whether or not it is supplied by the user. For both purposes, it must be the +// case that hashing a pointer-identical type produces consistent results +// (somewhat obviously). +// +// However, neither of these purposes require that our hash is perfect, and so +// this was not an explicit design goal of the context type. In fact, due to +// concurrent use it is convenient not to guarantee de-duplication. +// +// Nevertheless, in the future it could be helpful to allow users to leverage +// contexts to canonicalize instances, and it would probably be possible to +// achieve such a guarantee. + +// A Context is an opaque type checking context. It may be used to share +// identical type instances across type-checked packages or calls to +// Instantiate. Contexts are safe for concurrent use. +// +// The use of a shared context does not guarantee that identical instances are +// deduplicated in all cases. +type Context struct { + mu sync.Mutex + typeMap map[string][]ctxtEntry // type hash -> instances entries + nextID int // next unique ID + originIDs map[Type]int // origin type -> unique ID +} + +type ctxtEntry struct { + orig Type + targs []Type + instance Type // = orig[targs] +} + +// NewContext creates a new Context. +func NewContext() *Context { + return &Context{ + typeMap: make(map[string][]ctxtEntry), + originIDs: make(map[Type]int), + } +} + +// instanceHash returns a string representation of typ instantiated with targs. +// The hash should be a perfect hash, though out of caution the type checker +// does not assume this. The result is guaranteed to not contain blanks. +func (ctxt *Context) instanceHash(orig Type, targs []Type) string { + assert(ctxt != nil) + assert(orig != nil) + var buf bytes.Buffer + + h := newTypeHasher(&buf, ctxt) + h.string(strconv.Itoa(ctxt.getID(orig))) + // Because we've already written the unique origin ID this call to h.typ is + // unnecessary, but we leave it for hash readability. It can be removed later + // if performance is an issue. + h.typ(orig) + if len(targs) > 0 { + // TODO(rfindley): consider asserting on isGeneric(typ) here, if and when + // isGeneric handles *Signature types. + h.typeList(targs) + } + + return strings.Replace(buf.String(), " ", "#", -1) // ReplaceAll is not available in Go1.4 +} + +// lookup returns an existing instantiation of orig with targs, if it exists. +// Otherwise, it returns nil. +func (ctxt *Context) lookup(h string, orig Type, targs []Type) Type { + ctxt.mu.Lock() + defer ctxt.mu.Unlock() + + for _, e := range ctxt.typeMap[h] { + if identicalInstance(orig, targs, e.orig, e.targs) { + return e.instance + } + if debug { + // Panic during development to surface any imperfections in our hash. + panic(fmt.Sprintf("non-identical instances: (orig: %s, targs: %v) and %s", orig, targs, e.instance)) + } + } + + return nil +} + +// update de-duplicates n against previously seen types with the hash h. If an +// identical type is found with the type hash h, the previously seen type is +// returned. Otherwise, n is returned, and recorded in the Context for the hash +// h. +func (ctxt *Context) update(h string, orig Type, targs []Type, inst Type) Type { + assert(inst != nil) + + ctxt.mu.Lock() + defer ctxt.mu.Unlock() + + for _, e := range ctxt.typeMap[h] { + if inst == nil || Identical(inst, e.instance) { + return e.instance + } + if debug { + // Panic during development to surface any imperfections in our hash. + panic(fmt.Sprintf("%s and %s are not identical", inst, e.instance)) + } + } + + ctxt.typeMap[h] = append(ctxt.typeMap[h], ctxtEntry{ + orig: orig, + targs: targs, + instance: inst, + }) + + return inst +} + +// getID returns a unique ID for the type t. +func (ctxt *Context) getID(t Type) int { + ctxt.mu.Lock() + defer ctxt.mu.Unlock() + id, ok := ctxt.originIDs[t] + if !ok { + id = ctxt.nextID + ctxt.originIDs[t] = id + ctxt.nextID++ + } + return id +} diff --git a/src/go/types/context_test.go b/src/go/types/context_test.go new file mode 100644 index 0000000..ec30050 --- /dev/null +++ b/src/go/types/context_test.go @@ -0,0 +1,70 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "go/token" + "testing" +) + +func TestContextHashCollisions(t *testing.T) { + if debug { + t.Skip("hash collisions are expected, and would fail debug assertions") + } + // Unit test the de-duplication fall-back logic in Context. + // + // We can't test this via Instantiate because this is only a fall-back in + // case our hash is imperfect. + // + // These lookups and updates use reasonable looking types in an attempt to + // make them robust to internal type assertions, but could equally well use + // arbitrary types. + + // Create some distinct origin types. nullaryP and nullaryQ have no + // parameters and are identical (but have different type parameter names). + // unaryP has a parameter. + var nullaryP, nullaryQ, unaryP Type + { + // type nullaryP = func[P any]() + tparam := NewTypeParam(NewTypeName(token.NoPos, nil, "P", nil), &emptyInterface) + nullaryP = NewSignatureType(nil, nil, []*TypeParam{tparam}, nil, nil, false) + } + { + // type nullaryQ = func[Q any]() + tparam := NewTypeParam(NewTypeName(token.NoPos, nil, "Q", nil), &emptyInterface) + nullaryQ = NewSignatureType(nil, nil, []*TypeParam{tparam}, nil, nil, false) + } + { + // type unaryP = func[P any](_ P) + tparam := NewTypeParam(NewTypeName(token.NoPos, nil, "P", nil), &emptyInterface) + params := NewTuple(NewVar(token.NoPos, nil, "_", tparam)) + unaryP = NewSignatureType(nil, nil, []*TypeParam{tparam}, params, nil, false) + } + + ctxt := NewContext() + + // Update the context with an instantiation of nullaryP. + inst := NewSignatureType(nil, nil, nil, nil, nil, false) + if got := ctxt.update("", nullaryP, []Type{Typ[Int]}, inst); got != inst { + t.Error("bad") + } + + // unaryP is not identical to nullaryP, so we should not get inst when + // instantiated with identical type arguments. + if got := ctxt.lookup("", unaryP, []Type{Typ[Int]}); got != nil { + t.Error("bad") + } + + // nullaryQ is identical to nullaryP, so we *should* get inst when + // instantiated with identical type arguments. + if got := ctxt.lookup("", nullaryQ, []Type{Typ[Int]}); got != inst { + t.Error("bad") + } + + // ...but verify we don't get inst with different type arguments. + if got := ctxt.lookup("", nullaryQ, []Type{Typ[String]}); got != nil { + t.Error("bad") + } +} diff --git a/src/go/types/conversions.go b/src/go/types/conversions.go new file mode 100644 index 0000000..3cdbedb --- /dev/null +++ b/src/go/types/conversions.go @@ -0,0 +1,296 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements typechecking of conversions. + +package types + +import ( + "go/constant" + . "internal/types/errors" + "unicode" +) + +// Conversion type-checks the conversion T(x). +// The result is in x. +func (check *Checker) conversion(x *operand, T Type) { + constArg := x.mode == constant_ + + constConvertibleTo := func(T Type, val *constant.Value) bool { + switch t, _ := under(T).(*Basic); { + case t == nil: + // nothing to do + case representableConst(x.val, check, t, val): + return true + case isInteger(x.typ) && isString(t): + codepoint := unicode.ReplacementChar + if i, ok := constant.Uint64Val(x.val); ok && i <= unicode.MaxRune { + codepoint = rune(i) + } + if val != nil { + *val = constant.MakeString(string(codepoint)) + } + return true + } + return false + } + + var ok bool + var cause string + switch { + case constArg && isConstType(T): + // constant conversion + ok = constConvertibleTo(T, &x.val) + case constArg && isTypeParam(T): + // x is convertible to T if it is convertible + // to each specific type in the type set of T. + // If T's type set is empty, or if it doesn't + // have specific types, constant x cannot be + // converted. + ok = T.(*TypeParam).underIs(func(u Type) bool { + // u is nil if there are no specific type terms + if u == nil { + cause = check.sprintf("%s does not contain specific types", T) + return false + } + if isString(x.typ) && isBytesOrRunes(u) { + return true + } + if !constConvertibleTo(u, nil) { + cause = check.sprintf("cannot convert %s to type %s (in %s)", x, u, T) + return false + } + return true + }) + x.mode = value // type parameters are not constants + case x.convertibleTo(check, T, &cause): + // non-constant conversion + ok = true + x.mode = value + } + + if !ok { + if cause != "" { + check.errorf(x, InvalidConversion, "cannot convert %s to type %s: %s", x, T, cause) + } else { + check.errorf(x, InvalidConversion, "cannot convert %s to type %s", x, T) + } + x.mode = invalid + return + } + + // The conversion argument types are final. For untyped values the + // conversion provides the type, per the spec: "A constant may be + // given a type explicitly by a constant declaration or conversion,...". + if isUntyped(x.typ) { + final := T + // - For conversions to interfaces, use the argument's default type. + // - For conversions of untyped constants to non-constant types, also + // use the default type (e.g., []byte("foo") should report string + // not []byte as type for the constant "foo"). + // - Keep untyped nil for untyped nil arguments. + // - For constant integer to string conversions, keep the argument type. + // (See also the TODO below.) + if isNonTypeParamInterface(T) || constArg && !isConstType(T) || x.isNil() { + final = Default(x.typ) // default type of untyped nil is untyped nil + } else if x.mode == constant_ && isInteger(x.typ) && allString(T) { + final = x.typ + } + check.updateExprType(x.expr, final, true) + } + + x.typ = T +} + +// TODO(gri) convertibleTo checks if T(x) is valid. It assumes that the type +// of x is fully known, but that's not the case for say string(1<b-> ... ->g for a path [a, b, ... g]. +func pathString(path []Object) string { + var s string + for i, p := range path { + if i > 0 { + s += "->" + } + s += p.Name() + } + return s +} + +// objDecl type-checks the declaration of obj in its respective (file) environment. +// For the meaning of def, see Checker.definedType, in typexpr.go. +func (check *Checker) objDecl(obj Object, def *Named) { + if trace && obj.Type() == nil { + if check.indent == 0 { + fmt.Println() // empty line between top-level objects for readability + } + check.trace(obj.Pos(), "-- checking %s (%s, objPath = %s)", obj, obj.color(), pathString(check.objPath)) + check.indent++ + defer func() { + check.indent-- + check.trace(obj.Pos(), "=> %s (%s)", obj, obj.color()) + }() + } + + // Checking the declaration of obj means inferring its type + // (and possibly its value, for constants). + // An object's type (and thus the object) may be in one of + // three states which are expressed by colors: + // + // - an object whose type is not yet known is painted white (initial color) + // - an object whose type is in the process of being inferred is painted grey + // - an object whose type is fully inferred is painted black + // + // During type inference, an object's color changes from white to grey + // to black (pre-declared objects are painted black from the start). + // A black object (i.e., its type) can only depend on (refer to) other black + // ones. White and grey objects may depend on white and black objects. + // A dependency on a grey object indicates a cycle which may or may not be + // valid. + // + // When objects turn grey, they are pushed on the object path (a stack); + // they are popped again when they turn black. Thus, if a grey object (a + // cycle) is encountered, it is on the object path, and all the objects + // it depends on are the remaining objects on that path. Color encoding + // is such that the color value of a grey object indicates the index of + // that object in the object path. + + // During type-checking, white objects may be assigned a type without + // traversing through objDecl; e.g., when initializing constants and + // variables. Update the colors of those objects here (rather than + // everywhere where we set the type) to satisfy the color invariants. + if obj.color() == white && obj.Type() != nil { + obj.setColor(black) + return + } + + switch obj.color() { + case white: + assert(obj.Type() == nil) + // All color values other than white and black are considered grey. + // Because black and white are < grey, all values >= grey are grey. + // Use those values to encode the object's index into the object path. + obj.setColor(grey + color(check.push(obj))) + defer func() { + check.pop().setColor(black) + }() + + case black: + assert(obj.Type() != nil) + return + + default: + // Color values other than white or black are considered grey. + fallthrough + + case grey: + // We have a (possibly invalid) cycle. + // In the existing code, this is marked by a non-nil type + // for the object except for constants and variables whose + // type may be non-nil (known), or nil if it depends on the + // not-yet known initialization value. + // In the former case, set the type to Typ[Invalid] because + // we have an initialization cycle. The cycle error will be + // reported later, when determining initialization order. + // TODO(gri) Report cycle here and simplify initialization + // order code. + switch obj := obj.(type) { + case *Const: + if !check.validCycle(obj) || obj.typ == nil { + obj.typ = Typ[Invalid] + } + + case *Var: + if !check.validCycle(obj) || obj.typ == nil { + obj.typ = Typ[Invalid] + } + + case *TypeName: + if !check.validCycle(obj) { + // break cycle + // (without this, calling underlying() + // below may lead to an endless loop + // if we have a cycle for a defined + // (*Named) type) + obj.typ = Typ[Invalid] + } + + case *Func: + if !check.validCycle(obj) { + // Don't set obj.typ to Typ[Invalid] here + // because plenty of code type-asserts that + // functions have a *Signature type. Grey + // functions have their type set to an empty + // signature which makes it impossible to + // initialize a variable with the function. + } + + default: + unreachable() + } + assert(obj.Type() != nil) + return + } + + d := check.objMap[obj] + if d == nil { + check.dump("%v: %s should have been declared", obj.Pos(), obj) + unreachable() + } + + // save/restore current environment and set up object environment + defer func(env environment) { + check.environment = env + }(check.environment) + check.environment = environment{ + scope: d.file, + } + + // Const and var declarations must not have initialization + // cycles. We track them by remembering the current declaration + // in check.decl. Initialization expressions depending on other + // consts, vars, or functions, add dependencies to the current + // check.decl. + switch obj := obj.(type) { + case *Const: + check.decl = d // new package-level const decl + check.constDecl(obj, d.vtyp, d.init, d.inherited) + case *Var: + check.decl = d // new package-level var decl + check.varDecl(obj, d.lhs, d.vtyp, d.init) + case *TypeName: + // invalid recursive types are detected via path + check.typeDecl(obj, d.tdecl, def) + check.collectMethods(obj) // methods can only be added to top-level types + case *Func: + // functions may be recursive - no need to track dependencies + check.funcDecl(obj, d) + default: + unreachable() + } +} + +// validCycle checks if the cycle starting with obj is valid and +// reports an error if it is not. +func (check *Checker) validCycle(obj Object) (valid bool) { + // The object map contains the package scope objects and the non-interface methods. + if debug { + info := check.objMap[obj] + inObjMap := info != nil && (info.fdecl == nil || info.fdecl.Recv == nil) // exclude methods + isPkgObj := obj.Parent() == check.pkg.scope + if isPkgObj != inObjMap { + check.dump("%v: inconsistent object map for %s (isPkgObj = %v, inObjMap = %v)", obj.Pos(), obj, isPkgObj, inObjMap) + unreachable() + } + } + + // Count cycle objects. + assert(obj.color() >= grey) + start := obj.color() - grey // index of obj in objPath + cycle := check.objPath[start:] + tparCycle := false // if set, the cycle is through a type parameter list + nval := 0 // number of (constant or variable) values in the cycle; valid if !generic + ndef := 0 // number of type definitions in the cycle; valid if !generic +loop: + for _, obj := range cycle { + switch obj := obj.(type) { + case *Const, *Var: + nval++ + case *TypeName: + // If we reach a generic type that is part of a cycle + // and we are in a type parameter list, we have a cycle + // through a type parameter list, which is invalid. + if check.inTParamList && isGeneric(obj.typ) { + tparCycle = true + break loop + } + + // Determine if the type name is an alias or not. For + // package-level objects, use the object map which + // provides syntactic information (which doesn't rely + // on the order in which the objects are set up). For + // local objects, we can rely on the order, so use + // the object's predicate. + // TODO(gri) It would be less fragile to always access + // the syntactic information. We should consider storing + // this information explicitly in the object. + var alias bool + if d := check.objMap[obj]; d != nil { + alias = d.tdecl.Assign.IsValid() // package-level object + } else { + alias = obj.IsAlias() // function local object + } + if !alias { + ndef++ + } + case *Func: + // ignored for now + default: + unreachable() + } + } + + if trace { + check.trace(obj.Pos(), "## cycle detected: objPath = %s->%s (len = %d)", pathString(cycle), obj.Name(), len(cycle)) + if tparCycle { + check.trace(obj.Pos(), "## cycle contains: generic type in a type parameter list") + } else { + check.trace(obj.Pos(), "## cycle contains: %d values, %d type definitions", nval, ndef) + } + defer func() { + if valid { + check.trace(obj.Pos(), "=> cycle is valid") + } else { + check.trace(obj.Pos(), "=> error: cycle is invalid") + } + }() + } + + if !tparCycle { + // A cycle involving only constants and variables is invalid but we + // ignore them here because they are reported via the initialization + // cycle check. + if nval == len(cycle) { + return true + } + + // A cycle involving only types (and possibly functions) must have at least + // one type definition to be permitted: If there is no type definition, we + // have a sequence of alias type names which will expand ad infinitum. + if nval == 0 && ndef > 0 { + return true + } + } + + check.cycleError(cycle) + return false +} + +// cycleError reports a declaration cycle starting with +// the object in cycle that is "first" in the source. +func (check *Checker) cycleError(cycle []Object) { + // name returns the (possibly qualified) object name. + // This is needed because with generic types, cycles + // may refer to imported types. See issue #50788. + // TODO(gri) Thus functionality is used elsewhere. Factor it out. + name := func(obj Object) string { + return packagePrefix(obj.Pkg(), check.qualifier) + obj.Name() + } + + // TODO(gri) Should we start with the last (rather than the first) object in the cycle + // since that is the earliest point in the source where we start seeing the + // cycle? That would be more consistent with other error messages. + i := firstInSrc(cycle) + obj := cycle[i] + objName := name(obj) + // If obj is a type alias, mark it as valid (not broken) in order to avoid follow-on errors. + tname, _ := obj.(*TypeName) + if tname != nil && tname.IsAlias() { + check.validAlias(tname, Typ[Invalid]) + } + + // report a more concise error for self references + if len(cycle) == 1 { + if tname != nil { + check.errorf(obj, InvalidDeclCycle, "invalid recursive type: %s refers to itself", objName) + } else { + check.errorf(obj, InvalidDeclCycle, "invalid cycle in declaration: %s refers to itself", objName) + } + return + } + + if tname != nil { + check.errorf(obj, InvalidDeclCycle, "invalid recursive type %s", objName) + } else { + check.errorf(obj, InvalidDeclCycle, "invalid cycle in declaration of %s", objName) + } + for range cycle { + check.errorf(obj, InvalidDeclCycle, "\t%s refers to", objName) // secondary error, \t indented + i++ + if i >= len(cycle) { + i = 0 + } + obj = cycle[i] + objName = name(obj) + } + check.errorf(obj, InvalidDeclCycle, "\t%s", objName) +} + +// firstInSrc reports the index of the object with the "smallest" +// source position in path. path must not be empty. +func firstInSrc(path []Object) int { + fst, pos := 0, path[0].Pos() + for i, t := range path[1:] { + if t.Pos() < pos { + fst, pos = i+1, t.Pos() + } + } + return fst +} + +type ( + decl interface { + node() ast.Node + } + + importDecl struct{ spec *ast.ImportSpec } + constDecl struct { + spec *ast.ValueSpec + iota int + typ ast.Expr + init []ast.Expr + inherited bool + } + varDecl struct{ spec *ast.ValueSpec } + typeDecl struct{ spec *ast.TypeSpec } + funcDecl struct{ decl *ast.FuncDecl } +) + +func (d importDecl) node() ast.Node { return d.spec } +func (d constDecl) node() ast.Node { return d.spec } +func (d varDecl) node() ast.Node { return d.spec } +func (d typeDecl) node() ast.Node { return d.spec } +func (d funcDecl) node() ast.Node { return d.decl } + +func (check *Checker) walkDecls(decls []ast.Decl, f func(decl)) { + for _, d := range decls { + check.walkDecl(d, f) + } +} + +func (check *Checker) walkDecl(d ast.Decl, f func(decl)) { + switch d := d.(type) { + case *ast.BadDecl: + // ignore + case *ast.GenDecl: + var last *ast.ValueSpec // last ValueSpec with type or init exprs seen + for iota, s := range d.Specs { + switch s := s.(type) { + case *ast.ImportSpec: + f(importDecl{s}) + case *ast.ValueSpec: + switch d.Tok { + case token.CONST: + // determine which initialization expressions to use + inherited := true + switch { + case s.Type != nil || len(s.Values) > 0: + last = s + inherited = false + case last == nil: + last = new(ast.ValueSpec) // make sure last exists + inherited = false + } + check.arityMatch(s, last) + f(constDecl{spec: s, iota: iota, typ: last.Type, init: last.Values, inherited: inherited}) + case token.VAR: + check.arityMatch(s, nil) + f(varDecl{s}) + default: + check.errorf(s, InvalidSyntaxTree, "invalid token %s", d.Tok) + } + case *ast.TypeSpec: + f(typeDecl{s}) + default: + check.errorf(s, InvalidSyntaxTree, "unknown ast.Spec node %T", s) + } + } + case *ast.FuncDecl: + f(funcDecl{d}) + default: + check.errorf(d, InvalidSyntaxTree, "unknown ast.Decl node %T", d) + } +} + +func (check *Checker) constDecl(obj *Const, typ, init ast.Expr, inherited bool) { + assert(obj.typ == nil) + + // use the correct value of iota + defer func(iota constant.Value, errpos positioner) { + check.iota = iota + check.errpos = errpos + }(check.iota, check.errpos) + check.iota = obj.val + check.errpos = nil + + // provide valid constant value under all circumstances + obj.val = constant.MakeUnknown() + + // determine type, if any + if typ != nil { + t := check.typ(typ) + if !isConstType(t) { + // don't report an error if the type is an invalid C (defined) type + // (issue #22090) + if under(t) != Typ[Invalid] { + check.errorf(typ, InvalidConstType, "invalid constant type %s", t) + } + obj.typ = Typ[Invalid] + return + } + obj.typ = t + } + + // check initialization + var x operand + if init != nil { + if inherited { + // The initialization expression is inherited from a previous + // constant declaration, and (error) positions refer to that + // expression and not the current constant declaration. Use + // the constant identifier position for any errors during + // init expression evaluation since that is all we have + // (see issues #42991, #42992). + check.errpos = atPos(obj.pos) + } + check.expr(&x, init) + } + check.initConst(obj, &x) +} + +func (check *Checker) varDecl(obj *Var, lhs []*Var, typ, init ast.Expr) { + assert(obj.typ == nil) + + // determine type, if any + if typ != nil { + obj.typ = check.varType(typ) + // We cannot spread the type to all lhs variables if there + // are more than one since that would mark them as checked + // (see Checker.objDecl) and the assignment of init exprs, + // if any, would not be checked. + // + // TODO(gri) If we have no init expr, we should distribute + // a given type otherwise we need to re-evalate the type + // expr for each lhs variable, leading to duplicate work. + } + + // check initialization + if init == nil { + if typ == nil { + // error reported before by arityMatch + obj.typ = Typ[Invalid] + } + return + } + + if lhs == nil || len(lhs) == 1 { + assert(lhs == nil || lhs[0] == obj) + var x operand + check.expr(&x, init) + check.initVar(obj, &x, "variable declaration") + return + } + + if debug { + // obj must be one of lhs + found := false + for _, lhs := range lhs { + if obj == lhs { + found = true + break + } + } + if !found { + panic("inconsistent lhs") + } + } + + // We have multiple variables on the lhs and one init expr. + // Make sure all variables have been given the same type if + // one was specified, otherwise they assume the type of the + // init expression values (was issue #15755). + if typ != nil { + for _, lhs := range lhs { + lhs.typ = obj.typ + } + } + + check.initVars(lhs, []ast.Expr{init}, nil) +} + +// isImportedConstraint reports whether typ is an imported type constraint. +func (check *Checker) isImportedConstraint(typ Type) bool { + named, _ := typ.(*Named) + if named == nil || named.obj.pkg == check.pkg || named.obj.pkg == nil { + return false + } + u, _ := named.under().(*Interface) + return u != nil && !u.IsMethodSet() +} + +func (check *Checker) typeDecl(obj *TypeName, tdecl *ast.TypeSpec, def *Named) { + assert(obj.typ == nil) + + var rhs Type + check.later(func() { + if t, _ := obj.typ.(*Named); t != nil { // type may be invalid + check.validType(t) + } + // If typ is local, an error was already reported where typ is specified/defined. + if check.isImportedConstraint(rhs) && !check.allowVersion(check.pkg, 1, 18) { + check.errorf(tdecl.Type, UnsupportedFeature, "using type constraint %s requires go1.18 or later", rhs) + } + }).describef(obj, "validType(%s)", obj.Name()) + + alias := tdecl.Assign.IsValid() + if alias && tdecl.TypeParams.NumFields() != 0 { + // The parser will ensure this but we may still get an invalid AST. + // Complain and continue as regular type definition. + check.error(atPos(tdecl.Assign), BadDecl, "generic type cannot be alias") + alias = false + } + + // alias declaration + if alias { + if !check.allowVersion(check.pkg, 1, 9) { + check.error(atPos(tdecl.Assign), UnsupportedFeature, "type aliases requires go1.9 or later") + } + + check.brokenAlias(obj) + rhs = check.typ(tdecl.Type) + check.validAlias(obj, rhs) + return + } + + // type definition or generic type declaration + named := check.newNamed(obj, nil, nil) + def.setUnderlying(named) + + if tdecl.TypeParams != nil { + check.openScope(tdecl, "type parameters") + defer check.closeScope() + check.collectTypeParams(&named.tparams, tdecl.TypeParams) + } + + // determine underlying type of named + rhs = check.definedType(tdecl.Type, named) + assert(rhs != nil) + named.fromRHS = rhs + + // If the underlying type was not set while type-checking the right-hand + // side, it is invalid and an error should have been reported elsewhere. + if named.underlying == nil { + named.underlying = Typ[Invalid] + } + + // Disallow a lone type parameter as the RHS of a type declaration (issue #45639). + // We don't need this restriction anymore if we make the underlying type of a type + // parameter its constraint interface: if the RHS is a lone type parameter, we will + // use its underlying type (like we do for any RHS in a type declaration), and its + // underlying type is an interface and the type declaration is well defined. + if isTypeParam(rhs) { + check.error(tdecl.Type, MisplacedTypeParam, "cannot use a type parameter as RHS in type declaration") + named.underlying = Typ[Invalid] + } +} + +func (check *Checker) collectTypeParams(dst **TypeParamList, list *ast.FieldList) { + var tparams []*TypeParam + // Declare type parameters up-front, with empty interface as type bound. + // The scope of type parameters starts at the beginning of the type parameter + // list (so we can have mutually recursive parameterized interfaces). + for _, f := range list.List { + tparams = check.declareTypeParams(tparams, f.Names) + } + + // Set the type parameters before collecting the type constraints because + // the parameterized type may be used by the constraints (issue #47887). + // Example: type T[P T[P]] interface{} + *dst = bindTParams(tparams) + + // Signal to cycle detection that we are in a type parameter list. + // We can only be inside one type parameter list at any given time: + // function closures may appear inside a type parameter list but they + // cannot be generic, and their bodies are processed in delayed and + // sequential fashion. Note that with each new declaration, we save + // the existing environment and restore it when done; thus inTPList is + // true exactly only when we are in a specific type parameter list. + assert(!check.inTParamList) + check.inTParamList = true + defer func() { + check.inTParamList = false + }() + + index := 0 + for _, f := range list.List { + var bound Type + // NOTE: we may be able to assert that f.Type != nil here, but this is not + // an invariant of the AST, so we are cautious. + if f.Type != nil { + bound = check.bound(f.Type) + if isTypeParam(bound) { + // We may be able to allow this since it is now well-defined what + // the underlying type and thus type set of a type parameter is. + // But we may need some additional form of cycle detection within + // type parameter lists. + check.error(f.Type, MisplacedTypeParam, "cannot use a type parameter as constraint") + bound = Typ[Invalid] + } + } else { + bound = Typ[Invalid] + } + for i := range f.Names { + tparams[index+i].bound = bound + } + index += len(f.Names) + } +} + +func (check *Checker) bound(x ast.Expr) Type { + // A type set literal of the form ~T and A|B may only appear as constraint; + // embed it in an implicit interface so that only interface type-checking + // needs to take care of such type expressions. + wrap := false + switch op := x.(type) { + case *ast.UnaryExpr: + wrap = op.Op == token.TILDE + case *ast.BinaryExpr: + wrap = op.Op == token.OR + } + if wrap { + x = &ast.InterfaceType{Methods: &ast.FieldList{List: []*ast.Field{{Type: x}}}} + t := check.typ(x) + // mark t as implicit interface if all went well + if t, _ := t.(*Interface); t != nil { + t.implicit = true + } + return t + } + return check.typ(x) +} + +func (check *Checker) declareTypeParams(tparams []*TypeParam, names []*ast.Ident) []*TypeParam { + // Use Typ[Invalid] for the type constraint to ensure that a type + // is present even if the actual constraint has not been assigned + // yet. + // TODO(gri) Need to systematically review all uses of type parameter + // constraints to make sure we don't rely on them if they + // are not properly set yet. + for _, name := range names { + tname := NewTypeName(name.Pos(), check.pkg, name.Name, nil) + tpar := check.newTypeParam(tname, Typ[Invalid]) // assigns type to tpar as a side-effect + check.declare(check.scope, name, tname, check.scope.pos) // TODO(gri) check scope position + tparams = append(tparams, tpar) + } + + if trace && len(names) > 0 { + check.trace(names[0].Pos(), "type params = %v", tparams[len(tparams)-len(names):]) + } + + return tparams +} + +func (check *Checker) collectMethods(obj *TypeName) { + // get associated methods + // (Checker.collectObjects only collects methods with non-blank names; + // Checker.resolveBaseTypeName ensures that obj is not an alias name + // if it has attached methods.) + methods := check.methods[obj] + if methods == nil { + return + } + delete(check.methods, obj) + assert(!check.objMap[obj].tdecl.Assign.IsValid()) // don't use TypeName.IsAlias (requires fully set up object) + + // use an objset to check for name conflicts + var mset objset + + // spec: "If the base type is a struct type, the non-blank method + // and field names must be distinct." + base, _ := obj.typ.(*Named) // shouldn't fail but be conservative + if base != nil { + assert(base.TypeArgs().Len() == 0) // collectMethods should not be called on an instantiated type + + // See issue #52529: we must delay the expansion of underlying here, as + // base may not be fully set-up. + check.later(func() { + check.checkFieldUniqueness(base) + }).describef(obj, "verifying field uniqueness for %v", base) + + // Checker.Files may be called multiple times; additional package files + // may add methods to already type-checked types. Add pre-existing methods + // so that we can detect redeclarations. + for i := 0; i < base.NumMethods(); i++ { + m := base.Method(i) + assert(m.name != "_") + assert(mset.insert(m) == nil) + } + } + + // add valid methods + for _, m := range methods { + // spec: "For a base type, the non-blank names of methods bound + // to it must be unique." + assert(m.name != "_") + if alt := mset.insert(m); alt != nil { + if alt.Pos().IsValid() { + check.errorf(m, DuplicateMethod, "method %s.%s already declared at %s", obj.Name(), m.name, alt.Pos()) + } else { + check.errorf(m, DuplicateMethod, "method %s.%s already declared", obj.Name(), m.name) + } + continue + } + + if base != nil { + base.AddMethod(m) + } + } +} + +func (check *Checker) checkFieldUniqueness(base *Named) { + if t, _ := base.under().(*Struct); t != nil { + var mset objset + for i := 0; i < base.NumMethods(); i++ { + m := base.Method(i) + assert(m.name != "_") + assert(mset.insert(m) == nil) + } + + // Check that any non-blank field names of base are distinct from its + // method names. + for _, fld := range t.fields { + if fld.name != "_" { + if alt := mset.insert(fld); alt != nil { + // Struct fields should already be unique, so we should only + // encounter an alternate via collision with a method name. + _ = alt.(*Func) + + // For historical consistency, we report the primary error on the + // method, and the alt decl on the field. + check.errorf(alt, DuplicateFieldAndMethod, "field and method with the same name %s", fld.name) + check.reportAltDecl(fld) + } + } + } + } +} + +func (check *Checker) funcDecl(obj *Func, decl *declInfo) { + assert(obj.typ == nil) + + // func declarations cannot use iota + assert(check.iota == nil) + + sig := new(Signature) + obj.typ = sig // guard against cycles + + // Avoid cycle error when referring to method while type-checking the signature. + // This avoids a nuisance in the best case (non-parameterized receiver type) and + // since the method is not a type, we get an error. If we have a parameterized + // receiver type, instantiating the receiver type leads to the instantiation of + // its methods, and we don't want a cycle error in that case. + // TODO(gri) review if this is correct and/or whether we still need this? + saved := obj.color_ + obj.color_ = black + fdecl := decl.fdecl + check.funcType(sig, fdecl.Recv, fdecl.Type) + obj.color_ = saved + + if fdecl.Type.TypeParams.NumFields() > 0 && fdecl.Body == nil { + check.softErrorf(fdecl.Name, BadDecl, "generic function is missing function body") + } + + // function body must be type-checked after global declarations + // (functions implemented elsewhere have no body) + if !check.conf.IgnoreFuncBodies && fdecl.Body != nil { + check.later(func() { + check.funcBody(decl, obj.name, sig, fdecl.Body, nil) + }).describef(obj, "func %s", obj.name) + } +} + +func (check *Checker) declStmt(d ast.Decl) { + pkg := check.pkg + + check.walkDecl(d, func(d decl) { + switch d := d.(type) { + case constDecl: + top := len(check.delayed) + + // declare all constants + lhs := make([]*Const, len(d.spec.Names)) + for i, name := range d.spec.Names { + obj := NewConst(name.Pos(), pkg, name.Name, nil, constant.MakeInt64(int64(d.iota))) + lhs[i] = obj + + var init ast.Expr + if i < len(d.init) { + init = d.init[i] + } + + check.constDecl(obj, d.typ, init, d.inherited) + } + + // process function literals in init expressions before scope changes + check.processDelayed(top) + + // spec: "The scope of a constant or variable identifier declared + // inside a function begins at the end of the ConstSpec or VarSpec + // (ShortVarDecl for short variable declarations) and ends at the + // end of the innermost containing block." + scopePos := d.spec.End() + for i, name := range d.spec.Names { + check.declare(check.scope, name, lhs[i], scopePos) + } + + case varDecl: + top := len(check.delayed) + + lhs0 := make([]*Var, len(d.spec.Names)) + for i, name := range d.spec.Names { + lhs0[i] = NewVar(name.Pos(), pkg, name.Name, nil) + } + + // initialize all variables + for i, obj := range lhs0 { + var lhs []*Var + var init ast.Expr + switch len(d.spec.Values) { + case len(d.spec.Names): + // lhs and rhs match + init = d.spec.Values[i] + case 1: + // rhs is expected to be a multi-valued expression + lhs = lhs0 + init = d.spec.Values[0] + default: + if i < len(d.spec.Values) { + init = d.spec.Values[i] + } + } + check.varDecl(obj, lhs, d.spec.Type, init) + if len(d.spec.Values) == 1 { + // If we have a single lhs variable we are done either way. + // If we have a single rhs expression, it must be a multi- + // valued expression, in which case handling the first lhs + // variable will cause all lhs variables to have a type + // assigned, and we are done as well. + if debug { + for _, obj := range lhs0 { + assert(obj.typ != nil) + } + } + break + } + } + + // process function literals in init expressions before scope changes + check.processDelayed(top) + + // declare all variables + // (only at this point are the variable scopes (parents) set) + scopePos := d.spec.End() // see constant declarations + for i, name := range d.spec.Names { + // see constant declarations + check.declare(check.scope, name, lhs0[i], scopePos) + } + + case typeDecl: + obj := NewTypeName(d.spec.Name.Pos(), pkg, d.spec.Name.Name, nil) + // spec: "The scope of a type identifier declared inside a function + // begins at the identifier in the TypeSpec and ends at the end of + // the innermost containing block." + scopePos := d.spec.Name.Pos() + check.declare(check.scope, d.spec.Name, obj, scopePos) + // mark and unmark type before calling typeDecl; its type is still nil (see Checker.objDecl) + obj.setColor(grey + color(check.push(obj))) + check.typeDecl(obj, d.spec, nil) + check.pop().setColor(black) + default: + check.errorf(d.node(), InvalidSyntaxTree, "unknown ast.Decl node %T", d.node()) + } + }) +} diff --git a/src/go/types/errorcalls_test.go b/src/go/types/errorcalls_test.go new file mode 100644 index 0000000..6d6bd60 --- /dev/null +++ b/src/go/types/errorcalls_test.go @@ -0,0 +1,53 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE ast. + +package types_test + +import ( + "go/ast" + "go/token" + "testing" +) + +const errorfMinArgCount = 4 + +// TestErrorCalls makes sure that check.errorf calls have at least +// errorfMinArgCount arguments (otherwise we should use check.error). +func TestErrorCalls(t *testing.T) { + fset := token.NewFileSet() + files, err := pkgFiles(fset, ".") + if err != nil { + t.Fatal(err) + } + + for _, file := range files { + ast.Inspect(file, func(n ast.Node) bool { + call, _ := n.(*ast.CallExpr) + if call == nil { + return true + } + selx, _ := call.Fun.(*ast.SelectorExpr) + if selx == nil { + return true + } + if !(isName(selx.X, "check") && isName(selx.Sel, "errorf")) { + return true + } + // check.errorf calls should have at least errorfMinArgCount arguments: + // position, code, format string, and arguments to format + if n := len(call.Args); n < errorfMinArgCount { + t.Errorf("%s: got %d arguments, want at least %d", fset.Position(call.Pos()), n, errorfMinArgCount) + return false + } + return true + }) + } +} + +func isName(n ast.Node, name string) bool { + if n, ok := n.(*ast.Ident); ok { + return n.Name == name + } + return false +} diff --git a/src/go/types/errors.go b/src/go/types/errors.go new file mode 100644 index 0000000..b52019d --- /dev/null +++ b/src/go/types/errors.go @@ -0,0 +1,390 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements various error reporters. + +package types + +import ( + "bytes" + "fmt" + "go/ast" + "go/token" + . "internal/types/errors" + "runtime" + "strconv" + "strings" +) + +func assert(p bool) { + if !p { + msg := "assertion failed" + // Include information about the assertion location. Due to panic recovery, + // this location is otherwise buried in the middle of the panicking stack. + if _, file, line, ok := runtime.Caller(1); ok { + msg = fmt.Sprintf("%s:%d: %s", file, line, msg) + } + panic(msg) + } +} + +func unreachable() { + panic("unreachable") +} + +// An error_ represents a type-checking error. +// To report an error_, call Checker.report. +type error_ struct { + desc []errorDesc + code Code + soft bool // TODO(gri) eventually determine this from an error code +} + +// An errorDesc describes part of a type-checking error. +type errorDesc struct { + posn positioner + format string + args []interface{} +} + +func (err *error_) empty() bool { + return err.desc == nil +} + +func (err *error_) pos() token.Pos { + if err.empty() { + return token.NoPos + } + return err.desc[0].posn.Pos() +} + +func (err *error_) msg(fset *token.FileSet, qf Qualifier) string { + if err.empty() { + return "no error" + } + var buf strings.Builder + for i := range err.desc { + p := &err.desc[i] + if i > 0 { + fmt.Fprint(&buf, "\n\t") + if p.posn.Pos().IsValid() { + fmt.Fprintf(&buf, "%s: ", fset.Position(p.posn.Pos())) + } + } + buf.WriteString(sprintf(fset, qf, false, p.format, p.args...)) + } + return buf.String() +} + +// String is for testing. +func (err *error_) String() string { + if err.empty() { + return "no error" + } + return fmt.Sprintf("%d: %s", err.pos(), err.msg(nil, nil)) +} + +// errorf adds formatted error information to err. +// It may be called multiple times to provide additional information. +func (err *error_) errorf(at token.Pos, format string, args ...interface{}) { + err.desc = append(err.desc, errorDesc{atPos(at), format, args}) +} + +func (check *Checker) qualifier(pkg *Package) string { + // Qualify the package unless it's the package being type-checked. + if pkg != check.pkg { + if check.pkgPathMap == nil { + check.pkgPathMap = make(map[string]map[string]bool) + check.seenPkgMap = make(map[*Package]bool) + check.markImports(check.pkg) + } + // If the same package name was used by multiple packages, display the full path. + if len(check.pkgPathMap[pkg.name]) > 1 { + return strconv.Quote(pkg.path) + } + return pkg.name + } + return "" +} + +// markImports recursively walks pkg and its imports, to record unique import +// paths in pkgPathMap. +func (check *Checker) markImports(pkg *Package) { + if check.seenPkgMap[pkg] { + return + } + check.seenPkgMap[pkg] = true + + forName, ok := check.pkgPathMap[pkg.name] + if !ok { + forName = make(map[string]bool) + check.pkgPathMap[pkg.name] = forName + } + forName[pkg.path] = true + + for _, imp := range pkg.imports { + check.markImports(imp) + } +} + +// check may be nil. +func (check *Checker) sprintf(format string, args ...any) string { + var fset *token.FileSet + var qf Qualifier + if check != nil { + fset = check.fset + qf = check.qualifier + } + return sprintf(fset, qf, false, format, args...) +} + +func sprintf(fset *token.FileSet, qf Qualifier, tpSubscripts bool, format string, args ...any) string { + for i, arg := range args { + switch a := arg.(type) { + case nil: + arg = "" + case operand: + panic("got operand instead of *operand") + case *operand: + arg = operandString(a, qf) + case token.Pos: + if fset != nil { + arg = fset.Position(a).String() + } + case ast.Expr: + arg = ExprString(a) + case []ast.Expr: + var buf bytes.Buffer + buf.WriteByte('[') + writeExprList(&buf, a) + buf.WriteByte(']') + arg = buf.String() + case Object: + arg = ObjectString(a, qf) + case Type: + var buf bytes.Buffer + w := newTypeWriter(&buf, qf) + w.tpSubscripts = tpSubscripts + w.typ(a) + arg = buf.String() + case []Type: + var buf bytes.Buffer + w := newTypeWriter(&buf, qf) + w.tpSubscripts = tpSubscripts + buf.WriteByte('[') + for i, x := range a { + if i > 0 { + buf.WriteString(", ") + } + w.typ(x) + } + buf.WriteByte(']') + arg = buf.String() + case []*TypeParam: + var buf bytes.Buffer + w := newTypeWriter(&buf, qf) + w.tpSubscripts = tpSubscripts + buf.WriteByte('[') + for i, x := range a { + if i > 0 { + buf.WriteString(", ") + } + w.typ(x) + } + buf.WriteByte(']') + arg = buf.String() + } + args[i] = arg + } + return fmt.Sprintf(format, args...) +} + +func (check *Checker) trace(pos token.Pos, format string, args ...any) { + fmt.Printf("%s:\t%s%s\n", + check.fset.Position(pos), + strings.Repeat(". ", check.indent), + sprintf(check.fset, check.qualifier, true, format, args...), + ) +} + +// dump is only needed for debugging +func (check *Checker) dump(format string, args ...any) { + fmt.Println(sprintf(check.fset, check.qualifier, true, format, args...)) +} + +// Report records the error pointed to by errp, setting check.firstError if +// necessary. +func (check *Checker) report(errp *error_) { + if errp.empty() { + panic("empty error details") + } + + msg := errp.msg(check.fset, check.qualifier) + switch errp.code { + case InvalidSyntaxTree: + msg = "invalid AST: " + msg + case 0: + panic("no error code provided") + } + + span := spanOf(errp.desc[0].posn) + e := Error{ + Fset: check.fset, + Pos: span.pos, + Msg: msg, + Soft: errp.soft, + go116code: errp.code, + go116start: span.start, + go116end: span.end, + } + + // Cheap trick: Don't report errors with messages containing + // "invalid operand" or "invalid type" as those tend to be + // follow-on errors which don't add useful information. Only + // exclude them if these strings are not at the beginning, + // and only if we have at least one error already reported. + isInvalidErr := strings.Index(e.Msg, "invalid operand") > 0 || strings.Index(e.Msg, "invalid type") > 0 + if check.firstErr != nil && isInvalidErr { + return + } + + e.Msg = stripAnnotations(e.Msg) + if check.errpos != nil { + // If we have an internal error and the errpos override is set, use it to + // augment our error positioning. + // TODO(rFindley) we may also want to augment the error message and refer + // to the position (pos) in the original expression. + span := spanOf(check.errpos) + e.Pos = span.pos + e.go116start = span.start + e.go116end = span.end + } + err := e + + if check.firstErr == nil { + check.firstErr = err + } + + if trace { + pos := e.Pos + msg := e.Msg + check.trace(pos, "ERROR: %s", msg) + } + + f := check.conf.Error + if f == nil { + panic(bailout{}) // report only first error + } + f(err) +} + +const ( + invalidArg = "invalid argument: " + invalidOp = "invalid operation: " +) + +// newErrorf creates a new error_ for later reporting with check.report. +func newErrorf(at positioner, code Code, format string, args ...any) *error_ { + return &error_{ + desc: []errorDesc{{at, format, args}}, + code: code, + } +} + +func (check *Checker) error(at positioner, code Code, msg string) { + check.report(newErrorf(at, code, msg)) +} + +func (check *Checker) errorf(at positioner, code Code, format string, args ...any) { + check.report(newErrorf(at, code, format, args...)) +} + +func (check *Checker) softErrorf(at positioner, code Code, format string, args ...any) { + err := newErrorf(at, code, format, args...) + err.soft = true + check.report(err) +} + +func (check *Checker) versionErrorf(at positioner, goVersion string, format string, args ...interface{}) { + msg := check.sprintf(format, args...) + var err *error_ + err = newErrorf(at, UnsupportedFeature, "%s requires %s or later", msg, goVersion) + check.report(err) +} + +// The positioner interface is used to extract the position of type-checker +// errors. +type positioner interface { + Pos() token.Pos +} + +// posSpan holds a position range along with a highlighted position within that +// range. This is used for positioning errors, with pos by convention being the +// first position in the source where the error is known to exist, and start +// and end defining the full span of syntax being considered when the error was +// detected. Invariant: start <= pos < end || start == pos == end. +type posSpan struct { + start, pos, end token.Pos +} + +func (e posSpan) Pos() token.Pos { + return e.pos +} + +// inNode creates a posSpan for the given node. +// Invariant: node.Pos() <= pos < node.End() (node.End() is the position of the +// first byte after node within the source). +func inNode(node ast.Node, pos token.Pos) posSpan { + start, end := node.Pos(), node.End() + if debug { + assert(start <= pos && pos < end) + } + return posSpan{start, pos, end} +} + +// atPos wraps a token.Pos to implement the positioner interface. +type atPos token.Pos + +func (s atPos) Pos() token.Pos { + return token.Pos(s) +} + +// spanOf extracts an error span from the given positioner. By default this is +// the trivial span starting and ending at pos, but this span is expanded when +// the argument naturally corresponds to a span of source code. +func spanOf(at positioner) posSpan { + switch x := at.(type) { + case nil: + panic("nil positioner") + case posSpan: + return x + case ast.Node: + pos := x.Pos() + return posSpan{pos, pos, x.End()} + case *operand: + if x.expr != nil { + pos := x.Pos() + return posSpan{pos, pos, x.expr.End()} + } + return posSpan{token.NoPos, token.NoPos, token.NoPos} + default: + pos := at.Pos() + return posSpan{pos, pos, pos} + } +} + +// stripAnnotations removes internal (type) annotations from s. +func stripAnnotations(s string) string { + var buf strings.Builder + for _, r := range s { + // strip #'s and subscript digits + if r < 'ā‚€' || 'ā‚€'+10 <= r { // 'ā‚€' == U+2080 + buf.WriteRune(r) + } + } + if buf.Len() < len(s) { + return buf.String() + } + return s +} diff --git a/src/go/types/errors_test.go b/src/go/types/errors_test.go new file mode 100644 index 0000000..4b5dab6 --- /dev/null +++ b/src/go/types/errors_test.go @@ -0,0 +1,47 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "go/token" + "testing" +) + +func TestError(t *testing.T) { + var err error_ + want := "no error" + if got := err.String(); got != want { + t.Errorf("empty error: got %q, want %q", got, want) + } + + want = "0: foo 42" + err.errorf(token.NoPos, "foo %d", 42) + if got := err.String(); got != want { + t.Errorf("simple error: got %q, want %q", got, want) + } + + want = "0: foo 42\n\tbar 43" + err.errorf(token.NoPos, "bar %d", 43) + if got := err.String(); got != want { + t.Errorf("simple error: got %q, want %q", got, want) + } +} + +func TestStripAnnotations(t *testing.T) { + for _, test := range []struct { + in, want string + }{ + {"", ""}, + {" ", " "}, + {"foo", "foo"}, + {"fooā‚€", "foo"}, + {"foo(Tā‚€)", "foo(T)"}, + } { + got := stripAnnotations(test.in) + if got != test.want { + t.Errorf("%q: got %q; want %q", test.in, got, test.want) + } + } +} diff --git a/src/go/types/eval.go b/src/go/types/eval.go new file mode 100644 index 0000000..084f746 --- /dev/null +++ b/src/go/types/eval.go @@ -0,0 +1,99 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" +) + +// Eval returns the type and, if constant, the value for the +// expression expr, evaluated at position pos of package pkg, +// which must have been derived from type-checking an AST with +// complete position information relative to the provided file +// set. +// +// The meaning of the parameters fset, pkg, and pos is the +// same as in CheckExpr. An error is returned if expr cannot +// be parsed successfully, or the resulting expr AST cannot be +// type-checked. +func Eval(fset *token.FileSet, pkg *Package, pos token.Pos, expr string) (_ TypeAndValue, err error) { + // parse expressions + node, err := parser.ParseExprFrom(fset, "eval", expr, 0) + if err != nil { + return TypeAndValue{}, err + } + + info := &Info{ + Types: make(map[ast.Expr]TypeAndValue), + } + err = CheckExpr(fset, pkg, pos, node, info) + return info.Types[node], err +} + +// CheckExpr type checks the expression expr as if it had appeared at position +// pos of package pkg. Type information about the expression is recorded in +// info. The expression may be an identifier denoting an uninstantiated generic +// function or type. +// +// If pkg == nil, the Universe scope is used and the provided +// position pos is ignored. If pkg != nil, and pos is invalid, +// the package scope is used. Otherwise, pos must belong to the +// package. +// +// An error is returned if pos is not within the package or +// if the node cannot be type-checked. +// +// Note: Eval and CheckExpr should not be used instead of running Check +// to compute types and values, but in addition to Check, as these +// functions ignore the context in which an expression is used (e.g., an +// assignment). Thus, top-level untyped constants will return an +// untyped type rather then the respective context-specific type. +func CheckExpr(fset *token.FileSet, pkg *Package, pos token.Pos, expr ast.Expr, info *Info) (err error) { + // determine scope + var scope *Scope + if pkg == nil { + scope = Universe + pos = token.NoPos + } else if !pos.IsValid() { + scope = pkg.scope + } else { + // The package scope extent (position information) may be + // incorrect (files spread across a wide range of fset + // positions) - ignore it and just consider its children + // (file scopes). + for _, fscope := range pkg.scope.children { + if scope = fscope.Innermost(pos); scope != nil { + break + } + } + if scope == nil || debug { + s := scope + for s != nil && s != pkg.scope { + s = s.parent + } + // s == nil || s == pkg.scope + if s == nil { + return fmt.Errorf("no position %s found in package %s", fset.Position(pos), pkg.name) + } + } + } + + // initialize checker + check := NewChecker(nil, fset, pkg, info) + check.scope = scope + check.pos = pos + defer check.handleBailout(&err) + + // evaluate node + var x operand + check.rawExpr(&x, expr, nil, true) // allow generic expressions + check.processDelayed(0) // incl. all functions + check.recordUntyped() + + return nil +} diff --git a/src/go/types/eval_test.go b/src/go/types/eval_test.go new file mode 100644 index 0000000..b0745c1 --- /dev/null +++ b/src/go/types/eval_test.go @@ -0,0 +1,297 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains tests for Eval. + +package types_test + +import ( + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "internal/testenv" + "strings" + "testing" + + . "go/types" +) + +func testEval(t *testing.T, fset *token.FileSet, pkg *Package, pos token.Pos, expr string, typ Type, typStr, valStr string) { + gotTv, err := Eval(fset, pkg, pos, expr) + if err != nil { + t.Errorf("Eval(%q) failed: %s", expr, err) + return + } + if gotTv.Type == nil { + t.Errorf("Eval(%q) got nil type but no error", expr) + return + } + + // compare types + if typ != nil { + // we have a type, check identity + if !Identical(gotTv.Type, typ) { + t.Errorf("Eval(%q) got type %s, want %s", expr, gotTv.Type, typ) + return + } + } else { + // we have a string, compare type string + gotStr := gotTv.Type.String() + if gotStr != typStr { + t.Errorf("Eval(%q) got type %s, want %s", expr, gotStr, typStr) + return + } + } + + // compare values + gotStr := "" + if gotTv.Value != nil { + gotStr = gotTv.Value.ExactString() + } + if gotStr != valStr { + t.Errorf("Eval(%q) got value %s, want %s", expr, gotStr, valStr) + } +} + +func TestEvalBasic(t *testing.T) { + fset := token.NewFileSet() + for _, typ := range Typ[Bool : String+1] { + testEval(t, fset, nil, token.NoPos, typ.Name(), typ, "", "") + } +} + +func TestEvalComposite(t *testing.T) { + fset := token.NewFileSet() + for _, test := range independentTestTypes { + testEval(t, fset, nil, token.NoPos, test.src, nil, test.str, "") + } +} + +func TestEvalArith(t *testing.T) { + var tests = []string{ + `true`, + `false == false`, + `12345678 + 87654321 == 99999999`, + `10 * 20 == 200`, + `(1<<500)*2 >> 100 == 2<<400`, + `"foo" + "bar" == "foobar"`, + `"abc" <= "bcd"`, + `len([10]struct{}{}) == 2*5`, + } + fset := token.NewFileSet() + for _, test := range tests { + testEval(t, fset, nil, token.NoPos, test, Typ[UntypedBool], "", "true") + } +} + +func TestEvalPos(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // The contents of /*-style comments are of the form + // expr => value, type + // where value may be the empty string. + // Each expr is evaluated at the position of the comment + // and the result is compared with the expected value + // and type. + var sources = []string{ + ` + package p + import "fmt" + import m "math" + const c = 3.0 + type T []int + func f(a int, s string) float64 { + fmt.Println("calling f") + _ = m.Pi // use package math + const d int = c + 1 + var x int + x = a + len(s) + return float64(x) + /* true => true, untyped bool */ + /* fmt.Println => , func(a ...any) (n int, err error) */ + /* c => 3, untyped float */ + /* T => , p.T */ + /* a => , int */ + /* s => , string */ + /* d => 4, int */ + /* x => , int */ + /* d/c => 1, int */ + /* c/2 => 3/2, untyped float */ + /* m.Pi < m.E => false, untyped bool */ + } + `, + ` + package p + /* c => 3, untyped float */ + type T1 /* T1 => , p.T1 */ struct {} + var v1 /* v1 => , int */ = 42 + func /* f1 => , func(v1 float64) */ f1(v1 float64) { + /* f1 => , func(v1 float64) */ + /* v1 => , float64 */ + var c /* c => 3, untyped float */ = "foo" /* c => , string */ + { + var c struct { + c /* c => , string */ int + } + /* c => , struct{c int} */ + _ = c + } + _ = func(a, b, c int) /* c => , string */ { + /* c => , int */ + } + _ = c + type FT /* FT => , p.FT */ interface{} + } + `, + ` + package p + /* T => , p.T */ + `, + ` + package p + import "io" + type R = io.Reader + func _() { + /* interface{R}.Read => , func(_ interface{io.Reader}, p []byte) (n int, err error) */ + _ = func() { + /* interface{io.Writer}.Write => , func(_ interface{io.Writer}, p []byte) (n int, err error) */ + type io interface {} // must not shadow io in line above + } + type R interface {} // must not shadow R in first line of this function body + } + `, + } + + fset := token.NewFileSet() + var files []*ast.File + for i, src := range sources { + file, err := parser.ParseFile(fset, "p", src, parser.ParseComments) + if err != nil { + t.Fatalf("could not parse file %d: %s", i, err) + } + files = append(files, file) + } + + conf := Config{Importer: importer.Default()} + pkg, err := conf.Check("p", fset, files, nil) + if err != nil { + t.Fatal(err) + } + + for _, file := range files { + for _, group := range file.Comments { + for _, comment := range group.List { + s := comment.Text + if len(s) >= 4 && s[:2] == "/*" && s[len(s)-2:] == "*/" { + str, typ := split(s[2:len(s)-2], ", ") + str, val := split(str, "=>") + testEval(t, fset, pkg, comment.Pos(), str, nil, typ, val) + } + } + } + } +} + +// split splits string s at the first occurrence of s, trimming spaces. +func split(s, sep string) (string, string) { + before, after, _ := strings.Cut(s, sep) + return strings.TrimSpace(before), strings.TrimSpace(after) +} + +func TestCheckExpr(t *testing.T) { + testenv.MustHaveGoBuild(t) + + // Each comment has the form /* expr => object */: + // expr is an identifier or selector expression that is passed + // to CheckExpr at the position of the comment, and object is + // the string form of the object it denotes. + const src = ` +package p + +import "fmt" + +const c = 3.0 +type T []int +type S struct{ X int } + +func f(a int, s string) S { + /* fmt.Println => func fmt.Println(a ...any) (n int, err error) */ + /* fmt.Stringer.String => func (fmt.Stringer).String() string */ + fmt.Println("calling f") + + var fmt struct{ Println int } + /* fmt => var fmt struct{Println int} */ + /* fmt.Println => field Println int */ + /* f(1, "").X => field X int */ + fmt.Println = 1 + + /* append => builtin append */ + + /* new(S).X => field X int */ + + return S{} +}` + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "p", src, parser.ParseComments) + if err != nil { + t.Fatal(err) + } + + conf := Config{Importer: importer.Default()} + pkg, err := conf.Check("p", fset, []*ast.File{f}, nil) + if err != nil { + t.Fatal(err) + } + + checkExpr := func(pos token.Pos, str string) (Object, error) { + expr, err := parser.ParseExprFrom(fset, "eval", str, 0) + if err != nil { + return nil, err + } + + info := &Info{ + Uses: make(map[*ast.Ident]Object), + Selections: make(map[*ast.SelectorExpr]*Selection), + } + if err := CheckExpr(fset, pkg, pos, expr, info); err != nil { + return nil, fmt.Errorf("CheckExpr(%q) failed: %s", str, err) + } + switch expr := expr.(type) { + case *ast.Ident: + if obj, ok := info.Uses[expr]; ok { + return obj, nil + } + case *ast.SelectorExpr: + if sel, ok := info.Selections[expr]; ok { + return sel.Obj(), nil + } + if obj, ok := info.Uses[expr.Sel]; ok { + return obj, nil // qualified identifier + } + } + return nil, fmt.Errorf("no object for %s", str) + } + + for _, group := range f.Comments { + for _, comment := range group.List { + s := comment.Text + if len(s) >= 4 && strings.HasPrefix(s, "/*") && strings.HasSuffix(s, "*/") { + pos := comment.Pos() + expr, wantObj := split(s[2:len(s)-2], "=>") + obj, err := checkExpr(pos, expr) + if err != nil { + t.Errorf("%s: %s", fset.Position(pos), err) + continue + } + if obj.String() != wantObj { + t.Errorf("%s: checkExpr(%s) = %s, want %v", + fset.Position(pos), expr, obj, wantObj) + } + } + } + } +} diff --git a/src/go/types/example_test.go b/src/go/types/example_test.go new file mode 100644 index 0000000..605e987 --- /dev/null +++ b/src/go/types/example_test.go @@ -0,0 +1,326 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Only run where builders (build.golang.org) have +// access to compiled packages for import. +// +//go:build !android && !ios && !js + +package types_test + +// This file shows examples of basic usage of the go/types API. +// +// To locate a Go package, use (*go/build.Context).Import. +// To load, parse, and type-check a complete Go program +// from source, use golang.org/x/tools/go/loader. + +import ( + "fmt" + "go/ast" + "go/format" + "go/importer" + "go/parser" + "go/token" + "go/types" + "log" + "regexp" + "sort" + "strings" +) + +// ExampleScope prints the tree of Scopes of a package created from a +// set of parsed files. +func ExampleScope() { + // Parse the source files for a package. + fset := token.NewFileSet() + var files []*ast.File + for _, file := range []struct{ name, input string }{ + {"main.go", ` +package main +import "fmt" +func main() { + freezing := FToC(-18) + fmt.Println(freezing, Boiling) } +`}, + {"celsius.go", ` +package main +import "fmt" +type Celsius float64 +func (c Celsius) String() string { return fmt.Sprintf("%gĀ°C", c) } +func FToC(f float64) Celsius { return Celsius(f - 32 / 9 * 5) } +const Boiling Celsius = 100 +func Unused() { {}; {{ var x int; _ = x }} } // make sure empty block scopes get printed +`}, + } { + files = append(files, mustParse(fset, file.name, file.input)) + } + + // Type-check a package consisting of these files. + // Type information for the imported "fmt" package + // comes from $GOROOT/pkg/$GOOS_$GOOARCH/fmt.a. + conf := types.Config{Importer: importer.Default()} + pkg, err := conf.Check("temperature", fset, files, nil) + if err != nil { + log.Fatal(err) + } + + // Print the tree of scopes. + // For determinism, we redact addresses. + var buf strings.Builder + pkg.Scope().WriteTo(&buf, 0, true) + rx := regexp.MustCompile(` 0x[a-fA-F\d]*`) + fmt.Println(rx.ReplaceAllString(buf.String(), "")) + + // Output: + // package "temperature" scope { + // . const temperature.Boiling temperature.Celsius + // . type temperature.Celsius float64 + // . func temperature.FToC(f float64) temperature.Celsius + // . func temperature.Unused() + // . func temperature.main() + // . main.go scope { + // . . package fmt + // . . function scope { + // . . . var freezing temperature.Celsius + // . . } + // . } + // . celsius.go scope { + // . . package fmt + // . . function scope { + // . . . var c temperature.Celsius + // . . } + // . . function scope { + // . . . var f float64 + // . . } + // . . function scope { + // . . . block scope { + // . . . } + // . . . block scope { + // . . . . block scope { + // . . . . . var x int + // . . . . } + // . . . } + // . . } + // . } + // } +} + +// ExampleMethodSet prints the method sets of various types. +func ExampleMethodSet() { + // Parse a single source file. + const input = ` +package temperature +import "fmt" +type Celsius float64 +func (c Celsius) String() string { return fmt.Sprintf("%gĀ°C", c) } +func (c *Celsius) SetF(f float64) { *c = Celsius(f - 32 / 9 * 5) } + +type S struct { I; m int } +type I interface { m() byte } +` + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "celsius.go", input, 0) + if err != nil { + log.Fatal(err) + } + + // Type-check a package consisting of this file. + // Type information for the imported packages + // comes from $GOROOT/pkg/$GOOS_$GOOARCH/fmt.a. + conf := types.Config{Importer: importer.Default()} + pkg, err := conf.Check("temperature", fset, []*ast.File{f}, nil) + if err != nil { + log.Fatal(err) + } + + // Print the method sets of Celsius and *Celsius. + celsius := pkg.Scope().Lookup("Celsius").Type() + for _, t := range []types.Type{celsius, types.NewPointer(celsius)} { + fmt.Printf("Method set of %s:\n", t) + mset := types.NewMethodSet(t) + for i := 0; i < mset.Len(); i++ { + fmt.Println(mset.At(i)) + } + fmt.Println() + } + + // Print the method set of S. + styp := pkg.Scope().Lookup("S").Type() + fmt.Printf("Method set of %s:\n", styp) + fmt.Println(types.NewMethodSet(styp)) + + // Output: + // Method set of temperature.Celsius: + // method (temperature.Celsius) String() string + // + // Method set of *temperature.Celsius: + // method (*temperature.Celsius) SetF(f float64) + // method (*temperature.Celsius) String() string + // + // Method set of temperature.S: + // MethodSet {} +} + +// ExampleInfo prints various facts recorded by the type checker in a +// types.Info struct: definitions of and references to each named object, +// and the type, value, and mode of every expression in the package. +func ExampleInfo() { + // Parse a single source file. + const input = ` +package fib + +type S string + +var a, b, c = len(b), S(c), "hello" + +func fib(x int) int { + if x < 2 { + return x + } + return fib(x-1) - fib(x-2) +}` + fset := token.NewFileSet() + f := mustParse(fset, "fib.go", input) + + // Type-check the package. + // We create an empty map for each kind of input + // we're interested in, and Check populates them. + info := types.Info{ + Types: make(map[ast.Expr]types.TypeAndValue), + Defs: make(map[*ast.Ident]types.Object), + Uses: make(map[*ast.Ident]types.Object), + } + var conf types.Config + pkg, err := conf.Check("fib", fset, []*ast.File{f}, &info) + if err != nil { + log.Fatal(err) + } + + // Print package-level variables in initialization order. + fmt.Printf("InitOrder: %v\n\n", info.InitOrder) + + // For each named object, print the line and + // column of its definition and each of its uses. + fmt.Println("Defs and Uses of each named object:") + usesByObj := make(map[types.Object][]string) + for id, obj := range info.Uses { + posn := fset.Position(id.Pos()) + lineCol := fmt.Sprintf("%d:%d", posn.Line, posn.Column) + usesByObj[obj] = append(usesByObj[obj], lineCol) + } + var items []string + for obj, uses := range usesByObj { + sort.Strings(uses) + item := fmt.Sprintf("%s:\n defined at %s\n used at %s", + types.ObjectString(obj, types.RelativeTo(pkg)), + fset.Position(obj.Pos()), + strings.Join(uses, ", ")) + items = append(items, item) + } + sort.Strings(items) // sort by line:col, in effect + fmt.Println(strings.Join(items, "\n")) + fmt.Println() + + fmt.Println("Types and Values of each expression:") + items = nil + for expr, tv := range info.Types { + var buf strings.Builder + posn := fset.Position(expr.Pos()) + tvstr := tv.Type.String() + if tv.Value != nil { + tvstr += " = " + tv.Value.String() + } + // line:col | expr | mode : type = value + fmt.Fprintf(&buf, "%2d:%2d | %-19s | %-7s : %s", + posn.Line, posn.Column, exprString(fset, expr), + mode(tv), tvstr) + items = append(items, buf.String()) + } + sort.Strings(items) + fmt.Println(strings.Join(items, "\n")) + + // Output: + // InitOrder: [c = "hello" b = S(c) a = len(b)] + // + // Defs and Uses of each named object: + // builtin len: + // defined at - + // used at 6:15 + // func fib(x int) int: + // defined at fib.go:8:6 + // used at 12:20, 12:9 + // type S string: + // defined at fib.go:4:6 + // used at 6:23 + // type int: + // defined at - + // used at 8:12, 8:17 + // type string: + // defined at - + // used at 4:8 + // var b S: + // defined at fib.go:6:8 + // used at 6:19 + // var c string: + // defined at fib.go:6:11 + // used at 6:25 + // var x int: + // defined at fib.go:8:10 + // used at 10:10, 12:13, 12:24, 9:5 + // + // Types and Values of each expression: + // 4: 8 | string | type : string + // 6:15 | len | builtin : func(fib.S) int + // 6:15 | len(b) | value : int + // 6:19 | b | var : fib.S + // 6:23 | S | type : fib.S + // 6:23 | S(c) | value : fib.S + // 6:25 | c | var : string + // 6:29 | "hello" | value : string = "hello" + // 8:12 | int | type : int + // 8:17 | int | type : int + // 9: 5 | x | var : int + // 9: 5 | x < 2 | value : untyped bool + // 9: 9 | 2 | value : int = 2 + // 10:10 | x | var : int + // 12: 9 | fib | value : func(x int) int + // 12: 9 | fib(x - 1) | value : int + // 12: 9 | fib(x-1) - fib(x-2) | value : int + // 12:13 | x | var : int + // 12:13 | x - 1 | value : int + // 12:15 | 1 | value : int = 1 + // 12:20 | fib | value : func(x int) int + // 12:20 | fib(x - 2) | value : int + // 12:24 | x | var : int + // 12:24 | x - 2 | value : int + // 12:26 | 2 | value : int = 2 +} + +func mode(tv types.TypeAndValue) string { + switch { + case tv.IsVoid(): + return "void" + case tv.IsType(): + return "type" + case tv.IsBuiltin(): + return "builtin" + case tv.IsNil(): + return "nil" + case tv.Assignable(): + if tv.Addressable() { + return "var" + } + return "mapindex" + case tv.IsValue(): + return "value" + default: + return "unknown" + } +} + +func exprString(fset *token.FileSet, expr ast.Expr) string { + var buf strings.Builder + format.Node(&buf, fset, expr) + return buf.String() +} diff --git a/src/go/types/expr.go b/src/go/types/expr.go new file mode 100644 index 0000000..aa90145 --- /dev/null +++ b/src/go/types/expr.go @@ -0,0 +1,1826 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements typechecking of expressions. + +package types + +import ( + "fmt" + "go/ast" + "go/constant" + "go/internal/typeparams" + "go/token" + . "internal/types/errors" + "math" +) + +/* +Basic algorithm: + +Expressions are checked recursively, top down. Expression checker functions +are generally of the form: + + func f(x *operand, e *ast.Expr, ...) + +where e is the expression to be checked, and x is the result of the check. +The check performed by f may fail in which case x.mode == invalid, and +related error messages will have been issued by f. + +If a hint argument is present, it is the composite literal element type +of an outer composite literal; it is used to type-check composite literal +elements that have no explicit type specification in the source +(e.g.: []T{{...}, {...}}, the hint is the type T in this case). + +All expressions are checked via rawExpr, which dispatches according +to expression kind. Upon returning, rawExpr is recording the types and +constant values for all expressions that have an untyped type (those types +may change on the way up in the expression tree). Usually these are constants, +but the results of comparisons or non-constant shifts of untyped constants +may also be untyped, but not constant. + +Untyped expressions may eventually become fully typed (i.e., not untyped), +typically when the value is assigned to a variable, or is used otherwise. +The updateExprType method is used to record this final type and update +the recorded types: the type-checked expression tree is again traversed down, +and the new type is propagated as needed. Untyped constant expression values +that become fully typed must now be representable by the full type (constant +sub-expression trees are left alone except for their roots). This mechanism +ensures that a client sees the actual (run-time) type an untyped value would +have. It also permits type-checking of lhs shift operands "as if the shift +were not present": when updateExprType visits an untyped lhs shift operand +and assigns it it's final type, that type must be an integer type, and a +constant lhs must be representable as an integer. + +When an expression gets its final type, either on the way out from rawExpr, +on the way down in updateExprType, or at the end of the type checker run, +the type (and constant value, if any) is recorded via Info.Types, if present. +*/ + +type opPredicates map[token.Token]func(Type) bool + +var unaryOpPredicates opPredicates + +func init() { + // Setting unaryOpPredicates in init avoids declaration cycles. + unaryOpPredicates = opPredicates{ + token.ADD: allNumeric, + token.SUB: allNumeric, + token.XOR: allInteger, + token.NOT: allBoolean, + } +} + +func (check *Checker) op(m opPredicates, x *operand, op token.Token) bool { + if pred := m[op]; pred != nil { + if !pred(x.typ) { + check.errorf(x, UndefinedOp, invalidOp+"operator %s not defined on %s", op, x) + return false + } + } else { + check.errorf(x, InvalidSyntaxTree, "unknown operator %s", op) + return false + } + return true +} + +// overflow checks that the constant x is representable by its type. +// For untyped constants, it checks that the value doesn't become +// arbitrarily large. +func (check *Checker) overflow(x *operand, opPos token.Pos) { + assert(x.mode == constant_) + + if x.val.Kind() == constant.Unknown { + // TODO(gri) We should report exactly what went wrong. At the + // moment we don't have the (go/constant) API for that. + // See also TODO in go/constant/value.go. + check.error(atPos(opPos), InvalidConstVal, "constant result is not representable") + return + } + + // Typed constants must be representable in + // their type after each constant operation. + // x.typ cannot be a type parameter (type + // parameters cannot be constant types). + if isTyped(x.typ) { + check.representable(x, under(x.typ).(*Basic)) + return + } + + // Untyped integer values must not grow arbitrarily. + const prec = 512 // 512 is the constant precision + if x.val.Kind() == constant.Int && constant.BitLen(x.val) > prec { + op := opName(x.expr) + if op != "" { + op += " " + } + check.errorf(atPos(opPos), InvalidConstVal, "constant %soverflow", op) + x.val = constant.MakeUnknown() + } +} + +// opName returns the name of the operation if x is an operation +// that might overflow; otherwise it returns the empty string. +func opName(e ast.Expr) string { + switch e := e.(type) { + case *ast.BinaryExpr: + if int(e.Op) < len(op2str2) { + return op2str2[e.Op] + } + case *ast.UnaryExpr: + if int(e.Op) < len(op2str1) { + return op2str1[e.Op] + } + } + return "" +} + +var op2str1 = [...]string{ + token.XOR: "bitwise complement", +} + +// This is only used for operations that may cause overflow. +var op2str2 = [...]string{ + token.ADD: "addition", + token.SUB: "subtraction", + token.XOR: "bitwise XOR", + token.MUL: "multiplication", + token.SHL: "shift", +} + +// If typ is a type parameter, underIs returns the result of typ.underIs(f). +// Otherwise, underIs returns the result of f(under(typ)). +func underIs(typ Type, f func(Type) bool) bool { + if tpar, _ := typ.(*TypeParam); tpar != nil { + return tpar.underIs(f) + } + return f(under(typ)) +} + +// The unary expression e may be nil. It's passed in for better error messages only. +func (check *Checker) unary(x *operand, e *ast.UnaryExpr) { + check.expr(x, e.X) + if x.mode == invalid { + return + } + switch e.Op { + case token.AND: + // spec: "As an exception to the addressability + // requirement x may also be a composite literal." + if _, ok := unparen(e.X).(*ast.CompositeLit); !ok && x.mode != variable { + check.errorf(x, UnaddressableOperand, invalidOp+"cannot take address of %s", x) + x.mode = invalid + return + } + x.mode = value + x.typ = &Pointer{base: x.typ} + return + + case token.ARROW: + u := coreType(x.typ) + if u == nil { + check.errorf(x, InvalidReceive, invalidOp+"cannot receive from %s (no core type)", x) + x.mode = invalid + return + } + ch, _ := u.(*Chan) + if ch == nil { + check.errorf(x, InvalidReceive, invalidOp+"cannot receive from non-channel %s", x) + x.mode = invalid + return + } + if ch.dir == SendOnly { + check.errorf(x, InvalidReceive, invalidOp+"cannot receive from send-only channel %s", x) + x.mode = invalid + return + } + + x.mode = commaok + x.typ = ch.elem + check.hasCallOrRecv = true + return + + case token.TILDE: + // Provide a better error position and message than what check.op below could do. + check.error(e, UndefinedOp, "cannot use ~ outside of interface or type constraint") + x.mode = invalid + return + } + + if !check.op(unaryOpPredicates, x, e.Op) { + x.mode = invalid + return + } + + if x.mode == constant_ { + if x.val.Kind() == constant.Unknown { + // nothing to do (and don't cause an error below in the overflow check) + return + } + var prec uint + if isUnsigned(x.typ) { + prec = uint(check.conf.sizeof(x.typ) * 8) + } + x.val = constant.UnaryOp(e.Op, x.val, prec) + x.expr = e + check.overflow(x, x.Pos()) + return + } + + x.mode = value + // x.typ remains unchanged +} + +func isShift(op token.Token) bool { + return op == token.SHL || op == token.SHR +} + +func isComparison(op token.Token) bool { + // Note: tokens are not ordered well to make this much easier + switch op { + case token.EQL, token.NEQ, token.LSS, token.LEQ, token.GTR, token.GEQ: + return true + } + return false +} + +func fitsFloat32(x constant.Value) bool { + f32, _ := constant.Float32Val(x) + f := float64(f32) + return !math.IsInf(f, 0) +} + +func roundFloat32(x constant.Value) constant.Value { + f32, _ := constant.Float32Val(x) + f := float64(f32) + if !math.IsInf(f, 0) { + return constant.MakeFloat64(f) + } + return nil +} + +func fitsFloat64(x constant.Value) bool { + f, _ := constant.Float64Val(x) + return !math.IsInf(f, 0) +} + +func roundFloat64(x constant.Value) constant.Value { + f, _ := constant.Float64Val(x) + if !math.IsInf(f, 0) { + return constant.MakeFloat64(f) + } + return nil +} + +// representableConst reports whether x can be represented as +// value of the given basic type and for the configuration +// provided (only needed for int/uint sizes). +// +// If rounded != nil, *rounded is set to the rounded value of x for +// representable floating-point and complex values, and to an Int +// value for integer values; it is left alone otherwise. +// It is ok to provide the addressof the first argument for rounded. +// +// The check parameter may be nil if representableConst is invoked +// (indirectly) through an exported API call (AssignableTo, ConvertibleTo) +// because we don't need the Checker's config for those calls. +func representableConst(x constant.Value, check *Checker, typ *Basic, rounded *constant.Value) bool { + if x.Kind() == constant.Unknown { + return true // avoid follow-up errors + } + + var conf *Config + if check != nil { + conf = check.conf + } + + switch { + case isInteger(typ): + x := constant.ToInt(x) + if x.Kind() != constant.Int { + return false + } + if rounded != nil { + *rounded = x + } + if x, ok := constant.Int64Val(x); ok { + switch typ.kind { + case Int: + var s = uint(conf.sizeof(typ)) * 8 + return int64(-1)<<(s-1) <= x && x <= int64(1)<<(s-1)-1 + case Int8: + const s = 8 + return -1<<(s-1) <= x && x <= 1<<(s-1)-1 + case Int16: + const s = 16 + return -1<<(s-1) <= x && x <= 1<<(s-1)-1 + case Int32: + const s = 32 + return -1<<(s-1) <= x && x <= 1<<(s-1)-1 + case Int64, UntypedInt: + return true + case Uint, Uintptr: + if s := uint(conf.sizeof(typ)) * 8; s < 64 { + return 0 <= x && x <= int64(1)<= 0 && n <= int(s) + case Uint64: + return constant.Sign(x) >= 0 && n <= 64 + case UntypedInt: + return true + } + + case isFloat(typ): + x := constant.ToFloat(x) + if x.Kind() != constant.Float { + return false + } + switch typ.kind { + case Float32: + if rounded == nil { + return fitsFloat32(x) + } + r := roundFloat32(x) + if r != nil { + *rounded = r + return true + } + case Float64: + if rounded == nil { + return fitsFloat64(x) + } + r := roundFloat64(x) + if r != nil { + *rounded = r + return true + } + case UntypedFloat: + return true + default: + unreachable() + } + + case isComplex(typ): + x := constant.ToComplex(x) + if x.Kind() != constant.Complex { + return false + } + switch typ.kind { + case Complex64: + if rounded == nil { + return fitsFloat32(constant.Real(x)) && fitsFloat32(constant.Imag(x)) + } + re := roundFloat32(constant.Real(x)) + im := roundFloat32(constant.Imag(x)) + if re != nil && im != nil { + *rounded = constant.BinaryOp(re, token.ADD, constant.MakeImag(im)) + return true + } + case Complex128: + if rounded == nil { + return fitsFloat64(constant.Real(x)) && fitsFloat64(constant.Imag(x)) + } + re := roundFloat64(constant.Real(x)) + im := roundFloat64(constant.Imag(x)) + if re != nil && im != nil { + *rounded = constant.BinaryOp(re, token.ADD, constant.MakeImag(im)) + return true + } + case UntypedComplex: + return true + default: + unreachable() + } + + case isString(typ): + return x.Kind() == constant.String + + case isBoolean(typ): + return x.Kind() == constant.Bool + } + + return false +} + +// representable checks that a constant operand is representable in the given +// basic type. +func (check *Checker) representable(x *operand, typ *Basic) { + v, code := check.representation(x, typ) + if code != 0 { + check.invalidConversion(code, x, typ) + x.mode = invalid + return + } + assert(v != nil) + x.val = v +} + +// representation returns the representation of the constant operand x as the +// basic type typ. +// +// If no such representation is possible, it returns a non-zero error code. +func (check *Checker) representation(x *operand, typ *Basic) (constant.Value, Code) { + assert(x.mode == constant_) + v := x.val + if !representableConst(x.val, check, typ, &v) { + if isNumeric(x.typ) && isNumeric(typ) { + // numeric conversion : error msg + // + // integer -> integer : overflows + // integer -> float : overflows (actually not possible) + // float -> integer : truncated + // float -> float : overflows + // + if !isInteger(x.typ) && isInteger(typ) { + return nil, TruncatedFloat + } else { + return nil, NumericOverflow + } + } + return nil, InvalidConstVal + } + return v, 0 +} + +func (check *Checker) invalidConversion(code Code, x *operand, target Type) { + msg := "cannot convert %s to type %s" + switch code { + case TruncatedFloat: + msg = "%s truncated to %s" + case NumericOverflow: + msg = "%s overflows %s" + } + check.errorf(x, code, msg, x, target) +} + +// updateExprType updates the type of x to typ and invokes itself +// recursively for the operands of x, depending on expression kind. +// If typ is still an untyped and not the final type, updateExprType +// only updates the recorded untyped type for x and possibly its +// operands. Otherwise (i.e., typ is not an untyped type anymore, +// or it is the final type for x), the type and value are recorded. +// Also, if x is a constant, it must be representable as a value of typ, +// and if x is the (formerly untyped) lhs operand of a non-constant +// shift, it must be an integer value. +func (check *Checker) updateExprType(x ast.Expr, typ Type, final bool) { + check.updateExprType0(nil, x, typ, final) +} + +func (check *Checker) updateExprType0(parent, x ast.Expr, typ Type, final bool) { + old, found := check.untyped[x] + if !found { + return // nothing to do + } + + // update operands of x if necessary + switch x := x.(type) { + case *ast.BadExpr, + *ast.FuncLit, + *ast.CompositeLit, + *ast.IndexExpr, + *ast.SliceExpr, + *ast.TypeAssertExpr, + *ast.StarExpr, + *ast.KeyValueExpr, + *ast.ArrayType, + *ast.StructType, + *ast.FuncType, + *ast.InterfaceType, + *ast.MapType, + *ast.ChanType: + // These expression are never untyped - nothing to do. + // The respective sub-expressions got their final types + // upon assignment or use. + if debug { + check.dump("%v: found old type(%s): %s (new: %s)", x.Pos(), x, old.typ, typ) + unreachable() + } + return + + case *ast.CallExpr: + // Resulting in an untyped constant (e.g., built-in complex). + // The respective calls take care of calling updateExprType + // for the arguments if necessary. + + case *ast.Ident, *ast.BasicLit, *ast.SelectorExpr: + // An identifier denoting a constant, a constant literal, + // or a qualified identifier (imported untyped constant). + // No operands to take care of. + + case *ast.ParenExpr: + check.updateExprType0(x, x.X, typ, final) + + case *ast.UnaryExpr: + // If x is a constant, the operands were constants. + // The operands don't need to be updated since they + // never get "materialized" into a typed value. If + // left in the untyped map, they will be processed + // at the end of the type check. + if old.val != nil { + break + } + check.updateExprType0(x, x.X, typ, final) + + case *ast.BinaryExpr: + if old.val != nil { + break // see comment for unary expressions + } + if isComparison(x.Op) { + // The result type is independent of operand types + // and the operand types must have final types. + } else if isShift(x.Op) { + // The result type depends only on lhs operand. + // The rhs type was updated when checking the shift. + check.updateExprType0(x, x.X, typ, final) + } else { + // The operand types match the result type. + check.updateExprType0(x, x.X, typ, final) + check.updateExprType0(x, x.Y, typ, final) + } + + default: + unreachable() + } + + // If the new type is not final and still untyped, just + // update the recorded type. + if !final && isUntyped(typ) { + old.typ = under(typ).(*Basic) + check.untyped[x] = old + return + } + + // Otherwise we have the final (typed or untyped type). + // Remove it from the map of yet untyped expressions. + delete(check.untyped, x) + + if old.isLhs { + // If x is the lhs of a shift, its final type must be integer. + // We already know from the shift check that it is representable + // as an integer if it is a constant. + if !allInteger(typ) { + check.errorf(x, InvalidShiftOperand, invalidOp+"shifted operand %s (type %s) must be integer", x, typ) + return + } + // Even if we have an integer, if the value is a constant we + // still must check that it is representable as the specific + // int type requested (was issue #22969). Fall through here. + } + if old.val != nil { + // If x is a constant, it must be representable as a value of typ. + c := operand{old.mode, x, old.typ, old.val, 0} + check.convertUntyped(&c, typ) + if c.mode == invalid { + return + } + } + + // Everything's fine, record final type and value for x. + check.recordTypeAndValue(x, old.mode, typ, old.val) +} + +// updateExprVal updates the value of x to val. +func (check *Checker) updateExprVal(x ast.Expr, val constant.Value) { + if info, ok := check.untyped[x]; ok { + info.val = val + check.untyped[x] = info + } +} + +// convertUntyped attempts to set the type of an untyped value to the target type. +func (check *Checker) convertUntyped(x *operand, target Type) { + newType, val, code := check.implicitTypeAndValue(x, target) + if code != 0 { + t := target + if !isTypeParam(target) { + t = safeUnderlying(target) + } + check.invalidConversion(code, x, t) + x.mode = invalid + return + } + if val != nil { + x.val = val + check.updateExprVal(x.expr, val) + } + if newType != x.typ { + x.typ = newType + check.updateExprType(x.expr, newType, false) + } +} + +// implicitTypeAndValue returns the implicit type of x when used in a context +// where the target type is expected. If no such implicit conversion is +// possible, it returns a nil Type and non-zero error code. +// +// If x is a constant operand, the returned constant.Value will be the +// representation of x in this context. +func (check *Checker) implicitTypeAndValue(x *operand, target Type) (Type, constant.Value, Code) { + if x.mode == invalid || isTyped(x.typ) || target == Typ[Invalid] { + return x.typ, nil, 0 + } + + if isUntyped(target) { + // both x and target are untyped + xkind := x.typ.(*Basic).kind + tkind := target.(*Basic).kind + if isNumeric(x.typ) && isNumeric(target) { + if xkind < tkind { + return target, nil, 0 + } + } else if xkind != tkind { + return nil, nil, InvalidUntypedConversion + } + return x.typ, nil, 0 + } + + switch u := under(target).(type) { + case *Basic: + if x.mode == constant_ { + v, code := check.representation(x, u) + if code != 0 { + return nil, nil, code + } + return target, v, code + } + // Non-constant untyped values may appear as the + // result of comparisons (untyped bool), intermediate + // (delayed-checked) rhs operands of shifts, and as + // the value nil. + switch x.typ.(*Basic).kind { + case UntypedBool: + if !isBoolean(target) { + return nil, nil, InvalidUntypedConversion + } + case UntypedInt, UntypedRune, UntypedFloat, UntypedComplex: + if !isNumeric(target) { + return nil, nil, InvalidUntypedConversion + } + case UntypedString: + // Non-constant untyped string values are not permitted by the spec and + // should not occur during normal typechecking passes, but this path is + // reachable via the AssignableTo API. + if !isString(target) { + return nil, nil, InvalidUntypedConversion + } + case UntypedNil: + // Unsafe.Pointer is a basic type that includes nil. + if !hasNil(target) { + return nil, nil, InvalidUntypedConversion + } + // Preserve the type of nil as UntypedNil: see #13061. + return Typ[UntypedNil], nil, 0 + default: + return nil, nil, InvalidUntypedConversion + } + case *Interface: + if isTypeParam(target) { + if !u.typeSet().underIs(func(u Type) bool { + if u == nil { + return false + } + t, _, _ := check.implicitTypeAndValue(x, u) + return t != nil + }) { + return nil, nil, InvalidUntypedConversion + } + // keep nil untyped (was bug #39755) + if x.isNil() { + return Typ[UntypedNil], nil, 0 + } + break + } + // Values must have concrete dynamic types. If the value is nil, + // keep it untyped (this is important for tools such as go vet which + // need the dynamic type for argument checking of say, print + // functions) + if x.isNil() { + return Typ[UntypedNil], nil, 0 + } + // cannot assign untyped values to non-empty interfaces + if !u.Empty() { + return nil, nil, InvalidUntypedConversion + } + return Default(x.typ), nil, 0 + case *Pointer, *Signature, *Slice, *Map, *Chan: + if !x.isNil() { + return nil, nil, InvalidUntypedConversion + } + // Keep nil untyped - see comment for interfaces, above. + return Typ[UntypedNil], nil, 0 + default: + return nil, nil, InvalidUntypedConversion + } + return target, nil, 0 +} + +// If switchCase is true, the operator op is ignored. +func (check *Checker) comparison(x, y *operand, op token.Token, switchCase bool) { + // Avoid spurious errors if any of the operands has an invalid type (issue #54405). + if x.typ == Typ[Invalid] || y.typ == Typ[Invalid] { + x.mode = invalid + return + } + + if switchCase { + op = token.EQL + } + + errOp := x // operand for which error is reported, if any + cause := "" // specific error cause, if any + + // spec: "In any comparison, the first operand must be assignable + // to the type of the second operand, or vice versa." + code := MismatchedTypes + ok, _ := x.assignableTo(check, y.typ, nil) + if !ok { + ok, _ = y.assignableTo(check, x.typ, nil) + } + if !ok { + // Report the error on the 2nd operand since we only + // know after seeing the 2nd operand whether we have + // a type mismatch. + errOp = y + cause = check.sprintf("mismatched types %s and %s", x.typ, y.typ) + goto Error + } + + // check if comparison is defined for operands + code = UndefinedOp + switch op { + case token.EQL, token.NEQ: + // spec: "The equality operators == and != apply to operands that are comparable." + switch { + case x.isNil() || y.isNil(): + // Comparison against nil requires that the other operand type has nil. + typ := x.typ + if x.isNil() { + typ = y.typ + } + if !hasNil(typ) { + // This case should only be possible for "nil == nil". + // Report the error on the 2nd operand since we only + // know after seeing the 2nd operand whether we have + // an invalid comparison. + errOp = y + goto Error + } + + case !Comparable(x.typ): + errOp = x + cause = check.incomparableCause(x.typ) + goto Error + + case !Comparable(y.typ): + errOp = y + cause = check.incomparableCause(y.typ) + goto Error + } + + case token.LSS, token.LEQ, token.GTR, token.GEQ: + // spec: The ordering operators <, <=, >, and >= apply to operands that are ordered." + switch { + case !allOrdered(x.typ): + errOp = x + goto Error + case !allOrdered(y.typ): + errOp = y + goto Error + } + + default: + unreachable() + } + + // comparison is ok + if x.mode == constant_ && y.mode == constant_ { + x.val = constant.MakeBool(constant.Compare(x.val, op, y.val)) + // The operands are never materialized; no need to update + // their types. + } else { + x.mode = value + // The operands have now their final types, which at run- + // time will be materialized. Update the expression trees. + // If the current types are untyped, the materialized type + // is the respective default type. + check.updateExprType(x.expr, Default(x.typ), true) + check.updateExprType(y.expr, Default(y.typ), true) + } + + // spec: "Comparison operators compare two operands and yield + // an untyped boolean value." + x.typ = Typ[UntypedBool] + return + +Error: + // We have an offending operand errOp and possibly an error cause. + if cause == "" { + if isTypeParam(x.typ) || isTypeParam(y.typ) { + // TODO(gri) should report the specific type causing the problem, if any + if !isTypeParam(x.typ) { + errOp = y + } + cause = check.sprintf("type parameter %s is not comparable with %s", errOp.typ, op) + } else { + cause = check.sprintf("operator %s not defined on %s", op, check.kindString(errOp.typ)) // catch-all + } + } + if switchCase { + check.errorf(x, code, "invalid case %s in switch on %s (%s)", x.expr, y.expr, cause) // error position always at 1st operand + } else { + check.errorf(errOp, code, invalidOp+"%s %s %s (%s)", x.expr, op, y.expr, cause) + } + x.mode = invalid +} + +// incomparableCause returns a more specific cause why typ is not comparable. +// If there is no more specific cause, the result is "". +func (check *Checker) incomparableCause(typ Type) string { + switch under(typ).(type) { + case *Slice, *Signature, *Map: + return check.kindString(typ) + " can only be compared to nil" + } + // see if we can extract a more specific error + var cause string + comparable(typ, true, nil, func(format string, args ...interface{}) { + cause = check.sprintf(format, args...) + }) + return cause +} + +// kindString returns the type kind as a string. +func (check *Checker) kindString(typ Type) string { + switch under(typ).(type) { + case *Array: + return "array" + case *Slice: + return "slice" + case *Struct: + return "struct" + case *Pointer: + return "pointer" + case *Signature: + return "func" + case *Interface: + if isTypeParam(typ) { + return check.sprintf("type parameter %s", typ) + } + return "interface" + case *Map: + return "map" + case *Chan: + return "chan" + default: + return check.sprintf("%s", typ) // catch-all + } +} + +// If e != nil, it must be the shift expression; it may be nil for non-constant shifts. +func (check *Checker) shift(x, y *operand, e ast.Expr, op token.Token) { + // TODO(gri) This function seems overly complex. Revisit. + + var xval constant.Value + if x.mode == constant_ { + xval = constant.ToInt(x.val) + } + + if allInteger(x.typ) || isUntyped(x.typ) && xval != nil && xval.Kind() == constant.Int { + // The lhs is of integer type or an untyped constant representable + // as an integer. Nothing to do. + } else { + // shift has no chance + check.errorf(x, InvalidShiftOperand, invalidOp+"shifted operand %s must be integer", x) + x.mode = invalid + return + } + + // spec: "The right operand in a shift expression must have integer type + // or be an untyped constant representable by a value of type uint." + + // Check that constants are representable by uint, but do not convert them + // (see also issue #47243). + var yval constant.Value + if y.mode == constant_ { + // Provide a good error message for negative shift counts. + yval = constant.ToInt(y.val) // consider -1, 1.0, but not -1.1 + if yval.Kind() == constant.Int && constant.Sign(yval) < 0 { + check.errorf(y, InvalidShiftCount, invalidOp+"negative shift count %s", y) + x.mode = invalid + return + } + + if isUntyped(y.typ) { + // Caution: Check for representability here, rather than in the switch + // below, because isInteger includes untyped integers (was bug #43697). + check.representable(y, Typ[Uint]) + if y.mode == invalid { + x.mode = invalid + return + } + } + } else { + // Check that RHS is otherwise at least of integer type. + switch { + case allInteger(y.typ): + if !allUnsigned(y.typ) && !check.allowVersion(check.pkg, 1, 13) { + check.errorf(y, UnsupportedFeature, invalidOp+"signed shift count %s requires go1.13 or later", y) + x.mode = invalid + return + } + case isUntyped(y.typ): + // This is incorrect, but preserves pre-existing behavior. + // See also bug #47410. + check.convertUntyped(y, Typ[Uint]) + if y.mode == invalid { + x.mode = invalid + return + } + default: + check.errorf(y, InvalidShiftCount, invalidOp+"shift count %s must be integer", y) + x.mode = invalid + return + } + } + + if x.mode == constant_ { + if y.mode == constant_ { + // if either x or y has an unknown value, the result is unknown + if x.val.Kind() == constant.Unknown || y.val.Kind() == constant.Unknown { + x.val = constant.MakeUnknown() + // ensure the correct type - see comment below + if !isInteger(x.typ) { + x.typ = Typ[UntypedInt] + } + return + } + // rhs must be within reasonable bounds in constant shifts + const shiftBound = 1023 - 1 + 52 // so we can express smallestFloat64 (see issue #44057) + s, ok := constant.Uint64Val(yval) + if !ok || s > shiftBound { + check.errorf(y, InvalidShiftCount, invalidOp+"invalid shift count %s", y) + x.mode = invalid + return + } + // The lhs is representable as an integer but may not be an integer + // (e.g., 2.0, an untyped float) - this can only happen for untyped + // non-integer numeric constants. Correct the type so that the shift + // result is of integer type. + if !isInteger(x.typ) { + x.typ = Typ[UntypedInt] + } + // x is a constant so xval != nil and it must be of Int kind. + x.val = constant.Shift(xval, op, uint(s)) + x.expr = e + opPos := x.Pos() + if b, _ := e.(*ast.BinaryExpr); b != nil { + opPos = b.OpPos + } + check.overflow(x, opPos) + return + } + + // non-constant shift with constant lhs + if isUntyped(x.typ) { + // spec: "If the left operand of a non-constant shift + // expression is an untyped constant, the type of the + // constant is what it would be if the shift expression + // were replaced by its left operand alone.". + // + // Delay operand checking until we know the final type + // by marking the lhs expression as lhs shift operand. + // + // Usually (in correct programs), the lhs expression + // is in the untyped map. However, it is possible to + // create incorrect programs where the same expression + // is evaluated twice (via a declaration cycle) such + // that the lhs expression type is determined in the + // first round and thus deleted from the map, and then + // not found in the second round (double insertion of + // the same expr node still just leads to one entry for + // that node, and it can only be deleted once). + // Be cautious and check for presence of entry. + // Example: var e, f = int(1<<""[f]) // issue 11347 + if info, found := check.untyped[x.expr]; found { + info.isLhs = true + check.untyped[x.expr] = info + } + // keep x's type + x.mode = value + return + } + } + + // non-constant shift - lhs must be an integer + if !allInteger(x.typ) { + check.errorf(x, InvalidShiftOperand, invalidOp+"shifted operand %s must be integer", x) + x.mode = invalid + return + } + + x.mode = value +} + +var binaryOpPredicates opPredicates + +func init() { + // Setting binaryOpPredicates in init avoids declaration cycles. + binaryOpPredicates = opPredicates{ + token.ADD: allNumericOrString, + token.SUB: allNumeric, + token.MUL: allNumeric, + token.QUO: allNumeric, + token.REM: allInteger, + + token.AND: allInteger, + token.OR: allInteger, + token.XOR: allInteger, + token.AND_NOT: allInteger, + + token.LAND: allBoolean, + token.LOR: allBoolean, + } +} + +// If e != nil, it must be the binary expression; it may be nil for non-constant expressions +// (when invoked for an assignment operation where the binary expression is implicit). +func (check *Checker) binary(x *operand, e ast.Expr, lhs, rhs ast.Expr, op token.Token, opPos token.Pos) { + var y operand + + check.expr(x, lhs) + check.expr(&y, rhs) + + if x.mode == invalid { + return + } + if y.mode == invalid { + x.mode = invalid + x.expr = y.expr + return + } + + if isShift(op) { + check.shift(x, &y, e, op) + return + } + + // mayConvert reports whether the operands x and y may + // possibly have matching types after converting one + // untyped operand to the type of the other. + // If mayConvert returns true, we try to convert the + // operands to each other's types, and if that fails + // we report a conversion failure. + // If mayConvert returns false, we continue without an + // attempt at conversion, and if the operand types are + // not compatible, we report a type mismatch error. + mayConvert := func(x, y *operand) bool { + // If both operands are typed, there's no need for an implicit conversion. + if isTyped(x.typ) && isTyped(y.typ) { + return false + } + // An untyped operand may convert to its default type when paired with an empty interface + // TODO(gri) This should only matter for comparisons (the only binary operation that is + // valid with interfaces), but in that case the assignability check should take + // care of the conversion. Verify and possibly eliminate this extra test. + if isNonTypeParamInterface(x.typ) || isNonTypeParamInterface(y.typ) { + return true + } + // A boolean type can only convert to another boolean type. + if allBoolean(x.typ) != allBoolean(y.typ) { + return false + } + // A string type can only convert to another string type. + if allString(x.typ) != allString(y.typ) { + return false + } + // Untyped nil can only convert to a type that has a nil. + if x.isNil() { + return hasNil(y.typ) + } + if y.isNil() { + return hasNil(x.typ) + } + // An untyped operand cannot convert to a pointer. + // TODO(gri) generalize to type parameters + if isPointer(x.typ) || isPointer(y.typ) { + return false + } + return true + } + if mayConvert(x, &y) { + check.convertUntyped(x, y.typ) + if x.mode == invalid { + return + } + check.convertUntyped(&y, x.typ) + if y.mode == invalid { + x.mode = invalid + return + } + } + + if isComparison(op) { + check.comparison(x, &y, op, false) + return + } + + if !Identical(x.typ, y.typ) { + // only report an error if we have valid types + // (otherwise we had an error reported elsewhere already) + if x.typ != Typ[Invalid] && y.typ != Typ[Invalid] { + var posn positioner = x + if e != nil { + posn = e + } + if e != nil { + check.errorf(posn, MismatchedTypes, invalidOp+"%s (mismatched types %s and %s)", e, x.typ, y.typ) + } else { + check.errorf(posn, MismatchedTypes, invalidOp+"%s %s= %s (mismatched types %s and %s)", lhs, op, rhs, x.typ, y.typ) + } + } + x.mode = invalid + return + } + + if !check.op(binaryOpPredicates, x, op) { + x.mode = invalid + return + } + + if op == token.QUO || op == token.REM { + // check for zero divisor + if (x.mode == constant_ || allInteger(x.typ)) && y.mode == constant_ && constant.Sign(y.val) == 0 { + check.error(&y, DivByZero, invalidOp+"division by zero") + x.mode = invalid + return + } + + // check for divisor underflow in complex division (see issue 20227) + if x.mode == constant_ && y.mode == constant_ && isComplex(x.typ) { + re, im := constant.Real(y.val), constant.Imag(y.val) + re2, im2 := constant.BinaryOp(re, token.MUL, re), constant.BinaryOp(im, token.MUL, im) + if constant.Sign(re2) == 0 && constant.Sign(im2) == 0 { + check.error(&y, DivByZero, invalidOp+"division by zero") + x.mode = invalid + return + } + } + } + + if x.mode == constant_ && y.mode == constant_ { + // if either x or y has an unknown value, the result is unknown + if x.val.Kind() == constant.Unknown || y.val.Kind() == constant.Unknown { + x.val = constant.MakeUnknown() + // x.typ is unchanged + return + } + // force integer division of integer operands + if op == token.QUO && isInteger(x.typ) { + op = token.QUO_ASSIGN + } + x.val = constant.BinaryOp(x.val, op, y.val) + x.expr = e + check.overflow(x, opPos) + return + } + + x.mode = value + // x.typ is unchanged +} + +// exprKind describes the kind of an expression; the kind +// determines if an expression is valid in 'statement context'. +type exprKind int + +const ( + conversion exprKind = iota + expression + statement +) + +// rawExpr typechecks expression e and initializes x with the expression +// value or type. If an error occurred, x.mode is set to invalid. +// If hint != nil, it is the type of a composite literal element. +// If allowGeneric is set, the operand type may be an uninstantiated +// parameterized type or function value. +func (check *Checker) rawExpr(x *operand, e ast.Expr, hint Type, allowGeneric bool) exprKind { + if trace { + check.trace(e.Pos(), "-- expr %s", e) + check.indent++ + defer func() { + check.indent-- + check.trace(e.Pos(), "=> %s", x) + }() + } + + kind := check.exprInternal(x, e, hint) + + if !allowGeneric { + check.nonGeneric(x) + } + + check.record(x) + + return kind +} + +// If x is a generic function or type, nonGeneric reports an error and invalidates x.mode and x.typ. +// Otherwise it leaves x alone. +func (check *Checker) nonGeneric(x *operand) { + if x.mode == invalid || x.mode == novalue { + return + } + var what string + switch t := x.typ.(type) { + case *Named: + if isGeneric(t) { + what = "type" + } + case *Signature: + if t.tparams != nil { + what = "function" + } + } + if what != "" { + check.errorf(x.expr, WrongTypeArgCount, "cannot use generic %s %s without instantiation", what, x.expr) + x.mode = invalid + x.typ = Typ[Invalid] + } +} + +// exprInternal contains the core of type checking of expressions. +// Must only be called by rawExpr. +func (check *Checker) exprInternal(x *operand, e ast.Expr, hint Type) exprKind { + // make sure x has a valid state in case of bailout + // (was issue 5770) + x.mode = invalid + x.typ = Typ[Invalid] + + switch e := e.(type) { + case *ast.BadExpr: + goto Error // error was reported before + + case *ast.Ident: + check.ident(x, e, nil, false) + + case *ast.Ellipsis: + // ellipses are handled explicitly where they are legal + // (array composite literals and parameter lists) + check.error(e, BadDotDotDotSyntax, "invalid use of '...'") + goto Error + + case *ast.BasicLit: + switch e.Kind { + case token.INT, token.FLOAT, token.IMAG: + check.langCompat(e) + // The max. mantissa precision for untyped numeric values + // is 512 bits, or 4048 bits for each of the two integer + // parts of a fraction for floating-point numbers that are + // represented accurately in the go/constant package. + // Constant literals that are longer than this many bits + // are not meaningful; and excessively long constants may + // consume a lot of space and time for a useless conversion. + // Cap constant length with a generous upper limit that also + // allows for separators between all digits. + const limit = 10000 + if len(e.Value) > limit { + check.errorf(e, InvalidConstVal, "excessively long constant: %s... (%d chars)", e.Value[:10], len(e.Value)) + goto Error + } + } + x.setConst(e.Kind, e.Value) + if x.mode == invalid { + // The parser already establishes syntactic correctness. + // If we reach here it's because of number under-/overflow. + // TODO(gri) setConst (and in turn the go/constant package) + // should return an error describing the issue. + check.errorf(e, InvalidConstVal, "malformed constant: %s", e.Value) + goto Error + } + // Ensure that integer values don't overflow (issue #54280). + check.overflow(x, e.Pos()) + + case *ast.FuncLit: + if sig, ok := check.typ(e.Type).(*Signature); ok { + if !check.conf.IgnoreFuncBodies && e.Body != nil { + // Anonymous functions are considered part of the + // init expression/func declaration which contains + // them: use existing package-level declaration info. + decl := check.decl // capture for use in closure below + iota := check.iota // capture for use in closure below (#22345) + // Don't type-check right away because the function may + // be part of a type definition to which the function + // body refers. Instead, type-check as soon as possible, + // but before the enclosing scope contents changes (#22992). + check.later(func() { + check.funcBody(decl, "", sig, e.Body, iota) + }).describef(e, "func literal") + } + x.mode = value + x.typ = sig + } else { + check.errorf(e, InvalidSyntaxTree, "invalid function literal %s", e) + goto Error + } + + case *ast.CompositeLit: + var typ, base Type + + switch { + case e.Type != nil: + // composite literal type present - use it + // [...]T array types may only appear with composite literals. + // Check for them here so we don't have to handle ... in general. + if atyp, _ := e.Type.(*ast.ArrayType); atyp != nil && atyp.Len != nil { + if ellip, _ := atyp.Len.(*ast.Ellipsis); ellip != nil && ellip.Elt == nil { + // We have an "open" [...]T array type. + // Create a new ArrayType with unknown length (-1) + // and finish setting it up after analyzing the literal. + typ = &Array{len: -1, elem: check.varType(atyp.Elt)} + base = typ + break + } + } + typ = check.typ(e.Type) + base = typ + + case hint != nil: + // no composite literal type present - use hint (element type of enclosing type) + typ = hint + base, _ = deref(coreType(typ)) // *T implies &T{} + if base == nil { + check.errorf(e, InvalidLit, "invalid composite literal element type %s (no core type)", typ) + goto Error + } + + default: + // TODO(gri) provide better error messages depending on context + check.error(e, UntypedLit, "missing type in composite literal") + goto Error + } + + switch utyp := coreType(base).(type) { + case *Struct: + // Prevent crash if the struct referred to is not yet set up. + // See analogous comment for *Array. + if utyp.fields == nil { + check.error(e, InvalidTypeCycle, "invalid recursive type") + goto Error + } + if len(e.Elts) == 0 { + break + } + // Convention for error messages on invalid struct literals: + // we mention the struct type only if it clarifies the error + // (e.g., a duplicate field error doesn't need the struct type). + fields := utyp.fields + if _, ok := e.Elts[0].(*ast.KeyValueExpr); ok { + // all elements must have keys + visited := make([]bool, len(fields)) + for _, e := range e.Elts { + kv, _ := e.(*ast.KeyValueExpr) + if kv == nil { + check.error(e, MixedStructLit, "mixture of field:value and value elements in struct literal") + continue + } + key, _ := kv.Key.(*ast.Ident) + // do all possible checks early (before exiting due to errors) + // so we don't drop information on the floor + check.expr(x, kv.Value) + if key == nil { + check.errorf(kv, InvalidLitField, "invalid field name %s in struct literal", kv.Key) + continue + } + i := fieldIndex(utyp.fields, check.pkg, key.Name) + if i < 0 { + check.errorf(kv, MissingLitField, "unknown field %s in struct literal of type %s", key.Name, base) + continue + } + fld := fields[i] + check.recordUse(key, fld) + etyp := fld.typ + check.assignment(x, etyp, "struct literal") + // 0 <= i < len(fields) + if visited[i] { + check.errorf(kv, DuplicateLitField, "duplicate field name %s in struct literal", key.Name) + continue + } + visited[i] = true + } + } else { + // no element must have a key + for i, e := range e.Elts { + if kv, _ := e.(*ast.KeyValueExpr); kv != nil { + check.error(kv, MixedStructLit, "mixture of field:value and value elements in struct literal") + continue + } + check.expr(x, e) + if i >= len(fields) { + check.errorf(x, InvalidStructLit, "too many values in struct literal of type %s", base) + break // cannot continue + } + // i < len(fields) + fld := fields[i] + if !fld.Exported() && fld.pkg != check.pkg { + check.errorf(x, + UnexportedLitField, + "implicit assignment to unexported field %s in struct literal of type %s", fld.name, base) + continue + } + etyp := fld.typ + check.assignment(x, etyp, "struct literal") + } + if len(e.Elts) < len(fields) { + check.errorf(inNode(e, e.Rbrace), InvalidStructLit, "too few values in struct literal of type %s", base) + // ok to continue + } + } + + case *Array: + // Prevent crash if the array referred to is not yet set up. Was issue #18643. + // This is a stop-gap solution. Should use Checker.objPath to report entire + // path starting with earliest declaration in the source. TODO(gri) fix this. + if utyp.elem == nil { + check.error(e, InvalidTypeCycle, "invalid recursive type") + goto Error + } + n := check.indexedElts(e.Elts, utyp.elem, utyp.len) + // If we have an array of unknown length (usually [...]T arrays, but also + // arrays [n]T where n is invalid) set the length now that we know it and + // record the type for the array (usually done by check.typ which is not + // called for [...]T). We handle [...]T arrays and arrays with invalid + // length the same here because it makes sense to "guess" the length for + // the latter if we have a composite literal; e.g. for [n]int{1, 2, 3} + // where n is invalid for some reason, it seems fair to assume it should + // be 3 (see also Checked.arrayLength and issue #27346). + if utyp.len < 0 { + utyp.len = n + // e.Type is missing if we have a composite literal element + // that is itself a composite literal with omitted type. In + // that case there is nothing to record (there is no type in + // the source at that point). + if e.Type != nil { + check.recordTypeAndValue(e.Type, typexpr, utyp, nil) + } + } + + case *Slice: + // Prevent crash if the slice referred to is not yet set up. + // See analogous comment for *Array. + if utyp.elem == nil { + check.error(e, InvalidTypeCycle, "invalid recursive type") + goto Error + } + check.indexedElts(e.Elts, utyp.elem, -1) + + case *Map: + // Prevent crash if the map referred to is not yet set up. + // See analogous comment for *Array. + if utyp.key == nil || utyp.elem == nil { + check.error(e, InvalidTypeCycle, "invalid recursive type") + goto Error + } + // If the map key type is an interface (but not a type parameter), + // the type of a constant key must be considered when checking for + // duplicates. + keyIsInterface := isNonTypeParamInterface(utyp.key) + visited := make(map[any][]Type, len(e.Elts)) + for _, e := range e.Elts { + kv, _ := e.(*ast.KeyValueExpr) + if kv == nil { + check.error(e, MissingLitKey, "missing key in map literal") + continue + } + check.exprWithHint(x, kv.Key, utyp.key) + check.assignment(x, utyp.key, "map literal") + if x.mode == invalid { + continue + } + if x.mode == constant_ { + duplicate := false + xkey := keyVal(x.val) + if keyIsInterface { + for _, vtyp := range visited[xkey] { + if Identical(vtyp, x.typ) { + duplicate = true + break + } + } + visited[xkey] = append(visited[xkey], x.typ) + } else { + _, duplicate = visited[xkey] + visited[xkey] = nil + } + if duplicate { + check.errorf(x, DuplicateLitKey, "duplicate key %s in map literal", x.val) + continue + } + } + check.exprWithHint(x, kv.Value, utyp.elem) + check.assignment(x, utyp.elem, "map literal") + } + + default: + // when "using" all elements unpack KeyValueExpr + // explicitly because check.use doesn't accept them + for _, e := range e.Elts { + if kv, _ := e.(*ast.KeyValueExpr); kv != nil { + // Ideally, we should also "use" kv.Key but we can't know + // if it's an externally defined struct key or not. Going + // forward anyway can lead to other errors. Give up instead. + e = kv.Value + } + check.use(e) + } + // if utyp is invalid, an error was reported before + if utyp != Typ[Invalid] { + check.errorf(e, InvalidLit, "invalid composite literal type %s", typ) + goto Error + } + } + + x.mode = value + x.typ = typ + + case *ast.ParenExpr: + kind := check.rawExpr(x, e.X, nil, false) + x.expr = e + return kind + + case *ast.SelectorExpr: + check.selector(x, e, nil, false) + + case *ast.IndexExpr, *ast.IndexListExpr: + ix := typeparams.UnpackIndexExpr(e) + if check.indexExpr(x, ix) { + check.funcInst(x, ix) + } + if x.mode == invalid { + goto Error + } + + case *ast.SliceExpr: + check.sliceExpr(x, e) + if x.mode == invalid { + goto Error + } + + case *ast.TypeAssertExpr: + check.expr(x, e.X) + if x.mode == invalid { + goto Error + } + // TODO(gri) we may want to permit type assertions on type parameter values at some point + if isTypeParam(x.typ) { + check.errorf(x, InvalidAssert, invalidOp+"cannot use type assertion on type parameter value %s", x) + goto Error + } + if _, ok := under(x.typ).(*Interface); !ok { + check.errorf(x, InvalidAssert, invalidOp+"%s is not an interface", x) + goto Error + } + // x.(type) expressions are handled explicitly in type switches + if e.Type == nil { + // Don't use invalidAST because this can occur in the AST produced by + // go/parser. + check.error(e, BadTypeKeyword, "use of .(type) outside type switch") + goto Error + } + T := check.varType(e.Type) + if T == Typ[Invalid] { + goto Error + } + check.typeAssertion(e, x, T, false) + x.mode = commaok + x.typ = T + + case *ast.CallExpr: + return check.callExpr(x, e) + + case *ast.StarExpr: + check.exprOrType(x, e.X, false) + switch x.mode { + case invalid: + goto Error + case typexpr: + check.validVarType(e.X, x.typ) + x.typ = &Pointer{base: x.typ} + default: + var base Type + if !underIs(x.typ, func(u Type) bool { + p, _ := u.(*Pointer) + if p == nil { + check.errorf(x, InvalidIndirection, invalidOp+"cannot indirect %s", x) + return false + } + if base != nil && !Identical(p.base, base) { + check.errorf(x, InvalidIndirection, invalidOp+"pointers of %s must have identical base types", x) + return false + } + base = p.base + return true + }) { + goto Error + } + x.mode = variable + x.typ = base + } + + case *ast.UnaryExpr: + check.unary(x, e) + if x.mode == invalid { + goto Error + } + if e.Op == token.ARROW { + x.expr = e + return statement // receive operations may appear in statement context + } + + case *ast.BinaryExpr: + check.binary(x, e, e.X, e.Y, e.Op, e.OpPos) + if x.mode == invalid { + goto Error + } + + case *ast.KeyValueExpr: + // key:value expressions are handled in composite literals + check.error(e, InvalidSyntaxTree, "no key:value expected") + goto Error + + case *ast.ArrayType, *ast.StructType, *ast.FuncType, + *ast.InterfaceType, *ast.MapType, *ast.ChanType: + x.mode = typexpr + x.typ = check.typ(e) + // Note: rawExpr (caller of exprInternal) will call check.recordTypeAndValue + // even though check.typ has already called it. This is fine as both + // times the same expression and type are recorded. It is also not a + // performance issue because we only reach here for composite literal + // types, which are comparatively rare. + + default: + panic(fmt.Sprintf("%s: unknown expression type %T", check.fset.Position(e.Pos()), e)) + } + + // everything went well + x.expr = e + return expression + +Error: + x.mode = invalid + x.expr = e + return statement // avoid follow-up errors +} + +// keyVal maps a complex, float, integer, string or boolean constant value +// to the corresponding complex128, float64, int64, uint64, string, or bool +// Go value if possible; otherwise it returns x. +// A complex constant that can be represented as a float (such as 1.2 + 0i) +// is returned as a floating point value; if a floating point value can be +// represented as an integer (such as 1.0) it is returned as an integer value. +// This ensures that constants of different kind but equal value (such as +// 1.0 + 0i, 1.0, 1) result in the same value. +func keyVal(x constant.Value) interface{} { + switch x.Kind() { + case constant.Complex: + f := constant.ToFloat(x) + if f.Kind() != constant.Float { + r, _ := constant.Float64Val(constant.Real(x)) + i, _ := constant.Float64Val(constant.Imag(x)) + return complex(r, i) + } + x = f + fallthrough + case constant.Float: + i := constant.ToInt(x) + if i.Kind() != constant.Int { + v, _ := constant.Float64Val(x) + return v + } + x = i + fallthrough + case constant.Int: + if v, ok := constant.Int64Val(x); ok { + return v + } + if v, ok := constant.Uint64Val(x); ok { + return v + } + case constant.String: + return constant.StringVal(x) + case constant.Bool: + return constant.BoolVal(x) + } + return x +} + +// typeAssertion checks x.(T). The type of x must be an interface. +func (check *Checker) typeAssertion(e ast.Expr, x *operand, T Type, typeSwitch bool) { + method, alt := check.assertableTo(under(x.typ).(*Interface), T) + if method == nil { + return // success + } + + cause := check.missingMethodCause(T, x.typ, method, alt) + + if typeSwitch { + check.errorf(e, ImpossibleAssert, "impossible type switch case: %s\n\t%s cannot have dynamic type %s %s", e, x, T, cause) + return + } + + check.errorf(e, ImpossibleAssert, "impossible type assertion: %s\n\t%s does not implement %s %s", e, T, x.typ, cause) +} + +// expr typechecks expression e and initializes x with the expression value. +// The result must be a single value. +// If an error occurred, x.mode is set to invalid. +func (check *Checker) expr(x *operand, e ast.Expr) { + check.rawExpr(x, e, nil, false) + check.exclude(x, 1< 0 { + buf.WriteString("ā€¦") + } + buf.WriteByte('}') + + case *ast.ParenExpr: + buf.WriteByte('(') + WriteExpr(buf, x.X) + buf.WriteByte(')') + + case *ast.SelectorExpr: + WriteExpr(buf, x.X) + buf.WriteByte('.') + buf.WriteString(x.Sel.Name) + + case *ast.IndexExpr, *ast.IndexListExpr: + ix := typeparams.UnpackIndexExpr(x) + WriteExpr(buf, ix.X) + buf.WriteByte('[') + writeExprList(buf, ix.Indices) + buf.WriteByte(']') + + case *ast.SliceExpr: + WriteExpr(buf, x.X) + buf.WriteByte('[') + if x.Low != nil { + WriteExpr(buf, x.Low) + } + buf.WriteByte(':') + if x.High != nil { + WriteExpr(buf, x.High) + } + if x.Slice3 { + buf.WriteByte(':') + if x.Max != nil { + WriteExpr(buf, x.Max) + } + } + buf.WriteByte(']') + + case *ast.TypeAssertExpr: + WriteExpr(buf, x.X) + buf.WriteString(".(") + WriteExpr(buf, x.Type) + buf.WriteByte(')') + + case *ast.CallExpr: + WriteExpr(buf, x.Fun) + buf.WriteByte('(') + writeExprList(buf, x.Args) + if x.Ellipsis.IsValid() { + buf.WriteString("...") + } + buf.WriteByte(')') + + case *ast.StarExpr: + buf.WriteByte('*') + WriteExpr(buf, x.X) + + case *ast.UnaryExpr: + buf.WriteString(x.Op.String()) + WriteExpr(buf, x.X) + + case *ast.BinaryExpr: + WriteExpr(buf, x.X) + buf.WriteByte(' ') + buf.WriteString(x.Op.String()) + buf.WriteByte(' ') + WriteExpr(buf, x.Y) + + case *ast.ArrayType: + buf.WriteByte('[') + if x.Len != nil { + WriteExpr(buf, x.Len) + } + buf.WriteByte(']') + WriteExpr(buf, x.Elt) + + case *ast.StructType: + buf.WriteString("struct{") + writeFieldList(buf, x.Fields.List, "; ", false) + buf.WriteByte('}') + + case *ast.FuncType: + buf.WriteString("func") + writeSigExpr(buf, x) + + case *ast.InterfaceType: + buf.WriteString("interface{") + writeFieldList(buf, x.Methods.List, "; ", true) + buf.WriteByte('}') + + case *ast.MapType: + buf.WriteString("map[") + WriteExpr(buf, x.Key) + buf.WriteByte(']') + WriteExpr(buf, x.Value) + + case *ast.ChanType: + var s string + switch x.Dir { + case ast.SEND: + s = "chan<- " + case ast.RECV: + s = "<-chan " + default: + s = "chan " + } + buf.WriteString(s) + WriteExpr(buf, x.Value) + } +} + +func writeSigExpr(buf *bytes.Buffer, sig *ast.FuncType) { + buf.WriteByte('(') + writeFieldList(buf, sig.Params.List, ", ", false) + buf.WriteByte(')') + + res := sig.Results + n := res.NumFields() + if n == 0 { + // no result + return + } + + buf.WriteByte(' ') + if n == 1 && len(res.List[0].Names) == 0 { + // single unnamed result + WriteExpr(buf, res.List[0].Type) + return + } + + // multiple or named result(s) + buf.WriteByte('(') + writeFieldList(buf, res.List, ", ", false) + buf.WriteByte(')') +} + +func writeFieldList(buf *bytes.Buffer, list []*ast.Field, sep string, iface bool) { + for i, f := range list { + if i > 0 { + buf.WriteString(sep) + } + + // field list names + writeIdentList(buf, f.Names) + + // types of interface methods consist of signatures only + if sig, _ := f.Type.(*ast.FuncType); sig != nil && iface { + writeSigExpr(buf, sig) + continue + } + + // named fields are separated with a blank from the field type + if len(f.Names) > 0 { + buf.WriteByte(' ') + } + + WriteExpr(buf, f.Type) + + // ignore tag + } +} + +func writeIdentList(buf *bytes.Buffer, list []*ast.Ident) { + for i, x := range list { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(x.Name) + } +} + +func writeExprList(buf *bytes.Buffer, list []ast.Expr) { + for i, x := range list { + if i > 0 { + buf.WriteString(", ") + } + WriteExpr(buf, x) + } +} diff --git a/src/go/types/exprstring_test.go b/src/go/types/exprstring_test.go new file mode 100644 index 0000000..604ceb9 --- /dev/null +++ b/src/go/types/exprstring_test.go @@ -0,0 +1,121 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "go/parser" + "testing" + + . "go/types" +) + +var testExprs = []testEntry{ + // basic type literals + dup("x"), + dup("true"), + dup("42"), + dup("3.1415"), + dup("2.71828i"), + dup(`'a'`), + dup(`"foo"`), + dup("`bar`"), + + // func and composite literals + {"func(){}", "(func() literal)"}, + {"func(x int) complex128 {}", "(func(x int) complex128 literal)"}, + {"[]int{1, 2, 3}", "[]int{ā€¦}"}, + + // type expressions + dup("[1 << 10]byte"), + dup("[]int"), + dup("*int"), + dup("struct{x int}"), + dup("func()"), + dup("func(int, float32) string"), + dup("interface{m()}"), + dup("interface{m() string; n(x int)}"), + dup("interface{~int}"), + + dup("map[string]int"), + dup("chan E"), + dup("<-chan E"), + dup("chan<- E"), + + // new interfaces + dup("interface{int}"), + dup("interface{~int}"), + dup("interface{~int}"), + dup("interface{int | string}"), + dup("interface{~int | ~string; float64; m()}"), + + // See above. + // dup("interface{type a, b, c; ~int | ~string; float64; m()}"), + dup("interface{~T[int, string] | string}"), + + // non-type expressions + dup("(x)"), + dup("x.f"), + dup("a[i]"), + + dup("s[:]"), + dup("s[i:]"), + dup("s[:j]"), + dup("s[i:j]"), + dup("s[:j:k]"), + dup("s[i:j:k]"), + + dup("x.(T)"), + + dup("x.([10]int)"), + dup("x.([...]int)"), + + dup("x.(struct{})"), + dup("x.(struct{x int; y, z float32; E})"), + + dup("x.(func())"), + dup("x.(func(x int))"), + dup("x.(func() int)"), + dup("x.(func(x, y int, z float32) (r int))"), + dup("x.(func(a, b, c int))"), + dup("x.(func(x ...T))"), + + dup("x.(interface{})"), + dup("x.(interface{m(); n(x int); E})"), + dup("x.(interface{m(); n(x int) T; E; F})"), + + dup("x.(map[K]V)"), + + dup("x.(chan E)"), + dup("x.(<-chan E)"), + dup("x.(chan<- chan int)"), + dup("x.(chan<- <-chan int)"), + dup("x.(<-chan chan int)"), + dup("x.(chan (<-chan int))"), + + dup("f()"), + dup("f(x)"), + dup("int(x)"), + dup("f(x, x + y)"), + dup("f(s...)"), + dup("f(a, s...)"), + + dup("*x"), + dup("&x"), + dup("x + y"), + dup("x + y << (2 * s)"), +} + +func TestExprString(t *testing.T) { + for _, test := range testExprs { + x, err := parser.ParseExpr(test.src) + if err != nil { + t.Errorf("%s: %s", test.src, err) + continue + } + if got := ExprString(x); got != test.str { + t.Errorf("%s: got %s, want %s", test.src, got, test.str) + } + } +} diff --git a/src/go/types/gccgosizes.go b/src/go/types/gccgosizes.go new file mode 100644 index 0000000..9d077cc --- /dev/null +++ b/src/go/types/gccgosizes.go @@ -0,0 +1,41 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This is a copy of the file generated during the gccgo build process. +// Last update 2019-01-22. + +package types + +var gccgoArchSizes = map[string]*StdSizes{ + "386": {4, 4}, + "alpha": {8, 8}, + "amd64": {8, 8}, + "amd64p32": {4, 8}, + "arm": {4, 8}, + "armbe": {4, 8}, + "arm64": {8, 8}, + "arm64be": {8, 8}, + "ia64": {8, 8}, + "loong64": {8, 8}, + "m68k": {4, 2}, + "mips": {4, 8}, + "mipsle": {4, 8}, + "mips64": {8, 8}, + "mips64le": {8, 8}, + "mips64p32": {4, 8}, + "mips64p32le": {4, 8}, + "nios2": {4, 8}, + "ppc": {4, 8}, + "ppc64": {8, 8}, + "ppc64le": {8, 8}, + "riscv": {4, 8}, + "riscv64": {8, 8}, + "s390": {4, 8}, + "s390x": {8, 8}, + "sh": {4, 8}, + "shbe": {4, 8}, + "sparc": {4, 8}, + "sparc64": {8, 8}, + "wasm": {8, 8}, +} diff --git a/src/go/types/gotype.go b/src/go/types/gotype.go new file mode 100644 index 0000000..ee9c12c --- /dev/null +++ b/src/go/types/gotype.go @@ -0,0 +1,356 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build ignore + +// Build this command explicitly: go build gotype.go + +/* +The gotype command, like the front-end of a Go compiler, parses and +type-checks a single Go package. Errors are reported if the analysis +fails; otherwise gotype is quiet (unless -v is set). + +Without a list of paths, gotype reads from standard input, which +must provide a single Go source file defining a complete package. + +With a single directory argument, gotype checks the Go files in +that directory, comprising a single package. Use -t to include the +(in-package) _test.go files. Use -x to type check only external +test files. + +Otherwise, each path must be the filename of a Go file belonging +to the same package. + +Imports are processed by importing directly from the source of +imported packages (default), or by importing from compiled and +installed packages (by setting -c to the respective compiler). + +The -c flag must be set to a compiler ("gc", "gccgo") when type- +checking packages containing imports with relative import paths +(import "./mypkg") because the source importer cannot know which +files to include for such packages. + +Usage: + + gotype [flags] [path...] + +The flags are: + + -t + include local test files in a directory (ignored if -x is provided) + -x + consider only external test files in a directory + -e + report all errors (not just the first 10) + -v + verbose mode + -c + compiler used for installed packages (gc, gccgo, or source); default: source + +Flags controlling additional output: + + -ast + print AST + -trace + print parse trace + -comments + parse comments (ignored unless -ast or -trace is provided) + -panic + panic on first error + +Examples: + +To check the files a.go, b.go, and c.go: + + gotype a.go b.go c.go + +To check an entire package including (in-package) tests in the directory dir and print the processed files: + + gotype -t -v dir + +To check the external test package (if any) in the current directory, based on installed packages compiled with +cmd/compile: + + gotype -c=gc -x . + +To verify the output of a pipe: + + echo "package foo" | gotype +*/ +package main + +import ( + "flag" + "fmt" + "go/ast" + "go/build" + "go/importer" + "go/parser" + "go/scanner" + "go/token" + "go/types" + "io" + "os" + "path/filepath" + "sync" + "time" +) + +var ( + // main operation modes + testFiles = flag.Bool("t", false, "include in-package test files in a directory") + xtestFiles = flag.Bool("x", false, "consider only external test files in a directory") + allErrors = flag.Bool("e", false, "report all errors, not just the first 10") + verbose = flag.Bool("v", false, "verbose mode") + compiler = flag.String("c", "source", "compiler used for installed packages (gc, gccgo, or source)") + + // additional output control + printAST = flag.Bool("ast", false, "print AST") + printTrace = flag.Bool("trace", false, "print parse trace") + parseComments = flag.Bool("comments", false, "parse comments (ignored unless -ast or -trace is provided)") + panicOnError = flag.Bool("panic", false, "panic on first error") +) + +var ( + fset = token.NewFileSet() + errorCount = 0 + sequential = false + parserMode parser.Mode +) + +func initParserMode() { + if *allErrors { + parserMode |= parser.AllErrors + } + if *printAST { + sequential = true + } + if *printTrace { + parserMode |= parser.Trace + sequential = true + } + if *parseComments && (*printAST || *printTrace) { + parserMode |= parser.ParseComments + } +} + +const usageString = `usage: gotype [flags] [path ...] + +The gotype command, like the front-end of a Go compiler, parses and +type-checks a single Go package. Errors are reported if the analysis +fails; otherwise gotype is quiet (unless -v is set). + +Without a list of paths, gotype reads from standard input, which +must provide a single Go source file defining a complete package. + +With a single directory argument, gotype checks the Go files in +that directory, comprising a single package. Use -t to include the +(in-package) _test.go files. Use -x to type check only external +test files. + +Otherwise, each path must be the filename of a Go file belonging +to the same package. + +Imports are processed by importing directly from the source of +imported packages (default), or by importing from compiled and +installed packages (by setting -c to the respective compiler). + +The -c flag must be set to a compiler ("gc", "gccgo") when type- +checking packages containing imports with relative import paths +(import "./mypkg") because the source importer cannot know which +files to include for such packages. +` + +func usage() { + fmt.Fprintln(os.Stderr, usageString) + flag.PrintDefaults() + os.Exit(2) +} + +func report(err error) { + if *panicOnError { + panic(err) + } + scanner.PrintError(os.Stderr, err) + if list, ok := err.(scanner.ErrorList); ok { + errorCount += len(list) + return + } + errorCount++ +} + +// parse may be called concurrently. +func parse(filename string, src any) (*ast.File, error) { + if *verbose { + fmt.Println(filename) + } + file, err := parser.ParseFile(fset, filename, src, parserMode) // ok to access fset concurrently + if *printAST { + ast.Print(fset, file) + } + return file, err +} + +func parseStdin() (*ast.File, error) { + src, err := io.ReadAll(os.Stdin) + if err != nil { + return nil, err + } + return parse("", src) +} + +func parseFiles(dir string, filenames []string) ([]*ast.File, error) { + files := make([]*ast.File, len(filenames)) + errors := make([]error, len(filenames)) + + var wg sync.WaitGroup + for i, filename := range filenames { + wg.Add(1) + go func(i int, filepath string) { + defer wg.Done() + files[i], errors[i] = parse(filepath, nil) + }(i, filepath.Join(dir, filename)) + if sequential { + wg.Wait() + } + } + wg.Wait() + + // If there are errors, return the first one for deterministic results. + var first error + for _, err := range errors { + if err != nil { + first = err + // If we have an error, some files may be nil. + // Remove them. (The go/parser always returns + // a possibly partial AST even in the presence + // of errors, except if the file doesn't exist + // in the first place, in which case it cannot + // matter.) + i := 0 + for _, f := range files { + if f != nil { + files[i] = f + i++ + } + } + files = files[:i] + break + } + } + + return files, first +} + +func parseDir(dir string) ([]*ast.File, error) { + ctxt := build.Default + pkginfo, err := ctxt.ImportDir(dir, 0) + if _, nogo := err.(*build.NoGoError); err != nil && !nogo { + return nil, err + } + + if *xtestFiles { + return parseFiles(dir, pkginfo.XTestGoFiles) + } + + filenames := append(pkginfo.GoFiles, pkginfo.CgoFiles...) + if *testFiles { + filenames = append(filenames, pkginfo.TestGoFiles...) + } + return parseFiles(dir, filenames) +} + +func getPkgFiles(args []string) ([]*ast.File, error) { + if len(args) == 0 { + // stdin + file, err := parseStdin() + if err != nil { + return nil, err + } + return []*ast.File{file}, nil + } + + if len(args) == 1 { + // possibly a directory + path := args[0] + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if info.IsDir() { + return parseDir(path) + } + } + + // list of files + return parseFiles("", args) +} + +func checkPkgFiles(files []*ast.File) { + type bailout struct{} + + // if checkPkgFiles is called multiple times, set up conf only once + conf := types.Config{ + FakeImportC: true, + Error: func(err error) { + if !*allErrors && errorCount >= 10 { + panic(bailout{}) + } + report(err) + }, + Importer: importer.ForCompiler(fset, *compiler, nil), + Sizes: types.SizesFor(build.Default.Compiler, build.Default.GOARCH), + } + + defer func() { + switch p := recover().(type) { + case nil, bailout: + // normal return or early exit + default: + // re-panic + panic(p) + } + }() + + const path = "pkg" // any non-empty string will do for now + conf.Check(path, fset, files, nil) +} + +func printStats(d time.Duration) { + fileCount := 0 + lineCount := 0 + fset.Iterate(func(f *token.File) bool { + fileCount++ + lineCount += f.LineCount() + return true + }) + + fmt.Printf( + "%s (%d files, %d lines, %d lines/s)\n", + d, fileCount, lineCount, int64(float64(lineCount)/d.Seconds()), + ) +} + +func main() { + flag.Usage = usage + flag.Parse() + initParserMode() + + start := time.Now() + + files, err := getPkgFiles(flag.Args()) + if err != nil { + report(err) + // ok to continue (files may be empty, but not nil) + } + + checkPkgFiles(files) + if errorCount > 0 { + os.Exit(2) + } + + if *verbose { + printStats(time.Since(start)) + } +} diff --git a/src/go/types/hilbert_test.go b/src/go/types/hilbert_test.go new file mode 100644 index 0000000..7d0f58e --- /dev/null +++ b/src/go/types/hilbert_test.go @@ -0,0 +1,222 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "os" + "testing" + + . "go/types" +) + +var ( + H = flag.Int("H", 5, "Hilbert matrix size") + out = flag.String("out", "", "write generated program to out") +) + +func TestHilbert(t *testing.T) { + // generate source + src := program(*H, *out) + if *out != "" { + os.WriteFile(*out, src, 0666) + return + } + + // parse source + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "hilbert.go", src, 0) + if err != nil { + t.Fatal(err) + } + + // type-check file + DefPredeclaredTestFuncs() // define assert built-in + conf := Config{Importer: importer.Default()} + _, err = conf.Check(f.Name.Name, fset, []*ast.File{f}, nil) + if err != nil { + t.Fatal(err) + } +} + +func program(n int, out string) []byte { + var g gen + + g.p(`// Code generated by: go test -run=Hilbert -H=%d -out=%q. DO NOT EDIT. + +// +`+`build ignore + +// This program tests arbitrary precision constant arithmetic +// by generating the constant elements of a Hilbert matrix H, +// its inverse I, and the product P = H*I. The product should +// be the identity matrix. +package main + +func main() { + if !ok { + printProduct() + return + } + println("PASS") +} + +`, n, out) + g.hilbert(n) + g.inverse(n) + g.product(n) + g.verify(n) + g.printProduct(n) + g.binomials(2*n - 1) + g.factorials(2*n - 1) + + return g.Bytes() +} + +type gen struct { + bytes.Buffer +} + +func (g *gen) p(format string, args ...any) { + fmt.Fprintf(&g.Buffer, format, args...) +} + +func (g *gen) hilbert(n int) { + g.p(`// Hilbert matrix, n = %d +const ( +`, n) + for i := 0; i < n; i++ { + g.p("\t") + for j := 0; j < n; j++ { + if j > 0 { + g.p(", ") + } + g.p("h%d_%d", i, j) + } + if i == 0 { + g.p(" = ") + for j := 0; j < n; j++ { + if j > 0 { + g.p(", ") + } + g.p("1.0/(iota + %d)", j+1) + } + } + g.p("\n") + } + g.p(")\n\n") +} + +func (g *gen) inverse(n int) { + g.p(`// Inverse Hilbert matrix +const ( +`) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + s := "+" + if (i+j)&1 != 0 { + s = "-" + } + g.p("\ti%d_%d = %s%d * b%d_%d * b%d_%d * b%d_%d * b%d_%d\n", + i, j, s, i+j+1, n+i, n-j-1, n+j, n-i-1, i+j, i, i+j, i) + } + g.p("\n") + } + g.p(")\n\n") +} + +func (g *gen) product(n int) { + g.p(`// Product matrix +const ( +`) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + g.p("\tp%d_%d = ", i, j) + for k := 0; k < n; k++ { + if k > 0 { + g.p(" + ") + } + g.p("h%d_%d*i%d_%d", i, k, k, j) + } + g.p("\n") + } + g.p("\n") + } + g.p(")\n\n") +} + +func (g *gen) verify(n int) { + g.p(`// Verify that product is the identity matrix +const ok = +`) + for i := 0; i < n; i++ { + for j := 0; j < n; j++ { + if j == 0 { + g.p("\t") + } else { + g.p(" && ") + } + v := 0 + if i == j { + v = 1 + } + g.p("p%d_%d == %d", i, j, v) + } + g.p(" &&\n") + } + g.p("\ttrue\n\n") + + // verify ok at type-check time + if *out == "" { + g.p("const _ = assert(ok)\n\n") + } +} + +func (g *gen) printProduct(n int) { + g.p("func printProduct() {\n") + for i := 0; i < n; i++ { + g.p("\tprintln(") + for j := 0; j < n; j++ { + if j > 0 { + g.p(", ") + } + g.p("p%d_%d", i, j) + } + g.p(")\n") + } + g.p("}\n\n") +} + +func (g *gen) binomials(n int) { + g.p(`// Binomials +const ( +`) + for j := 0; j <= n; j++ { + if j > 0 { + g.p("\n") + } + for k := 0; k <= j; k++ { + g.p("\tb%d_%d = f%d / (f%d*f%d)\n", j, k, j, k, j-k) + } + } + g.p(")\n\n") +} + +func (g *gen) factorials(n int) { + g.p(`// Factorials +const ( + f0 = 1 + f1 = 1 +`) + for i := 2; i <= n; i++ { + g.p("\tf%d = f%d * %d\n", i, i-1, i) + } + g.p(")\n\n") +} diff --git a/src/go/types/index.go b/src/go/types/index.go new file mode 100644 index 0000000..45d591e --- /dev/null +++ b/src/go/types/index.go @@ -0,0 +1,456 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements typechecking of index/slice expressions. + +package types + +import ( + "go/ast" + "go/constant" + "go/internal/typeparams" + . "internal/types/errors" +) + +// If e is a valid function instantiation, indexExpr returns true. +// In that case x represents the uninstantiated function value and +// it is the caller's responsibility to instantiate the function. +func (check *Checker) indexExpr(x *operand, e *typeparams.IndexExpr) (isFuncInst bool) { + check.exprOrType(x, e.X, true) + // x may be generic + + switch x.mode { + case invalid: + check.use(e.Indices...) + return false + + case typexpr: + // type instantiation + x.mode = invalid + // TODO(gri) here we re-evaluate e.X - try to avoid this + x.typ = check.varType(e.Orig) + if x.typ != Typ[Invalid] { + x.mode = typexpr + } + return false + + case value: + if sig, _ := under(x.typ).(*Signature); sig != nil && sig.TypeParams().Len() > 0 { + // function instantiation + return true + } + } + + // x should not be generic at this point, but be safe and check + check.nonGeneric(x) + if x.mode == invalid { + return false + } + + // ordinary index expression + valid := false + length := int64(-1) // valid if >= 0 + switch typ := under(x.typ).(type) { + case *Basic: + if isString(typ) { + valid = true + if x.mode == constant_ { + length = int64(len(constant.StringVal(x.val))) + } + // an indexed string always yields a byte value + // (not a constant) even if the string and the + // index are constant + x.mode = value + x.typ = universeByte // use 'byte' name + } + + case *Array: + valid = true + length = typ.len + if x.mode != variable { + x.mode = value + } + x.typ = typ.elem + + case *Pointer: + if typ, _ := under(typ.base).(*Array); typ != nil { + valid = true + length = typ.len + x.mode = variable + x.typ = typ.elem + } + + case *Slice: + valid = true + x.mode = variable + x.typ = typ.elem + + case *Map: + index := check.singleIndex(e) + if index == nil { + x.mode = invalid + return false + } + var key operand + check.expr(&key, index) + check.assignment(&key, typ.key, "map index") + // ok to continue even if indexing failed - map element type is known + x.mode = mapindex + x.typ = typ.elem + x.expr = e.Orig + return false + + case *Interface: + if !isTypeParam(x.typ) { + break + } + // TODO(gri) report detailed failure cause for better error messages + var key, elem Type // key != nil: we must have all maps + mode := variable // non-maps result mode + // TODO(gri) factor out closure and use it for non-typeparam cases as well + if typ.typeSet().underIs(func(u Type) bool { + l := int64(-1) // valid if >= 0 + var k, e Type // k is only set for maps + switch t := u.(type) { + case *Basic: + if isString(t) { + e = universeByte + mode = value + } + case *Array: + l = t.len + e = t.elem + if x.mode != variable { + mode = value + } + case *Pointer: + if t, _ := under(t.base).(*Array); t != nil { + l = t.len + e = t.elem + } + case *Slice: + e = t.elem + case *Map: + k = t.key + e = t.elem + } + if e == nil { + return false + } + if elem == nil { + // first type + length = l + key, elem = k, e + return true + } + // all map keys must be identical (incl. all nil) + // (that is, we cannot mix maps with other types) + if !Identical(key, k) { + return false + } + // all element types must be identical + if !Identical(elem, e) { + return false + } + // track the minimal length for arrays, if any + if l >= 0 && l < length { + length = l + } + return true + }) { + // For maps, the index expression must be assignable to the map key type. + if key != nil { + index := check.singleIndex(e) + if index == nil { + x.mode = invalid + return false + } + var k operand + check.expr(&k, index) + check.assignment(&k, key, "map index") + // ok to continue even if indexing failed - map element type is known + x.mode = mapindex + x.typ = elem + x.expr = e + return false + } + + // no maps + valid = true + x.mode = mode + x.typ = elem + } + } + + if !valid { + // types2 uses the position of '[' for the error + check.errorf(x, NonIndexableOperand, invalidOp+"cannot index %s", x) + x.mode = invalid + return false + } + + index := check.singleIndex(e) + if index == nil { + x.mode = invalid + return false + } + + // In pathological (invalid) cases (e.g.: type T1 [][[]T1{}[0][0]]T0) + // the element type may be accessed before it's set. Make sure we have + // a valid type. + if x.typ == nil { + x.typ = Typ[Invalid] + } + + check.index(index, length) + return false +} + +func (check *Checker) sliceExpr(x *operand, e *ast.SliceExpr) { + check.expr(x, e.X) + if x.mode == invalid { + check.use(e.Low, e.High, e.Max) + return + } + + valid := false + length := int64(-1) // valid if >= 0 + switch u := coreString(x.typ).(type) { + case nil: + check.errorf(x, NonSliceableOperand, invalidOp+"cannot slice %s: %s has no core type", x, x.typ) + x.mode = invalid + return + + case *Basic: + if isString(u) { + if e.Slice3 { + at := e.Max + if at == nil { + at = e // e.Index[2] should be present but be careful + } + check.error(at, InvalidSliceExpr, invalidOp+"3-index slice of string") + x.mode = invalid + return + } + valid = true + if x.mode == constant_ { + length = int64(len(constant.StringVal(x.val))) + } + // spec: "For untyped string operands the result + // is a non-constant value of type string." + if isUntyped(x.typ) { + x.typ = Typ[String] + } + } + + case *Array: + valid = true + length = u.len + if x.mode != variable { + check.errorf(x, NonSliceableOperand, invalidOp+"cannot slice %s (value not addressable)", x) + x.mode = invalid + return + } + x.typ = &Slice{elem: u.elem} + + case *Pointer: + if u, _ := under(u.base).(*Array); u != nil { + valid = true + length = u.len + x.typ = &Slice{elem: u.elem} + } + + case *Slice: + valid = true + // x.typ doesn't change + } + + if !valid { + check.errorf(x, NonSliceableOperand, invalidOp+"cannot slice %s", x) + x.mode = invalid + return + } + + x.mode = value + + // spec: "Only the first index may be omitted; it defaults to 0." + if e.Slice3 && (e.High == nil || e.Max == nil) { + check.error(inNode(e, e.Rbrack), InvalidSyntaxTree, "2nd and 3rd index required in 3-index slice") + x.mode = invalid + return + } + + // check indices + var ind [3]int64 + for i, expr := range []ast.Expr{e.Low, e.High, e.Max} { + x := int64(-1) + switch { + case expr != nil: + // The "capacity" is only known statically for strings, arrays, + // and pointers to arrays, and it is the same as the length for + // those types. + max := int64(-1) + if length >= 0 { + max = length + 1 + } + if _, v := check.index(expr, max); v >= 0 { + x = v + } + case i == 0: + // default is 0 for the first index + x = 0 + case length >= 0: + // default is length (== capacity) otherwise + x = length + } + ind[i] = x + } + + // constant indices must be in range + // (check.index already checks that existing indices >= 0) +L: + for i, x := range ind[:len(ind)-1] { + if x > 0 { + for j, y := range ind[i+1:] { + if y >= 0 && y < x { + // The value y corresponds to the expression e.Index[i+1+j]. + // Because y >= 0, it must have been set from the expression + // when checking indices and thus e.Index[i+1+j] is not nil. + at := []ast.Expr{e.Low, e.High, e.Max}[i+1+j] + check.errorf(at, SwappedSliceIndices, "invalid slice indices: %d < %d", y, x) + break L // only report one error, ok to continue + } + } + } + } +} + +// singleIndex returns the (single) index from the index expression e. +// If the index is missing, or if there are multiple indices, an error +// is reported and the result is nil. +func (check *Checker) singleIndex(expr *typeparams.IndexExpr) ast.Expr { + if len(expr.Indices) == 0 { + check.errorf(expr.Orig, InvalidSyntaxTree, "index expression %v with 0 indices", expr) + return nil + } + if len(expr.Indices) > 1 { + // TODO(rFindley) should this get a distinct error code? + check.error(expr.Indices[1], InvalidIndex, invalidOp+"more than one index") + } + return expr.Indices[0] +} + +// index checks an index expression for validity. +// If max >= 0, it is the upper bound for index. +// If the result typ is != Typ[Invalid], index is valid and typ is its (possibly named) integer type. +// If the result val >= 0, index is valid and val is its constant int value. +func (check *Checker) index(index ast.Expr, max int64) (typ Type, val int64) { + typ = Typ[Invalid] + val = -1 + + var x operand + check.expr(&x, index) + if !check.isValidIndex(&x, InvalidIndex, "index", false) { + return + } + + if x.mode != constant_ { + return x.typ, -1 + } + + if x.val.Kind() == constant.Unknown { + return + } + + v, ok := constant.Int64Val(x.val) + assert(ok) + if max >= 0 && v >= max { + check.errorf(&x, InvalidIndex, invalidArg+"index %s out of bounds [0:%d]", x.val.String(), max) + return + } + + // 0 <= v [ && v < max ] + return x.typ, v +} + +func (check *Checker) isValidIndex(x *operand, code Code, what string, allowNegative bool) bool { + if x.mode == invalid { + return false + } + + // spec: "a constant index that is untyped is given type int" + check.convertUntyped(x, Typ[Int]) + if x.mode == invalid { + return false + } + + // spec: "the index x must be of integer type or an untyped constant" + if !allInteger(x.typ) { + check.errorf(x, code, invalidArg+"%s %s must be integer", what, x) + return false + } + + if x.mode == constant_ { + // spec: "a constant index must be non-negative ..." + if !allowNegative && constant.Sign(x.val) < 0 { + check.errorf(x, code, invalidArg+"%s %s must not be negative", what, x) + return false + } + + // spec: "... and representable by a value of type int" + if !representableConst(x.val, check, Typ[Int], &x.val) { + check.errorf(x, code, invalidArg+"%s %s overflows int", what, x) + return false + } + } + + return true +} + +// indexElts checks the elements (elts) of an array or slice composite literal +// against the literal's element type (typ), and the element indices against +// the literal length if known (length >= 0). It returns the length of the +// literal (maximum index value + 1). +func (check *Checker) indexedElts(elts []ast.Expr, typ Type, length int64) int64 { + visited := make(map[int64]bool, len(elts)) + var index, max int64 + for _, e := range elts { + // determine and check index + validIndex := false + eval := e + if kv, _ := e.(*ast.KeyValueExpr); kv != nil { + if typ, i := check.index(kv.Key, length); typ != Typ[Invalid] { + if i >= 0 { + index = i + validIndex = true + } else { + check.errorf(e, InvalidLitIndex, "index %s must be integer constant", kv.Key) + } + } + eval = kv.Value + } else if length >= 0 && index >= length { + check.errorf(e, OversizeArrayLit, "index %d is out of bounds (>= %d)", index, length) + } else { + validIndex = true + } + + // if we have a valid index, check for duplicate entries + if validIndex { + if visited[index] { + check.errorf(e, DuplicateLitKey, "duplicate index %d in array or slice literal", index) + } + visited[index] = true + } + index++ + if index > max { + max = index + } + + // check element against composite literal element type + var x operand + check.exprWithHint(&x, eval, typ) + check.assignment(&x, typ, "array or slice literal") + } + return max +} diff --git a/src/go/types/infer.go b/src/go/types/infer.go new file mode 100644 index 0000000..dc87902 --- /dev/null +++ b/src/go/types/infer.go @@ -0,0 +1,773 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements type parameter inference given +// a list of concrete arguments and a parameter list. + +package types + +import ( + "fmt" + "go/token" + . "internal/types/errors" + "strings" +) + +// infer attempts to infer the complete set of type arguments for generic function instantiation/call +// based on the given type parameters tparams, type arguments targs, function parameters params, and +// function arguments args, if any. There must be at least one type parameter, no more type arguments +// than type parameters, and params and args must match in number (incl. zero). +// If successful, infer returns the complete list of type arguments, one for each type parameter. +// Otherwise the result is nil and appropriate errors will be reported. +// +// Inference proceeds as follows: +// +// Starting with given type arguments +// 1) apply FTI (function type inference) with typed arguments, +// 2) apply CTI (constraint type inference), +// 3) apply FTI with untyped function arguments, +// 4) apply CTI. +// +// The process stops as soon as all type arguments are known or an error occurs. +func (check *Checker) infer(posn positioner, tparams []*TypeParam, targs []Type, params *Tuple, args []*operand) (result []Type) { + if debug { + defer func() { + assert(result == nil || len(result) == len(tparams)) + for _, targ := range result { + assert(targ != nil) + } + //check.dump("### inferred targs = %s", result) + }() + } + + if traceInference { + check.dump("-- inferA %s%s āžž %s", tparams, params, targs) + defer func() { + check.dump("=> inferA %s āžž %s", tparams, result) + }() + } + + // There must be at least one type parameter, and no more type arguments than type parameters. + n := len(tparams) + assert(n > 0 && len(targs) <= n) + + // Function parameters and arguments must match in number. + assert(params.Len() == len(args)) + + // If we already have all type arguments, we're done. + if len(targs) == n { + return targs + } + // len(targs) < n + + const enableTparamRenaming = true + if enableTparamRenaming { + // For the purpose of type inference we must differentiate type parameters + // occurring in explicit type or value function arguments from the type + // parameters we are solving for via unification, because they may be the + // same in self-recursive calls. For example: + // + // func f[P *Q, Q any](p P, q Q) { + // f(p) + // } + // + // In this example, the fact that the P used in the instantation f[P] has + // the same pointer identity as the P we are trying to solve for via + // unification is coincidental: there is nothing special about recursive + // calls that should cause them to conflate the identity of type arguments + // with type parameters. To put it another way: any such self-recursive + // call is equivalent to a mutually recursive call, which does not run into + // any problems of type parameter identity. For example, the following code + // is equivalent to the code above. + // + // func f[P interface{*Q}, Q any](p P, q Q) { + // f2(p) + // } + // + // func f2[P interface{*Q}, Q any](p P, q Q) { + // f(p) + // } + // + // We turn the first example into the second example by renaming type + // parameters in the original signature to give them a new identity. + tparams2 := make([]*TypeParam, len(tparams)) + for i, tparam := range tparams { + tname := NewTypeName(tparam.Obj().Pos(), tparam.Obj().Pkg(), tparam.Obj().Name(), nil) + tparams2[i] = NewTypeParam(tname, nil) + tparams2[i].index = tparam.index // == i + } + + renameMap := makeRenameMap(tparams, tparams2) + for i, tparam := range tparams { + tparams2[i].bound = check.subst(posn.Pos(), tparam.bound, renameMap, nil, check.context()) + } + + tparams = tparams2 + params = check.subst(posn.Pos(), params, renameMap, nil, check.context()).(*Tuple) + } + + // If we have more than 2 arguments, we may have arguments with named and unnamed types. + // If that is the case, permutate params and args such that the arguments with named + // types are first in the list. This doesn't affect type inference if all types are taken + // as is. But when we have inexact unification enabled (as is the case for function type + // inference), when a named type is unified with an unnamed type, unification proceeds + // with the underlying type of the named type because otherwise unification would fail + // right away. This leads to an asymmetry in type inference: in cases where arguments of + // named and unnamed types are passed to parameters with identical type, different types + // (named vs underlying) may be inferred depending on the order of the arguments. + // By ensuring that named types are seen first, order dependence is avoided and unification + // succeeds where it can (issue #43056). + const enableArgSorting = true + if m := len(args); m >= 2 && enableArgSorting { + // Determine indices of arguments with named and unnamed types. + var named, unnamed []int + for i, arg := range args { + if hasName(arg.typ) { + named = append(named, i) + } else { + unnamed = append(unnamed, i) + } + } + + // If we have named and unnamed types, move the arguments with + // named types first. Update the parameter list accordingly. + // Make copies so as not to clobber the incoming slices. + if len(named) != 0 && len(unnamed) != 0 { + params2 := make([]*Var, m) + args2 := make([]*operand, m) + i := 0 + for _, j := range named { + params2[i] = params.At(j) + args2[i] = args[j] + i++ + } + for _, j := range unnamed { + params2[i] = params.At(j) + args2[i] = args[j] + i++ + } + params = NewTuple(params2...) + args = args2 + } + } + + // --- 1 --- + // Continue with the type arguments we have. Avoid matching generic + // parameters that already have type arguments against function arguments: + // It may fail because matching uses type identity while parameter passing + // uses assignment rules. Instantiate the parameter list with the type + // arguments we have, and continue with that parameter list. + + // First, make sure we have a "full" list of type arguments, some of which + // may be nil (unknown). Make a copy so as to not clobber the incoming slice. + if len(targs) < n { + targs2 := make([]Type, n) + copy(targs2, targs) + targs = targs2 + } + // len(targs) == n + + // Substitute type arguments for their respective type parameters in params, + // if any. Note that nil targs entries are ignored by check.subst. + // TODO(gri) Can we avoid this (we're setting known type arguments below, + // but that doesn't impact the isParameterized check for now). + if params.Len() > 0 { + smap := makeSubstMap(tparams, targs) + params = check.subst(token.NoPos, params, smap, nil, check.context()).(*Tuple) + } + + // Unify parameter and argument types for generic parameters with typed arguments + // and collect the indices of generic parameters with untyped arguments. + // Terminology: generic parameter = function parameter with a type-parameterized type + u := newUnifier(false) + u.x.init(tparams) + + // Set the type arguments which we know already. + for i, targ := range targs { + if targ != nil { + u.x.set(i, targ) + } + } + + errorf := func(kind string, tpar, targ Type, arg *operand) { + // provide a better error message if we can + targs, index := u.x.types() + if index == 0 { + // The first type parameter couldn't be inferred. + // If none of them could be inferred, don't try + // to provide the inferred type in the error msg. + allFailed := true + for _, targ := range targs { + if targ != nil { + allFailed = false + break + } + } + if allFailed { + check.errorf(arg, CannotInferTypeArgs, "%s %s of %s does not match %s (cannot infer %s)", kind, targ, arg.expr, tpar, typeParamsString(tparams)) + return + } + } + smap := makeSubstMap(tparams, targs) + // TODO(rFindley): pass a positioner here, rather than arg.Pos(). + inferred := check.subst(arg.Pos(), tpar, smap, nil, check.context()) + // _CannotInferTypeArgs indicates a failure of inference, though the actual + // error may be better attributed to a user-provided type argument (hence + // _InvalidTypeArg). We can't differentiate these cases, so fall back on + // the more general _CannotInferTypeArgs. + if inferred != tpar { + check.errorf(arg, CannotInferTypeArgs, "%s %s of %s does not match inferred type %s for %s", kind, targ, arg.expr, inferred, tpar) + } else { + check.errorf(arg, CannotInferTypeArgs, "%s %s of %s does not match %s", kind, targ, arg.expr, tpar) + } + } + + // indices of the generic parameters with untyped arguments - save for later + var indices []int + for i, arg := range args { + par := params.At(i) + // If we permit bidirectional unification, this conditional code needs to be + // executed even if par.typ is not parameterized since the argument may be a + // generic function (for which we want to infer its type arguments). + if isParameterized(tparams, par.typ) { + if arg.mode == invalid { + // An error was reported earlier. Ignore this targ + // and continue, we may still be able to infer all + // targs resulting in fewer follow-on errors. + continue + } + if targ := arg.typ; isTyped(targ) { + // If we permit bidirectional unification, and targ is + // a generic function, we need to initialize u.y with + // the respective type parameters of targ. + if !u.unify(par.typ, targ) { + errorf("type", par.typ, targ, arg) + return nil + } + } else if _, ok := par.typ.(*TypeParam); ok { + // Since default types are all basic (i.e., non-composite) types, an + // untyped argument will never match a composite parameter type; the + // only parameter type it can possibly match against is a *TypeParam. + // Thus, for untyped arguments we only need to look at parameter types + // that are single type parameters. + indices = append(indices, i) + } + } + } + + // If we've got all type arguments, we're done. + var index int + targs, index = u.x.types() + if index < 0 { + return targs + } + + // --- 2 --- + // See how far we get with constraint type inference. + // Note that even if we don't have any type arguments, constraint type inference + // may produce results for constraints that explicitly specify a type. + targs, index = check.inferB(posn, tparams, targs) + if targs == nil || index < 0 { + return targs + } + + // --- 3 --- + // Use any untyped arguments to infer additional type arguments. + // Some generic parameters with untyped arguments may have been given + // a type by now, we can ignore them. + for _, i := range indices { + tpar := params.At(i).typ.(*TypeParam) // is type parameter by construction of indices + // Only consider untyped arguments for which the corresponding type + // parameter doesn't have an inferred type yet. + if targs[tpar.index] == nil { + arg := args[i] + targ := Default(arg.typ) + // The default type for an untyped nil is untyped nil. We must not + // infer an untyped nil type as type parameter type. Ignore untyped + // nil by making sure all default argument types are typed. + if isTyped(targ) && !u.unify(tpar, targ) { + errorf("default type", tpar, targ, arg) + return nil + } + } + } + + // If we've got all type arguments, we're done. + targs, index = u.x.types() + if index < 0 { + return targs + } + + // --- 4 --- + // Again, follow up with constraint type inference. + targs, index = check.inferB(posn, tparams, targs) + if targs == nil || index < 0 { + return targs + } + + // At least one type argument couldn't be inferred. + assert(index >= 0 && targs[index] == nil) + tpar := tparams[index] + check.errorf(posn, CannotInferTypeArgs, "cannot infer %s (%v)", tpar.obj.name, tpar.obj.pos) + return nil +} + +// typeParamsString produces a string containing all the type parameter names +// in list suitable for human consumption. +func typeParamsString(list []*TypeParam) string { + // common cases + n := len(list) + switch n { + case 0: + return "" + case 1: + return list[0].obj.name + case 2: + return list[0].obj.name + " and " + list[1].obj.name + } + + // general case (n > 2) + var buf strings.Builder + for i, tname := range list[:n-1] { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(tname.obj.name) + } + buf.WriteString(", and ") + buf.WriteString(list[n-1].obj.name) + return buf.String() +} + +// isParameterized reports whether typ contains any of the type parameters of tparams. +func isParameterized(tparams []*TypeParam, typ Type) bool { + w := tpWalker{ + seen: make(map[Type]bool), + tparams: tparams, + } + return w.isParameterized(typ) +} + +type tpWalker struct { + seen map[Type]bool + tparams []*TypeParam +} + +func (w *tpWalker) isParameterized(typ Type) (res bool) { + // detect cycles + if x, ok := w.seen[typ]; ok { + return x + } + w.seen[typ] = false + defer func() { + w.seen[typ] = res + }() + + switch t := typ.(type) { + case nil, *Basic: // TODO(gri) should nil be handled here? + break + + case *Array: + return w.isParameterized(t.elem) + + case *Slice: + return w.isParameterized(t.elem) + + case *Struct: + for _, fld := range t.fields { + if w.isParameterized(fld.typ) { + return true + } + } + + case *Pointer: + return w.isParameterized(t.base) + + case *Tuple: + n := t.Len() + for i := 0; i < n; i++ { + if w.isParameterized(t.At(i).typ) { + return true + } + } + + case *Signature: + // t.tparams may not be nil if we are looking at a signature + // of a generic function type (or an interface method) that is + // part of the type we're testing. We don't care about these type + // parameters. + // Similarly, the receiver of a method may declare (rather then + // use) type parameters, we don't care about those either. + // Thus, we only need to look at the input and result parameters. + return w.isParameterized(t.params) || w.isParameterized(t.results) + + case *Interface: + tset := t.typeSet() + for _, m := range tset.methods { + if w.isParameterized(m.typ) { + return true + } + } + return tset.is(func(t *term) bool { + return t != nil && w.isParameterized(t.typ) + }) + + case *Map: + return w.isParameterized(t.key) || w.isParameterized(t.elem) + + case *Chan: + return w.isParameterized(t.elem) + + case *Named: + return w.isParameterizedTypeList(t.TypeArgs().list()) + + case *TypeParam: + // t must be one of w.tparams + return tparamIndex(w.tparams, t) >= 0 + + default: + unreachable() + } + + return false +} + +func (w *tpWalker) isParameterizedTypeList(list []Type) bool { + for _, t := range list { + if w.isParameterized(t) { + return true + } + } + return false +} + +// inferB returns the list of actual type arguments inferred from the type parameters' +// bounds and an initial set of type arguments. If type inference is impossible because +// unification fails, an error is reported if report is set to true, the resulting types +// list is nil, and index is 0. +// Otherwise, types is the list of inferred type arguments, and index is the index of the +// first type argument in that list that couldn't be inferred (and thus is nil). If all +// type arguments were inferred successfully, index is < 0. The number of type arguments +// provided may be less than the number of type parameters, but there must be at least one. +func (check *Checker) inferB(posn positioner, tparams []*TypeParam, targs []Type) (types []Type, index int) { + assert(len(tparams) >= len(targs) && len(targs) > 0) + + if traceInference { + check.dump("-- inferB %s āžž %s", tparams, targs) + defer func() { + check.dump("=> inferB %s āžž %s", tparams, types) + }() + } + + // Setup bidirectional unification between constraints + // and the corresponding type arguments (which may be nil!). + u := newUnifier(false) + u.x.init(tparams) + u.y = u.x // type parameters between LHS and RHS of unification are identical + + // Set the type arguments which we know already. + for i, targ := range targs { + if targ != nil { + u.x.set(i, targ) + } + } + + // Repeatedly apply constraint type inference as long as + // there are still unknown type arguments and progress is + // being made. + // + // This is an O(n^2) algorithm where n is the number of + // type parameters: if there is progress (and iteration + // continues), at least one type argument is inferred + // per iteration and we have a doubly nested loop. + // In practice this is not a problem because the number + // of type parameters tends to be very small (< 5 or so). + // (It should be possible for unification to efficiently + // signal newly inferred type arguments; then the loops + // here could handle the respective type parameters only, + // but that will come at a cost of extra complexity which + // may not be worth it.) + for n := u.x.unknowns(); n > 0; { + nn := n + + for i, tpar := range tparams { + // If there is a core term (i.e., a core type with tilde information) + // unify the type parameter with the core type. + if core, single := coreTerm(tpar); core != nil { + // A type parameter can be unified with its core type in two cases. + tx := u.x.at(i) + switch { + case tx != nil: + // The corresponding type argument tx is known. + // In this case, if the core type has a tilde, the type argument's underlying + // type must match the core type, otherwise the type argument and the core type + // must match. + // If tx is an external type parameter, don't consider its underlying type + // (which is an interface). Core type unification will attempt to unify against + // core.typ. + // Note also that even with inexact unification we cannot leave away the under + // call here because it's possible that both tx and core.typ are named types, + // with under(tx) being a (named) basic type matching core.typ. Such cases do + // not match with inexact unification. + if core.tilde && !isTypeParam(tx) { + tx = under(tx) + } + if !u.unify(tx, core.typ) { + // TODO(gri) improve error message by providing the type arguments + // which we know already + // Don't use term.String() as it always qualifies types, even if they + // are in the current package. + tilde := "" + if core.tilde { + tilde = "~" + } + check.errorf(posn, InvalidTypeArg, "%s does not match %s%s", tx, tilde, core.typ) + return nil, 0 + } + + case single && !core.tilde: + // The corresponding type argument tx is unknown and there's a single + // specific type and no tilde. + // In this case the type argument must be that single type; set it. + u.x.set(i, core.typ) + + default: + // Unification is not possible and no progress was made. + continue + } + + // The number of known type arguments may have changed. + nn = u.x.unknowns() + if nn == 0 { + break // all type arguments are known + } + } + } + + assert(nn <= n) + if nn == n { + break // no progress + } + n = nn + } + + // u.x.types() now contains the incoming type arguments plus any additional type + // arguments which were inferred from core terms. The newly inferred non-nil + // entries may still contain references to other type parameters. + // For instance, for [A any, B interface{ []C }, C interface{ *A }], if A == int + // was given, unification produced the type list [int, []C, *A]. We eliminate the + // remaining type parameters by substituting the type parameters in this type list + // until nothing changes anymore. + types, _ = u.x.types() + if debug { + for i, targ := range targs { + assert(targ == nil || types[i] == targ) + } + } + + // The data structure of each (provided or inferred) type represents a graph, where + // each node corresponds to a type and each (directed) vertex points to a component + // type. The substitution process described above repeatedly replaces type parameter + // nodes in these graphs with the graphs of the types the type parameters stand for, + // which creates a new (possibly bigger) graph for each type. + // The substitution process will not stop if the replacement graph for a type parameter + // also contains that type parameter. + // For instance, for [A interface{ *A }], without any type argument provided for A, + // unification produces the type list [*A]. Substituting A in *A with the value for + // A will lead to infinite expansion by producing [**A], [****A], [********A], etc., + // because the graph A -> *A has a cycle through A. + // Generally, cycles may occur across multiple type parameters and inferred types + // (for instance, consider [P interface{ *Q }, Q interface{ func(P) }]). + // We eliminate cycles by walking the graphs for all type parameters. If a cycle + // through a type parameter is detected, cycleFinder nils out the respective type + // which kills the cycle; this also means that the respective type could not be + // inferred. + // + // TODO(gri) If useful, we could report the respective cycle as an error. We don't + // do this now because type inference will fail anyway, and furthermore, + // constraints with cycles of this kind cannot currently be satisfied by + // any user-supplied type. But should that change, reporting an error + // would be wrong. + w := cycleFinder{tparams, types, make(map[Type]bool)} + for _, t := range tparams { + w.typ(t) // t != nil + } + + // dirty tracks the indices of all types that may still contain type parameters. + // We know that nil type entries and entries corresponding to provided (non-nil) + // type arguments are clean, so exclude them from the start. + var dirty []int + for i, typ := range types { + if typ != nil && (i >= len(targs) || targs[i] == nil) { + dirty = append(dirty, i) + } + } + + for len(dirty) > 0 { + // TODO(gri) Instead of creating a new substMap for each iteration, + // provide an update operation for substMaps and only change when + // needed. Optimization. + smap := makeSubstMap(tparams, types) + n := 0 + for _, index := range dirty { + t0 := types[index] + if t1 := check.subst(token.NoPos, t0, smap, nil, check.context()); t1 != t0 { + types[index] = t1 + dirty[n] = index + n++ + } + } + dirty = dirty[:n] + } + + // Once nothing changes anymore, we may still have type parameters left; + // e.g., a constraint with core type *P may match a type parameter Q but + // we don't have any type arguments to fill in for *P or Q (issue #45548). + // Don't let such inferences escape, instead nil them out. + for i, typ := range types { + if typ != nil && isParameterized(tparams, typ) { + types[i] = nil + } + } + + // update index + index = -1 + for i, typ := range types { + if typ == nil { + index = i + break + } + } + + return +} + +// If the type parameter has a single specific type S, coreTerm returns (S, true). +// Otherwise, if tpar has a core type T, it returns a term corresponding to that +// core type and false. In that case, if any term of tpar has a tilde, the core +// term has a tilde. In all other cases coreTerm returns (nil, false). +func coreTerm(tpar *TypeParam) (*term, bool) { + n := 0 + var single *term // valid if n == 1 + var tilde bool + tpar.is(func(t *term) bool { + if t == nil { + assert(n == 0) + return false // no terms + } + n++ + single = t + if t.tilde { + tilde = true + } + return true + }) + if n == 1 { + if debug { + assert(debug && under(single.typ) == coreType(tpar)) + } + return single, true + } + if typ := coreType(tpar); typ != nil { + // A core type is always an underlying type. + // If any term of tpar has a tilde, we don't + // have a precise core type and we must return + // a tilde as well. + return &term{tilde, typ}, false + } + return nil, false +} + +type cycleFinder struct { + tparams []*TypeParam + types []Type + seen map[Type]bool +} + +func (w *cycleFinder) typ(typ Type) { + if w.seen[typ] { + // We have seen typ before. If it is one of the type parameters + // in tparams, iterative substitution will lead to infinite expansion. + // Nil out the corresponding type which effectively kills the cycle. + if tpar, _ := typ.(*TypeParam); tpar != nil { + if i := tparamIndex(w.tparams, tpar); i >= 0 { + // cycle through tpar + w.types[i] = nil + } + } + // If we don't have one of our type parameters, the cycle is due + // to an ordinary recursive type and we can just stop walking it. + return + } + w.seen[typ] = true + defer delete(w.seen, typ) + + switch t := typ.(type) { + case *Basic: + // nothing to do + + case *Array: + w.typ(t.elem) + + case *Slice: + w.typ(t.elem) + + case *Struct: + w.varList(t.fields) + + case *Pointer: + w.typ(t.base) + + // case *Tuple: + // This case should not occur because tuples only appear + // in signatures where they are handled explicitly. + + case *Signature: + if t.params != nil { + w.varList(t.params.vars) + } + if t.results != nil { + w.varList(t.results.vars) + } + + case *Union: + for _, t := range t.terms { + w.typ(t.typ) + } + + case *Interface: + for _, m := range t.methods { + w.typ(m.typ) + } + for _, t := range t.embeddeds { + w.typ(t) + } + + case *Map: + w.typ(t.key) + w.typ(t.elem) + + case *Chan: + w.typ(t.elem) + + case *Named: + for _, tpar := range t.TypeArgs().list() { + w.typ(tpar) + } + + case *TypeParam: + if i := tparamIndex(w.tparams, t); i >= 0 && w.types[i] != nil { + w.typ(w.types[i]) + } + + default: + panic(fmt.Sprintf("unexpected %T", typ)) + } +} + +func (w *cycleFinder) varList(list []*Var) { + for _, v := range list { + w.typ(v.typ) + } +} diff --git a/src/go/types/initorder.go b/src/go/types/initorder.go new file mode 100644 index 0000000..9ee176f --- /dev/null +++ b/src/go/types/initorder.go @@ -0,0 +1,325 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "container/heap" + "fmt" + . "internal/types/errors" + "sort" +) + +// initOrder computes the Info.InitOrder for package variables. +func (check *Checker) initOrder() { + // An InitOrder may already have been computed if a package is + // built from several calls to (*Checker).Files. Clear it. + check.Info.InitOrder = check.Info.InitOrder[:0] + + // Compute the object dependency graph and initialize + // a priority queue with the list of graph nodes. + pq := nodeQueue(dependencyGraph(check.objMap)) + heap.Init(&pq) + + const debug = false + if debug { + fmt.Printf("Computing initialization order for %s\n\n", check.pkg) + fmt.Println("Object dependency graph:") + for obj, d := range check.objMap { + // only print objects that may appear in the dependency graph + if obj, _ := obj.(dependency); obj != nil { + if len(d.deps) > 0 { + fmt.Printf("\t%s depends on\n", obj.Name()) + for dep := range d.deps { + fmt.Printf("\t\t%s\n", dep.Name()) + } + } else { + fmt.Printf("\t%s has no dependencies\n", obj.Name()) + } + } + } + fmt.Println() + + fmt.Println("Transposed object dependency graph (functions eliminated):") + for _, n := range pq { + fmt.Printf("\t%s depends on %d nodes\n", n.obj.Name(), n.ndeps) + for p := range n.pred { + fmt.Printf("\t\t%s is dependent\n", p.obj.Name()) + } + } + fmt.Println() + + fmt.Println("Processing nodes:") + } + + // Determine initialization order by removing the highest priority node + // (the one with the fewest dependencies) and its edges from the graph, + // repeatedly, until there are no nodes left. + // In a valid Go program, those nodes always have zero dependencies (after + // removing all incoming dependencies), otherwise there are initialization + // cycles. + emitted := make(map[*declInfo]bool) + for len(pq) > 0 { + // get the next node + n := heap.Pop(&pq).(*graphNode) + + if debug { + fmt.Printf("\t%s (src pos %d) depends on %d nodes now\n", + n.obj.Name(), n.obj.order(), n.ndeps) + } + + // if n still depends on other nodes, we have a cycle + if n.ndeps > 0 { + cycle := findPath(check.objMap, n.obj, n.obj, make(map[Object]bool)) + // If n.obj is not part of the cycle (e.g., n.obj->b->c->d->c), + // cycle will be nil. Don't report anything in that case since + // the cycle is reported when the algorithm gets to an object + // in the cycle. + // Furthermore, once an object in the cycle is encountered, + // the cycle will be broken (dependency count will be reduced + // below), and so the remaining nodes in the cycle don't trigger + // another error (unless they are part of multiple cycles). + if cycle != nil { + check.reportCycle(cycle) + } + // Ok to continue, but the variable initialization order + // will be incorrect at this point since it assumes no + // cycle errors. + } + + // reduce dependency count of all dependent nodes + // and update priority queue + for p := range n.pred { + p.ndeps-- + heap.Fix(&pq, p.index) + } + + // record the init order for variables with initializers only + v, _ := n.obj.(*Var) + info := check.objMap[v] + if v == nil || !info.hasInitializer() { + continue + } + + // n:1 variable declarations such as: a, b = f() + // introduce a node for each lhs variable (here: a, b); + // but they all have the same initializer - emit only + // one, for the first variable seen + if emitted[info] { + continue // initializer already emitted, if any + } + emitted[info] = true + + infoLhs := info.lhs // possibly nil (see declInfo.lhs field comment) + if infoLhs == nil { + infoLhs = []*Var{v} + } + init := &Initializer{infoLhs, info.init} + check.Info.InitOrder = append(check.Info.InitOrder, init) + } + + if debug { + fmt.Println() + fmt.Println("Initialization order:") + for _, init := range check.Info.InitOrder { + fmt.Printf("\t%s\n", init) + } + fmt.Println() + } +} + +// findPath returns the (reversed) list of objects []Object{to, ... from} +// such that there is a path of object dependencies from 'from' to 'to'. +// If there is no such path, the result is nil. +func findPath(objMap map[Object]*declInfo, from, to Object, seen map[Object]bool) []Object { + if seen[from] { + return nil + } + seen[from] = true + + for d := range objMap[from].deps { + if d == to { + return []Object{d} + } + if P := findPath(objMap, d, to, seen); P != nil { + return append(P, d) + } + } + + return nil +} + +// reportCycle reports an error for the given cycle. +func (check *Checker) reportCycle(cycle []Object) { + obj := cycle[0] + + // report a more concise error for self references + if len(cycle) == 1 { + check.errorf(obj, InvalidInitCycle, "initialization cycle: %s refers to itself", obj.Name()) + return + } + + check.errorf(obj, InvalidInitCycle, "initialization cycle for %s", obj.Name()) + // subtle loop: print cycle[i] for i = 0, n-1, n-2, ... 1 for len(cycle) = n + for i := len(cycle) - 1; i >= 0; i-- { + check.errorf(obj, InvalidInitCycle, "\t%s refers to", obj.Name()) // secondary error, \t indented + obj = cycle[i] + } + // print cycle[0] again to close the cycle + check.errorf(obj, InvalidInitCycle, "\t%s", obj.Name()) +} + +// ---------------------------------------------------------------------------- +// Object dependency graph + +// A dependency is an object that may be a dependency in an initialization +// expression. Only constants, variables, and functions can be dependencies. +// Constants are here because constant expression cycles are reported during +// initialization order computation. +type dependency interface { + Object + isDependency() +} + +// A graphNode represents a node in the object dependency graph. +// Each node p in n.pred represents an edge p->n, and each node +// s in n.succ represents an edge n->s; with a->b indicating that +// a depends on b. +type graphNode struct { + obj dependency // object represented by this node + pred, succ nodeSet // consumers and dependencies of this node (lazily initialized) + index int // node index in graph slice/priority queue + ndeps int // number of outstanding dependencies before this object can be initialized +} + +// cost returns the cost of removing this node, which involves copying each +// predecessor to each successor (and vice-versa). +func (n *graphNode) cost() int { + return len(n.pred) * len(n.succ) +} + +type nodeSet map[*graphNode]bool + +func (s *nodeSet) add(p *graphNode) { + if *s == nil { + *s = make(nodeSet) + } + (*s)[p] = true +} + +// dependencyGraph computes the object dependency graph from the given objMap, +// with any function nodes removed. The resulting graph contains only constants +// and variables. +func dependencyGraph(objMap map[Object]*declInfo) []*graphNode { + // M is the dependency (Object) -> graphNode mapping + M := make(map[dependency]*graphNode) + for obj := range objMap { + // only consider nodes that may be an initialization dependency + if obj, _ := obj.(dependency); obj != nil { + M[obj] = &graphNode{obj: obj} + } + } + + // compute edges for graph M + // (We need to include all nodes, even isolated ones, because they still need + // to be scheduled for initialization in correct order relative to other nodes.) + for obj, n := range M { + // for each dependency obj -> d (= deps[i]), create graph edges n->s and s->n + for d := range objMap[obj].deps { + // only consider nodes that may be an initialization dependency + if d, _ := d.(dependency); d != nil { + d := M[d] + n.succ.add(d) + d.pred.add(n) + } + } + } + + var G, funcG []*graphNode // separate non-functions and functions + for _, n := range M { + if _, ok := n.obj.(*Func); ok { + funcG = append(funcG, n) + } else { + G = append(G, n) + } + } + + // remove function nodes and collect remaining graph nodes in G + // (Mutually recursive functions may introduce cycles among themselves + // which are permitted. Yet such cycles may incorrectly inflate the dependency + // count for variables which in turn may not get scheduled for initialization + // in correct order.) + // + // Note that because we recursively copy predecessors and successors + // throughout the function graph, the cost of removing a function at + // position X is proportional to cost * (len(funcG)-X). Therefore, we should + // remove high-cost functions last. + sort.Slice(funcG, func(i, j int) bool { + return funcG[i].cost() < funcG[j].cost() + }) + for _, n := range funcG { + // connect each predecessor p of n with each successor s + // and drop the function node (don't collect it in G) + for p := range n.pred { + // ignore self-cycles + if p != n { + // Each successor s of n becomes a successor of p, and + // each predecessor p of n becomes a predecessor of s. + for s := range n.succ { + // ignore self-cycles + if s != n { + p.succ.add(s) + s.pred.add(p) + } + } + delete(p.succ, n) // remove edge to n + } + } + for s := range n.succ { + delete(s.pred, n) // remove edge to n + } + } + + // fill in index and ndeps fields + for i, n := range G { + n.index = i + n.ndeps = len(n.succ) + } + + return G +} + +// ---------------------------------------------------------------------------- +// Priority queue + +// nodeQueue implements the container/heap interface; +// a nodeQueue may be used as a priority queue. +type nodeQueue []*graphNode + +func (a nodeQueue) Len() int { return len(a) } + +func (a nodeQueue) Swap(i, j int) { + x, y := a[i], a[j] + a[i], a[j] = y, x + x.index, y.index = j, i +} + +func (a nodeQueue) Less(i, j int) bool { + x, y := a[i], a[j] + // nodes are prioritized by number of incoming dependencies (1st key) + // and source order (2nd key) + return x.ndeps < y.ndeps || x.ndeps == y.ndeps && x.obj.order() < y.obj.order() +} + +func (a *nodeQueue) Push(x any) { + panic("unreachable") +} + +func (a *nodeQueue) Pop() any { + n := len(*a) + x := (*a)[n-1] + x.index = -1 // for safety + *a = (*a)[:n-1] + return x +} diff --git a/src/go/types/instantiate.go b/src/go/types/instantiate.go new file mode 100644 index 0000000..2cf48c1 --- /dev/null +++ b/src/go/types/instantiate.go @@ -0,0 +1,366 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements instantiation of generic types +// through substitution of type parameters by type arguments. + +package types + +import ( + "errors" + "fmt" + "go/token" + . "internal/types/errors" +) + +// Instantiate instantiates the type orig with the given type arguments targs. +// orig must be a *Named or a *Signature type. If there is no error, the +// resulting Type is an instantiated type of the same kind (either a *Named or +// a *Signature). Methods attached to a *Named type are also instantiated, and +// associated with a new *Func that has the same position as the original +// method, but nil function scope. +// +// If ctxt is non-nil, it may be used to de-duplicate the instance against +// previous instances with the same identity. As a special case, generic +// *Signature origin types are only considered identical if they are pointer +// equivalent, so that instantiating distinct (but possibly identical) +// signatures will yield different instances. The use of a shared context does +// not guarantee that identical instances are deduplicated in all cases. +// +// If validate is set, Instantiate verifies that the number of type arguments +// and parameters match, and that the type arguments satisfy their +// corresponding type constraints. If verification fails, the resulting error +// may wrap an *ArgumentError indicating which type argument did not satisfy +// its corresponding type parameter constraint, and why. +// +// If validate is not set, Instantiate does not verify the type argument count +// or whether the type arguments satisfy their constraints. Instantiate is +// guaranteed to not return an error, but may panic. Specifically, for +// *Signature types, Instantiate will panic immediately if the type argument +// count is incorrect; for *Named types, a panic may occur later inside the +// *Named API. +func Instantiate(ctxt *Context, orig Type, targs []Type, validate bool) (Type, error) { + if ctxt == nil { + ctxt = NewContext() + } + if validate { + var tparams []*TypeParam + switch t := orig.(type) { + case *Named: + tparams = t.TypeParams().list() + case *Signature: + tparams = t.TypeParams().list() + } + if len(targs) != len(tparams) { + return nil, fmt.Errorf("got %d type arguments but %s has %d type parameters", len(targs), orig, len(tparams)) + } + if i, err := (*Checker)(nil).verify(token.NoPos, tparams, targs, ctxt); err != nil { + return nil, &ArgumentError{i, err} + } + } + + inst := (*Checker)(nil).instance(token.NoPos, orig, targs, nil, ctxt) + return inst, nil +} + +// instance instantiates the given original (generic) function or type with the +// provided type arguments and returns the resulting instance. If an identical +// instance exists already in the given contexts, it returns that instance, +// otherwise it creates a new one. +// +// If expanding is non-nil, it is the Named instance type currently being +// expanded. If ctxt is non-nil, it is the context associated with the current +// type-checking pass or call to Instantiate. At least one of expanding or ctxt +// must be non-nil. +// +// For Named types the resulting instance may be unexpanded. +func (check *Checker) instance(pos token.Pos, orig Type, targs []Type, expanding *Named, ctxt *Context) (res Type) { + // The order of the contexts below matters: we always prefer instances in the + // expanding instance context in order to preserve reference cycles. + // + // Invariant: if expanding != nil, the returned instance will be the instance + // recorded in expanding.inst.ctxt. + var ctxts []*Context + if expanding != nil { + ctxts = append(ctxts, expanding.inst.ctxt) + } + if ctxt != nil { + ctxts = append(ctxts, ctxt) + } + assert(len(ctxts) > 0) + + // Compute all hashes; hashes may differ across contexts due to different + // unique IDs for Named types within the hasher. + hashes := make([]string, len(ctxts)) + for i, ctxt := range ctxts { + hashes[i] = ctxt.instanceHash(orig, targs) + } + + // If local is non-nil, updateContexts return the type recorded in + // local. + updateContexts := func(res Type) Type { + for i := len(ctxts) - 1; i >= 0; i-- { + res = ctxts[i].update(hashes[i], orig, targs, res) + } + return res + } + + // typ may already have been instantiated with identical type arguments. In + // that case, re-use the existing instance. + for i, ctxt := range ctxts { + if inst := ctxt.lookup(hashes[i], orig, targs); inst != nil { + return updateContexts(inst) + } + } + + switch orig := orig.(type) { + case *Named: + res = check.newNamedInstance(pos, orig, targs, expanding) // substituted lazily + + case *Signature: + assert(expanding == nil) // function instances cannot be reached from Named types + + tparams := orig.TypeParams() + if !check.validateTArgLen(pos, tparams.Len(), len(targs)) { + return Typ[Invalid] + } + if tparams.Len() == 0 { + return orig // nothing to do (minor optimization) + } + sig := check.subst(pos, orig, makeSubstMap(tparams.list(), targs), nil, ctxt).(*Signature) + // If the signature doesn't use its type parameters, subst + // will not make a copy. In that case, make a copy now (so + // we can set tparams to nil w/o causing side-effects). + if sig == orig { + copy := *sig + sig = © + } + // After instantiating a generic signature, it is not generic + // anymore; we need to set tparams to nil. + sig.tparams = nil + res = sig + + default: + // only types and functions can be generic + panic(fmt.Sprintf("%v: cannot instantiate %v", pos, orig)) + } + + // Update all contexts; it's possible that we've lost a race. + return updateContexts(res) +} + +// validateTArgLen verifies that the length of targs and tparams matches, +// reporting an error if not. If validation fails and check is nil, +// validateTArgLen panics. +func (check *Checker) validateTArgLen(pos token.Pos, ntparams, ntargs int) bool { + if ntargs != ntparams { + // TODO(gri) provide better error message + if check != nil { + check.errorf(atPos(pos), WrongTypeArgCount, "got %d arguments but %d type parameters", ntargs, ntparams) + return false + } + panic(fmt.Sprintf("%v: got %d arguments but %d type parameters", pos, ntargs, ntparams)) + } + return true +} + +func (check *Checker) verify(pos token.Pos, tparams []*TypeParam, targs []Type, ctxt *Context) (int, error) { + smap := makeSubstMap(tparams, targs) + for i, tpar := range tparams { + // Ensure that we have a (possibly implicit) interface as type bound (issue #51048). + tpar.iface() + // The type parameter bound is parameterized with the same type parameters + // as the instantiated type; before we can use it for bounds checking we + // need to instantiate it with the type arguments with which we instantiated + // the parameterized type. + bound := check.subst(pos, tpar.bound, smap, nil, ctxt) + var cause string + if !check.implements(targs[i], bound, true, &cause) { + return i, errors.New(cause) + } + } + return -1, nil +} + +// implements checks if V implements T. The receiver may be nil if implements +// is called through an exported API call such as AssignableTo. If constraint +// is set, T is a type constraint. +// +// If the provided cause is non-nil, it may be set to an error string +// explaining why V does not implement (or satisfy, for constraints) T. +func (check *Checker) implements(V, T Type, constraint bool, cause *string) bool { + Vu := under(V) + Tu := under(T) + if Vu == Typ[Invalid] || Tu == Typ[Invalid] { + return true // avoid follow-on errors + } + if p, _ := Vu.(*Pointer); p != nil && under(p.base) == Typ[Invalid] { + return true // avoid follow-on errors (see issue #49541 for an example) + } + + verb := "implement" + if constraint { + verb = "satisfy" + } + + Ti, _ := Tu.(*Interface) + if Ti == nil { + if cause != nil { + var detail string + if isInterfacePtr(Tu) { + detail = check.sprintf("type %s is pointer to interface, not interface", T) + } else { + detail = check.sprintf("%s is not an interface", T) + } + *cause = check.sprintf("%s does not %s %s (%s)", V, verb, T, detail) + } + return false + } + + // Every type satisfies the empty interface. + if Ti.Empty() { + return true + } + // T is not the empty interface (i.e., the type set of T is restricted) + + // An interface V with an empty type set satisfies any interface. + // (The empty set is a subset of any set.) + Vi, _ := Vu.(*Interface) + if Vi != nil && Vi.typeSet().IsEmpty() { + return true + } + // type set of V is not empty + + // No type with non-empty type set satisfies the empty type set. + if Ti.typeSet().IsEmpty() { + if cause != nil { + *cause = check.sprintf("cannot %s %s (empty type set)", verb, T) + } + return false + } + + // V must implement T's methods, if any. + if m, wrong := check.missingMethod(V, Ti, true); m != nil /* !Implements(V, Ti) */ { + if cause != nil { + *cause = check.sprintf("%s does not %s %s %s", V, verb, T, check.missingMethodCause(V, T, m, wrong)) + } + return false + } + + // Only check comparability if we don't have a more specific error. + checkComparability := func() bool { + if !Ti.IsComparable() { + return true + } + // If T is comparable, V must be comparable. + // If V is strictly comparable, we're done. + if comparable(V, false /* strict comparability */, nil, nil) { + return true + } + // If check.conf.OldComparableSemantics is set (by the compiler or + // a test), we only consider strict comparability and we're done. + // TODO(gri) remove this check for Go 1.21 + if check != nil && check.conf.oldComparableSemantics { + if cause != nil { + *cause = check.sprintf("%s does not %s comparable", V, verb) + } + return false + } + // For constraint satisfaction, use dynamic (spec) comparability + // so that ordinary, non-type parameter interfaces implement comparable. + if constraint && comparable(V, true /* spec comparability */, nil, nil) { + // V is comparable if we are at Go 1.20 or higher. + if check == nil || check.allowVersion(check.pkg, 1, 20) { + return true + } + if cause != nil { + *cause = check.sprintf("%s to %s comparable requires go1.20 or later", V, verb) + } + return false + } + if cause != nil { + *cause = check.sprintf("%s does not %s comparable", V, verb) + } + return false + } + + // V must also be in the set of types of T, if any. + // Constraints with empty type sets were already excluded above. + if !Ti.typeSet().hasTerms() { + return checkComparability() // nothing to do + } + + // If V is itself an interface, each of its possible types must be in the set + // of T types (i.e., the V type set must be a subset of the T type set). + // Interfaces V with empty type sets were already excluded above. + if Vi != nil { + if !Vi.typeSet().subsetOf(Ti.typeSet()) { + // TODO(gri) report which type is missing + if cause != nil { + *cause = check.sprintf("%s does not %s %s", V, verb, T) + } + return false + } + return checkComparability() + } + + // Otherwise, V's type must be included in the iface type set. + var alt Type + if Ti.typeSet().is(func(t *term) bool { + if !t.includes(V) { + // If V āˆ‰ t.typ but V āˆˆ ~t.typ then remember this type + // so we can suggest it as an alternative in the error + // message. + if alt == nil && !t.tilde && Identical(t.typ, under(t.typ)) { + tt := *t + tt.tilde = true + if tt.includes(V) { + alt = t.typ + } + } + return true + } + return false + }) { + if cause != nil { + var detail string + switch { + case alt != nil: + detail = check.sprintf("possibly missing ~ for %s in %s", alt, T) + case mentions(Ti, V): + detail = check.sprintf("%s mentions %s, but %s is not in the type set of %s", T, V, V, T) + default: + detail = check.sprintf("%s missing in %s", V, Ti.typeSet().terms) + } + *cause = check.sprintf("%s does not %s %s (%s)", V, verb, T, detail) + } + return false + } + + return checkComparability() +} + +// mentions reports whether type T "mentions" typ in an (embedded) element or term +// of T (whether typ is in the type set of T or not). For better error messages. +func mentions(T, typ Type) bool { + switch T := T.(type) { + case *Interface: + for _, e := range T.embeddeds { + if mentions(e, typ) { + return true + } + } + case *Union: + for _, t := range T.terms { + if mentions(t.typ, typ) { + return true + } + } + default: + if Identical(T, typ) { + return true + } + } + return false +} diff --git a/src/go/types/instantiate_test.go b/src/go/types/instantiate_test.go new file mode 100644 index 0000000..0b44a1a --- /dev/null +++ b/src/go/types/instantiate_test.go @@ -0,0 +1,238 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "go/token" + . "go/types" + "strings" + "testing" +) + +func TestInstantiateEquality(t *testing.T) { + emptySignature := NewSignatureType(nil, nil, nil, nil, nil, false) + tests := []struct { + src string + name1 string + targs1 []Type + name2 string + targs2 []Type + wantEqual bool + }{ + { + "package basictype; type T[P any] int", + "T", []Type{Typ[Int]}, + "T", []Type{Typ[Int]}, + true, + }, + { + "package differenttypeargs; type T[P any] int", + "T", []Type{Typ[Int]}, + "T", []Type{Typ[String]}, + false, + }, + { + "package typeslice; type T[P any] int", + "T", []Type{NewSlice(Typ[Int])}, + "T", []Type{NewSlice(Typ[Int])}, + true, + }, + { + // interface{interface{...}} is equivalent to interface{...} + "package equivalentinterfaces; type T[P any] int", + "T", []Type{ + NewInterfaceType([]*Func{NewFunc(token.NoPos, nil, "M", emptySignature)}, nil), + }, + "T", []Type{ + NewInterfaceType( + nil, + []Type{ + NewInterfaceType([]*Func{NewFunc(token.NoPos, nil, "M", emptySignature)}, nil), + }, + ), + }, + true, + }, + { + // int|string is equivalent to string|int + "package equivalenttypesets; type T[P any] int", + "T", []Type{ + NewInterfaceType(nil, []Type{ + NewUnion([]*Term{NewTerm(false, Typ[Int]), NewTerm(false, Typ[String])}), + }), + }, + "T", []Type{ + NewInterfaceType(nil, []Type{ + NewUnion([]*Term{NewTerm(false, Typ[String]), NewTerm(false, Typ[Int])}), + }), + }, + true, + }, + { + "package basicfunc; func F[P any]() {}", + "F", []Type{Typ[Int]}, + "F", []Type{Typ[Int]}, + true, + }, + { + "package funcslice; func F[P any]() {}", + "F", []Type{NewSlice(Typ[Int])}, + "F", []Type{NewSlice(Typ[Int])}, + true, + }, + { + "package funcwithparams; func F[P any](x string) float64 { return 0 }", + "F", []Type{Typ[Int]}, + "F", []Type{Typ[Int]}, + true, + }, + { + "package differentfuncargs; func F[P any](x string) float64 { return 0 }", + "F", []Type{Typ[Int]}, + "F", []Type{Typ[String]}, + false, + }, + { + "package funcequality; func F1[P any](x int) {}; func F2[Q any](x int) {}", + "F1", []Type{Typ[Int]}, + "F2", []Type{Typ[Int]}, + false, + }, + { + "package funcsymmetry; func F1[P any](x P) {}; func F2[Q any](x Q) {}", + "F1", []Type{Typ[Int]}, + "F2", []Type{Typ[Int]}, + false, + }, + } + + for _, test := range tests { + pkg := mustTypecheck(".", test.src, nil) + + t.Run(pkg.Name(), func(t *testing.T) { + ctxt := NewContext() + + T1 := pkg.Scope().Lookup(test.name1).Type() + res1, err := Instantiate(ctxt, T1, test.targs1, false) + if err != nil { + t.Fatal(err) + } + + T2 := pkg.Scope().Lookup(test.name2).Type() + res2, err := Instantiate(ctxt, T2, test.targs2, false) + if err != nil { + t.Fatal(err) + } + + if gotEqual := res1 == res2; gotEqual != test.wantEqual { + t.Errorf("%s == %s: %t, want %t", res1, res2, gotEqual, test.wantEqual) + } + }) + } +} + +func TestInstantiateNonEquality(t *testing.T) { + const src = "package p; type T[P any] int" + + pkg1 := mustTypecheck(".", src, nil) + pkg2 := mustTypecheck(".", src, nil) + + // We consider T1 and T2 to be distinct types, so their instances should not + // be deduplicated by the context. + T1 := pkg1.Scope().Lookup("T").Type().(*Named) + T2 := pkg2.Scope().Lookup("T").Type().(*Named) + + ctxt := NewContext() + res1, err := Instantiate(ctxt, T1, []Type{Typ[Int]}, false) + if err != nil { + t.Fatal(err) + } + res2, err := Instantiate(ctxt, T2, []Type{Typ[Int]}, false) + if err != nil { + t.Fatal(err) + } + + if res1 == res2 { + t.Errorf("instance from pkg1 (%s) is pointer-equivalent to instance from pkg2 (%s)", res1, res2) + } + if Identical(res1, res2) { + t.Errorf("instance from pkg1 (%s) is identical to instance from pkg2 (%s)", res1, res2) + } +} + +func TestMethodInstantiation(t *testing.T) { + const prefix = `package p + +type T[P any] struct{} + +var X T[int] + +` + tests := []struct { + decl string + want string + }{ + {"func (r T[P]) m() P", "func (T[int]).m() int"}, + {"func (r T[P]) m(P)", "func (T[int]).m(int)"}, + {"func (r *T[P]) m(P)", "func (*T[int]).m(int)"}, + {"func (r T[P]) m() T[P]", "func (T[int]).m() T[int]"}, + {"func (r T[P]) m(T[P])", "func (T[int]).m(T[int])"}, + {"func (r T[P]) m(T[P], P, string)", "func (T[int]).m(T[int], int, string)"}, + {"func (r T[P]) m(T[P], T[string], T[int])", "func (T[int]).m(T[int], T[string], T[int])"}, + } + + for _, test := range tests { + src := prefix + test.decl + pkg := mustTypecheck(".", src, nil) + typ := NewPointer(pkg.Scope().Lookup("X").Type()) + obj, _, _ := LookupFieldOrMethod(typ, false, pkg, "m") + m, _ := obj.(*Func) + if m == nil { + t.Fatalf(`LookupFieldOrMethod(%s, "m") = %v, want func m`, typ, obj) + } + if got := ObjectString(m, RelativeTo(pkg)); got != test.want { + t.Errorf("instantiated %q, want %q", got, test.want) + } + } +} + +func TestImmutableSignatures(t *testing.T) { + const src = `package p + +type T[P any] struct{} + +func (T[P]) m() {} + +var _ T[int] +` + pkg := mustTypecheck(".", src, nil) + typ := pkg.Scope().Lookup("T").Type().(*Named) + obj, _, _ := LookupFieldOrMethod(typ, false, pkg, "m") + if obj == nil { + t.Fatalf(`LookupFieldOrMethod(%s, "m") = %v, want func m`, typ, obj) + } + + // Verify that the original method is not mutated by instantiating T (this + // bug manifested when subst did not return a new signature). + want := "func (T[P]).m()" + if got := stripAnnotations(ObjectString(obj, RelativeTo(pkg))); got != want { + t.Errorf("instantiated %q, want %q", got, want) + } +} + +// Copied from errors.go. +func stripAnnotations(s string) string { + var buf strings.Builder + for _, r := range s { + // strip #'s and subscript digits + if r < 'ā‚€' || 'ā‚€'+10 <= r { // 'ā‚€' == U+2080 + buf.WriteRune(r) + } + } + if buf.Len() < len(s) { + return buf.String() + } + return s +} diff --git a/src/go/types/interface.go b/src/go/types/interface.go new file mode 100644 index 0000000..83538d2 --- /dev/null +++ b/src/go/types/interface.go @@ -0,0 +1,233 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "go/ast" + "go/token" + . "internal/types/errors" +) + +// ---------------------------------------------------------------------------- +// API + +// An Interface represents an interface type. +type Interface struct { + check *Checker // for error reporting; nil once type set is computed + methods []*Func // ordered list of explicitly declared methods + embeddeds []Type // ordered list of explicitly embedded elements + embedPos *[]token.Pos // positions of embedded elements; or nil (for error messages) - use pointer to save space + implicit bool // interface is wrapper for type set literal (non-interface T, ~T, or A|B) + complete bool // indicates that obj, methods, and embeddeds are set and type set can be computed + + tset *_TypeSet // type set described by this interface, computed lazily +} + +// typeSet returns the type set for interface t. +func (t *Interface) typeSet() *_TypeSet { return computeInterfaceTypeSet(t.check, token.NoPos, t) } + +// emptyInterface represents the empty (completed) interface +var emptyInterface = Interface{complete: true, tset: &topTypeSet} + +// NewInterface returns a new interface for the given methods and embedded types. +// NewInterface takes ownership of the provided methods and may modify their types +// by setting missing receivers. +// +// Deprecated: Use NewInterfaceType instead which allows arbitrary embedded types. +func NewInterface(methods []*Func, embeddeds []*Named) *Interface { + tnames := make([]Type, len(embeddeds)) + for i, t := range embeddeds { + tnames[i] = t + } + return NewInterfaceType(methods, tnames) +} + +// NewInterfaceType returns a new interface for the given methods and embedded +// types. NewInterfaceType takes ownership of the provided methods and may +// modify their types by setting missing receivers. +// +// To avoid race conditions, the interface's type set should be computed before +// concurrent use of the interface, by explicitly calling Complete. +func NewInterfaceType(methods []*Func, embeddeds []Type) *Interface { + if len(methods) == 0 && len(embeddeds) == 0 { + return &emptyInterface + } + + // set method receivers if necessary + typ := (*Checker)(nil).newInterface() + for _, m := range methods { + if sig := m.typ.(*Signature); sig.recv == nil { + sig.recv = NewVar(m.pos, m.pkg, "", typ) + } + } + + // sort for API stability + sortMethods(methods) + + typ.methods = methods + typ.embeddeds = embeddeds + typ.complete = true + + return typ +} + +// check may be nil +func (check *Checker) newInterface() *Interface { + typ := &Interface{check: check} + if check != nil { + check.needsCleanup(typ) + } + return typ +} + +// MarkImplicit marks the interface t as implicit, meaning this interface +// corresponds to a constraint literal such as ~T or A|B without explicit +// interface embedding. MarkImplicit should be called before any concurrent use +// of implicit interfaces. +func (t *Interface) MarkImplicit() { + t.implicit = true +} + +// NumExplicitMethods returns the number of explicitly declared methods of interface t. +func (t *Interface) NumExplicitMethods() int { return len(t.methods) } + +// ExplicitMethod returns the i'th explicitly declared method of interface t for 0 <= i < t.NumExplicitMethods(). +// The methods are ordered by their unique Id. +func (t *Interface) ExplicitMethod(i int) *Func { return t.methods[i] } + +// NumEmbeddeds returns the number of embedded types in interface t. +func (t *Interface) NumEmbeddeds() int { return len(t.embeddeds) } + +// Embedded returns the i'th embedded defined (*Named) type of interface t for 0 <= i < t.NumEmbeddeds(). +// The result is nil if the i'th embedded type is not a defined type. +// +// Deprecated: Use EmbeddedType which is not restricted to defined (*Named) types. +func (t *Interface) Embedded(i int) *Named { tname, _ := t.embeddeds[i].(*Named); return tname } + +// EmbeddedType returns the i'th embedded type of interface t for 0 <= i < t.NumEmbeddeds(). +func (t *Interface) EmbeddedType(i int) Type { return t.embeddeds[i] } + +// NumMethods returns the total number of methods of interface t. +func (t *Interface) NumMethods() int { return t.typeSet().NumMethods() } + +// Method returns the i'th method of interface t for 0 <= i < t.NumMethods(). +// The methods are ordered by their unique Id. +func (t *Interface) Method(i int) *Func { return t.typeSet().Method(i) } + +// Empty reports whether t is the empty interface. +func (t *Interface) Empty() bool { return t.typeSet().IsAll() } + +// IsComparable reports whether each type in interface t's type set is comparable. +func (t *Interface) IsComparable() bool { return t.typeSet().IsComparable(nil) } + +// IsMethodSet reports whether the interface t is fully described by its method +// set. +func (t *Interface) IsMethodSet() bool { return t.typeSet().IsMethodSet() } + +// IsImplicit reports whether the interface t is a wrapper for a type set literal. +func (t *Interface) IsImplicit() bool { return t.implicit } + +// Complete computes the interface's type set. It must be called by users of +// NewInterfaceType and NewInterface after the interface's embedded types are +// fully defined and before using the interface type in any way other than to +// form other types. The interface must not contain duplicate methods or a +// panic occurs. Complete returns the receiver. +// +// Interface types that have been completed are safe for concurrent use. +func (t *Interface) Complete() *Interface { + if !t.complete { + t.complete = true + } + t.typeSet() // checks if t.tset is already set + return t +} + +func (t *Interface) Underlying() Type { return t } +func (t *Interface) String() string { return TypeString(t, nil) } + +// ---------------------------------------------------------------------------- +// Implementation + +func (t *Interface) cleanup() { + t.check = nil + t.embedPos = nil +} + +func (check *Checker) interfaceType(ityp *Interface, iface *ast.InterfaceType, def *Named) { + addEmbedded := func(pos token.Pos, typ Type) { + ityp.embeddeds = append(ityp.embeddeds, typ) + if ityp.embedPos == nil { + ityp.embedPos = new([]token.Pos) + } + *ityp.embedPos = append(*ityp.embedPos, pos) + } + + for _, f := range iface.Methods.List { + if len(f.Names) == 0 { + addEmbedded(f.Type.Pos(), parseUnion(check, f.Type)) + continue + } + // f.Name != nil + + // We have a method with name f.Names[0]. + name := f.Names[0] + if name.Name == "_" { + check.error(name, BlankIfaceMethod, "methods must have a unique non-blank name") + continue // ignore + } + + typ := check.typ(f.Type) + sig, _ := typ.(*Signature) + if sig == nil { + if typ != Typ[Invalid] { + check.errorf(f.Type, InvalidSyntaxTree, "%s is not a method signature", typ) + } + continue // ignore + } + + // Always type-check method type parameters but complain if they are not enabled. + // (This extra check is needed here because interface method signatures don't have + // a receiver specification.) + if sig.tparams != nil { + var at positioner = f.Type + if ftyp, _ := f.Type.(*ast.FuncType); ftyp != nil && ftyp.TypeParams != nil { + at = ftyp.TypeParams + } + check.error(at, InvalidMethodTypeParams, "methods cannot have type parameters") + } + + // use named receiver type if available (for better error messages) + var recvTyp Type = ityp + if def != nil { + recvTyp = def + } + sig.recv = NewVar(name.Pos(), check.pkg, "", recvTyp) + + m := NewFunc(name.Pos(), check.pkg, name.Name, sig) + check.recordDef(name, m) + ityp.methods = append(ityp.methods, m) + } + + // All methods and embedded elements for this interface are collected; + // i.e., this interface may be used in a type set computation. + ityp.complete = true + + if len(ityp.methods) == 0 && len(ityp.embeddeds) == 0 { + // empty interface + ityp.tset = &topTypeSet + return + } + + // sort for API stability + sortMethods(ityp.methods) + // (don't sort embeddeds: they must correspond to *embedPos entries) + + // Compute type set as soon as possible to report any errors. + // Subsequent uses of type sets will use this computed type + // set and won't need to pass in a *Checker. + check.later(func() { + computeInterfaceTypeSet(check, iface.Pos(), ityp) + }).describef(iface, "compute type set for %s", ityp) +} diff --git a/src/go/types/issues_test.go b/src/go/types/issues_test.go new file mode 100644 index 0000000..b4845b1 --- /dev/null +++ b/src/go/types/issues_test.go @@ -0,0 +1,854 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements tests for various issues. + +package types_test + +import ( + "fmt" + "go/ast" + "go/importer" + "go/token" + "internal/testenv" + "regexp" + "sort" + "strings" + "testing" + + . "go/types" +) + +func TestIssue5770(t *testing.T) { + f := mustParse(fset, "", `package p; type S struct{T}`) + conf := Config{Importer: importer.Default()} + _, err := conf.Check(f.Name.Name, fset, []*ast.File{f}, nil) // do not crash + const want = "undefined: T" + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("got: %v; want: %s", err, want) + } +} + +func TestIssue5849(t *testing.T) { + src := ` +package p +var ( + s uint + _ = uint8(8) + _ = uint16(16) << s + _ = uint32(32 << s) + _ = uint64(64 << s + s) + _ = (interface{})("foo") + _ = (interface{})(nil) +)` + types := make(map[ast.Expr]TypeAndValue) + mustTypecheck("p", src, &Info{Types: types}) + + for x, tv := range types { + var want Type + switch x := x.(type) { + case *ast.BasicLit: + switch x.Value { + case `8`: + want = Typ[Uint8] + case `16`: + want = Typ[Uint16] + case `32`: + want = Typ[Uint32] + case `64`: + want = Typ[Uint] // because of "+ s", s is of type uint + case `"foo"`: + want = Typ[String] + } + case *ast.Ident: + if x.Name == "nil" { + want = Typ[UntypedNil] + } + } + if want != nil && !Identical(tv.Type, want) { + t.Errorf("got %s; want %s", tv.Type, want) + } + } +} + +func TestIssue6413(t *testing.T) { + src := ` +package p +func f() int { + defer f() + go f() + return 0 +} +` + types := make(map[ast.Expr]TypeAndValue) + mustTypecheck("p", src, &Info{Types: types}) + + want := Typ[Int] + n := 0 + for x, tv := range types { + if _, ok := x.(*ast.CallExpr); ok { + if tv.Type != want { + t.Errorf("%s: got %s; want %s", fset.Position(x.Pos()), tv.Type, want) + } + n++ + } + } + + if n != 2 { + t.Errorf("got %d CallExprs; want 2", n) + } +} + +func TestIssue7245(t *testing.T) { + src := ` +package p +func (T) m() (res bool) { return } +type T struct{} // receiver type after method declaration +` + f := mustParse(fset, "", src) + + var conf Config + defs := make(map[*ast.Ident]Object) + _, err := conf.Check(f.Name.Name, fset, []*ast.File{f}, &Info{Defs: defs}) + if err != nil { + t.Fatal(err) + } + + m := f.Decls[0].(*ast.FuncDecl) + res1 := defs[m.Name].(*Func).Type().(*Signature).Results().At(0) + res2 := defs[m.Type.Results.List[0].Names[0]].(*Var) + + if res1 != res2 { + t.Errorf("got %s (%p) != %s (%p)", res1, res2, res1, res2) + } +} + +// This tests that uses of existing vars on the LHS of an assignment +// are Uses, not Defs; and also that the (illegal) use of a non-var on +// the LHS of an assignment is a Use nonetheless. +func TestIssue7827(t *testing.T) { + const src = ` +package p +func _() { + const w = 1 // defs w + x, y := 2, 3 // defs x, y + w, x, z := 4, 5, 6 // uses w, x, defs z; error: cannot assign to w + _, _, _ = x, y, z // uses x, y, z +} +` + f := mustParse(fset, "", src) + + const want = `L3 defs func p._() +L4 defs const w untyped int +L5 defs var x int +L5 defs var y int +L6 defs var z int +L6 uses const w untyped int +L6 uses var x int +L7 uses var x int +L7 uses var y int +L7 uses var z int` + + // don't abort at the first error + conf := Config{Error: func(err error) { t.Log(err) }} + defs := make(map[*ast.Ident]Object) + uses := make(map[*ast.Ident]Object) + _, err := conf.Check(f.Name.Name, fset, []*ast.File{f}, &Info{Defs: defs, Uses: uses}) + if s := fmt.Sprint(err); !strings.HasSuffix(s, "cannot assign to w") { + t.Errorf("Check: unexpected error: %s", s) + } + + var facts []string + for id, obj := range defs { + if obj != nil { + fact := fmt.Sprintf("L%d defs %s", fset.Position(id.Pos()).Line, obj) + facts = append(facts, fact) + } + } + for id, obj := range uses { + fact := fmt.Sprintf("L%d uses %s", fset.Position(id.Pos()).Line, obj) + facts = append(facts, fact) + } + sort.Strings(facts) + + got := strings.Join(facts, "\n") + if got != want { + t.Errorf("Unexpected defs/uses\ngot:\n%s\nwant:\n%s", got, want) + } +} + +// This tests that the package associated with the types.Object.Pkg method +// is the type's package independent of the order in which the imports are +// listed in the sources src1, src2 below. +// The actual issue is in go/internal/gcimporter which has a corresponding +// test; we leave this test here to verify correct behavior at the go/types +// level. +func TestIssue13898(t *testing.T) { + testenv.MustHaveGoBuild(t) + + const src0 = ` +package main + +import "go/types" + +func main() { + var info types.Info + for _, obj := range info.Uses { + _ = obj.Pkg() + } +} +` + // like src0, but also imports go/importer + const src1 = ` +package main + +import ( + "go/types" + _ "go/importer" +) + +func main() { + var info types.Info + for _, obj := range info.Uses { + _ = obj.Pkg() + } +} +` + // like src1 but with different import order + // (used to fail with this issue) + const src2 = ` +package main + +import ( + _ "go/importer" + "go/types" +) + +func main() { + var info types.Info + for _, obj := range info.Uses { + _ = obj.Pkg() + } +} +` + f := func(test, src string) { + f := mustParse(fset, "", src) + cfg := Config{Importer: importer.Default()} + info := Info{Uses: make(map[*ast.Ident]Object)} + _, err := cfg.Check("main", fset, []*ast.File{f}, &info) + if err != nil { + t.Fatal(err) + } + + var pkg *Package + count := 0 + for id, obj := range info.Uses { + if id.Name == "Pkg" { + pkg = obj.Pkg() + count++ + } + } + if count != 1 { + t.Fatalf("%s: got %d entries named Pkg; want 1", test, count) + } + if pkg.Name() != "types" { + t.Fatalf("%s: got %v; want package types", test, pkg) + } + } + + f("src0", src0) + f("src1", src1) + f("src2", src2) +} + +func TestIssue22525(t *testing.T) { + f := mustParse(fset, "", `package p; func f() { var a, b, c, d, e int }`) + + got := "\n" + conf := Config{Error: func(err error) { got += err.Error() + "\n" }} + conf.Check(f.Name.Name, fset, []*ast.File{f}, nil) // do not crash + want := ` +1:27: a declared and not used +1:30: b declared and not used +1:33: c declared and not used +1:36: d declared and not used +1:39: e declared and not used +` + if got != want { + t.Errorf("got: %swant: %s", got, want) + } +} + +func TestIssue25627(t *testing.T) { + const prefix = `package p; import "unsafe"; type P *struct{}; type I interface{}; type T ` + // The src strings (without prefix) are constructed such that the number of semicolons + // plus one corresponds to the number of fields expected in the respective struct. + for _, src := range []string{ + `struct { x Missing }`, + `struct { Missing }`, + `struct { *Missing }`, + `struct { unsafe.Pointer }`, + `struct { P }`, + `struct { *I }`, + `struct { a int; b Missing; *Missing }`, + } { + f := mustParse(fset, "", prefix+src) + + cfg := Config{Importer: importer.Default(), Error: func(err error) {}} + info := &Info{Types: make(map[ast.Expr]TypeAndValue)} + _, err := cfg.Check(f.Name.Name, fset, []*ast.File{f}, info) + if err != nil { + if _, ok := err.(Error); !ok { + t.Fatal(err) + } + } + + ast.Inspect(f, func(n ast.Node) bool { + if spec, _ := n.(*ast.TypeSpec); spec != nil { + if tv, ok := info.Types[spec.Type]; ok && spec.Name.Name == "T" { + want := strings.Count(src, ";") + 1 + if got := tv.Type.(*Struct).NumFields(); got != want { + t.Errorf("%s: got %d fields; want %d", src, got, want) + } + } + } + return true + }) + } +} + +func TestIssue28005(t *testing.T) { + // method names must match defining interface name for this test + // (see last comment in this function) + sources := [...]string{ + "package p; type A interface{ A() }", + "package p; type B interface{ B() }", + "package p; type X interface{ A; B }", + } + + // compute original file ASTs + var orig [len(sources)]*ast.File + for i, src := range sources { + orig[i] = mustParse(fset, "", src) + } + + // run the test for all order permutations of the incoming files + for _, perm := range [][len(sources)]int{ + {0, 1, 2}, + {0, 2, 1}, + {1, 0, 2}, + {1, 2, 0}, + {2, 0, 1}, + {2, 1, 0}, + } { + // create file order permutation + files := make([]*ast.File, len(sources)) + for i := range perm { + files[i] = orig[perm[i]] + } + + // type-check package with given file order permutation + var conf Config + info := &Info{Defs: make(map[*ast.Ident]Object)} + _, err := conf.Check("", fset, files, info) + if err != nil { + t.Fatal(err) + } + + // look for interface object X + var obj Object + for name, def := range info.Defs { + if name.Name == "X" { + obj = def + break + } + } + if obj == nil { + t.Fatal("object X not found") + } + iface := obj.Type().Underlying().(*Interface) // object X must be an interface + + // Each iface method m is embedded; and m's receiver base type name + // must match the method's name per the choice in the source file. + for i := 0; i < iface.NumMethods(); i++ { + m := iface.Method(i) + recvName := m.Type().(*Signature).Recv().Type().(*Named).Obj().Name() + if recvName != m.Name() { + t.Errorf("perm %v: got recv %s; want %s", perm, recvName, m.Name()) + } + } + } +} + +func TestIssue28282(t *testing.T) { + // create type interface { error } + et := Universe.Lookup("error").Type() + it := NewInterfaceType(nil, []Type{et}) + it.Complete() + // verify that after completing the interface, the embedded method remains unchanged + want := et.Underlying().(*Interface).Method(0) + got := it.Method(0) + if got != want { + t.Fatalf("%s.Method(0): got %q (%p); want %q (%p)", it, got, got, want, want) + } + // verify that lookup finds the same method in both interfaces (redundant check) + obj, _, _ := LookupFieldOrMethod(et, false, nil, "Error") + if obj != want { + t.Fatalf("%s.Lookup: got %q (%p); want %q (%p)", et, obj, obj, want, want) + } + obj, _, _ = LookupFieldOrMethod(it, false, nil, "Error") + if obj != want { + t.Fatalf("%s.Lookup: got %q (%p); want %q (%p)", it, obj, obj, want, want) + } +} + +func TestIssue29029(t *testing.T) { + f1 := mustParse(fset, "", `package p; type A interface { M() }`) + f2 := mustParse(fset, "", `package p; var B interface { A }`) + + // printInfo prints the *Func definitions recorded in info, one *Func per line. + printInfo := func(info *Info) string { + var buf strings.Builder + for _, obj := range info.Defs { + if fn, ok := obj.(*Func); ok { + fmt.Fprintln(&buf, fn) + } + } + return buf.String() + } + + // The *Func (method) definitions for package p must be the same + // independent on whether f1 and f2 are type-checked together, or + // incrementally. + + // type-check together + var conf Config + info := &Info{Defs: make(map[*ast.Ident]Object)} + check := NewChecker(&conf, fset, NewPackage("", "p"), info) + if err := check.Files([]*ast.File{f1, f2}); err != nil { + t.Fatal(err) + } + want := printInfo(info) + + // type-check incrementally + info = &Info{Defs: make(map[*ast.Ident]Object)} + check = NewChecker(&conf, fset, NewPackage("", "p"), info) + if err := check.Files([]*ast.File{f1}); err != nil { + t.Fatal(err) + } + if err := check.Files([]*ast.File{f2}); err != nil { + t.Fatal(err) + } + got := printInfo(info) + + if got != want { + t.Errorf("\ngot : %swant: %s", got, want) + } +} + +func TestIssue34151(t *testing.T) { + const asrc = `package a; type I interface{ M() }; type T struct { F interface { I } }` + const bsrc = `package b; import "a"; type T struct { F interface { a.I } }; var _ = a.T(T{})` + + a := mustTypecheck("a", asrc, nil) + + bast := mustParse(fset, "", bsrc) + conf := Config{Importer: importHelper{pkg: a}} + b, err := conf.Check(bast.Name.Name, fset, []*ast.File{bast}, nil) + if err != nil { + t.Errorf("package %s failed to typecheck: %v", b.Name(), err) + } +} + +type importHelper struct { + pkg *Package + fallback Importer +} + +func (h importHelper) Import(path string) (*Package, error) { + if path == h.pkg.Path() { + return h.pkg, nil + } + if h.fallback == nil { + return nil, fmt.Errorf("got package path %q; want %q", path, h.pkg.Path()) + } + return h.fallback.Import(path) +} + +// TestIssue34921 verifies that we don't update an imported type's underlying +// type when resolving an underlying type. Specifically, when determining the +// underlying type of b.T (which is the underlying type of a.T, which is int) +// we must not set the underlying type of a.T again since that would lead to +// a race condition if package b is imported elsewhere, in a package that is +// concurrently type-checked. +func TestIssue34921(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Error(r) + } + }() + + var sources = []string{ + `package a; type T int`, + `package b; import "a"; type T a.T`, + } + + var pkg *Package + for _, src := range sources { + f := mustParse(fset, "", src) + conf := Config{Importer: importHelper{pkg: pkg}} + res, err := conf.Check(f.Name.Name, fset, []*ast.File{f}, nil) + if err != nil { + t.Errorf("%q failed to typecheck: %v", src, err) + } + pkg = res // res is imported by the next package in this test + } +} + +func TestIssue43088(t *testing.T) { + // type T1 struct { + // _ T2 + // } + // + // type T2 struct { + // _ struct { + // _ T2 + // } + // } + n1 := NewTypeName(token.NoPos, nil, "T1", nil) + T1 := NewNamed(n1, nil, nil) + n2 := NewTypeName(token.NoPos, nil, "T2", nil) + T2 := NewNamed(n2, nil, nil) + s1 := NewStruct([]*Var{NewField(token.NoPos, nil, "_", T2, false)}, nil) + T1.SetUnderlying(s1) + s2 := NewStruct([]*Var{NewField(token.NoPos, nil, "_", T2, false)}, nil) + s3 := NewStruct([]*Var{NewField(token.NoPos, nil, "_", s2, false)}, nil) + T2.SetUnderlying(s3) + + // These calls must terminate (no endless recursion). + Comparable(T1) + Comparable(T2) +} + +func TestIssue44515(t *testing.T) { + typ := Unsafe.Scope().Lookup("Pointer").Type() + + got := TypeString(typ, nil) + want := "unsafe.Pointer" + if got != want { + t.Errorf("got %q; want %q", got, want) + } + + qf := func(pkg *Package) string { + if pkg == Unsafe { + return "foo" + } + return "" + } + got = TypeString(typ, qf) + want = "foo.Pointer" + if got != want { + t.Errorf("got %q; want %q", got, want) + } +} + +func TestIssue43124(t *testing.T) { + // TODO(rFindley) move this to testdata by enhancing support for importing. + + testenv.MustHaveGoBuild(t) // The go command is needed for the importer to determine the locations of stdlib .a files. + + // All involved packages have the same name (template). Error messages should + // disambiguate between text/template and html/template by printing the full + // path. + const ( + asrc = `package a; import "text/template"; func F(template.Template) {}; func G(int) {}` + bsrc = ` +package b + +import ( + "a" + "html/template" +) + +func _() { + // Packages should be fully qualified when there is ambiguity within the + // error string itself. + a.F(template /* ERROR cannot use.*html/template.* as .*text/template */ .Template{}) +} +` + csrc = ` +package c + +import ( + "a" + "fmt" + "html/template" +) + +// Issue #46905: make sure template is not the first package qualified. +var _ fmt.Stringer = 1 // ERROR cannot use 1.*as fmt\.Stringer + +// Packages should be fully qualified when there is ambiguity in reachable +// packages. In this case both a (and for that matter html/template) import +// text/template. +func _() { a.G(template /* ERROR cannot use .*html/template.*Template */ .Template{}) } +` + + tsrc = ` +package template + +import "text/template" + +type T int + +// Verify that the current package name also causes disambiguation. +var _ T = template /* ERROR cannot use.*text/template.* as T value */.Template{} +` + ) + + a := mustTypecheck("a", asrc, nil) + imp := importHelper{pkg: a, fallback: importer.Default()} + + testFiles(t, nil, []string{"b.go"}, [][]byte{[]byte(bsrc)}, false, imp) + testFiles(t, nil, []string{"c.go"}, [][]byte{[]byte(csrc)}, false, imp) + testFiles(t, nil, []string{"t.go"}, [][]byte{[]byte(tsrc)}, false, imp) +} + +func TestIssue50646(t *testing.T) { + anyType := Universe.Lookup("any").Type() + comparableType := Universe.Lookup("comparable").Type() + + if !Comparable(anyType) { + t.Error("any is not a comparable type") + } + if !Comparable(comparableType) { + t.Error("comparable is not a comparable type") + } + + if Implements(anyType, comparableType.Underlying().(*Interface)) { + t.Error("any implements comparable") + } + if !Implements(comparableType, anyType.(*Interface)) { + t.Error("comparable does not implement any") + } + + if AssignableTo(anyType, comparableType) { + t.Error("any assignable to comparable") + } + if !AssignableTo(comparableType, anyType) { + t.Error("comparable not assignable to any") + } +} + +func TestIssue55030(t *testing.T) { + // makeSig makes the signature func(typ...) + makeSig := func(typ Type) { + par := NewVar(token.NoPos, nil, "", typ) + params := NewTuple(par) + NewSignatureType(nil, nil, nil, params, nil, true) + } + + // makeSig must not panic for the following (example) types: + // []int + makeSig(NewSlice(Typ[Int])) + + // string + makeSig(Typ[String]) + + // P where P's core type is string + { + P := NewTypeName(token.NoPos, nil, "P", nil) // [P string] + makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{Typ[String]}))) + } + + // P where P's core type is an (unnamed) slice + { + P := NewTypeName(token.NoPos, nil, "P", nil) // [P []int] + makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{NewSlice(Typ[Int])}))) + } + + // P where P's core type is bytestring (i.e., string or []byte) + { + t1 := NewTerm(true, Typ[String]) // ~string + t2 := NewTerm(false, NewSlice(Typ[Byte])) // []byte + u := NewUnion([]*Term{t1, t2}) // ~string | []byte + P := NewTypeName(token.NoPos, nil, "P", nil) // [P ~string | []byte] + makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{u}))) + } +} + +func TestIssue51093(t *testing.T) { + // Each test stands for a conversion of the form P(val) + // where P is a type parameter with typ as constraint. + // The test ensures that P(val) has the correct type P + // and is not a constant. + var tests = []struct { + typ string + val string + }{ + {"bool", "false"}, + {"int", "-1"}, + {"uint", "1.0"}, + {"rune", "'a'"}, + {"float64", "3.5"}, + {"complex64", "1.25"}, + {"string", "\"foo\""}, + + // some more complex constraints + {"~byte", "1"}, + {"~int | ~float64 | complex128", "1"}, + {"~uint64 | ~rune", "'X'"}, + } + + for _, test := range tests { + src := fmt.Sprintf("package p; func _[P %s]() { _ = P(%s) }", test.typ, test.val) + types := make(map[ast.Expr]TypeAndValue) + mustTypecheck("p", src, &Info{Types: types}) + + var n int + for x, tv := range types { + if x, _ := x.(*ast.CallExpr); x != nil { + // there must be exactly one CallExpr which is the P(val) conversion + n++ + tpar, _ := tv.Type.(*TypeParam) + if tpar == nil { + t.Fatalf("%s: got type %s, want type parameter", ExprString(x), tv.Type) + } + if name := tpar.Obj().Name(); name != "P" { + t.Fatalf("%s: got type parameter name %s, want P", ExprString(x), name) + } + // P(val) must not be constant + if tv.Value != nil { + t.Errorf("%s: got constant value %s (%s), want no constant", ExprString(x), tv.Value, tv.Value.String()) + } + } + } + + if n != 1 { + t.Fatalf("%s: got %d CallExpr nodes; want 1", src, 1) + } + } +} + +func TestIssue54258(t *testing.T) { + + tests := []struct{ main, b, want string }{ + { //--------------------------------------------------------------- + `package main +import "b" +type I0 interface { + M0(w struct{ f string }) +} +var _ I0 = b.S{} +`, + `package b +type S struct{} +func (S) M0(struct{ f string }) {} +`, + `6:12: cannot use b[.]S{} [(]value of type b[.]S[)] as I0 value in variable declaration: b[.]S does not implement I0 [(]wrong type for method M0[)] +.*have M0[(]struct{f string /[*] package b [*]/ }[)] +.*want M0[(]struct{f string /[*] package main [*]/ }[)]`}, + + { //--------------------------------------------------------------- + `package main +import "b" +type I1 interface { + M1(struct{ string }) +} +var _ I1 = b.S{} +`, + `package b +type S struct{} +func (S) M1(struct{ string }) {} +`, + `6:12: cannot use b[.]S{} [(]value of type b[.]S[)] as I1 value in variable declaration: b[.]S does not implement I1 [(]wrong type for method M1[)] +.*have M1[(]struct{string /[*] package b [*]/ }[)] +.*want M1[(]struct{string /[*] package main [*]/ }[)]`}, + + { //--------------------------------------------------------------- + `package main +import "b" +type I2 interface { + M2(y struct{ f struct{ f string } }) +} +var _ I2 = b.S{} +`, + `package b +type S struct{} +func (S) M2(struct{ f struct{ f string } }) {} +`, + `6:12: cannot use b[.]S{} [(]value of type b[.]S[)] as I2 value in variable declaration: b[.]S does not implement I2 [(]wrong type for method M2[)] +.*have M2[(]struct{f struct{f string} /[*] package b [*]/ }[)] +.*want M2[(]struct{f struct{f string} /[*] package main [*]/ }[)]`}, + + { //--------------------------------------------------------------- + `package main +import "b" +type I3 interface { + M3(z struct{ F struct{ f string } }) +} +var _ I3 = b.S{} +`, + `package b +type S struct{} +func (S) M3(struct{ F struct{ f string } }) {} +`, + `6:12: cannot use b[.]S{} [(]value of type b[.]S[)] as I3 value in variable declaration: b[.]S does not implement I3 [(]wrong type for method M3[)] +.*have M3[(]struct{F struct{f string /[*] package b [*]/ }}[)] +.*want M3[(]struct{F struct{f string /[*] package main [*]/ }}[)]`}, + + { //--------------------------------------------------------------- + `package main +import "b" +type I4 interface { + M4(_ struct { *string }) +} +var _ I4 = b.S{} +`, + `package b +type S struct{} +func (S) M4(struct { *string }) {} +`, + `6:12: cannot use b[.]S{} [(]value of type b[.]S[)] as I4 value in variable declaration: b[.]S does not implement I4 [(]wrong type for method M4[)] +.*have M4[(]struct{[*]string /[*] package b [*]/ }[)] +.*want M4[(]struct{[*]string /[*] package main [*]/ }[)]`}, + + { //--------------------------------------------------------------- + `package main +import "b" +type t struct{ A int } +type I5 interface { + M5(_ struct {b.S;t}) +} +var _ I5 = b.S{} +`, + `package b +type S struct{} +type t struct{ A int } +func (S) M5(struct {S;t}) {} +`, + `7:12: cannot use b[.]S{} [(]value of type b[.]S[)] as I5 value in variable declaration: b[.]S does not implement I5 [(]wrong type for method M5[)] +.*have M5[(]struct{b[.]S; b[.]t}[)] +.*want M5[(]struct{b[.]S; t}[)]`}, + } + + fset := token.NewFileSet() + test := func(main, b, want string) { + re := regexp.MustCompile(want) + bpkg := mustTypecheck("b", b, nil) + mast := mustParse(fset, "main.go", main) + conf := Config{Importer: importHelper{pkg: bpkg}} + _, err := conf.Check(mast.Name.Name, fset, []*ast.File{mast}, nil) + if err == nil { + t.Error("Expected failure, but it did not") + } else if got := err.Error(); !re.MatchString(got) { + t.Errorf("Wanted match for\n\t%s\n but got\n\t%s", want, got) + } else if testing.Verbose() { + t.Logf("Saw expected\n\t%s", err.Error()) + } + } + for _, t := range tests { + test(t.main, t.b, t.want) + } +} diff --git a/src/go/types/labels.go b/src/go/types/labels.go new file mode 100644 index 0000000..5ee941e --- /dev/null +++ b/src/go/types/labels.go @@ -0,0 +1,274 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "go/ast" + "go/token" + . "internal/types/errors" +) + +// labels checks correct label use in body. +func (check *Checker) labels(body *ast.BlockStmt) { + // set of all labels in this body + all := NewScope(nil, body.Pos(), body.End(), "label") + + fwdJumps := check.blockBranches(all, nil, nil, body.List) + + // If there are any forward jumps left, no label was found for + // the corresponding goto statements. Either those labels were + // never defined, or they are inside blocks and not reachable + // for the respective gotos. + for _, jmp := range fwdJumps { + var msg string + var code Code + name := jmp.Label.Name + if alt := all.Lookup(name); alt != nil { + msg = "goto %s jumps into block" + alt.(*Label).used = true // avoid another error + code = JumpIntoBlock + } else { + msg = "label %s not declared" + code = UndeclaredLabel + } + check.errorf(jmp.Label, code, msg, name) + } + + // spec: "It is illegal to define a label that is never used." + for name, obj := range all.elems { + obj = resolve(name, obj) + if lbl := obj.(*Label); !lbl.used { + check.softErrorf(lbl, UnusedLabel, "label %s declared and not used", lbl.name) + } + } +} + +// A block tracks label declarations in a block and its enclosing blocks. +type block struct { + parent *block // enclosing block + lstmt *ast.LabeledStmt // labeled statement to which this block belongs, or nil + labels map[string]*ast.LabeledStmt // allocated lazily +} + +// insert records a new label declaration for the current block. +// The label must not have been declared before in any block. +func (b *block) insert(s *ast.LabeledStmt) { + name := s.Label.Name + if debug { + assert(b.gotoTarget(name) == nil) + } + labels := b.labels + if labels == nil { + labels = make(map[string]*ast.LabeledStmt) + b.labels = labels + } + labels[name] = s +} + +// gotoTarget returns the labeled statement in the current +// or an enclosing block with the given label name, or nil. +func (b *block) gotoTarget(name string) *ast.LabeledStmt { + for s := b; s != nil; s = s.parent { + if t := s.labels[name]; t != nil { + return t + } + } + return nil +} + +// enclosingTarget returns the innermost enclosing labeled +// statement with the given label name, or nil. +func (b *block) enclosingTarget(name string) *ast.LabeledStmt { + for s := b; s != nil; s = s.parent { + if t := s.lstmt; t != nil && t.Label.Name == name { + return t + } + } + return nil +} + +// blockBranches processes a block's statement list and returns the set of outgoing forward jumps. +// all is the scope of all declared labels, parent the set of labels declared in the immediately +// enclosing block, and lstmt is the labeled statement this block is associated with (or nil). +func (check *Checker) blockBranches(all *Scope, parent *block, lstmt *ast.LabeledStmt, list []ast.Stmt) []*ast.BranchStmt { + b := &block{parent: parent, lstmt: lstmt} + + var ( + varDeclPos token.Pos + fwdJumps, badJumps []*ast.BranchStmt + ) + + // All forward jumps jumping over a variable declaration are possibly + // invalid (they may still jump out of the block and be ok). + // recordVarDecl records them for the given position. + recordVarDecl := func(pos token.Pos) { + varDeclPos = pos + badJumps = append(badJumps[:0], fwdJumps...) // copy fwdJumps to badJumps + } + + jumpsOverVarDecl := func(jmp *ast.BranchStmt) bool { + if varDeclPos.IsValid() { + for _, bad := range badJumps { + if jmp == bad { + return true + } + } + } + return false + } + + blockBranches := func(lstmt *ast.LabeledStmt, list []ast.Stmt) { + // Unresolved forward jumps inside the nested block + // become forward jumps in the current block. + fwdJumps = append(fwdJumps, check.blockBranches(all, b, lstmt, list)...) + } + + var stmtBranches func(ast.Stmt) + stmtBranches = func(s ast.Stmt) { + switch s := s.(type) { + case *ast.DeclStmt: + if d, _ := s.Decl.(*ast.GenDecl); d != nil && d.Tok == token.VAR { + recordVarDecl(d.Pos()) + } + + case *ast.LabeledStmt: + // declare non-blank label + if name := s.Label.Name; name != "_" { + lbl := NewLabel(s.Label.Pos(), check.pkg, name) + if alt := all.Insert(lbl); alt != nil { + check.softErrorf(lbl, DuplicateLabel, "label %s already declared", name) + check.reportAltDecl(alt) + // ok to continue + } else { + b.insert(s) + check.recordDef(s.Label, lbl) + } + // resolve matching forward jumps and remove them from fwdJumps + i := 0 + for _, jmp := range fwdJumps { + if jmp.Label.Name == name { + // match + lbl.used = true + check.recordUse(jmp.Label, lbl) + if jumpsOverVarDecl(jmp) { + check.softErrorf( + jmp.Label, + JumpOverDecl, + "goto %s jumps over variable declaration at line %d", + name, + check.fset.Position(varDeclPos).Line, + ) + // ok to continue + } + } else { + // no match - record new forward jump + fwdJumps[i] = jmp + i++ + } + } + fwdJumps = fwdJumps[:i] + lstmt = s + } + stmtBranches(s.Stmt) + + case *ast.BranchStmt: + if s.Label == nil { + return // checked in 1st pass (check.stmt) + } + + // determine and validate target + name := s.Label.Name + switch s.Tok { + case token.BREAK: + // spec: "If there is a label, it must be that of an enclosing + // "for", "switch", or "select" statement, and that is the one + // whose execution terminates." + valid := false + if t := b.enclosingTarget(name); t != nil { + switch t.Stmt.(type) { + case *ast.SwitchStmt, *ast.TypeSwitchStmt, *ast.SelectStmt, *ast.ForStmt, *ast.RangeStmt: + valid = true + } + } + if !valid { + check.errorf(s.Label, MisplacedLabel, "invalid break label %s", name) + return + } + + case token.CONTINUE: + // spec: "If there is a label, it must be that of an enclosing + // "for" statement, and that is the one whose execution advances." + valid := false + if t := b.enclosingTarget(name); t != nil { + switch t.Stmt.(type) { + case *ast.ForStmt, *ast.RangeStmt: + valid = true + } + } + if !valid { + check.errorf(s.Label, MisplacedLabel, "invalid continue label %s", name) + return + } + + case token.GOTO: + if b.gotoTarget(name) == nil { + // label may be declared later - add branch to forward jumps + fwdJumps = append(fwdJumps, s) + return + } + + default: + check.errorf(s, InvalidSyntaxTree, "branch statement: %s %s", s.Tok, name) + return + } + + // record label use + obj := all.Lookup(name) + obj.(*Label).used = true + check.recordUse(s.Label, obj) + + case *ast.AssignStmt: + if s.Tok == token.DEFINE { + recordVarDecl(s.Pos()) + } + + case *ast.BlockStmt: + blockBranches(lstmt, s.List) + + case *ast.IfStmt: + stmtBranches(s.Body) + if s.Else != nil { + stmtBranches(s.Else) + } + + case *ast.CaseClause: + blockBranches(nil, s.Body) + + case *ast.SwitchStmt: + stmtBranches(s.Body) + + case *ast.TypeSwitchStmt: + stmtBranches(s.Body) + + case *ast.CommClause: + blockBranches(nil, s.Body) + + case *ast.SelectStmt: + stmtBranches(s.Body) + + case *ast.ForStmt: + stmtBranches(s.Body) + + case *ast.RangeStmt: + stmtBranches(s.Body) + } + } + + for _, s := range list { + stmtBranches(s) + } + + return fwdJumps +} diff --git a/src/go/types/lookup.go b/src/go/types/lookup.go new file mode 100644 index 0000000..4eedcc2 --- /dev/null +++ b/src/go/types/lookup.go @@ -0,0 +1,543 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements various field and method lookup functions. + +package types + +import ( + "bytes" + "strings" +) + +// Internal use of LookupFieldOrMethod: If the obj result is a method +// associated with a concrete (non-interface) type, the method's signature +// may not be fully set up. Call Checker.objDecl(obj, nil) before accessing +// the method's type. + +// LookupFieldOrMethod looks up a field or method with given package and name +// in T and returns the corresponding *Var or *Func, an index sequence, and a +// bool indicating if there were any pointer indirections on the path to the +// field or method. If addressable is set, T is the type of an addressable +// variable (only matters for method lookups). T must not be nil. +// +// The last index entry is the field or method index in the (possibly embedded) +// type where the entry was found, either: +// +// 1. the list of declared methods of a named type; or +// 2. the list of all methods (method set) of an interface type; or +// 3. the list of fields of a struct type. +// +// The earlier index entries are the indices of the embedded struct fields +// traversed to get to the found entry, starting at depth 0. +// +// If no entry is found, a nil object is returned. In this case, the returned +// index and indirect values have the following meaning: +// +// - If index != nil, the index sequence points to an ambiguous entry +// (the same name appeared more than once at the same embedding level). +// +// - If indirect is set, a method with a pointer receiver type was found +// but there was no pointer on the path from the actual receiver type to +// the method's formal receiver base type, nor was the receiver addressable. +func LookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string) (obj Object, index []int, indirect bool) { + if T == nil { + panic("LookupFieldOrMethod on nil type") + } + + // Methods cannot be associated to a named pointer type. + // (spec: "The type denoted by T is called the receiver base type; + // it must not be a pointer or interface type and it must be declared + // in the same package as the method."). + // Thus, if we have a named pointer type, proceed with the underlying + // pointer type but discard the result if it is a method since we would + // not have found it for T (see also issue 8590). + if t, _ := T.(*Named); t != nil { + if p, _ := t.Underlying().(*Pointer); p != nil { + obj, index, indirect = lookupFieldOrMethod(p, false, pkg, name, false) + if _, ok := obj.(*Func); ok { + return nil, nil, false + } + return + } + } + + obj, index, indirect = lookupFieldOrMethod(T, addressable, pkg, name, false) + + // If we didn't find anything and if we have a type parameter with a core type, + // see if there is a matching field (but not a method, those need to be declared + // explicitly in the constraint). If the constraint is a named pointer type (see + // above), we are ok here because only fields are accepted as results. + const enableTParamFieldLookup = false // see issue #51576 + if enableTParamFieldLookup && obj == nil && isTypeParam(T) { + if t := coreType(T); t != nil { + obj, index, indirect = lookupFieldOrMethod(t, addressable, pkg, name, false) + if _, ok := obj.(*Var); !ok { + obj, index, indirect = nil, nil, false // accept fields (variables) only + } + } + } + return +} + +// lookupFieldOrMethod should only be called by LookupFieldOrMethod and missingMethod. +// If foldCase is true, the lookup for methods will include looking for any method +// which case-folds to the same as 'name' (used for giving helpful error messages). +// +// The resulting object may not be fully type-checked. +func lookupFieldOrMethod(T Type, addressable bool, pkg *Package, name string, foldCase bool) (obj Object, index []int, indirect bool) { + // WARNING: The code in this function is extremely subtle - do not modify casually! + + if name == "_" { + return // blank fields/methods are never found + } + + typ, isPtr := deref(T) + + // *typ where typ is an interface (incl. a type parameter) has no methods. + if isPtr { + if _, ok := under(typ).(*Interface); ok { + return + } + } + + // Start with typ as single entry at shallowest depth. + current := []embeddedType{{typ, nil, isPtr, false}} + + // seen tracks named types that we have seen already, allocated lazily. + // Used to avoid endless searches in case of recursive types. + // + // We must use a lookup on identity rather than a simple map[*Named]bool as + // instantiated types may be identical but not equal. + var seen instanceLookup + + // search current depth + for len(current) > 0 { + var next []embeddedType // embedded types found at current depth + + // look for (pkg, name) in all types at current depth + for _, e := range current { + typ := e.typ + + // If we have a named type, we may have associated methods. + // Look for those first. + if named, _ := typ.(*Named); named != nil { + if alt := seen.lookup(named); alt != nil { + // We have seen this type before, at a more shallow depth + // (note that multiples of this type at the current depth + // were consolidated before). The type at that depth shadows + // this same type at the current depth, so we can ignore + // this one. + continue + } + seen.add(named) + + // look for a matching attached method + if i, m := named.lookupMethod(pkg, name, foldCase); m != nil { + // potential match + // caution: method may not have a proper signature yet + index = concat(e.index, i) + if obj != nil || e.multiples { + return nil, index, false // collision + } + obj = m + indirect = e.indirect + continue // we can't have a matching field or interface method + } + } + + switch t := under(typ).(type) { + case *Struct: + // look for a matching field and collect embedded types + for i, f := range t.fields { + if f.sameId(pkg, name) { + assert(f.typ != nil) + index = concat(e.index, i) + if obj != nil || e.multiples { + return nil, index, false // collision + } + obj = f + indirect = e.indirect + continue // we can't have a matching interface method + } + // Collect embedded struct fields for searching the next + // lower depth, but only if we have not seen a match yet + // (if we have a match it is either the desired field or + // we have a name collision on the same depth; in either + // case we don't need to look further). + // Embedded fields are always of the form T or *T where + // T is a type name. If e.typ appeared multiple times at + // this depth, f.typ appears multiple times at the next + // depth. + if obj == nil && f.embedded { + typ, isPtr := deref(f.typ) + // TODO(gri) optimization: ignore types that can't + // have fields or methods (only Named, Struct, and + // Interface types need to be considered). + next = append(next, embeddedType{typ, concat(e.index, i), e.indirect || isPtr, e.multiples}) + } + } + + case *Interface: + // look for a matching method (interface may be a type parameter) + if i, m := t.typeSet().LookupMethod(pkg, name, foldCase); m != nil { + assert(m.typ != nil) + index = concat(e.index, i) + if obj != nil || e.multiples { + return nil, index, false // collision + } + obj = m + indirect = e.indirect + } + } + } + + if obj != nil { + // found a potential match + // spec: "A method call x.m() is valid if the method set of (the type of) x + // contains m and the argument list can be assigned to the parameter + // list of m. If x is addressable and &x's method set contains m, x.m() + // is shorthand for (&x).m()". + if f, _ := obj.(*Func); f != nil { + // determine if method has a pointer receiver + if f.hasPtrRecv() && !indirect && !addressable { + return nil, nil, true // pointer/addressable receiver required + } + } + return + } + + current = consolidateMultiples(next) + } + + return nil, nil, false // not found +} + +// embeddedType represents an embedded type +type embeddedType struct { + typ Type + index []int // embedded field indices, starting with index at depth 0 + indirect bool // if set, there was a pointer indirection on the path to this field + multiples bool // if set, typ appears multiple times at this depth +} + +// consolidateMultiples collects multiple list entries with the same type +// into a single entry marked as containing multiples. The result is the +// consolidated list. +func consolidateMultiples(list []embeddedType) []embeddedType { + if len(list) <= 1 { + return list // at most one entry - nothing to do + } + + n := 0 // number of entries w/ unique type + prev := make(map[Type]int) // index at which type was previously seen + for _, e := range list { + if i, found := lookupType(prev, e.typ); found { + list[i].multiples = true + // ignore this entry + } else { + prev[e.typ] = n + list[n] = e + n++ + } + } + return list[:n] +} + +func lookupType(m map[Type]int, typ Type) (int, bool) { + // fast path: maybe the types are equal + if i, found := m[typ]; found { + return i, true + } + + for t, i := range m { + if Identical(t, typ) { + return i, true + } + } + + return 0, false +} + +type instanceLookup struct { + // buf is used to avoid allocating the map m in the common case of a small + // number of instances. + buf [3]*Named + m map[*Named][]*Named +} + +func (l *instanceLookup) lookup(inst *Named) *Named { + for _, t := range l.buf { + if t != nil && Identical(inst, t) { + return t + } + } + for _, t := range l.m[inst.Origin()] { + if Identical(inst, t) { + return t + } + } + return nil +} + +func (l *instanceLookup) add(inst *Named) { + for i, t := range l.buf { + if t == nil { + l.buf[i] = inst + return + } + } + if l.m == nil { + l.m = make(map[*Named][]*Named) + } + insts := l.m[inst.Origin()] + l.m[inst.Origin()] = append(insts, inst) +} + +// MissingMethod returns (nil, false) if V implements T, otherwise it +// returns a missing method required by T and whether it is missing or +// just has the wrong type. +// +// For non-interface types V, or if static is set, V implements T if all +// methods of T are present in V. Otherwise (V is an interface and static +// is not set), MissingMethod only checks that methods of T which are also +// present in V have matching types (e.g., for a type assertion x.(T) where +// x is of interface type V). +func MissingMethod(V Type, T *Interface, static bool) (method *Func, wrongType bool) { + m, alt := (*Checker)(nil).missingMethod(V, T, static) + // Only report a wrong type if the alternative method has the same name as m. + return m, alt != nil && alt.name == m.name // alt != nil implies m != nil +} + +// missingMethod is like MissingMethod but accepts a *Checker as receiver. +// The receiver may be nil if missingMethod is invoked through an exported +// API call (such as MissingMethod), i.e., when all methods have been type- +// checked. +// +// If a method is missing on T but is found on *T, or if a method is found +// on T when looked up with case-folding, this alternative method is returned +// as the second result. +func (check *Checker) missingMethod(V Type, T *Interface, static bool) (method, alt *Func) { + if T.NumMethods() == 0 { + return + } + + // V is an interface + if u, _ := under(V).(*Interface); u != nil { + tset := u.typeSet() + for _, m := range T.typeSet().methods { + _, f := tset.LookupMethod(m.pkg, m.name, false) + + if f == nil { + if !static { + continue + } + return m, nil + } + + if !Identical(f.typ, m.typ) { + return m, f + } + } + + return + } + + // V is not an interface + for _, m := range T.typeSet().methods { + // TODO(gri) should this be calling LookupFieldOrMethod instead (and why not)? + obj, _, _ := lookupFieldOrMethod(V, false, m.pkg, m.name, false) + + // check if m is on *V, or on V with case-folding + found := obj != nil + if !found { + // TODO(gri) Instead of NewPointer(V) below, can we just set the "addressable" argument? + obj, _, _ = lookupFieldOrMethod(NewPointer(V), false, m.pkg, m.name, false) + if obj == nil { + obj, _, _ = lookupFieldOrMethod(V, false, m.pkg, m.name, true /* fold case */) + } + } + + // we must have a method (not a struct field) + f, _ := obj.(*Func) + if f == nil { + return m, nil + } + + // methods may not have a fully set up signature yet + if check != nil { + check.objDecl(f, nil) + } + + if !found || !Identical(f.typ, m.typ) { + return m, f + } + } + + return +} + +// missingMethodCause returns a string giving the detailed cause for a missing method m, +// where m is missing from V, but required by T. It puts the cause in parentheses, +// and may include more have/want info after that. If non-nil, alt is a relevant +// method that matches in some way. It may have the correct name, but wrong type, or +// it may have a pointer receiver, or it may have the correct name except wrong case. +// check may be nil. +func (check *Checker) missingMethodCause(V, T Type, m, alt *Func) string { + mname := "method " + m.Name() + + if alt != nil { + if m.Name() != alt.Name() { + return check.sprintf("(missing %s)\n\t\thave %s\n\t\twant %s", + mname, check.funcString(alt, false), check.funcString(m, false)) + } + + if Identical(m.typ, alt.typ) { + return check.sprintf("(%s has pointer receiver)", mname) + } + + altS, mS := check.funcString(alt, false), check.funcString(m, false) + if altS == mS { + // Would tell the user that Foo isn't a Foo, add package information to disambiguate. See #54258. + altS, mS = check.funcString(alt, true), check.funcString(m, true) + } + + return check.sprintf("(wrong type for %s)\n\t\thave %s\n\t\twant %s", + mname, altS, mS) + } + + if isInterfacePtr(V) { + return "(" + check.interfacePtrError(V) + ")" + } + + if isInterfacePtr(T) { + return "(" + check.interfacePtrError(T) + ")" + } + + obj, _, _ := lookupFieldOrMethod(V, true /* auto-deref */, m.pkg, m.name, false) + if fld, _ := obj.(*Var); fld != nil { + return check.sprintf("(%s.%s is a field, not a method)", V, fld.Name()) + } + + return check.sprintf("(missing %s)", mname) +} + +func isInterfacePtr(T Type) bool { + p, _ := under(T).(*Pointer) + return p != nil && IsInterface(p.base) +} + +// check may be nil. +func (check *Checker) interfacePtrError(T Type) string { + assert(isInterfacePtr(T)) + if p, _ := under(T).(*Pointer); isTypeParam(p.base) { + return check.sprintf("type %s is pointer to type parameter, not type parameter", T) + } + return check.sprintf("type %s is pointer to interface, not interface", T) +} + +// funcString returns a string of the form name + signature for f. +// check may be nil. +func (check *Checker) funcString(f *Func, pkgInfo bool) string { + buf := bytes.NewBufferString(f.name) + var qf Qualifier + if check != nil && !pkgInfo { + qf = check.qualifier + } + w := newTypeWriter(buf, qf) + w.pkgInfo = pkgInfo + w.paramNames = false + w.signature(f.typ.(*Signature)) + return buf.String() +} + +// assertableTo reports whether a value of type V can be asserted to have type T. +// It returns (nil, false) as affirmative answer. Otherwise it returns a missing +// method required by V and whether it is missing or just has the wrong type. +// The receiver may be nil if assertableTo is invoked through an exported API call +// (such as AssertableTo), i.e., when all methods have been type-checked. +// TODO(gri) replace calls to this function with calls to newAssertableTo. +func (check *Checker) assertableTo(V *Interface, T Type) (method, wrongType *Func) { + // no static check is required if T is an interface + // spec: "If T is an interface type, x.(T) asserts that the + // dynamic type of x implements the interface T." + if IsInterface(T) { + return + } + // TODO(gri) fix this for generalized interfaces + return check.missingMethod(T, V, false) +} + +// newAssertableTo reports whether a value of type V can be asserted to have type T. +// It also implements behavior for interfaces that currently are only permitted +// in constraint position (we have not yet defined that behavior in the spec). +func (check *Checker) newAssertableTo(V *Interface, T Type) bool { + // no static check is required if T is an interface + // spec: "If T is an interface type, x.(T) asserts that the + // dynamic type of x implements the interface T." + if IsInterface(T) { + return true + } + return check.implements(T, V, false, nil) +} + +// deref dereferences typ if it is a *Pointer and returns its base and true. +// Otherwise it returns (typ, false). +func deref(typ Type) (Type, bool) { + if p, _ := typ.(*Pointer); p != nil { + // p.base should never be nil, but be conservative + if p.base == nil { + if debug { + panic("pointer with nil base type (possibly due to an invalid cyclic declaration)") + } + return Typ[Invalid], true + } + return p.base, true + } + return typ, false +} + +// derefStructPtr dereferences typ if it is a (named or unnamed) pointer to a +// (named or unnamed) struct and returns its base. Otherwise it returns typ. +func derefStructPtr(typ Type) Type { + if p, _ := under(typ).(*Pointer); p != nil { + if _, ok := under(p.base).(*Struct); ok { + return p.base + } + } + return typ +} + +// concat returns the result of concatenating list and i. +// The result does not share its underlying array with list. +func concat(list []int, i int) []int { + var t []int + t = append(t, list...) + return append(t, i) +} + +// fieldIndex returns the index for the field with matching package and name, or a value < 0. +func fieldIndex(fields []*Var, pkg *Package, name string) int { + if name != "_" { + for i, f := range fields { + if f.sameId(pkg, name) { + return i + } + } + } + return -1 +} + +// lookupMethod returns the index of and method with matching package and name, or (-1, nil). +// If foldCase is true, method names are considered equal if they are equal with case folding. +func lookupMethod(methods []*Func, pkg *Package, name string, foldCase bool) (int, *Func) { + if name != "_" { + for i, m := range methods { + if (m.name == name || foldCase && strings.EqualFold(m.name, name)) && m.sameId(pkg, m.name) { + return i, m + } + } + } + return -1, nil +} diff --git a/src/go/types/lookup_test.go b/src/go/types/lookup_test.go new file mode 100644 index 0000000..d3ca58b --- /dev/null +++ b/src/go/types/lookup_test.go @@ -0,0 +1,58 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "go/importer" + "go/token" + "path/filepath" + "runtime" + "testing" + + . "go/types" +) + +// BenchmarkLookupFieldOrMethod measures types.LookupFieldOrMethod performance. +// LookupFieldOrMethod is a performance hotspot for both type-checking and +// external API calls. +func BenchmarkLookupFieldOrMethod(b *testing.B) { + // Choose an arbitrary, large package. + path := filepath.Join(runtime.GOROOT(), "src", "net", "http") + + fset := token.NewFileSet() + files, err := pkgFiles(fset, path) + if err != nil { + b.Fatal(err) + } + + conf := Config{ + Importer: importer.Default(), + } + + pkg, err := conf.Check("http", fset, files, nil) + if err != nil { + b.Fatal(err) + } + + scope := pkg.Scope() + names := scope.Names() + + // Look up an arbitrary name for each type referenced in the package scope. + lookup := func() { + for _, name := range names { + typ := scope.Lookup(name).Type() + LookupFieldOrMethod(typ, true, pkg, "m") + } + } + + // Perform a lookup once, to ensure that any lazily-evaluated state is + // complete. + lookup() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + lookup() + } +} diff --git a/src/go/types/main_test.go b/src/go/types/main_test.go new file mode 100644 index 0000000..73d7d18 --- /dev/null +++ b/src/go/types/main_test.go @@ -0,0 +1,17 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "go/build" + "internal/testenv" + "os" + "testing" +) + +func TestMain(m *testing.M) { + build.Default.GOROOT = testenv.GOROOT(nil) + os.Exit(m.Run()) +} diff --git a/src/go/types/map.go b/src/go/types/map.go new file mode 100644 index 0000000..01e13b2 --- /dev/null +++ b/src/go/types/map.go @@ -0,0 +1,24 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +// A Map represents a map type. +type Map struct { + key, elem Type +} + +// NewMap returns a new map for the given key and element types. +func NewMap(key, elem Type) *Map { + return &Map{key: key, elem: elem} +} + +// Key returns the key type of map m. +func (m *Map) Key() Type { return m.key } + +// Elem returns the element type of map m. +func (m *Map) Elem() Type { return m.elem } + +func (t *Map) Underlying() Type { return t } +func (t *Map) String() string { return TypeString(t, nil) } diff --git a/src/go/types/methodset.go b/src/go/types/methodset.go new file mode 100644 index 0000000..2bf3028 --- /dev/null +++ b/src/go/types/methodset.go @@ -0,0 +1,238 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements method sets. + +package types + +import ( + "fmt" + "sort" + "strings" +) + +// A MethodSet is an ordered set of concrete or abstract (interface) methods; +// a method is a MethodVal selection, and they are ordered by ascending m.Obj().Id(). +// The zero value for a MethodSet is a ready-to-use empty method set. +type MethodSet struct { + list []*Selection +} + +func (s *MethodSet) String() string { + if s.Len() == 0 { + return "MethodSet {}" + } + + var buf strings.Builder + fmt.Fprintln(&buf, "MethodSet {") + for _, f := range s.list { + fmt.Fprintf(&buf, "\t%s\n", f) + } + fmt.Fprintln(&buf, "}") + return buf.String() +} + +// Len returns the number of methods in s. +func (s *MethodSet) Len() int { return len(s.list) } + +// At returns the i'th method in s for 0 <= i < s.Len(). +func (s *MethodSet) At(i int) *Selection { return s.list[i] } + +// Lookup returns the method with matching package and name, or nil if not found. +func (s *MethodSet) Lookup(pkg *Package, name string) *Selection { + if s.Len() == 0 { + return nil + } + + key := Id(pkg, name) + i := sort.Search(len(s.list), func(i int) bool { + m := s.list[i] + return m.obj.Id() >= key + }) + if i < len(s.list) { + m := s.list[i] + if m.obj.Id() == key { + return m + } + } + return nil +} + +// Shared empty method set. +var emptyMethodSet MethodSet + +// Note: NewMethodSet is intended for external use only as it +// requires interfaces to be complete. It may be used +// internally if LookupFieldOrMethod completed the same +// interfaces beforehand. + +// NewMethodSet returns the method set for the given type T. +// It always returns a non-nil method set, even if it is empty. +func NewMethodSet(T Type) *MethodSet { + // WARNING: The code in this function is extremely subtle - do not modify casually! + // This function and lookupFieldOrMethod should be kept in sync. + + // TODO(rfindley) confirm that this code is in sync with lookupFieldOrMethod + // with respect to type params. + + // method set up to the current depth, allocated lazily + var base methodSet + + typ, isPtr := deref(T) + + // *typ where typ is an interface has no methods. + if isPtr && IsInterface(typ) { + return &emptyMethodSet + } + + // Start with typ as single entry at shallowest depth. + current := []embeddedType{{typ, nil, isPtr, false}} + + // seen tracks named types that we have seen already, allocated lazily. + // Used to avoid endless searches in case of recursive types. + // + // We must use a lookup on identity rather than a simple map[*Named]bool as + // instantiated types may be identical but not equal. + var seen instanceLookup + + // collect methods at current depth + for len(current) > 0 { + var next []embeddedType // embedded types found at current depth + + // field and method sets at current depth, indexed by names (Id's), and allocated lazily + var fset map[string]bool // we only care about the field names + var mset methodSet + + for _, e := range current { + typ := e.typ + + // If we have a named type, we may have associated methods. + // Look for those first. + if named, _ := typ.(*Named); named != nil { + if alt := seen.lookup(named); alt != nil { + // We have seen this type before, at a more shallow depth + // (note that multiples of this type at the current depth + // were consolidated before). The type at that depth shadows + // this same type at the current depth, so we can ignore + // this one. + continue + } + seen.add(named) + + for i := 0; i < named.NumMethods(); i++ { + mset = mset.addOne(named.Method(i), concat(e.index, i), e.indirect, e.multiples) + } + } + + switch t := under(typ).(type) { + case *Struct: + for i, f := range t.fields { + if fset == nil { + fset = make(map[string]bool) + } + fset[f.Id()] = true + + // Embedded fields are always of the form T or *T where + // T is a type name. If typ appeared multiple times at + // this depth, f.Type appears multiple times at the next + // depth. + if f.embedded { + typ, isPtr := deref(f.typ) + // TODO(gri) optimization: ignore types that can't + // have fields or methods (only Named, Struct, and + // Interface types need to be considered). + next = append(next, embeddedType{typ, concat(e.index, i), e.indirect || isPtr, e.multiples}) + } + } + + case *Interface: + mset = mset.add(t.typeSet().methods, e.index, true, e.multiples) + } + } + + // Add methods and collisions at this depth to base if no entries with matching + // names exist already. + for k, m := range mset { + if _, found := base[k]; !found { + // Fields collide with methods of the same name at this depth. + if fset[k] { + m = nil // collision + } + if base == nil { + base = make(methodSet) + } + base[k] = m + } + } + + // Add all (remaining) fields at this depth as collisions (since they will + // hide any method further down) if no entries with matching names exist already. + for k := range fset { + if _, found := base[k]; !found { + if base == nil { + base = make(methodSet) + } + base[k] = nil // collision + } + } + + current = consolidateMultiples(next) + } + + if len(base) == 0 { + return &emptyMethodSet + } + + // collect methods + var list []*Selection + for _, m := range base { + if m != nil { + m.recv = T + list = append(list, m) + } + } + // sort by unique name + sort.Slice(list, func(i, j int) bool { + return list[i].obj.Id() < list[j].obj.Id() + }) + return &MethodSet{list} +} + +// A methodSet is a set of methods and name collisions. +// A collision indicates that multiple methods with the +// same unique id, or a field with that id appeared. +type methodSet map[string]*Selection // a nil entry indicates a name collision + +// Add adds all functions in list to the method set s. +// If multiples is set, every function in list appears multiple times +// and is treated as a collision. +func (s methodSet) add(list []*Func, index []int, indirect bool, multiples bool) methodSet { + if len(list) == 0 { + return s + } + for i, f := range list { + s = s.addOne(f, concat(index, i), indirect, multiples) + } + return s +} + +func (s methodSet) addOne(f *Func, index []int, indirect bool, multiples bool) methodSet { + if s == nil { + s = make(methodSet) + } + key := f.Id() + // if f is not in the set, add it + if !multiples { + // TODO(gri) A found method may not be added because it's not in the method set + // (!indirect && f.hasPtrRecv()). A 2nd method on the same level may be in the method + // set and may not collide with the first one, thus leading to a false positive. + // Is that possible? Investigate. + if _, found := s[key]; !found && (indirect || !f.hasPtrRecv()) { + s[key] = &Selection{MethodVal, nil, f, index, indirect} + return s + } + } + s[key] = nil // collision + return s +} diff --git a/src/go/types/methodset_test.go b/src/go/types/methodset_test.go new file mode 100644 index 0000000..443994b --- /dev/null +++ b/src/go/types/methodset_test.go @@ -0,0 +1,156 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "testing" + + "go/ast" + "go/parser" + "go/token" + . "go/types" +) + +func TestNewMethodSet(t *testing.T) { + type method struct { + name string + index []int + indirect bool + } + + // Tests are expressed src -> methods, for simplifying the composite literal. + // Should be kept in sync with TestLookupFieldOrMethod. + tests := map[string][]method{ + // Named types + "var a T; type T struct{}; func (T) f() {}": {{"f", []int{0}, false}}, + "var a *T; type T struct{}; func (T) f() {}": {{"f", []int{0}, true}}, + "var a T; type T struct{}; func (*T) f() {}": {}, + "var a *T; type T struct{}; func (*T) f() {}": {{"f", []int{0}, true}}, + + // Generic named types + "var a T[int]; type T[P any] struct{}; func (T[P]) f() {}": {{"f", []int{0}, false}}, + "var a *T[int]; type T[P any] struct{}; func (T[P]) f() {}": {{"f", []int{0}, true}}, + "var a T[int]; type T[P any] struct{}; func (*T[P]) f() {}": {}, + "var a *T[int]; type T[P any] struct{}; func (*T[P]) f() {}": {{"f", []int{0}, true}}, + + // Interfaces + "var a T; type T interface{ f() }": {{"f", []int{0}, true}}, + "var a T1; type ( T1 T2; T2 interface{ f() } )": {{"f", []int{0}, true}}, + "var a T1; type ( T1 interface{ T2 }; T2 interface{ f() } )": {{"f", []int{0}, true}}, + + // Generic interfaces + "var a T[int]; type T[P any] interface{ f() }": {{"f", []int{0}, true}}, + "var a T1[int]; type ( T1[P any] T2[P]; T2[P any] interface{ f() } )": {{"f", []int{0}, true}}, + "var a T1[int]; type ( T1[P any] interface{ T2[P] }; T2[P any] interface{ f() } )": {{"f", []int{0}, true}}, + + // Embedding + "var a struct{ E }; type E interface{ f() }": {{"f", []int{0, 0}, true}}, + "var a *struct{ E }; type E interface{ f() }": {{"f", []int{0, 0}, true}}, + "var a struct{ E }; type E struct{}; func (E) f() {}": {{"f", []int{0, 0}, false}}, + "var a struct{ *E }; type E struct{}; func (E) f() {}": {{"f", []int{0, 0}, true}}, + "var a struct{ E }; type E struct{}; func (*E) f() {}": {}, + "var a struct{ *E }; type E struct{}; func (*E) f() {}": {{"f", []int{0, 0}, true}}, + + // Embedding of generic types + "var a struct{ E[int] }; type E[P any] interface{ f() }": {{"f", []int{0, 0}, true}}, + "var a *struct{ E[int] }; type E[P any] interface{ f() }": {{"f", []int{0, 0}, true}}, + "var a struct{ E[int] }; type E[P any] struct{}; func (E[P]) f() {}": {{"f", []int{0, 0}, false}}, + "var a struct{ *E[int] }; type E[P any] struct{}; func (E[P]) f() {}": {{"f", []int{0, 0}, true}}, + "var a struct{ E[int] }; type E[P any] struct{}; func (*E[P]) f() {}": {}, + "var a struct{ *E[int] }; type E[P any] struct{}; func (*E[P]) f() {}": {{"f", []int{0, 0}, true}}, + + // collisions + "var a struct{ E1; *E2 }; type ( E1 interface{ f() }; E2 struct{ f int })": {}, + "var a struct{ E1; *E2 }; type ( E1 struct{ f int }; E2 struct{} ); func (E2) f() {}": {}, + + // recursive generic types; see golang/go#52715 + "var a T[int]; type ( T[P any] struct { *N[P] }; N[P any] struct { *T[P] } ); func (N[P]) m() {}": {{"m", []int{0, 0}, true}}, + "var a T[int]; type ( T[P any] struct { *N[P] }; N[P any] struct { *T[P] } ); func (T[P]) m() {}": {{"m", []int{0}, false}}, + } + + tParamTests := map[string][]method{ + // By convention, look up a in the scope of "g" + "type C interface{ f() }; func g[T C](a T){}": {{"f", []int{0}, true}}, + "type C interface{ f() }; func g[T C]() { var a T; _ = a }": {{"f", []int{0}, true}}, + + // Issue #43621: We don't allow this anymore. Keep this code in case we + // decide to revisit this decision. + // "type C interface{ f() }; func g[T C]() { var a struct{T}; _ = a }": {{"f", []int{0, 0}, true}}, + + // Issue #45639: We also don't allow this anymore. + // "type C interface{ f() }; func g[T C]() { type Y T; var a Y; _ = a }": {}, + } + + check := func(src string, methods []method, generic bool) { + pkg := mustTypecheck("test", "package p;"+src, nil) + + scope := pkg.Scope() + if generic { + fn := pkg.Scope().Lookup("g").(*Func) + scope = fn.Scope() + } + obj := scope.Lookup("a") + if obj == nil { + t.Errorf("%s: incorrect test case - no object a", src) + return + } + + ms := NewMethodSet(obj.Type()) + if got, want := ms.Len(), len(methods); got != want { + t.Errorf("%s: got %d methods, want %d", src, got, want) + return + } + for i, m := range methods { + sel := ms.At(i) + if got, want := sel.Obj().Name(), m.name; got != want { + t.Errorf("%s [method %d]: got name = %q at, want %q", src, i, got, want) + } + if got, want := sel.Index(), m.index; !sameSlice(got, want) { + t.Errorf("%s [method %d]: got index = %v, want %v", src, i, got, want) + } + if got, want := sel.Indirect(), m.indirect; got != want { + t.Errorf("%s [method %d]: got indirect = %v, want %v", src, i, got, want) + } + } + } + + for src, methods := range tests { + check(src, methods, false) + } + + for src, methods := range tParamTests { + check(src, methods, true) + } +} + +// Test for golang/go#52715 +func TestNewMethodSet_RecursiveGeneric(t *testing.T) { + const src = ` +package pkg + +type Tree[T any] struct { + *Node[T] +} + +type Node[T any] struct { + *Tree[T] +} + +type Instance = *Tree[int] +` + + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "foo.go", src, 0) + if err != nil { + panic(err) + } + pkg := NewPackage("pkg", f.Name.Name) + if err := NewChecker(nil, fset, pkg, nil).Files([]*ast.File{f}); err != nil { + panic(err) + } + + T := pkg.Scope().Lookup("Instance").Type() + _ = NewMethodSet(T) // verify that NewMethodSet terminates +} diff --git a/src/go/types/mono.go b/src/go/types/mono.go new file mode 100644 index 0000000..cf3f5a8 --- /dev/null +++ b/src/go/types/mono.go @@ -0,0 +1,337 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "go/ast" + "go/token" + . "internal/types/errors" +) + +// This file implements a check to validate that a Go package doesn't +// have unbounded recursive instantiation, which is not compatible +// with compilers using static instantiation (such as +// monomorphization). +// +// It implements a sort of "type flow" analysis by detecting which +// type parameters are instantiated with other type parameters (or +// types derived thereof). A package cannot be statically instantiated +// if the graph has any cycles involving at least one derived type. +// +// Concretely, we construct a directed, weighted graph. Vertices are +// used to represent type parameters as well as some defined +// types. Edges are used to represent how types depend on each other: +// +// * Everywhere a type-parameterized function or type is instantiated, +// we add edges to each type parameter from the vertices (if any) +// representing each type parameter or defined type referenced by +// the type argument. If the type argument is just the referenced +// type itself, then the edge has weight 0, otherwise 1. +// +// * For every defined type declared within a type-parameterized +// function or method, we add an edge of weight 1 to the defined +// type from each ambient type parameter. +// +// For example, given: +// +// func f[A, B any]() { +// type T int +// f[T, map[A]B]() +// } +// +// we construct vertices representing types A, B, and T. Because of +// declaration "type T int", we construct edges T<-A and T<-B with +// weight 1; and because of instantiation "f[T, map[A]B]" we construct +// edges A<-T with weight 0, and B<-A and B<-B with weight 1. +// +// Finally, we look for any positive-weight cycles. Zero-weight cycles +// are allowed because static instantiation will reach a fixed point. + +type monoGraph struct { + vertices []monoVertex + edges []monoEdge + + // canon maps method receiver type parameters to their respective + // receiver type's type parameters. + canon map[*TypeParam]*TypeParam + + // nameIdx maps a defined type or (canonical) type parameter to its + // vertex index. + nameIdx map[*TypeName]int +} + +type monoVertex struct { + weight int // weight of heaviest known path to this vertex + pre int // previous edge (if any) in the above path + len int // length of the above path + + // obj is the defined type or type parameter represented by this + // vertex. + obj *TypeName +} + +type monoEdge struct { + dst, src int + weight int + + pos token.Pos + typ Type +} + +func (check *Checker) monomorph() { + // We detect unbounded instantiation cycles using a variant of + // Bellman-Ford's algorithm. Namely, instead of always running |V| + // iterations, we run until we either reach a fixed point or we've + // found a path of length |V|. This allows us to terminate earlier + // when there are no cycles, which should be the common case. + + again := true + for again { + again = false + + for i, edge := range check.mono.edges { + src := &check.mono.vertices[edge.src] + dst := &check.mono.vertices[edge.dst] + + // N.B., we're looking for the greatest weight paths, unlike + // typical Bellman-Ford. + w := src.weight + edge.weight + if w <= dst.weight { + continue + } + + dst.pre = i + dst.len = src.len + 1 + if dst.len == len(check.mono.vertices) { + check.reportInstanceLoop(edge.dst) + return + } + + dst.weight = w + again = true + } + } +} + +func (check *Checker) reportInstanceLoop(v int) { + var stack []int + seen := make([]bool, len(check.mono.vertices)) + + // We have a path that contains a cycle and ends at v, but v may + // only be reachable from the cycle, not on the cycle itself. We + // start by walking backwards along the path until we find a vertex + // that appears twice. + for !seen[v] { + stack = append(stack, v) + seen[v] = true + v = check.mono.edges[check.mono.vertices[v].pre].src + } + + // Trim any vertices we visited before visiting v the first + // time. Since v is the first vertex we found within the cycle, any + // vertices we visited earlier cannot be part of the cycle. + for stack[0] != v { + stack = stack[1:] + } + + // TODO(mdempsky): Pivot stack so we report the cycle from the top? + + obj0 := check.mono.vertices[v].obj + check.error(obj0, InvalidInstanceCycle, "instantiation cycle:") + + qf := RelativeTo(check.pkg) + for _, v := range stack { + edge := check.mono.edges[check.mono.vertices[v].pre] + obj := check.mono.vertices[edge.dst].obj + + switch obj.Type().(type) { + default: + panic("unexpected type") + case *Named: + check.errorf(atPos(edge.pos), InvalidInstanceCycle, "\t%s implicitly parameterized by %s", obj.Name(), TypeString(edge.typ, qf)) // secondary error, \t indented + case *TypeParam: + check.errorf(atPos(edge.pos), InvalidInstanceCycle, "\t%s instantiated as %s", obj.Name(), TypeString(edge.typ, qf)) // secondary error, \t indented + } + } +} + +// recordCanon records that tpar is the canonical type parameter +// corresponding to method type parameter mpar. +func (w *monoGraph) recordCanon(mpar, tpar *TypeParam) { + if w.canon == nil { + w.canon = make(map[*TypeParam]*TypeParam) + } + w.canon[mpar] = tpar +} + +// recordInstance records that the given type parameters were +// instantiated with the corresponding type arguments. +func (w *monoGraph) recordInstance(pkg *Package, pos token.Pos, tparams []*TypeParam, targs []Type, xlist []ast.Expr) { + for i, tpar := range tparams { + pos := pos + if i < len(xlist) { + pos = xlist[i].Pos() + } + w.assign(pkg, pos, tpar, targs[i]) + } +} + +// assign records that tpar was instantiated as targ at pos. +func (w *monoGraph) assign(pkg *Package, pos token.Pos, tpar *TypeParam, targ Type) { + // Go generics do not have an analog to C++`s template-templates, + // where a template parameter can itself be an instantiable + // template. So any instantiation cycles must occur within a single + // package. Accordingly, we can ignore instantiations of imported + // type parameters. + // + // TODO(mdempsky): Push this check up into recordInstance? All type + // parameters in a list will appear in the same package. + if tpar.Obj().Pkg() != pkg { + return + } + + // flow adds an edge from vertex src representing that typ flows to tpar. + flow := func(src int, typ Type) { + weight := 1 + if typ == targ { + weight = 0 + } + + w.addEdge(w.typeParamVertex(tpar), src, weight, pos, targ) + } + + // Recursively walk the type argument to find any defined types or + // type parameters. + var do func(typ Type) + do = func(typ Type) { + switch typ := typ.(type) { + default: + panic("unexpected type") + + case *TypeParam: + assert(typ.Obj().Pkg() == pkg) + flow(w.typeParamVertex(typ), typ) + + case *Named: + if src := w.localNamedVertex(pkg, typ.Origin()); src >= 0 { + flow(src, typ) + } + + targs := typ.TypeArgs() + for i := 0; i < targs.Len(); i++ { + do(targs.At(i)) + } + + case *Array: + do(typ.Elem()) + case *Basic: + // ok + case *Chan: + do(typ.Elem()) + case *Map: + do(typ.Key()) + do(typ.Elem()) + case *Pointer: + do(typ.Elem()) + case *Slice: + do(typ.Elem()) + + case *Interface: + for i := 0; i < typ.NumMethods(); i++ { + do(typ.Method(i).Type()) + } + case *Signature: + tuple := func(tup *Tuple) { + for i := 0; i < tup.Len(); i++ { + do(tup.At(i).Type()) + } + } + tuple(typ.Params()) + tuple(typ.Results()) + case *Struct: + for i := 0; i < typ.NumFields(); i++ { + do(typ.Field(i).Type()) + } + } + } + do(targ) +} + +// localNamedVertex returns the index of the vertex representing +// named, or -1 if named doesn't need representation. +func (w *monoGraph) localNamedVertex(pkg *Package, named *Named) int { + obj := named.Obj() + if obj.Pkg() != pkg { + return -1 // imported type + } + + root := pkg.Scope() + if obj.Parent() == root { + return -1 // package scope, no ambient type parameters + } + + if idx, ok := w.nameIdx[obj]; ok { + return idx + } + + idx := -1 + + // Walk the type definition's scope to find any ambient type + // parameters that it's implicitly parameterized by. + for scope := obj.Parent(); scope != root; scope = scope.Parent() { + for _, elem := range scope.elems { + if elem, ok := elem.(*TypeName); ok && !elem.IsAlias() && elem.Pos() < obj.Pos() { + if tpar, ok := elem.Type().(*TypeParam); ok { + if idx < 0 { + idx = len(w.vertices) + w.vertices = append(w.vertices, monoVertex{obj: obj}) + } + + w.addEdge(idx, w.typeParamVertex(tpar), 1, obj.Pos(), tpar) + } + } + } + } + + if w.nameIdx == nil { + w.nameIdx = make(map[*TypeName]int) + } + w.nameIdx[obj] = idx + return idx +} + +// typeParamVertex returns the index of the vertex representing tpar. +func (w *monoGraph) typeParamVertex(tpar *TypeParam) int { + if x, ok := w.canon[tpar]; ok { + tpar = x + } + + obj := tpar.Obj() + + if idx, ok := w.nameIdx[obj]; ok { + return idx + } + + if w.nameIdx == nil { + w.nameIdx = make(map[*TypeName]int) + } + + idx := len(w.vertices) + w.vertices = append(w.vertices, monoVertex{obj: obj}) + w.nameIdx[obj] = idx + return idx +} + +func (w *monoGraph) addEdge(dst, src, weight int, pos token.Pos, typ Type) { + // TODO(mdempsky): Deduplicate redundant edges? + w.edges = append(w.edges, monoEdge{ + dst: dst, + src: src, + weight: weight, + + pos: pos, + typ: typ, + }) +} diff --git a/src/go/types/mono_test.go b/src/go/types/mono_test.go new file mode 100644 index 0000000..02daa4f --- /dev/null +++ b/src/go/types/mono_test.go @@ -0,0 +1,91 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "errors" + "fmt" + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "strings" + "testing" +) + +func checkMono(t *testing.T, body string) error { + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "x.go", "package x; import `unsafe`; var _ unsafe.Pointer;\n"+body, 0) + if err != nil { + t.Fatal(err) + } + files := []*ast.File{file} + + var buf strings.Builder + conf := types.Config{ + Error: func(err error) { fmt.Fprintln(&buf, err) }, + Importer: importer.Default(), + } + conf.Check("x", fset, files, nil) + if buf.Len() == 0 { + return nil + } + return errors.New(strings.TrimRight(buf.String(), "\n")) +} + +func TestMonoGood(t *testing.T) { + for i, good := range goods { + if err := checkMono(t, good); err != nil { + t.Errorf("%d: unexpected failure: %v", i, err) + } + } +} + +func TestMonoBad(t *testing.T) { + for i, bad := range bads { + if err := checkMono(t, bad); err == nil { + t.Errorf("%d: unexpected success", i) + } else { + t.Log(err) + } + } +} + +var goods = []string{ + "func F[T any](x T) { F(x) }", + "func F[T, U, V any]() { F[U, V, T](); F[V, T, U]() }", + "type Ring[A, B, C any] struct { L *Ring[B, C, A]; R *Ring[C, A, B] }", + "func F[T any]() { type U[T any] [unsafe.Sizeof(F[*T])]byte }", + "func F[T any]() { type U[T any] [unsafe.Sizeof(F[*T])]byte; var _ U[int] }", + "type U[T any] [unsafe.Sizeof(F[*T])]byte; func F[T any]() { var _ U[U[int]] }", + "func F[T any]() { type A = int; F[A]() }", +} + +// TODO(mdempsky): Validate specific error messages and positioning. + +var bads = []string{ + "func F[T any](x T) { F(&x) }", + "func F[T any]() { F[*T]() }", + "func F[T any]() { F[[]T]() }", + "func F[T any]() { F[[1]T]() }", + "func F[T any]() { F[chan T]() }", + "func F[T any]() { F[map[*T]int]() }", + "func F[T any]() { F[map[error]T]() }", + "func F[T any]() { F[func(T)]() }", + "func F[T any]() { F[func() T]() }", + "func F[T any]() { F[struct{ t T }]() }", + "func F[T any]() { F[interface{ t() T }]() }", + "type U[_ any] int; func F[T any]() { F[U[T]]() }", + "func F[T any]() { type U int; F[U]() }", + "func F[T any]() { type U int; F[*U]() }", + "type U[T any] int; func (U[T]) m() { var _ U[*T] }", + "type U[T any] int; func (*U[T]) m() { var _ U[*T] }", + "type U[T1 any] [unsafe.Sizeof(F[*T1])]byte; func F[T2 any]() { var _ U[T2] }", + "func F[A, B, C, D, E any]() { F[B, C, D, E, *A]() }", + "type U[_ any] int; const X = unsafe.Sizeof(func() { type A[T any] U[A[*T]] })", + "func F[T any]() { type A = *T; F[A]() }", + "type A[T any] struct { _ A[*T] }", +} diff --git a/src/go/types/named.go b/src/go/types/named.go new file mode 100644 index 0000000..c08997a --- /dev/null +++ b/src/go/types/named.go @@ -0,0 +1,656 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "go/token" + "sync" + "sync/atomic" +) + +// Type-checking Named types is subtle, because they may be recursively +// defined, and because their full details may be spread across multiple +// declarations (via methods). For this reason they are type-checked lazily, +// to avoid information being accessed before it is complete. +// +// Conceptually, it is helpful to think of named types as having two distinct +// sets of information: +// - "LHS" information, defining their identity: Obj() and TypeArgs() +// - "RHS" information, defining their details: TypeParams(), Underlying(), +// and methods. +// +// In this taxonomy, LHS information is available immediately, but RHS +// information is lazy. Specifically, a named type N may be constructed in any +// of the following ways: +// 1. type-checked from the source +// 2. loaded eagerly from export data +// 3. loaded lazily from export data (when using unified IR) +// 4. instantiated from a generic type +// +// In cases 1, 3, and 4, it is possible that the underlying type or methods of +// N may not be immediately available. +// - During type-checking, we allocate N before type-checking its underlying +// type or methods, so that we may resolve recursive references. +// - When loading from export data, we may load its methods and underlying +// type lazily using a provided load function. +// - After instantiating, we lazily expand the underlying type and methods +// (note that instances may be created while still in the process of +// type-checking the original type declaration). +// +// In cases 3 and 4 this lazy construction may also occur concurrently, due to +// concurrent use of the type checker API (after type checking or importing has +// finished). It is critical that we keep track of state, so that Named types +// are constructed exactly once and so that we do not access their details too +// soon. +// +// We achieve this by tracking state with an atomic state variable, and +// guarding potentially concurrent calculations with a mutex. At any point in +// time this state variable determines which data on N may be accessed. As +// state monotonically progresses, any data available at state M may be +// accessed without acquiring the mutex at state N, provided N >= M. +// +// GLOSSARY: Here are a few terms used in this file to describe Named types: +// - We say that a Named type is "instantiated" if it has been constructed by +// instantiating a generic named type with type arguments. +// - We say that a Named type is "declared" if it corresponds to a type +// declaration in the source. Instantiated named types correspond to a type +// instantiation in the source, not a declaration. But their Origin type is +// a declared type. +// - We say that a Named type is "resolved" if its RHS information has been +// loaded or fully type-checked. For Named types constructed from export +// data, this may involve invoking a loader function to extract information +// from export data. For instantiated named types this involves reading +// information from their origin. +// - We say that a Named type is "expanded" if it is an instantiated type and +// type parameters in its underlying type and methods have been substituted +// with the type arguments from the instantiation. A type may be partially +// expanded if some but not all of these details have been substituted. +// Similarly, we refer to these individual details (underlying type or +// method) as being "expanded". +// - When all information is known for a named type, we say it is "complete". +// +// Some invariants to keep in mind: each declared Named type has a single +// corresponding object, and that object's type is the (possibly generic) Named +// type. Declared Named types are identical if and only if their pointers are +// identical. On the other hand, multiple instantiated Named types may be +// identical even though their pointers are not identical. One has to use +// Identical to compare them. For instantiated named types, their obj is a +// synthetic placeholder that records their position of the corresponding +// instantiation in the source (if they were constructed during type checking). +// +// To prevent infinite expansion of named instances that are created outside of +// type-checking, instances share a Context with other instances created during +// their expansion. Via the pidgeonhole principle, this guarantees that in the +// presence of a cycle of named types, expansion will eventually find an +// existing instance in the Context and short-circuit the expansion. +// +// Once an instance is complete, we can nil out this shared Context to unpin +// memory, though this Context may still be held by other incomplete instances +// in its "lineage". + +// A Named represents a named (defined) type. +type Named struct { + check *Checker // non-nil during type-checking; nil otherwise + obj *TypeName // corresponding declared object for declared types; see above for instantiated types + + // fromRHS holds the type (on RHS of declaration) this *Named type is derived + // from (for cycle reporting). Only used by validType, and therefore does not + // require synchronization. + fromRHS Type + + // information for instantiated types; nil otherwise + inst *instance + + mu sync.Mutex // guards all fields below + state_ uint32 // the current state of this type; must only be accessed atomically + underlying Type // possibly a *Named during setup; never a *Named once set up completely + tparams *TypeParamList // type parameters, or nil + + // methods declared for this type (not the method set of this type) + // Signatures are type-checked lazily. + // For non-instantiated types, this is a fully populated list of methods. For + // instantiated types, methods are individually expanded when they are first + // accessed. + methods []*Func + + // loader may be provided to lazily load type parameters, underlying type, and methods. + loader func(*Named) (tparams []*TypeParam, underlying Type, methods []*Func) +} + +// instance holds information that is only necessary for instantiated named +// types. +type instance struct { + orig *Named // original, uninstantiated type + targs *TypeList // type arguments + expandedMethods int // number of expanded methods; expandedMethods <= len(orig.methods) + ctxt *Context // local Context; set to nil after full expansion +} + +// namedState represents the possible states that a named type may assume. +type namedState uint32 + +const ( + unresolved namedState = iota // tparams, underlying type and methods might be unavailable + resolved // resolve has run; methods might be incomplete (for instances) + complete // all data is known +) + +// NewNamed returns a new named type for the given type name, underlying type, and associated methods. +// If the given type name obj doesn't have a type yet, its type is set to the returned named type. +// The underlying type must not be a *Named. +func NewNamed(obj *TypeName, underlying Type, methods []*Func) *Named { + if _, ok := underlying.(*Named); ok { + panic("underlying type must not be *Named") + } + return (*Checker)(nil).newNamed(obj, underlying, methods) +} + +// resolve resolves the type parameters, methods, and underlying type of n. +// This information may be loaded from a provided loader function, or computed +// from an origin type (in the case of instances). +// +// After resolution, the type parameters, methods, and underlying type of n are +// accessible; but if n is an instantiated type, its methods may still be +// unexpanded. +func (n *Named) resolve() *Named { + if n.state() >= resolved { // avoid locking below + return n + } + + // TODO(rfindley): if n.check is non-nil we can avoid locking here, since + // type-checking is not concurrent. Evaluate if this is worth doing. + n.mu.Lock() + defer n.mu.Unlock() + + if n.state() >= resolved { + return n + } + + if n.inst != nil { + assert(n.underlying == nil) // n is an unresolved instance + assert(n.loader == nil) // instances are created by instantiation, in which case n.loader is nil + + orig := n.inst.orig + orig.resolve() + underlying := n.expandUnderlying() + + n.tparams = orig.tparams + n.underlying = underlying + n.fromRHS = orig.fromRHS // for cycle detection + + if len(orig.methods) == 0 { + n.setState(complete) // nothing further to do + n.inst.ctxt = nil + } else { + n.setState(resolved) + } + return n + } + + // TODO(mdempsky): Since we're passing n to the loader anyway + // (necessary because types2 expects the receiver type for methods + // on defined interface types to be the Named rather than the + // underlying Interface), maybe it should just handle calling + // SetTypeParams, SetUnderlying, and AddMethod instead? Those + // methods would need to support reentrant calls though. It would + // also make the API more future-proof towards further extensions. + if n.loader != nil { + assert(n.underlying == nil) + assert(n.TypeArgs().Len() == 0) // instances are created by instantiation, in which case n.loader is nil + + tparams, underlying, methods := n.loader(n) + + n.tparams = bindTParams(tparams) + n.underlying = underlying + n.fromRHS = underlying // for cycle detection + n.methods = methods + n.loader = nil + } + + n.setState(complete) + return n +} + +// state atomically accesses the current state of the receiver. +func (n *Named) state() namedState { + return namedState(atomic.LoadUint32(&n.state_)) +} + +// setState atomically stores the given state for n. +// Must only be called while holding n.mu. +func (n *Named) setState(state namedState) { + atomic.StoreUint32(&n.state_, uint32(state)) +} + +// newNamed is like NewNamed but with a *Checker receiver and additional orig argument. +func (check *Checker) newNamed(obj *TypeName, underlying Type, methods []*Func) *Named { + typ := &Named{check: check, obj: obj, fromRHS: underlying, underlying: underlying, methods: methods} + if obj.typ == nil { + obj.typ = typ + } + // Ensure that typ is always sanity-checked. + if check != nil { + check.needsCleanup(typ) + } + return typ +} + +// newNamedInstance creates a new named instance for the given origin and type +// arguments, recording pos as the position of its synthetic object (for error +// reporting). +// +// If set, expanding is the named type instance currently being expanded, that +// led to the creation of this instance. +func (check *Checker) newNamedInstance(pos token.Pos, orig *Named, targs []Type, expanding *Named) *Named { + assert(len(targs) > 0) + + obj := NewTypeName(pos, orig.obj.pkg, orig.obj.name, nil) + inst := &instance{orig: orig, targs: newTypeList(targs)} + + // Only pass the expanding context to the new instance if their packages + // match. Since type reference cycles are only possible within a single + // package, this is sufficient for the purposes of short-circuiting cycles. + // Avoiding passing the context in other cases prevents unnecessary coupling + // of types across packages. + if expanding != nil && expanding.Obj().pkg == obj.pkg { + inst.ctxt = expanding.inst.ctxt + } + typ := &Named{check: check, obj: obj, inst: inst} + obj.typ = typ + // Ensure that typ is always sanity-checked. + if check != nil { + check.needsCleanup(typ) + } + return typ +} + +func (t *Named) cleanup() { + assert(t.inst == nil || t.inst.orig.inst == nil) + // Ensure that every defined type created in the course of type-checking has + // either non-*Named underlying type, or is unexpanded. + // + // This guarantees that we don't leak any types whose underlying type is + // *Named, because any unexpanded instances will lazily compute their + // underlying type by substituting in the underlying type of their origin. + // The origin must have either been imported or type-checked and expanded + // here, and in either case its underlying type will be fully expanded. + switch t.underlying.(type) { + case nil: + if t.TypeArgs().Len() == 0 { + panic("nil underlying") + } + case *Named: + t.under() // t.under may add entries to check.cleaners + } + t.check = nil +} + +// Obj returns the type name for the declaration defining the named type t. For +// instantiated types, this is same as the type name of the origin type. +func (t *Named) Obj() *TypeName { + if t.inst == nil { + return t.obj + } + return t.inst.orig.obj +} + +// Origin returns the generic type from which the named type t is +// instantiated. If t is not an instantiated type, the result is t. +func (t *Named) Origin() *Named { + if t.inst == nil { + return t + } + return t.inst.orig +} + +// TypeParams returns the type parameters of the named type t, or nil. +// The result is non-nil for an (originally) generic type even if it is instantiated. +func (t *Named) TypeParams() *TypeParamList { return t.resolve().tparams } + +// SetTypeParams sets the type parameters of the named type t. +// t must not have type arguments. +func (t *Named) SetTypeParams(tparams []*TypeParam) { + assert(t.inst == nil) + t.resolve().tparams = bindTParams(tparams) +} + +// TypeArgs returns the type arguments used to instantiate the named type t. +func (t *Named) TypeArgs() *TypeList { + if t.inst == nil { + return nil + } + return t.inst.targs +} + +// NumMethods returns the number of explicit methods defined for t. +func (t *Named) NumMethods() int { + return len(t.Origin().resolve().methods) +} + +// Method returns the i'th method of named type t for 0 <= i < t.NumMethods(). +// +// For an ordinary or instantiated type t, the receiver base type of this +// method is the named type t. For an uninstantiated generic type t, each +// method receiver is instantiated with its receiver type parameters. +func (t *Named) Method(i int) *Func { + t.resolve() + + if t.state() >= complete { + return t.methods[i] + } + + assert(t.inst != nil) // only instances should have incomplete methods + orig := t.inst.orig + + t.mu.Lock() + defer t.mu.Unlock() + + if len(t.methods) != len(orig.methods) { + assert(len(t.methods) == 0) + t.methods = make([]*Func, len(orig.methods)) + } + + if t.methods[i] == nil { + assert(t.inst.ctxt != nil) // we should still have a context remaining from the resolution phase + t.methods[i] = t.expandMethod(i) + t.inst.expandedMethods++ + + // Check if we've created all methods at this point. If we have, mark the + // type as fully expanded. + if t.inst.expandedMethods == len(orig.methods) { + t.setState(complete) + t.inst.ctxt = nil // no need for a context anymore + } + } + + return t.methods[i] +} + +// expandMethod substitutes type arguments in the i'th method for an +// instantiated receiver. +func (t *Named) expandMethod(i int) *Func { + // t.orig.methods is not lazy. origm is the method instantiated with its + // receiver type parameters (the "origin" method). + origm := t.inst.orig.Method(i) + assert(origm != nil) + + check := t.check + // Ensure that the original method is type-checked. + if check != nil { + check.objDecl(origm, nil) + } + + origSig := origm.typ.(*Signature) + rbase, _ := deref(origSig.Recv().Type()) + + // If rbase is t, then origm is already the instantiated method we're looking + // for. In this case, we return origm to preserve the invariant that + // traversing Method->Receiver Type->Method should get back to the same + // method. + // + // This occurs if t is instantiated with the receiver type parameters, as in + // the use of m in func (r T[_]) m() { r.m() }. + if rbase == t { + return origm + } + + sig := origSig + // We can only substitute if we have a correspondence between type arguments + // and type parameters. This check is necessary in the presence of invalid + // code. + if origSig.RecvTypeParams().Len() == t.inst.targs.Len() { + smap := makeSubstMap(origSig.RecvTypeParams().list(), t.inst.targs.list()) + var ctxt *Context + if check != nil { + ctxt = check.context() + } + sig = check.subst(origm.pos, origSig, smap, t, ctxt).(*Signature) + } + + if sig == origSig { + // No substitution occurred, but we still need to create a new signature to + // hold the instantiated receiver. + copy := *origSig + sig = © + } + + var rtyp Type + if origm.hasPtrRecv() { + rtyp = NewPointer(t) + } else { + rtyp = t + } + + sig.recv = substVar(origSig.recv, rtyp) + return substFunc(origm, sig) +} + +// SetUnderlying sets the underlying type and marks t as complete. +// t must not have type arguments. +func (t *Named) SetUnderlying(underlying Type) { + assert(t.inst == nil) + if underlying == nil { + panic("underlying type must not be nil") + } + if _, ok := underlying.(*Named); ok { + panic("underlying type must not be *Named") + } + t.resolve().underlying = underlying + if t.fromRHS == nil { + t.fromRHS = underlying // for cycle detection + } +} + +// AddMethod adds method m unless it is already in the method list. +// t must not have type arguments. +func (t *Named) AddMethod(m *Func) { + assert(t.inst == nil) + t.resolve() + if i, _ := lookupMethod(t.methods, m.pkg, m.name, false); i < 0 { + t.methods = append(t.methods, m) + } +} + +func (t *Named) Underlying() Type { return t.resolve().underlying } +func (t *Named) String() string { return TypeString(t, nil) } + +// ---------------------------------------------------------------------------- +// Implementation +// +// TODO(rfindley): reorganize the loading and expansion methods under this +// heading. + +// under returns the expanded underlying type of n0; possibly by following +// forward chains of named types. If an underlying type is found, resolve +// the chain by setting the underlying type for each defined type in the +// chain before returning it. If no underlying type is found or a cycle +// is detected, the result is Typ[Invalid]. If a cycle is detected and +// n0.check != nil, the cycle is reported. +// +// This is necessary because the underlying type of named may be itself a +// named type that is incomplete: +// +// type ( +// A B +// B *C +// C A +// ) +// +// The type of C is the (named) type of A which is incomplete, +// and which has as its underlying type the named type B. +func (n0 *Named) under() Type { + u := n0.Underlying() + + // If the underlying type of a defined type is not a defined + // (incl. instance) type, then that is the desired underlying + // type. + var n1 *Named + switch u1 := u.(type) { + case nil: + // After expansion via Underlying(), we should never encounter a nil + // underlying. + panic("nil underlying") + default: + // common case + return u + case *Named: + // handled below + n1 = u1 + } + + if n0.check == nil { + panic("Named.check == nil but type is incomplete") + } + + // Invariant: after this point n0 as well as any named types in its + // underlying chain should be set up when this function exits. + check := n0.check + n := n0 + + seen := make(map[*Named]int) // types that need their underlying type resolved + var path []Object // objects encountered, for cycle reporting + +loop: + for { + seen[n] = len(seen) + path = append(path, n.obj) + n = n1 + if i, ok := seen[n]; ok { + // cycle + check.cycleError(path[i:]) + u = Typ[Invalid] + break + } + u = n.Underlying() + switch u1 := u.(type) { + case nil: + u = Typ[Invalid] + break loop + default: + break loop + case *Named: + // Continue collecting *Named types in the chain. + n1 = u1 + } + } + + for n := range seen { + // We should never have to update the underlying type of an imported type; + // those underlying types should have been resolved during the import. + // Also, doing so would lead to a race condition (was issue #31749). + // Do this check always, not just in debug mode (it's cheap). + if n.obj.pkg != check.pkg { + panic("imported type with unresolved underlying type") + } + n.underlying = u + } + + return u +} + +func (n *Named) setUnderlying(typ Type) { + if n != nil { + n.underlying = typ + } +} + +func (n *Named) lookupMethod(pkg *Package, name string, foldCase bool) (int, *Func) { + n.resolve() + // If n is an instance, we may not have yet instantiated all of its methods. + // Look up the method index in orig, and only instantiate method at the + // matching index (if any). + i, _ := lookupMethod(n.Origin().methods, pkg, name, foldCase) + if i < 0 { + return -1, nil + } + // For instances, m.Method(i) will be different from the orig method. + return i, n.Method(i) +} + +// context returns the type-checker context. +func (check *Checker) context() *Context { + if check.ctxt == nil { + check.ctxt = NewContext() + } + return check.ctxt +} + +// expandUnderlying substitutes type arguments in the underlying type n.orig, +// returning the result. Returns Typ[Invalid] if there was an error. +func (n *Named) expandUnderlying() Type { + check := n.check + if check != nil && trace { + check.trace(n.obj.pos, "-- Named.expandUnderlying %s", n) + check.indent++ + defer func() { + check.indent-- + check.trace(n.obj.pos, "=> %s (tparams = %s, under = %s)", n, n.tparams.list(), n.underlying) + }() + } + + assert(n.inst.orig.underlying != nil) + if n.inst.ctxt == nil { + n.inst.ctxt = NewContext() + } + + orig := n.inst.orig + targs := n.inst.targs + + if _, unexpanded := orig.underlying.(*Named); unexpanded { + // We should only get a Named underlying type here during type checking + // (for example, in recursive type declarations). + assert(check != nil) + } + + if orig.tparams.Len() != targs.Len() { + // Mismatching arg and tparam length may be checked elsewhere. + return Typ[Invalid] + } + + // Ensure that an instance is recorded before substituting, so that we + // resolve n for any recursive references. + h := n.inst.ctxt.instanceHash(orig, targs.list()) + n2 := n.inst.ctxt.update(h, orig, n.TypeArgs().list(), n) + assert(n == n2) + + smap := makeSubstMap(orig.tparams.list(), targs.list()) + var ctxt *Context + if check != nil { + ctxt = check.context() + } + underlying := n.check.subst(n.obj.pos, orig.underlying, smap, n, ctxt) + // If the underlying type of n is an interface, we need to set the receiver of + // its methods accurately -- we set the receiver of interface methods on + // the RHS of a type declaration to the defined type. + if iface, _ := underlying.(*Interface); iface != nil { + if methods, copied := replaceRecvType(iface.methods, orig, n); copied { + // If the underlying type doesn't actually use type parameters, it's + // possible that it wasn't substituted. In this case we need to create + // a new *Interface before modifying receivers. + if iface == orig.underlying { + old := iface + iface = check.newInterface() + iface.embeddeds = old.embeddeds + iface.complete = old.complete + iface.implicit = old.implicit // should be false but be conservative + underlying = iface + } + iface.methods = methods + } + } + + return underlying +} + +// safeUnderlying returns the underlying type of typ without expanding +// instances, to avoid infinite recursion. +// +// TODO(rfindley): eliminate this function or give it a better name. +func safeUnderlying(typ Type) Type { + if t, _ := typ.(*Named); t != nil { + return t.underlying + } + return typ.Underlying() +} diff --git a/src/go/types/named_test.go b/src/go/types/named_test.go new file mode 100644 index 0000000..92f17e5 --- /dev/null +++ b/src/go/types/named_test.go @@ -0,0 +1,129 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "go/ast" + "go/token" + "testing" + + . "go/types" +) + +func BenchmarkNamed(b *testing.B) { + const src = ` +package p + +type T struct { + P int +} + +func (T) M(int) {} +func (T) N() (i int) { return } + +type G[P any] struct { + F P +} + +func (G[P]) M(P) {} +func (G[P]) N() (p P) { return } + +type Inst = G[int] + ` + pkg := mustTypecheck("p", src, nil) + + var ( + T = pkg.Scope().Lookup("T").Type() + G = pkg.Scope().Lookup("G").Type() + SrcInst = pkg.Scope().Lookup("Inst").Type() + UserInst = mustInstantiate(b, G, Typ[Int]) + ) + + tests := []struct { + name string + typ Type + }{ + {"nongeneric", T}, + {"generic", G}, + {"src instance", SrcInst}, + {"user instance", UserInst}, + } + + b.Run("Underlying", func(b *testing.B) { + for _, test := range tests { + b.Run(test.name, func(b *testing.B) { + // Access underlying once, to trigger any lazy calculation. + _ = test.typ.Underlying() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = test.typ.Underlying() + } + }) + } + }) + + b.Run("NewMethodSet", func(b *testing.B) { + for _, test := range tests { + b.Run(test.name, func(b *testing.B) { + // Access underlying once, to trigger any lazy calculation. + _ = NewMethodSet(test.typ) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = NewMethodSet(test.typ) + } + }) + } + }) +} + +func mustInstantiate(tb testing.TB, orig Type, targs ...Type) Type { + inst, err := Instantiate(nil, orig, targs, true) + if err != nil { + tb.Fatal(err) + } + return inst +} + +// Test that types do not expand infinitely, as in golang/go#52715. +func TestFiniteTypeExpansion(t *testing.T) { + const src = ` +package p + +type Tree[T any] struct { + *Node[T] +} + +func (*Tree[R]) N(r R) R { return r } + +type Node[T any] struct { + *Tree[T] +} + +func (Node[Q]) M(Q) {} + +type Inst = *Tree[int] +` + + fset := token.NewFileSet() + f := mustParse(fset, "foo.go", src) + pkg := NewPackage("p", f.Name.Name) + if err := NewChecker(nil, fset, pkg, nil).Files([]*ast.File{f}); err != nil { + t.Fatal(err) + } + + firstFieldType := func(n *Named) *Named { + return n.Underlying().(*Struct).Field(0).Type().(*Pointer).Elem().(*Named) + } + + Inst := pkg.Scope().Lookup("Inst").Type().(*Pointer).Elem().(*Named) + Node := firstFieldType(Inst) + Tree := firstFieldType(Node) + if !Identical(Inst, Tree) { + t.Fatalf("Not a cycle: got %v, want %v", Tree, Inst) + } + if Inst != Tree { + t.Errorf("Duplicate instances in cycle: %s (%p) -> %s (%p) -> %s (%p)", Inst, Inst, Node, Node, Tree, Tree) + } +} diff --git a/src/go/types/object.go b/src/go/types/object.go new file mode 100644 index 0000000..6e63948 --- /dev/null +++ b/src/go/types/object.go @@ -0,0 +1,568 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "bytes" + "fmt" + "go/constant" + "go/token" +) + +// An Object describes a named language entity such as a package, +// constant, type, variable, function (incl. methods), or label. +// All objects implement the Object interface. +type Object interface { + Parent() *Scope // scope in which this object is declared; nil for methods and struct fields + Pos() token.Pos // position of object identifier in declaration + Pkg() *Package // package to which this object belongs; nil for labels and objects in the Universe scope + Name() string // package local object name + Type() Type // object type + Exported() bool // reports whether the name starts with a capital letter + Id() string // object name if exported, qualified name if not exported (see func Id) + + // String returns a human-readable string of the object. + String() string + + // order reflects a package-level object's source order: if object + // a is before object b in the source, then a.order() < b.order(). + // order returns a value > 0 for package-level objects; it returns + // 0 for all other objects (including objects in file scopes). + order() uint32 + + // color returns the object's color. + color() color + + // setType sets the type of the object. + setType(Type) + + // setOrder sets the order number of the object. It must be > 0. + setOrder(uint32) + + // setColor sets the object's color. It must not be white. + setColor(color color) + + // setParent sets the parent scope of the object. + setParent(*Scope) + + // sameId reports whether obj.Id() and Id(pkg, name) are the same. + sameId(pkg *Package, name string) bool + + // scopePos returns the start position of the scope of this Object + scopePos() token.Pos + + // setScopePos sets the start position of the scope for this Object. + setScopePos(pos token.Pos) +} + +// Id returns name if it is exported, otherwise it +// returns the name qualified with the package path. +func Id(pkg *Package, name string) string { + if token.IsExported(name) { + return name + } + // unexported names need the package path for differentiation + // (if there's no package, make sure we don't start with '.' + // as that may change the order of methods between a setup + // inside a package and outside a package - which breaks some + // tests) + path := "_" + // pkg is nil for objects in Universe scope and possibly types + // introduced via Eval (see also comment in object.sameId) + if pkg != nil && pkg.path != "" { + path = pkg.path + } + return path + "." + name +} + +// An object implements the common parts of an Object. +type object struct { + parent *Scope + pos token.Pos + pkg *Package + name string + typ Type + order_ uint32 + color_ color + scopePos_ token.Pos +} + +// color encodes the color of an object (see Checker.objDecl for details). +type color uint32 + +// An object may be painted in one of three colors. +// Color values other than white or black are considered grey. +const ( + white color = iota + black + grey // must be > white and black +) + +func (c color) String() string { + switch c { + case white: + return "white" + case black: + return "black" + default: + return "grey" + } +} + +// colorFor returns the (initial) color for an object depending on +// whether its type t is known or not. +func colorFor(t Type) color { + if t != nil { + return black + } + return white +} + +// Parent returns the scope in which the object is declared. +// The result is nil for methods and struct fields. +func (obj *object) Parent() *Scope { return obj.parent } + +// Pos returns the declaration position of the object's identifier. +func (obj *object) Pos() token.Pos { return obj.pos } + +// Pkg returns the package to which the object belongs. +// The result is nil for labels and objects in the Universe scope. +func (obj *object) Pkg() *Package { return obj.pkg } + +// Name returns the object's (package-local, unqualified) name. +func (obj *object) Name() string { return obj.name } + +// Type returns the object's type. +func (obj *object) Type() Type { return obj.typ } + +// Exported reports whether the object is exported (starts with a capital letter). +// It doesn't take into account whether the object is in a local (function) scope +// or not. +func (obj *object) Exported() bool { return token.IsExported(obj.name) } + +// Id is a wrapper for Id(obj.Pkg(), obj.Name()). +func (obj *object) Id() string { return Id(obj.pkg, obj.name) } + +func (obj *object) String() string { panic("abstract") } +func (obj *object) order() uint32 { return obj.order_ } +func (obj *object) color() color { return obj.color_ } +func (obj *object) scopePos() token.Pos { return obj.scopePos_ } + +func (obj *object) setParent(parent *Scope) { obj.parent = parent } +func (obj *object) setType(typ Type) { obj.typ = typ } +func (obj *object) setOrder(order uint32) { assert(order > 0); obj.order_ = order } +func (obj *object) setColor(color color) { assert(color != white); obj.color_ = color } +func (obj *object) setScopePos(pos token.Pos) { obj.scopePos_ = pos } + +func (obj *object) sameId(pkg *Package, name string) bool { + // spec: + // "Two identifiers are different if they are spelled differently, + // or if they appear in different packages and are not exported. + // Otherwise, they are the same." + if name != obj.name { + return false + } + // obj.Name == name + if obj.Exported() { + return true + } + // not exported, so packages must be the same (pkg == nil for + // fields in Universe scope; this can only happen for types + // introduced via Eval) + if pkg == nil || obj.pkg == nil { + return pkg == obj.pkg + } + // pkg != nil && obj.pkg != nil + return pkg.path == obj.pkg.path +} + +// A PkgName represents an imported Go package. +// PkgNames don't have a type. +type PkgName struct { + object + imported *Package + used bool // set if the package was used +} + +// NewPkgName returns a new PkgName object representing an imported package. +// The remaining arguments set the attributes found with all Objects. +func NewPkgName(pos token.Pos, pkg *Package, name string, imported *Package) *PkgName { + return &PkgName{object{nil, pos, pkg, name, Typ[Invalid], 0, black, token.NoPos}, imported, false} +} + +// Imported returns the package that was imported. +// It is distinct from Pkg(), which is the package containing the import statement. +func (obj *PkgName) Imported() *Package { return obj.imported } + +// A Const represents a declared constant. +type Const struct { + object + val constant.Value +} + +// NewConst returns a new constant with value val. +// The remaining arguments set the attributes found with all Objects. +func NewConst(pos token.Pos, pkg *Package, name string, typ Type, val constant.Value) *Const { + return &Const{object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}, val} +} + +// Val returns the constant's value. +func (obj *Const) Val() constant.Value { return obj.val } + +func (*Const) isDependency() {} // a constant may be a dependency of an initialization expression + +// A TypeName represents a name for a (defined or alias) type. +type TypeName struct { + object +} + +// NewTypeName returns a new type name denoting the given typ. +// The remaining arguments set the attributes found with all Objects. +// +// The typ argument may be a defined (Named) type or an alias type. +// It may also be nil such that the returned TypeName can be used as +// argument for NewNamed, which will set the TypeName's type as a side- +// effect. +func NewTypeName(pos token.Pos, pkg *Package, name string, typ Type) *TypeName { + return &TypeName{object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}} +} + +// _NewTypeNameLazy returns a new defined type like NewTypeName, but it +// lazily calls resolve to finish constructing the Named object. +func _NewTypeNameLazy(pos token.Pos, pkg *Package, name string, load func(named *Named) (tparams []*TypeParam, underlying Type, methods []*Func)) *TypeName { + obj := NewTypeName(pos, pkg, name, nil) + NewNamed(obj, nil, nil).loader = load + return obj +} + +// IsAlias reports whether obj is an alias name for a type. +func (obj *TypeName) IsAlias() bool { + switch t := obj.typ.(type) { + case nil: + return false + case *Basic: + // unsafe.Pointer is not an alias. + if obj.pkg == Unsafe { + return false + } + // Any user-defined type name for a basic type is an alias for a + // basic type (because basic types are pre-declared in the Universe + // scope, outside any package scope), and so is any type name with + // a different name than the name of the basic type it refers to. + // Additionally, we need to look for "byte" and "rune" because they + // are aliases but have the same names (for better error messages). + return obj.pkg != nil || t.name != obj.name || t == universeByte || t == universeRune + case *Named: + return obj != t.obj + case *TypeParam: + return obj != t.obj + default: + return true + } +} + +// A Variable represents a declared variable (including function parameters and results, and struct fields). +type Var struct { + object + embedded bool // if set, the variable is an embedded struct field, and name is the type name + isField bool // var is struct field + used bool // set if the variable was used + origin *Var // if non-nil, the Var from which this one was instantiated +} + +// NewVar returns a new variable. +// The arguments set the attributes found with all Objects. +func NewVar(pos token.Pos, pkg *Package, name string, typ Type) *Var { + return &Var{object: object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}} +} + +// NewParam returns a new variable representing a function parameter. +func NewParam(pos token.Pos, pkg *Package, name string, typ Type) *Var { + return &Var{object: object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}, used: true} // parameters are always 'used' +} + +// NewField returns a new variable representing a struct field. +// For embedded fields, the name is the unqualified type name +// under which the field is accessible. +func NewField(pos token.Pos, pkg *Package, name string, typ Type, embedded bool) *Var { + return &Var{object: object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}, embedded: embedded, isField: true} +} + +// Anonymous reports whether the variable is an embedded field. +// Same as Embedded; only present for backward-compatibility. +func (obj *Var) Anonymous() bool { return obj.embedded } + +// Embedded reports whether the variable is an embedded field. +func (obj *Var) Embedded() bool { return obj.embedded } + +// IsField reports whether the variable is a struct field. +func (obj *Var) IsField() bool { return obj.isField } + +// Origin returns the canonical Var for its receiver, i.e. the Var object +// recorded in Info.Defs. +// +// For synthetic Vars created during instantiation (such as struct fields or +// function parameters that depend on type arguments), this will be the +// corresponding Var on the generic (uninstantiated) type. For all other Vars +// Origin returns the receiver. +func (obj *Var) Origin() *Var { + if obj.origin != nil { + return obj.origin + } + return obj +} + +func (*Var) isDependency() {} // a variable may be a dependency of an initialization expression + +// A Func represents a declared function, concrete method, or abstract +// (interface) method. Its Type() is always a *Signature. +// An abstract method may belong to many interfaces due to embedding. +type Func struct { + object + hasPtrRecv_ bool // only valid for methods that don't have a type yet; use hasPtrRecv() to read + origin *Func // if non-nil, the Func from which this one was instantiated +} + +// NewFunc returns a new function with the given signature, representing +// the function's type. +func NewFunc(pos token.Pos, pkg *Package, name string, sig *Signature) *Func { + // don't store a (typed) nil signature + var typ Type + if sig != nil { + typ = sig + } + return &Func{object{nil, pos, pkg, name, typ, 0, colorFor(typ), token.NoPos}, false, nil} +} + +// FullName returns the package- or receiver-type-qualified name of +// function or method obj. +func (obj *Func) FullName() string { + var buf bytes.Buffer + writeFuncName(&buf, obj, nil) + return buf.String() +} + +// Scope returns the scope of the function's body block. +// The result is nil for imported or instantiated functions and methods +// (but there is also no mechanism to get to an instantiated function). +func (obj *Func) Scope() *Scope { return obj.typ.(*Signature).scope } + +// Origin returns the canonical Func for its receiver, i.e. the Func object +// recorded in Info.Defs. +// +// For synthetic functions created during instantiation (such as methods on an +// instantiated Named type or interface methods that depend on type arguments), +// this will be the corresponding Func on the generic (uninstantiated) type. +// For all other Funcs Origin returns the receiver. +func (obj *Func) Origin() *Func { + if obj.origin != nil { + return obj.origin + } + return obj +} + +// hasPtrRecv reports whether the receiver is of the form *T for the given method obj. +func (obj *Func) hasPtrRecv() bool { + // If a method's receiver type is set, use that as the source of truth for the receiver. + // Caution: Checker.funcDecl (decl.go) marks a function by setting its type to an empty + // signature. We may reach here before the signature is fully set up: we must explicitly + // check if the receiver is set (we cannot just look for non-nil obj.typ). + if sig, _ := obj.typ.(*Signature); sig != nil && sig.recv != nil { + _, isPtr := deref(sig.recv.typ) + return isPtr + } + + // If a method's type is not set it may be a method/function that is: + // 1) client-supplied (via NewFunc with no signature), or + // 2) internally created but not yet type-checked. + // For case 1) we can't do anything; the client must know what they are doing. + // For case 2) we can use the information gathered by the resolver. + return obj.hasPtrRecv_ +} + +func (*Func) isDependency() {} // a function may be a dependency of an initialization expression + +// A Label represents a declared label. +// Labels don't have a type. +type Label struct { + object + used bool // set if the label was used +} + +// NewLabel returns a new label. +func NewLabel(pos token.Pos, pkg *Package, name string) *Label { + return &Label{object{pos: pos, pkg: pkg, name: name, typ: Typ[Invalid], color_: black}, false} +} + +// A Builtin represents a built-in function. +// Builtins don't have a valid type. +type Builtin struct { + object + id builtinId +} + +func newBuiltin(id builtinId) *Builtin { + return &Builtin{object{name: predeclaredFuncs[id].name, typ: Typ[Invalid], color_: black}, id} +} + +// Nil represents the predeclared value nil. +type Nil struct { + object +} + +func writeObject(buf *bytes.Buffer, obj Object, qf Qualifier) { + var tname *TypeName + typ := obj.Type() + + switch obj := obj.(type) { + case *PkgName: + fmt.Fprintf(buf, "package %s", obj.Name()) + if path := obj.imported.path; path != "" && path != obj.name { + fmt.Fprintf(buf, " (%q)", path) + } + return + + case *Const: + buf.WriteString("const") + + case *TypeName: + tname = obj + buf.WriteString("type") + if isTypeParam(typ) { + buf.WriteString(" parameter") + } + + case *Var: + if obj.isField { + buf.WriteString("field") + } else { + buf.WriteString("var") + } + + case *Func: + buf.WriteString("func ") + writeFuncName(buf, obj, qf) + if typ != nil { + WriteSignature(buf, typ.(*Signature), qf) + } + return + + case *Label: + buf.WriteString("label") + typ = nil + + case *Builtin: + buf.WriteString("builtin") + typ = nil + + case *Nil: + buf.WriteString("nil") + return + + default: + panic(fmt.Sprintf("writeObject(%T)", obj)) + } + + buf.WriteByte(' ') + + // For package-level objects, qualify the name. + if obj.Pkg() != nil && obj.Pkg().scope.Lookup(obj.Name()) == obj { + buf.WriteString(packagePrefix(obj.Pkg(), qf)) + } + buf.WriteString(obj.Name()) + + if typ == nil { + return + } + + if tname != nil { + switch t := typ.(type) { + case *Basic: + // Don't print anything more for basic types since there's + // no more information. + return + case *Named: + if t.TypeParams().Len() > 0 { + newTypeWriter(buf, qf).tParamList(t.TypeParams().list()) + } + } + if tname.IsAlias() { + buf.WriteString(" =") + } else if t, _ := typ.(*TypeParam); t != nil { + typ = t.bound + } else { + // TODO(gri) should this be fromRHS for *Named? + typ = under(typ) + } + } + + // Special handling for any: because WriteType will format 'any' as 'any', + // resulting in the object string `type any = any` rather than `type any = + // interface{}`. To avoid this, swap in a different empty interface. + if obj == universeAny { + assert(Identical(typ, &emptyInterface)) + typ = &emptyInterface + } + + buf.WriteByte(' ') + WriteType(buf, typ, qf) +} + +func packagePrefix(pkg *Package, qf Qualifier) string { + if pkg == nil { + return "" + } + var s string + if qf != nil { + s = qf(pkg) + } else { + s = pkg.Path() + } + if s != "" { + s += "." + } + return s +} + +// ObjectString returns the string form of obj. +// The Qualifier controls the printing of +// package-level objects, and may be nil. +func ObjectString(obj Object, qf Qualifier) string { + var buf bytes.Buffer + writeObject(&buf, obj, qf) + return buf.String() +} + +func (obj *PkgName) String() string { return ObjectString(obj, nil) } +func (obj *Const) String() string { return ObjectString(obj, nil) } +func (obj *TypeName) String() string { return ObjectString(obj, nil) } +func (obj *Var) String() string { return ObjectString(obj, nil) } +func (obj *Func) String() string { return ObjectString(obj, nil) } +func (obj *Label) String() string { return ObjectString(obj, nil) } +func (obj *Builtin) String() string { return ObjectString(obj, nil) } +func (obj *Nil) String() string { return ObjectString(obj, nil) } + +func writeFuncName(buf *bytes.Buffer, f *Func, qf Qualifier) { + if f.typ != nil { + sig := f.typ.(*Signature) + if recv := sig.Recv(); recv != nil { + buf.WriteByte('(') + if _, ok := recv.Type().(*Interface); ok { + // gcimporter creates abstract methods of + // named interfaces using the interface type + // (not the named type) as the receiver. + // Don't print it in full. + buf.WriteString("interface") + } else { + WriteType(buf, recv.Type(), qf) + } + buf.WriteByte(')') + buf.WriteByte('.') + } else if f.pkg != nil { + buf.WriteString(packagePrefix(f.pkg, qf)) + } + } + buf.WriteString(f.name) +} diff --git a/src/go/types/object_test.go b/src/go/types/object_test.go new file mode 100644 index 0000000..60e7a84 --- /dev/null +++ b/src/go/types/object_test.go @@ -0,0 +1,166 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "go/ast" + "go/token" + "internal/testenv" + "strings" + "testing" + + . "go/types" +) + +func TestIsAlias(t *testing.T) { + check := func(obj *TypeName, want bool) { + if got := obj.IsAlias(); got != want { + t.Errorf("%v: got IsAlias = %v; want %v", obj, got, want) + } + } + + // predeclared types + check(Unsafe.Scope().Lookup("Pointer").(*TypeName), false) + for _, name := range Universe.Names() { + if obj, _ := Universe.Lookup(name).(*TypeName); obj != nil { + check(obj, name == "any" || name == "byte" || name == "rune") + } + } + + // various other types + pkg := NewPackage("p", "p") + t1 := NewTypeName(0, pkg, "t1", nil) + n1 := NewNamed(t1, new(Struct), nil) + t5 := NewTypeName(0, pkg, "t5", nil) + NewTypeParam(t5, nil) + for _, test := range []struct { + name *TypeName + alias bool + }{ + {NewTypeName(0, nil, "t0", nil), false}, // no type yet + {NewTypeName(0, pkg, "t0", nil), false}, // no type yet + {t1, false}, // type name refers to named type and vice versa + {NewTypeName(0, nil, "t2", NewInterfaceType(nil, nil)), true}, // type name refers to unnamed type + {NewTypeName(0, pkg, "t3", n1), true}, // type name refers to named type with different type name + {NewTypeName(0, nil, "t4", Typ[Int32]), true}, // type name refers to basic type with different name + {NewTypeName(0, nil, "int32", Typ[Int32]), false}, // type name refers to basic type with same name + {NewTypeName(0, pkg, "int32", Typ[Int32]), true}, // type name is declared in user-defined package (outside Universe) + {NewTypeName(0, nil, "rune", Typ[Rune]), true}, // type name refers to basic type rune which is an alias already + {t5, false}, // type name refers to type parameter and vice versa + } { + check(test.name, test.alias) + } +} + +// TestEmbeddedMethod checks that an embedded method is represented by +// the same Func Object as the original method. See also issue #34421. +func TestEmbeddedMethod(t *testing.T) { + const src = `package p; type I interface { error }` + + // type-check src + fset := token.NewFileSet() + f := mustParse(fset, "", src) + var conf Config + pkg, err := conf.Check(f.Name.Name, fset, []*ast.File{f}, nil) + if err != nil { + t.Fatalf("typecheck failed: %s", err) + } + + // get original error.Error method + eface := Universe.Lookup("error") + orig, _, _ := LookupFieldOrMethod(eface.Type(), false, nil, "Error") + if orig == nil { + t.Fatalf("original error.Error not found") + } + + // get embedded error.Error method + iface := pkg.Scope().Lookup("I") + embed, _, _ := LookupFieldOrMethod(iface.Type(), false, nil, "Error") + if embed == nil { + t.Fatalf("embedded error.Error not found") + } + + // original and embedded Error object should be identical + if orig != embed { + t.Fatalf("%s (%p) != %s (%p)", orig, orig, embed, embed) + } +} + +var testObjects = []struct { + src string + obj string + want string +}{ + {"import \"io\"; var r io.Reader", "r", "var p.r io.Reader"}, + + {"const c = 1.2", "c", "const p.c untyped float"}, + {"const c float64 = 3.14", "c", "const p.c float64"}, + + {"type t struct{f int}", "t", "type p.t struct{f int}"}, + {"type t func(int)", "t", "type p.t func(int)"}, + {"type t[P any] struct{f P}", "t", "type p.t[P any] struct{f P}"}, + {"type t[P any] struct{f P}", "t.P", "type parameter P any"}, + {"type C interface{m()}; type t[P C] struct{}", "t.P", "type parameter P p.C"}, + + {"type t = struct{f int}", "t", "type p.t = struct{f int}"}, + {"type t = func(int)", "t", "type p.t = func(int)"}, + + {"var v int", "v", "var p.v int"}, + + {"func f(int) string", "f", "func p.f(int) string"}, + {"func g[P any](x P){}", "g", "func p.g[P any](x P)"}, + {"func g[P interface{~int}](x P){}", "g.P", "type parameter P interface{~int}"}, + {"", "any", "type any = interface{}"}, +} + +func TestObjectString(t *testing.T) { + testenv.MustHaveGoBuild(t) + + for _, test := range testObjects { + src := "package p; " + test.src + pkg, err := typecheck(filename, src, nil) + if err != nil { + t.Errorf("%s: %s", src, err) + continue + } + + names := strings.Split(test.obj, ".") + if len(names) != 1 && len(names) != 2 { + t.Errorf("%s: invalid object path %s", test.src, test.obj) + continue + } + _, obj := pkg.Scope().LookupParent(names[0], token.NoPos) + if obj == nil { + t.Errorf("%s: %s not found", test.src, names[0]) + continue + } + if len(names) == 2 { + if typ, ok := obj.Type().(interface{ TypeParams() *TypeParamList }); ok { + obj = lookupTypeParamObj(typ.TypeParams(), names[1]) + if obj == nil { + t.Errorf("%s: %s not found", test.src, test.obj) + continue + } + } else { + t.Errorf("%s: %s has no type parameters", test.src, names[0]) + continue + } + } + + if got := obj.String(); got != test.want { + t.Errorf("%s: got %s, want %s", test.src, got, test.want) + } + } +} + +func lookupTypeParamObj(list *TypeParamList, name string) Object { + for i := 0; i < list.Len(); i++ { + tpar := list.At(i) + if tpar.Obj().Name() == name { + return tpar.Obj() + } + } + return nil +} diff --git a/src/go/types/objset.go b/src/go/types/objset.go new file mode 100644 index 0000000..55eb74a --- /dev/null +++ b/src/go/types/objset.go @@ -0,0 +1,31 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements objsets. +// +// An objset is similar to a Scope but objset elements +// are identified by their unique id, instead of their +// object name. + +package types + +// An objset is a set of objects identified by their unique id. +// The zero value for objset is a ready-to-use empty objset. +type objset map[string]Object // initialized lazily + +// insert attempts to insert an object obj into objset s. +// If s already contains an alternative object alt with +// the same name, insert leaves s unchanged and returns alt. +// Otherwise it inserts obj and returns nil. +func (s *objset) insert(obj Object) Object { + id := obj.Id() + if alt := (*s)[id]; alt != nil { + return alt + } + if *s == nil { + *s = make(map[string]Object) + } + (*s)[id] = obj + return nil +} diff --git a/src/go/types/operand.go b/src/go/types/operand.go new file mode 100644 index 0000000..819c99e --- /dev/null +++ b/src/go/types/operand.go @@ -0,0 +1,368 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file defines operands and associated operations. + +package types + +import ( + "bytes" + "go/ast" + "go/constant" + "go/token" + . "internal/types/errors" +) + +// An operandMode specifies the (addressing) mode of an operand. +type operandMode byte + +const ( + invalid operandMode = iota // operand is invalid + novalue // operand represents no value (result of a function call w/o result) + builtin // operand is a built-in function + typexpr // operand is a type + constant_ // operand is a constant; the operand's typ is a Basic type + variable // operand is an addressable variable + mapindex // operand is a map index expression (acts like a variable on lhs, commaok on rhs of an assignment) + value // operand is a computed value + commaok // like value, but operand may be used in a comma,ok expression + commaerr // like commaok, but second value is error, not boolean + cgofunc // operand is a cgo function +) + +var operandModeString = [...]string{ + invalid: "invalid operand", + novalue: "no value", + builtin: "built-in", + typexpr: "type", + constant_: "constant", + variable: "variable", + mapindex: "map index expression", + value: "value", + commaok: "comma, ok expression", + commaerr: "comma, error expression", + cgofunc: "cgo function", +} + +// An operand represents an intermediate value during type checking. +// Operands have an (addressing) mode, the expression evaluating to +// the operand, the operand's type, a value for constants, and an id +// for built-in functions. +// The zero value of operand is a ready to use invalid operand. +type operand struct { + mode operandMode + expr ast.Expr + typ Type + val constant.Value + id builtinId +} + +// Pos returns the position of the expression corresponding to x. +// If x is invalid the position is token.NoPos. +func (x *operand) Pos() token.Pos { + // x.expr may not be set if x is invalid + if x.expr == nil { + return token.NoPos + } + return x.expr.Pos() +} + +// Operand string formats +// (not all "untyped" cases can appear due to the type system, +// but they fall out naturally here) +// +// mode format +// +// invalid ( ) +// novalue ( ) +// builtin ( ) +// typexpr ( ) +// +// constant ( ) +// constant ( of type ) +// constant ( ) +// constant ( of type ) +// +// variable ( ) +// variable ( of type ) +// +// mapindex ( ) +// mapindex ( of type ) +// +// value ( ) +// value ( of type ) +// +// commaok ( ) +// commaok ( of type ) +// +// commaerr ( ) +// commaerr ( of type ) +// +// cgofunc ( ) +// cgofunc ( of type ) +func operandString(x *operand, qf Qualifier) string { + // special-case nil + if x.mode == value && x.typ == Typ[UntypedNil] { + return "nil" + } + + var buf bytes.Buffer + + var expr string + if x.expr != nil { + expr = ExprString(x.expr) + } else { + switch x.mode { + case builtin: + expr = predeclaredFuncs[x.id].name + case typexpr: + expr = TypeString(x.typ, qf) + case constant_: + expr = x.val.String() + } + } + + // ( + if expr != "" { + buf.WriteString(expr) + buf.WriteString(" (") + } + + // + hasType := false + switch x.mode { + case invalid, novalue, builtin, typexpr: + // no type + default: + // should have a type, but be cautious (don't crash during printing) + if x.typ != nil { + if isUntyped(x.typ) { + buf.WriteString(x.typ.(*Basic).name) + buf.WriteByte(' ') + break + } + hasType = true + } + } + + // + buf.WriteString(operandModeString[x.mode]) + + // + if x.mode == constant_ { + if s := x.val.String(); s != expr { + buf.WriteByte(' ') + buf.WriteString(s) + } + } + + // + if hasType { + if x.typ != Typ[Invalid] { + var intro string + if isGeneric(x.typ) { + intro = " of generic type " + } else { + intro = " of type " + } + buf.WriteString(intro) + WriteType(&buf, x.typ, qf) + if tpar, _ := x.typ.(*TypeParam); tpar != nil { + buf.WriteString(" constrained by ") + WriteType(&buf, tpar.bound, qf) // do not compute interface type sets here + // If we have the type set and it's empty, say so for better error messages. + if hasEmptyTypeset(tpar) { + buf.WriteString(" with empty type set") + } + } + } else { + buf.WriteString(" with invalid type") + } + } + + // ) + if expr != "" { + buf.WriteByte(')') + } + + return buf.String() +} + +func (x *operand) String() string { + return operandString(x, nil) +} + +// setConst sets x to the untyped constant for literal lit. +func (x *operand) setConst(tok token.Token, lit string) { + var kind BasicKind + switch tok { + case token.INT: + kind = UntypedInt + case token.FLOAT: + kind = UntypedFloat + case token.IMAG: + kind = UntypedComplex + case token.CHAR: + kind = UntypedRune + case token.STRING: + kind = UntypedString + default: + unreachable() + } + + val := constant.MakeFromLiteral(lit, tok, 0) + if val.Kind() == constant.Unknown { + x.mode = invalid + x.typ = Typ[Invalid] + return + } + x.mode = constant_ + x.typ = Typ[kind] + x.val = val +} + +// isNil reports whether x is the nil value. +func (x *operand) isNil() bool { + return x.mode == value && x.typ == Typ[UntypedNil] +} + +// assignableTo reports whether x is assignable to a variable of type T. If the +// result is false and a non-nil cause is provided, it may be set to a more +// detailed explanation of the failure (result != ""). The returned error code +// is only valid if the (first) result is false. The check parameter may be nil +// if assignableTo is invoked through an exported API call, i.e., when all +// methods have been type-checked. +func (x *operand) assignableTo(check *Checker, T Type, cause *string) (bool, Code) { + if x.mode == invalid || T == Typ[Invalid] { + return true, 0 // avoid spurious errors + } + + V := x.typ + + // x's type is identical to T + if Identical(V, T) { + return true, 0 + } + + Vu := under(V) + Tu := under(T) + Vp, _ := V.(*TypeParam) + Tp, _ := T.(*TypeParam) + + // x is an untyped value representable by a value of type T. + if isUntyped(Vu) { + assert(Vp == nil) + if Tp != nil { + // T is a type parameter: x is assignable to T if it is + // representable by each specific type in the type set of T. + return Tp.is(func(t *term) bool { + if t == nil { + return false + } + // A term may be a tilde term but the underlying + // type of an untyped value doesn't change so we + // don't need to do anything special. + newType, _, _ := check.implicitTypeAndValue(x, t.typ) + return newType != nil + }), IncompatibleAssign + } + newType, _, _ := check.implicitTypeAndValue(x, T) + return newType != nil, IncompatibleAssign + } + // Vu is typed + + // x's type V and T have identical underlying types + // and at least one of V or T is not a named type + // and neither V nor T is a type parameter. + if Identical(Vu, Tu) && (!hasName(V) || !hasName(T)) && Vp == nil && Tp == nil { + return true, 0 + } + + // T is an interface type and x implements T and T is not a type parameter. + // Also handle the case where T is a pointer to an interface. + if _, ok := Tu.(*Interface); ok && Tp == nil || isInterfacePtr(Tu) { + if !check.implements(V, T, false, cause) { + return false, InvalidIfaceAssign + } + return true, 0 + } + + // If V is an interface, check if a missing type assertion is the problem. + if Vi, _ := Vu.(*Interface); Vi != nil && Vp == nil { + if check.implements(T, V, false, nil) { + // T implements V, so give hint about type assertion. + if cause != nil { + *cause = "need type assertion" + } + return false, IncompatibleAssign + } + } + + // x is a bidirectional channel value, T is a channel + // type, x's type V and T have identical element types, + // and at least one of V or T is not a named type. + if Vc, ok := Vu.(*Chan); ok && Vc.dir == SendRecv { + if Tc, ok := Tu.(*Chan); ok && Identical(Vc.elem, Tc.elem) { + return !hasName(V) || !hasName(T), InvalidChanAssign + } + } + + // optimization: if we don't have type parameters, we're done + if Vp == nil && Tp == nil { + return false, IncompatibleAssign + } + + errorf := func(format string, args ...any) { + if check != nil && cause != nil { + msg := check.sprintf(format, args...) + if *cause != "" { + msg += "\n\t" + *cause + } + *cause = msg + } + } + + // x's type V is not a named type and T is a type parameter, and + // x is assignable to each specific type in T's type set. + if !hasName(V) && Tp != nil { + ok := false + code := IncompatibleAssign + Tp.is(func(T *term) bool { + if T == nil { + return false // no specific types + } + ok, code = x.assignableTo(check, T.typ, cause) + if !ok { + errorf("cannot assign %s to %s (in %s)", x.typ, T.typ, Tp) + return false + } + return true + }) + return ok, code + } + + // x's type V is a type parameter and T is not a named type, + // and values x' of each specific type in V's type set are + // assignable to T. + if Vp != nil && !hasName(T) { + x := *x // don't clobber outer x + ok := false + code := IncompatibleAssign + Vp.is(func(V *term) bool { + if V == nil { + return false // no specific types + } + x.typ = V.typ + ok, code = x.assignableTo(check, T, cause) + if !ok { + errorf("cannot assign %s (in %s) to %s", V.typ, Vp, T) + return false + } + return true + }) + return ok, code + } + + return false, IncompatibleAssign +} diff --git a/src/go/types/package.go b/src/go/types/package.go new file mode 100644 index 0000000..2b72ff1 --- /dev/null +++ b/src/go/types/package.go @@ -0,0 +1,74 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "fmt" + "go/token" +) + +// A Package describes a Go package. +type Package struct { + path string + name string + scope *Scope + complete bool + imports []*Package + fake bool // scope lookup errors are silently dropped if package is fake (internal use only) + cgo bool // uses of this package will be rewritten into uses of declarations from _cgo_gotypes.go +} + +// NewPackage returns a new Package for the given package path and name. +// The package is not complete and contains no explicit imports. +func NewPackage(path, name string) *Package { + scope := NewScope(Universe, token.NoPos, token.NoPos, fmt.Sprintf("package %q", path)) + return &Package{path: path, name: name, scope: scope} +} + +// Path returns the package path. +func (pkg *Package) Path() string { return pkg.path } + +// Name returns the package name. +func (pkg *Package) Name() string { return pkg.name } + +// SetName sets the package name. +func (pkg *Package) SetName(name string) { pkg.name = name } + +// Scope returns the (complete or incomplete) package scope +// holding the objects declared at package level (TypeNames, +// Consts, Vars, and Funcs). +// For a nil pkg receiver, Scope returns the Universe scope. +func (pkg *Package) Scope() *Scope { + if pkg != nil { + return pkg.scope + } + return Universe +} + +// A package is complete if its scope contains (at least) all +// exported objects; otherwise it is incomplete. +func (pkg *Package) Complete() bool { return pkg.complete } + +// MarkComplete marks a package as complete. +func (pkg *Package) MarkComplete() { pkg.complete = true } + +// Imports returns the list of packages directly imported by +// pkg; the list is in source order. +// +// If pkg was loaded from export data, Imports includes packages that +// provide package-level objects referenced by pkg. This may be more or +// less than the set of packages directly imported by pkg's source code. +// +// If pkg uses cgo and the FakeImportC configuration option +// was enabled, the imports list may contain a fake "C" package. +func (pkg *Package) Imports() []*Package { return pkg.imports } + +// SetImports sets the list of explicitly imported packages to list. +// It is the caller's responsibility to make sure list elements are unique. +func (pkg *Package) SetImports(list []*Package) { pkg.imports = list } + +func (pkg *Package) String() string { + return fmt.Sprintf("package %s (%q)", pkg.name, pkg.path) +} diff --git a/src/go/types/pointer.go b/src/go/types/pointer.go new file mode 100644 index 0000000..6352ee5 --- /dev/null +++ b/src/go/types/pointer.go @@ -0,0 +1,19 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +// A Pointer represents a pointer type. +type Pointer struct { + base Type // element type +} + +// NewPointer returns a new pointer type for the given element (base) type. +func NewPointer(elem Type) *Pointer { return &Pointer{base: elem} } + +// Elem returns the element type for the given pointer p. +func (p *Pointer) Elem() Type { return p.base } + +func (t *Pointer) Underlying() Type { return t } +func (t *Pointer) String() string { return TypeString(t, nil) } diff --git a/src/go/types/predicates.go b/src/go/types/predicates.go new file mode 100644 index 0000000..e9a0e43 --- /dev/null +++ b/src/go/types/predicates.go @@ -0,0 +1,497 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements commonly used type predicates. + +package types + +import "go/token" + +// The isX predicates below report whether t is an X. +// If t is a type parameter the result is false; i.e., +// these predicates don't look inside a type parameter. + +func isBoolean(t Type) bool { return isBasic(t, IsBoolean) } +func isInteger(t Type) bool { return isBasic(t, IsInteger) } +func isUnsigned(t Type) bool { return isBasic(t, IsUnsigned) } +func isFloat(t Type) bool { return isBasic(t, IsFloat) } +func isComplex(t Type) bool { return isBasic(t, IsComplex) } +func isNumeric(t Type) bool { return isBasic(t, IsNumeric) } +func isString(t Type) bool { return isBasic(t, IsString) } +func isIntegerOrFloat(t Type) bool { return isBasic(t, IsInteger|IsFloat) } +func isConstType(t Type) bool { return isBasic(t, IsConstType) } + +// isBasic reports whether under(t) is a basic type with the specified info. +// If t is a type parameter the result is false; i.e., +// isBasic does not look inside a type parameter. +func isBasic(t Type, info BasicInfo) bool { + u, _ := under(t).(*Basic) + return u != nil && u.info&info != 0 +} + +// The allX predicates below report whether t is an X. +// If t is a type parameter the result is true if isX is true +// for all specified types of the type parameter's type set. +// allX is an optimized version of isX(coreType(t)) (which +// is the same as underIs(t, isX)). + +func allBoolean(typ Type) bool { return allBasic(typ, IsBoolean) } +func allInteger(typ Type) bool { return allBasic(typ, IsInteger) } +func allUnsigned(typ Type) bool { return allBasic(typ, IsUnsigned) } +func allNumeric(typ Type) bool { return allBasic(typ, IsNumeric) } +func allString(typ Type) bool { return allBasic(typ, IsString) } +func allOrdered(typ Type) bool { return allBasic(typ, IsOrdered) } +func allNumericOrString(typ Type) bool { return allBasic(typ, IsNumeric|IsString) } + +// allBasic reports whether under(t) is a basic type with the specified info. +// If t is a type parameter, the result is true if isBasic(t, info) is true +// for all specific types of the type parameter's type set. +// allBasic(t, info) is an optimized version of isBasic(coreType(t), info). +func allBasic(t Type, info BasicInfo) bool { + if tpar, _ := t.(*TypeParam); tpar != nil { + return tpar.is(func(t *term) bool { return t != nil && isBasic(t.typ, info) }) + } + return isBasic(t, info) +} + +// hasName reports whether t has a name. This includes +// predeclared types, defined types, and type parameters. +// hasName may be called with types that are not fully set up. +func hasName(t Type) bool { + switch t.(type) { + case *Basic, *Named, *TypeParam: + return true + } + return false +} + +// isTyped reports whether t is typed; i.e., not an untyped +// constant or boolean. isTyped may be called with types that +// are not fully set up. +func isTyped(t Type) bool { + // isTyped is called with types that are not fully + // set up. Must not call under()! + b, _ := t.(*Basic) + return b == nil || b.info&IsUntyped == 0 +} + +// isUntyped(t) is the same as !isTyped(t). +func isUntyped(t Type) bool { + return !isTyped(t) +} + +// IsInterface reports whether t is an interface type. +func IsInterface(t Type) bool { + _, ok := under(t).(*Interface) + return ok +} + +// isNonTypeParamInterface reports whether t is an interface type but not a type parameter. +func isNonTypeParamInterface(t Type) bool { + return !isTypeParam(t) && IsInterface(t) +} + +// isTypeParam reports whether t is a type parameter. +func isTypeParam(t Type) bool { + _, ok := t.(*TypeParam) + return ok +} + +// hasEmptyTypeset reports whether t is a type parameter with an empty type set. +// The function does not force the computation of the type set and so is safe to +// use anywhere, but it may report a false negative if the type set has not been +// computed yet. +func hasEmptyTypeset(t Type) bool { + if tpar, _ := t.(*TypeParam); tpar != nil && tpar.bound != nil { + iface, _ := safeUnderlying(tpar.bound).(*Interface) + return iface != nil && iface.tset != nil && iface.tset.IsEmpty() + } + return false +} + +// isGeneric reports whether a type is a generic, uninstantiated type +// (generic signatures are not included). +// TODO(gri) should we include signatures or assert that they are not present? +func isGeneric(t Type) bool { + // A parameterized type is only generic if it doesn't have an instantiation already. + named, _ := t.(*Named) + return named != nil && named.obj != nil && named.inst == nil && named.TypeParams().Len() > 0 +} + +// Comparable reports whether values of type T are comparable. +func Comparable(T Type) bool { + return comparable(T, true, nil, nil) +} + +// If dynamic is set, non-type parameter interfaces are always comparable. +// If reportf != nil, it may be used to report why T is not comparable. +func comparable(T Type, dynamic bool, seen map[Type]bool, reportf func(string, ...interface{})) bool { + if seen[T] { + return true + } + if seen == nil { + seen = make(map[Type]bool) + } + seen[T] = true + + switch t := under(T).(type) { + case *Basic: + // assume invalid types to be comparable + // to avoid follow-up errors + return t.kind != UntypedNil + case *Pointer, *Chan: + return true + case *Struct: + for _, f := range t.fields { + if !comparable(f.typ, dynamic, seen, nil) { + if reportf != nil { + reportf("struct containing %s cannot be compared", f.typ) + } + return false + } + } + return true + case *Array: + if !comparable(t.elem, dynamic, seen, nil) { + if reportf != nil { + reportf("%s cannot be compared", t) + } + return false + } + return true + case *Interface: + if dynamic && !isTypeParam(T) || t.typeSet().IsComparable(seen) { + return true + } + if reportf != nil { + if t.typeSet().IsEmpty() { + reportf("empty type set") + } else { + reportf("incomparable types in type set") + } + } + // fallthrough + } + return false +} + +// hasNil reports whether type t includes the nil value. +func hasNil(t Type) bool { + switch u := under(t).(type) { + case *Basic: + return u.kind == UnsafePointer + case *Slice, *Pointer, *Signature, *Map, *Chan: + return true + case *Interface: + return !isTypeParam(t) || u.typeSet().underIs(func(u Type) bool { + return u != nil && hasNil(u) + }) + } + return false +} + +// An ifacePair is a node in a stack of interface type pairs compared for identity. +type ifacePair struct { + x, y *Interface + prev *ifacePair +} + +func (p *ifacePair) identical(q *ifacePair) bool { + return p.x == q.x && p.y == q.y || p.x == q.y && p.y == q.x +} + +// For changes to this code the corresponding changes should be made to unifier.nify. +func identical(x, y Type, cmpTags bool, p *ifacePair) bool { + if x == y { + return true + } + + switch x := x.(type) { + case *Basic: + // Basic types are singletons except for the rune and byte + // aliases, thus we cannot solely rely on the x == y check + // above. See also comment in TypeName.IsAlias. + if y, ok := y.(*Basic); ok { + return x.kind == y.kind + } + + case *Array: + // Two array types are identical if they have identical element types + // and the same array length. + if y, ok := y.(*Array); ok { + // If one or both array lengths are unknown (< 0) due to some error, + // assume they are the same to avoid spurious follow-on errors. + return (x.len < 0 || y.len < 0 || x.len == y.len) && identical(x.elem, y.elem, cmpTags, p) + } + + case *Slice: + // Two slice types are identical if they have identical element types. + if y, ok := y.(*Slice); ok { + return identical(x.elem, y.elem, cmpTags, p) + } + + case *Struct: + // Two struct types are identical if they have the same sequence of fields, + // and if corresponding fields have the same names, and identical types, + // and identical tags. Two embedded fields are considered to have the same + // name. Lower-case field names from different packages are always different. + if y, ok := y.(*Struct); ok { + if x.NumFields() == y.NumFields() { + for i, f := range x.fields { + g := y.fields[i] + if f.embedded != g.embedded || + cmpTags && x.Tag(i) != y.Tag(i) || + !f.sameId(g.pkg, g.name) || + !identical(f.typ, g.typ, cmpTags, p) { + return false + } + } + return true + } + } + + case *Pointer: + // Two pointer types are identical if they have identical base types. + if y, ok := y.(*Pointer); ok { + return identical(x.base, y.base, cmpTags, p) + } + + case *Tuple: + // Two tuples types are identical if they have the same number of elements + // and corresponding elements have identical types. + if y, ok := y.(*Tuple); ok { + if x.Len() == y.Len() { + if x != nil { + for i, v := range x.vars { + w := y.vars[i] + if !identical(v.typ, w.typ, cmpTags, p) { + return false + } + } + } + return true + } + } + + case *Signature: + y, _ := y.(*Signature) + if y == nil { + return false + } + + // Two function types are identical if they have the same number of + // parameters and result values, corresponding parameter and result types + // are identical, and either both functions are variadic or neither is. + // Parameter and result names are not required to match, and type + // parameters are considered identical modulo renaming. + + if x.TypeParams().Len() != y.TypeParams().Len() { + return false + } + + // In the case of generic signatures, we will substitute in yparams and + // yresults. + yparams := y.params + yresults := y.results + + if x.TypeParams().Len() > 0 { + // We must ignore type parameter names when comparing x and y. The + // easiest way to do this is to substitute x's type parameters for y's. + xtparams := x.TypeParams().list() + ytparams := y.TypeParams().list() + + var targs []Type + for i := range xtparams { + targs = append(targs, x.TypeParams().At(i)) + } + smap := makeSubstMap(ytparams, targs) + + var check *Checker // ok to call subst on a nil *Checker + ctxt := NewContext() // need a non-nil Context for the substitution below + + // Constraints must be pair-wise identical, after substitution. + for i, xtparam := range xtparams { + ybound := check.subst(token.NoPos, ytparams[i].bound, smap, nil, ctxt) + if !identical(xtparam.bound, ybound, cmpTags, p) { + return false + } + } + + yparams = check.subst(token.NoPos, y.params, smap, nil, ctxt).(*Tuple) + yresults = check.subst(token.NoPos, y.results, smap, nil, ctxt).(*Tuple) + } + + return x.variadic == y.variadic && + identical(x.params, yparams, cmpTags, p) && + identical(x.results, yresults, cmpTags, p) + + case *Union: + if y, _ := y.(*Union); y != nil { + // TODO(rfindley): can this be reached during type checking? If so, + // consider passing a type set map. + unionSets := make(map[*Union]*_TypeSet) + xset := computeUnionTypeSet(nil, unionSets, token.NoPos, x) + yset := computeUnionTypeSet(nil, unionSets, token.NoPos, y) + return xset.terms.equal(yset.terms) + } + + case *Interface: + // Two interface types are identical if they describe the same type sets. + // With the existing implementation restriction, this simplifies to: + // + // Two interface types are identical if they have the same set of methods with + // the same names and identical function types, and if any type restrictions + // are the same. Lower-case method names from different packages are always + // different. The order of the methods is irrelevant. + if y, ok := y.(*Interface); ok { + xset := x.typeSet() + yset := y.typeSet() + if xset.comparable != yset.comparable { + return false + } + if !xset.terms.equal(yset.terms) { + return false + } + a := xset.methods + b := yset.methods + if len(a) == len(b) { + // Interface types are the only types where cycles can occur + // that are not "terminated" via named types; and such cycles + // can only be created via method parameter types that are + // anonymous interfaces (directly or indirectly) embedding + // the current interface. Example: + // + // type T interface { + // m() interface{T} + // } + // + // If two such (differently named) interfaces are compared, + // endless recursion occurs if the cycle is not detected. + // + // If x and y were compared before, they must be equal + // (if they were not, the recursion would have stopped); + // search the ifacePair stack for the same pair. + // + // This is a quadratic algorithm, but in practice these stacks + // are extremely short (bounded by the nesting depth of interface + // type declarations that recur via parameter types, an extremely + // rare occurrence). An alternative implementation might use a + // "visited" map, but that is probably less efficient overall. + q := &ifacePair{x, y, p} + for p != nil { + if p.identical(q) { + return true // same pair was compared before + } + p = p.prev + } + if debug { + assertSortedMethods(a) + assertSortedMethods(b) + } + for i, f := range a { + g := b[i] + if f.Id() != g.Id() || !identical(f.typ, g.typ, cmpTags, q) { + return false + } + } + return true + } + } + + case *Map: + // Two map types are identical if they have identical key and value types. + if y, ok := y.(*Map); ok { + return identical(x.key, y.key, cmpTags, p) && identical(x.elem, y.elem, cmpTags, p) + } + + case *Chan: + // Two channel types are identical if they have identical value types + // and the same direction. + if y, ok := y.(*Chan); ok { + return x.dir == y.dir && identical(x.elem, y.elem, cmpTags, p) + } + + case *Named: + // Two named types are identical if their type names originate + // in the same type declaration. + if y, ok := y.(*Named); ok { + xargs := x.TypeArgs().list() + yargs := y.TypeArgs().list() + + if len(xargs) != len(yargs) { + return false + } + + if len(xargs) > 0 { + // Instances are identical if their original type and type arguments + // are identical. + if !Identical(x.Origin(), y.Origin()) { + return false + } + for i, xa := range xargs { + if !Identical(xa, yargs[i]) { + return false + } + } + return true + } + + // TODO(gri) Why is x == y not sufficient? And if it is, + // we can just return false here because x == y + // is caught in the very beginning of this function. + return x.obj == y.obj + } + + case *TypeParam: + // nothing to do (x and y being equal is caught in the very beginning of this function) + + case nil: + // avoid a crash in case of nil type + + default: + unreachable() + } + + return false +} + +// identicalInstance reports if two type instantiations are identical. +// Instantiations are identical if their origin and type arguments are +// identical. +func identicalInstance(xorig Type, xargs []Type, yorig Type, yargs []Type) bool { + if len(xargs) != len(yargs) { + return false + } + + for i, xa := range xargs { + if !Identical(xa, yargs[i]) { + return false + } + } + + return Identical(xorig, yorig) +} + +// Default returns the default "typed" type for an "untyped" type; +// it returns the incoming type for all other types. The default type +// for untyped nil is untyped nil. +func Default(t Type) Type { + if t, ok := t.(*Basic); ok { + switch t.kind { + case UntypedBool: + return Typ[Bool] + case UntypedInt: + return Typ[Int] + case UntypedRune: + return universeRune // use 'rune' name + case UntypedFloat: + return Typ[Float64] + case UntypedComplex: + return Typ[Complex128] + case UntypedString: + return Typ[String] + } + } + return t +} diff --git a/src/go/types/resolver.go b/src/go/types/resolver.go new file mode 100644 index 0000000..075bd91 --- /dev/null +++ b/src/go/types/resolver.go @@ -0,0 +1,726 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "fmt" + "go/ast" + "go/constant" + "go/internal/typeparams" + "go/token" + . "internal/types/errors" + "sort" + "strconv" + "strings" + "unicode" +) + +// A declInfo describes a package-level const, type, var, or func declaration. +type declInfo struct { + file *Scope // scope of file containing this declaration + lhs []*Var // lhs of n:1 variable declarations, or nil + vtyp ast.Expr // type, or nil (for const and var declarations only) + init ast.Expr // init/orig expression, or nil (for const and var declarations only) + inherited bool // if set, the init expression is inherited from a previous constant declaration + tdecl *ast.TypeSpec // type declaration, or nil + fdecl *ast.FuncDecl // func declaration, or nil + + // The deps field tracks initialization expression dependencies. + deps map[Object]bool // lazily initialized +} + +// hasInitializer reports whether the declared object has an initialization +// expression or function body. +func (d *declInfo) hasInitializer() bool { + return d.init != nil || d.fdecl != nil && d.fdecl.Body != nil +} + +// addDep adds obj to the set of objects d's init expression depends on. +func (d *declInfo) addDep(obj Object) { + m := d.deps + if m == nil { + m = make(map[Object]bool) + d.deps = m + } + m[obj] = true +} + +// arityMatch checks that the lhs and rhs of a const or var decl +// have the appropriate number of names and init exprs. For const +// decls, init is the value spec providing the init exprs; for +// var decls, init is nil (the init exprs are in s in this case). +func (check *Checker) arityMatch(s, init *ast.ValueSpec) { + l := len(s.Names) + r := len(s.Values) + if init != nil { + r = len(init.Values) + } + + const code = WrongAssignCount + switch { + case init == nil && r == 0: + // var decl w/o init expr + if s.Type == nil { + check.error(s, code, "missing type or init expr") + } + case l < r: + if l < len(s.Values) { + // init exprs from s + n := s.Values[l] + check.errorf(n, code, "extra init expr %s", n) + // TODO(gri) avoid declared and not used error here + } else { + // init exprs "inherited" + check.errorf(s, code, "extra init expr at %s", check.fset.Position(init.Pos())) + // TODO(gri) avoid declared and not used error here + } + case l > r && (init != nil || r != 1): + n := s.Names[r] + check.errorf(n, code, "missing init expr for %s", n) + } +} + +func validatedImportPath(path string) (string, error) { + s, err := strconv.Unquote(path) + if err != nil { + return "", err + } + if s == "" { + return "", fmt.Errorf("empty string") + } + const illegalChars = `!"#$%&'()*,:;<=>?[\]^{|}` + "`\uFFFD" + for _, r := range s { + if !unicode.IsGraphic(r) || unicode.IsSpace(r) || strings.ContainsRune(illegalChars, r) { + return s, fmt.Errorf("invalid character %#U", r) + } + } + return s, nil +} + +// declarePkgObj declares obj in the package scope, records its ident -> obj mapping, +// and updates check.objMap. The object must not be a function or method. +func (check *Checker) declarePkgObj(ident *ast.Ident, obj Object, d *declInfo) { + assert(ident.Name == obj.Name()) + + // spec: "A package-scope or file-scope identifier with name init + // may only be declared to be a function with this (func()) signature." + if ident.Name == "init" { + check.error(ident, InvalidInitDecl, "cannot declare init - must be func") + return + } + + // spec: "The main package must have package name main and declare + // a function main that takes no arguments and returns no value." + if ident.Name == "main" && check.pkg.name == "main" { + check.error(ident, InvalidMainDecl, "cannot declare main - must be func") + return + } + + check.declare(check.pkg.scope, ident, obj, token.NoPos) + check.objMap[obj] = d + obj.setOrder(uint32(len(check.objMap))) +} + +// filename returns a filename suitable for debugging output. +func (check *Checker) filename(fileNo int) string { + file := check.files[fileNo] + if pos := file.Pos(); pos.IsValid() { + return check.fset.File(pos).Name() + } + return fmt.Sprintf("file[%d]", fileNo) +} + +func (check *Checker) importPackage(at positioner, path, dir string) *Package { + // If we already have a package for the given (path, dir) + // pair, use it instead of doing a full import. + // Checker.impMap only caches packages that are marked Complete + // or fake (dummy packages for failed imports). Incomplete but + // non-fake packages do require an import to complete them. + key := importKey{path, dir} + imp := check.impMap[key] + if imp != nil { + return imp + } + + // no package yet => import it + if path == "C" && (check.conf.FakeImportC || check.conf.go115UsesCgo) { + imp = NewPackage("C", "C") + imp.fake = true // package scope is not populated + imp.cgo = check.conf.go115UsesCgo + } else { + // ordinary import + var err error + if importer := check.conf.Importer; importer == nil { + err = fmt.Errorf("Config.Importer not installed") + } else if importerFrom, ok := importer.(ImporterFrom); ok { + imp, err = importerFrom.ImportFrom(path, dir, 0) + if imp == nil && err == nil { + err = fmt.Errorf("Config.Importer.ImportFrom(%s, %s, 0) returned nil but no error", path, dir) + } + } else { + imp, err = importer.Import(path) + if imp == nil && err == nil { + err = fmt.Errorf("Config.Importer.Import(%s) returned nil but no error", path) + } + } + // make sure we have a valid package name + // (errors here can only happen through manipulation of packages after creation) + if err == nil && imp != nil && (imp.name == "_" || imp.name == "") { + err = fmt.Errorf("invalid package name: %q", imp.name) + imp = nil // create fake package below + } + if err != nil { + check.errorf(at, BrokenImport, "could not import %s (%s)", path, err) + if imp == nil { + // create a new fake package + // come up with a sensible package name (heuristic) + name := path + if i := len(name); i > 0 && name[i-1] == '/' { + name = name[:i-1] + } + if i := strings.LastIndex(name, "/"); i >= 0 { + name = name[i+1:] + } + imp = NewPackage(path, name) + } + // continue to use the package as best as we can + imp.fake = true // avoid follow-up lookup failures + } + } + + // package should be complete or marked fake, but be cautious + if imp.complete || imp.fake { + check.impMap[key] = imp + // Once we've formatted an error message, keep the pkgPathMap + // up-to-date on subsequent imports. It is used for package + // qualification in error messages. + if check.pkgPathMap != nil { + check.markImports(imp) + } + return imp + } + + // something went wrong (importer may have returned incomplete package without error) + return nil +} + +// collectObjects collects all file and package objects and inserts them +// into their respective scopes. It also performs imports and associates +// methods with receiver base type names. +func (check *Checker) collectObjects() { + pkg := check.pkg + + // pkgImports is the set of packages already imported by any package file seen + // so far. Used to avoid duplicate entries in pkg.imports. Allocate and populate + // it (pkg.imports may not be empty if we are checking test files incrementally). + // Note that pkgImports is keyed by package (and thus package path), not by an + // importKey value. Two different importKey values may map to the same package + // which is why we cannot use the check.impMap here. + var pkgImports = make(map[*Package]bool) + for _, imp := range pkg.imports { + pkgImports[imp] = true + } + + type methodInfo struct { + obj *Func // method + ptr bool // true if pointer receiver + recv *ast.Ident // receiver type name + } + var methods []methodInfo // collected methods with valid receivers and non-blank _ names + var fileScopes []*Scope + for fileNo, file := range check.files { + // The package identifier denotes the current package, + // but there is no corresponding package object. + check.recordDef(file.Name, nil) + + // Use the actual source file extent rather than *ast.File extent since the + // latter doesn't include comments which appear at the start or end of the file. + // Be conservative and use the *ast.File extent if we don't have a *token.File. + pos, end := file.Pos(), file.End() + if f := check.fset.File(file.Pos()); f != nil { + pos, end = token.Pos(f.Base()), token.Pos(f.Base()+f.Size()) + } + fileScope := NewScope(check.pkg.scope, pos, end, check.filename(fileNo)) + fileScopes = append(fileScopes, fileScope) + check.recordScope(file, fileScope) + + // determine file directory, necessary to resolve imports + // FileName may be "" (typically for tests) in which case + // we get "." as the directory which is what we would want. + fileDir := dir(check.fset.Position(file.Name.Pos()).Filename) + + check.walkDecls(file.Decls, func(d decl) { + switch d := d.(type) { + case importDecl: + // import package + if d.spec.Path.Value == "" { + return // error reported by parser + } + path, err := validatedImportPath(d.spec.Path.Value) + if err != nil { + check.errorf(d.spec.Path, BadImportPath, "invalid import path (%s)", err) + return + } + + imp := check.importPackage(d.spec.Path, path, fileDir) + if imp == nil { + return + } + + // local name overrides imported package name + name := imp.name + if d.spec.Name != nil { + name = d.spec.Name.Name + if path == "C" { + // match 1.17 cmd/compile (not prescribed by spec) + check.error(d.spec.Name, ImportCRenamed, `cannot rename import "C"`) + return + } + } + + if name == "init" { + check.error(d.spec, InvalidInitDecl, "cannot import package as init - init must be a func") + return + } + + // add package to list of explicit imports + // (this functionality is provided as a convenience + // for clients; it is not needed for type-checking) + if !pkgImports[imp] { + pkgImports[imp] = true + pkg.imports = append(pkg.imports, imp) + } + + pkgName := NewPkgName(d.spec.Pos(), pkg, name, imp) + if d.spec.Name != nil { + // in a dot-import, the dot represents the package + check.recordDef(d.spec.Name, pkgName) + } else { + check.recordImplicit(d.spec, pkgName) + } + + if imp.fake { + // match 1.17 cmd/compile (not prescribed by spec) + pkgName.used = true + } + + // add import to file scope + check.imports = append(check.imports, pkgName) + if name == "." { + // dot-import + if check.dotImportMap == nil { + check.dotImportMap = make(map[dotImportKey]*PkgName) + } + // merge imported scope with file scope + for name, obj := range imp.scope.elems { + // Note: Avoid eager resolve(name, obj) here, so we only + // resolve dot-imported objects as needed. + + // A package scope may contain non-exported objects, + // do not import them! + if token.IsExported(name) { + // declare dot-imported object + // (Do not use check.declare because it modifies the object + // via Object.setScopePos, which leads to a race condition; + // the object may be imported into more than one file scope + // concurrently. See issue #32154.) + if alt := fileScope.Lookup(name); alt != nil { + check.errorf(d.spec.Name, DuplicateDecl, "%s redeclared in this block", alt.Name()) + check.reportAltDecl(alt) + } else { + fileScope.insert(name, obj) + check.dotImportMap[dotImportKey{fileScope, name}] = pkgName + } + } + } + } else { + // declare imported package object in file scope + // (no need to provide s.Name since we called check.recordDef earlier) + check.declare(fileScope, nil, pkgName, token.NoPos) + } + case constDecl: + // declare all constants + for i, name := range d.spec.Names { + obj := NewConst(name.Pos(), pkg, name.Name, nil, constant.MakeInt64(int64(d.iota))) + + var init ast.Expr + if i < len(d.init) { + init = d.init[i] + } + + d := &declInfo{file: fileScope, vtyp: d.typ, init: init, inherited: d.inherited} + check.declarePkgObj(name, obj, d) + } + + case varDecl: + lhs := make([]*Var, len(d.spec.Names)) + // If there's exactly one rhs initializer, use + // the same declInfo d1 for all lhs variables + // so that each lhs variable depends on the same + // rhs initializer (n:1 var declaration). + var d1 *declInfo + if len(d.spec.Values) == 1 { + // The lhs elements are only set up after the for loop below, + // but that's ok because declareVar only collects the declInfo + // for a later phase. + d1 = &declInfo{file: fileScope, lhs: lhs, vtyp: d.spec.Type, init: d.spec.Values[0]} + } + + // declare all variables + for i, name := range d.spec.Names { + obj := NewVar(name.Pos(), pkg, name.Name, nil) + lhs[i] = obj + + di := d1 + if di == nil { + // individual assignments + var init ast.Expr + if i < len(d.spec.Values) { + init = d.spec.Values[i] + } + di = &declInfo{file: fileScope, vtyp: d.spec.Type, init: init} + } + + check.declarePkgObj(name, obj, di) + } + case typeDecl: + if d.spec.TypeParams.NumFields() != 0 && !check.allowVersion(pkg, 1, 18) { + check.softErrorf(d.spec.TypeParams.List[0], UnsupportedFeature, "type parameter requires go1.18 or later") + } + obj := NewTypeName(d.spec.Name.Pos(), pkg, d.spec.Name.Name, nil) + check.declarePkgObj(d.spec.Name, obj, &declInfo{file: fileScope, tdecl: d.spec}) + case funcDecl: + name := d.decl.Name.Name + obj := NewFunc(d.decl.Name.Pos(), pkg, name, nil) + hasTParamError := false // avoid duplicate type parameter errors + if d.decl.Recv.NumFields() == 0 { + // regular function + if d.decl.Recv != nil { + check.error(d.decl.Recv, BadRecv, "method has no receiver") + // treat as function + } + if name == "init" || (name == "main" && check.pkg.name == "main") { + code := InvalidInitDecl + if name == "main" { + code = InvalidMainDecl + } + if d.decl.Type.TypeParams.NumFields() != 0 { + check.softErrorf(d.decl.Type.TypeParams.List[0], code, "func %s must have no type parameters", name) + hasTParamError = true + } + if t := d.decl.Type; t.Params.NumFields() != 0 || t.Results != nil { + // TODO(rFindley) Should this be a hard error? + check.softErrorf(d.decl.Name, code, "func %s must have no arguments and no return values", name) + } + } + if name == "init" { + // don't declare init functions in the package scope - they are invisible + obj.parent = pkg.scope + check.recordDef(d.decl.Name, obj) + // init functions must have a body + if d.decl.Body == nil { + // TODO(gri) make this error message consistent with the others above + check.softErrorf(obj, MissingInitBody, "missing function body") + } + } else { + check.declare(pkg.scope, d.decl.Name, obj, token.NoPos) + } + } else { + // method + + // TODO(rFindley) earlier versions of this code checked that methods + // have no type parameters, but this is checked later + // when type checking the function type. Confirm that + // we don't need to check tparams here. + + ptr, recv, _ := check.unpackRecv(d.decl.Recv.List[0].Type, false) + // (Methods with invalid receiver cannot be associated to a type, and + // methods with blank _ names are never found; no need to collect any + // of them. They will still be type-checked with all the other functions.) + if recv != nil && name != "_" { + methods = append(methods, methodInfo{obj, ptr, recv}) + } + check.recordDef(d.decl.Name, obj) + } + if d.decl.Type.TypeParams.NumFields() != 0 && !check.allowVersion(pkg, 1, 18) && !hasTParamError { + check.softErrorf(d.decl.Type.TypeParams.List[0], UnsupportedFeature, "type parameter requires go1.18 or later") + } + info := &declInfo{file: fileScope, fdecl: d.decl} + // Methods are not package-level objects but we still track them in the + // object map so that we can handle them like regular functions (if the + // receiver is invalid); also we need their fdecl info when associating + // them with their receiver base type, below. + check.objMap[obj] = info + obj.setOrder(uint32(len(check.objMap))) + } + }) + } + + // verify that objects in package and file scopes have different names + for _, scope := range fileScopes { + for name, obj := range scope.elems { + if alt := pkg.scope.Lookup(name); alt != nil { + obj = resolve(name, obj) + if pkg, ok := obj.(*PkgName); ok { + check.errorf(alt, DuplicateDecl, "%s already declared through import of %s", alt.Name(), pkg.Imported()) + check.reportAltDecl(pkg) + } else { + check.errorf(alt, DuplicateDecl, "%s already declared through dot-import of %s", alt.Name(), obj.Pkg()) + // TODO(gri) dot-imported objects don't have a position; reportAltDecl won't print anything + check.reportAltDecl(obj) + } + } + } + } + + // Now that we have all package scope objects and all methods, + // associate methods with receiver base type name where possible. + // Ignore methods that have an invalid receiver. They will be + // type-checked later, with regular functions. + if methods == nil { + return // nothing to do + } + check.methods = make(map[*TypeName][]*Func) + for i := range methods { + m := &methods[i] + // Determine the receiver base type and associate m with it. + ptr, base := check.resolveBaseTypeName(m.ptr, m.recv) + if base != nil { + m.obj.hasPtrRecv_ = ptr + check.methods[base] = append(check.methods[base], m.obj) + } + } +} + +// unpackRecv unpacks a receiver type and returns its components: ptr indicates whether +// rtyp is a pointer receiver, rname is the receiver type name, and tparams are its +// type parameters, if any. The type parameters are only unpacked if unpackParams is +// set. If rname is nil, the receiver is unusable (i.e., the source has a bug which we +// cannot easily work around). +func (check *Checker) unpackRecv(rtyp ast.Expr, unpackParams bool) (ptr bool, rname *ast.Ident, tparams []*ast.Ident) { +L: // unpack receiver type + // This accepts invalid receivers such as ***T and does not + // work for other invalid receivers, but we don't care. The + // validity of receiver expressions is checked elsewhere. + for { + switch t := rtyp.(type) { + case *ast.ParenExpr: + rtyp = t.X + case *ast.StarExpr: + ptr = true + rtyp = t.X + default: + break L + } + } + + // unpack type parameters, if any + switch rtyp.(type) { + case *ast.IndexExpr, *ast.IndexListExpr: + ix := typeparams.UnpackIndexExpr(rtyp) + rtyp = ix.X + if unpackParams { + for _, arg := range ix.Indices { + var par *ast.Ident + switch arg := arg.(type) { + case *ast.Ident: + par = arg + case *ast.BadExpr: + // ignore - error already reported by parser + case nil: + check.error(ix.Orig, InvalidSyntaxTree, "parameterized receiver contains nil parameters") + default: + check.errorf(arg, BadDecl, "receiver type parameter %s must be an identifier", arg) + } + if par == nil { + par = &ast.Ident{NamePos: arg.Pos(), Name: "_"} + } + tparams = append(tparams, par) + } + } + } + + // unpack receiver name + if name, _ := rtyp.(*ast.Ident); name != nil { + rname = name + } + + return +} + +// resolveBaseTypeName returns the non-alias base type name for typ, and whether +// there was a pointer indirection to get to it. The base type name must be declared +// in package scope, and there can be at most one pointer indirection. If no such type +// name exists, the returned base is nil. +func (check *Checker) resolveBaseTypeName(seenPtr bool, name *ast.Ident) (ptr bool, base *TypeName) { + // Algorithm: Starting from a type expression, which may be a name, + // we follow that type through alias declarations until we reach a + // non-alias type name. If we encounter anything but pointer types or + // parentheses we're done. If we encounter more than one pointer type + // we're done. + ptr = seenPtr + var seen map[*TypeName]bool + var typ ast.Expr = name + for { + typ = unparen(typ) + + // check if we have a pointer type + if pexpr, _ := typ.(*ast.StarExpr); pexpr != nil { + // if we've already seen a pointer, we're done + if ptr { + return false, nil + } + ptr = true + typ = unparen(pexpr.X) // continue with pointer base type + } + + // typ must be a name + name, _ := typ.(*ast.Ident) + if name == nil { + return false, nil + } + + // name must denote an object found in the current package scope + // (note that dot-imported objects are not in the package scope!) + obj := check.pkg.scope.Lookup(name.Name) + if obj == nil { + return false, nil + } + + // the object must be a type name... + tname, _ := obj.(*TypeName) + if tname == nil { + return false, nil + } + + // ... which we have not seen before + if seen[tname] { + return false, nil + } + + // we're done if tdecl defined tname as a new type + // (rather than an alias) + tdecl := check.objMap[tname].tdecl // must exist for objects in package scope + if !tdecl.Assign.IsValid() { + return ptr, tname + } + + // otherwise, continue resolving + typ = tdecl.Type + if seen == nil { + seen = make(map[*TypeName]bool) + } + seen[tname] = true + } +} + +// packageObjects typechecks all package objects, but not function bodies. +func (check *Checker) packageObjects() { + // process package objects in source order for reproducible results + objList := make([]Object, len(check.objMap)) + i := 0 + for obj := range check.objMap { + objList[i] = obj + i++ + } + sort.Sort(inSourceOrder(objList)) + + // add new methods to already type-checked types (from a prior Checker.Files call) + for _, obj := range objList { + if obj, _ := obj.(*TypeName); obj != nil && obj.typ != nil { + check.collectMethods(obj) + } + } + + // We process non-alias type declarations first, followed by alias declarations, + // and then everything else. This appears to avoid most situations where the type + // of an alias is needed before it is available. + // There may still be cases where this is not good enough (see also issue #25838). + // In those cases Checker.ident will report an error ("invalid use of type alias"). + var aliasList []*TypeName + var othersList []Object // everything that's not a type + // phase 1: non-alias type declarations + for _, obj := range objList { + if tname, _ := obj.(*TypeName); tname != nil { + if check.objMap[tname].tdecl.Assign.IsValid() { + aliasList = append(aliasList, tname) + } else { + check.objDecl(obj, nil) + } + } else { + othersList = append(othersList, obj) + } + } + // phase 2: alias type declarations + for _, obj := range aliasList { + check.objDecl(obj, nil) + } + // phase 3: all other declarations + for _, obj := range othersList { + check.objDecl(obj, nil) + } + + // At this point we may have a non-empty check.methods map; this means that not all + // entries were deleted at the end of typeDecl because the respective receiver base + // types were not found. In that case, an error was reported when declaring those + // methods. We can now safely discard this map. + check.methods = nil +} + +// inSourceOrder implements the sort.Sort interface. +type inSourceOrder []Object + +func (a inSourceOrder) Len() int { return len(a) } +func (a inSourceOrder) Less(i, j int) bool { return a[i].order() < a[j].order() } +func (a inSourceOrder) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// unusedImports checks for unused imports. +func (check *Checker) unusedImports() { + // If function bodies are not checked, packages' uses are likely missing - don't check. + if check.conf.IgnoreFuncBodies { + return + } + + // spec: "It is illegal (...) to directly import a package without referring to + // any of its exported identifiers. To import a package solely for its side-effects + // (initialization), use the blank identifier as explicit package name." + + for _, obj := range check.imports { + if !obj.used && obj.name != "_" { + check.errorUnusedPkg(obj) + } + } +} + +func (check *Checker) errorUnusedPkg(obj *PkgName) { + // If the package was imported with a name other than the final + // import path element, show it explicitly in the error message. + // Note that this handles both renamed imports and imports of + // packages containing unconventional package declarations. + // Note that this uses / always, even on Windows, because Go import + // paths always use forward slashes. + path := obj.imported.path + elem := path + if i := strings.LastIndex(elem, "/"); i >= 0 { + elem = elem[i+1:] + } + if obj.name == "" || obj.name == "." || obj.name == elem { + check.softErrorf(obj, UnusedImport, "%q imported and not used", path) + } else { + check.softErrorf(obj, UnusedImport, "%q imported as %s and not used", path, obj.name) + } +} + +// dir makes a good-faith attempt to return the directory +// portion of path. If path is empty, the result is ".". +// (Per the go/build package dependency tests, we cannot import +// path/filepath and simply use filepath.Dir.) +func dir(path string) string { + if i := strings.LastIndexAny(path, `/\`); i > 0 { + return path[:i] + } + // i <= 0 + return "." +} diff --git a/src/go/types/resolver_test.go b/src/go/types/resolver_test.go new file mode 100644 index 0000000..376ecfb --- /dev/null +++ b/src/go/types/resolver_test.go @@ -0,0 +1,211 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "fmt" + "go/ast" + "go/importer" + "go/token" + "internal/testenv" + "sort" + "testing" + + . "go/types" +) + +type resolveTestImporter struct { + importer ImporterFrom + imported map[string]bool +} + +func (imp *resolveTestImporter) Import(string) (*Package, error) { + panic("should not be called") +} + +func (imp *resolveTestImporter) ImportFrom(path, srcDir string, mode ImportMode) (*Package, error) { + if mode != 0 { + panic("mode must be 0") + } + if imp.importer == nil { + imp.importer = importer.Default().(ImporterFrom) + imp.imported = make(map[string]bool) + } + pkg, err := imp.importer.ImportFrom(path, srcDir, mode) + if err != nil { + return nil, err + } + imp.imported[path] = true + return pkg, nil +} + +func TestResolveIdents(t *testing.T) { + testenv.MustHaveGoBuild(t) + + sources := []string{ + ` + package p + import "fmt" + import "math" + const pi = math.Pi + func sin(x float64) float64 { + return math.Sin(x) + } + var Println = fmt.Println + `, + ` + package p + import "fmt" + type errorStringer struct { fmt.Stringer; error } + func f() string { + _ = "foo" + return fmt.Sprintf("%d", g()) + } + func g() (x int) { return } + `, + ` + package p + import . "go/parser" + import "sync" + func h() Mode { return ImportsOnly } + var _, x int = 1, 2 + func init() {} + type T struct{ *sync.Mutex; a, b, c int} + type I interface{ m() } + var _ = T{a: 1, b: 2, c: 3} + func (_ T) m() {} + func (T) _() {} + var i I + var _ = i.m + func _(s []int) { for i, x := range s { _, _ = i, x } } + func _(x interface{}) { + switch x := x.(type) { + case int: + _ = x + } + switch {} // implicit 'true' tag + } + `, + ` + package p + type S struct{} + func (T) _() {} + func (T) _() {} + `, + ` + package p + func _() { + L0: + L1: + goto L0 + for { + goto L1 + } + if true { + goto L2 + } + L2: + } + `, + } + + pkgnames := []string{ + "fmt", + "math", + } + + // parse package files + fset := token.NewFileSet() + var files []*ast.File + for i, src := range sources { + files = append(files, mustParse(fset, fmt.Sprintf("sources[%d]", i), src)) + } + + // resolve and type-check package AST + importer := new(resolveTestImporter) + conf := Config{Importer: importer} + uses := make(map[*ast.Ident]Object) + defs := make(map[*ast.Ident]Object) + _, err := conf.Check("testResolveIdents", fset, files, &Info{Defs: defs, Uses: uses}) + if err != nil { + t.Fatal(err) + } + + // check that all packages were imported + for _, name := range pkgnames { + if !importer.imported[name] { + t.Errorf("package %s not imported", name) + } + } + + // check that qualified identifiers are resolved + for _, f := range files { + ast.Inspect(f, func(n ast.Node) bool { + if s, ok := n.(*ast.SelectorExpr); ok { + if x, ok := s.X.(*ast.Ident); ok { + obj := uses[x] + if obj == nil { + t.Errorf("%s: unresolved qualified identifier %s", fset.Position(x.Pos()), x.Name) + return false + } + if _, ok := obj.(*PkgName); ok && uses[s.Sel] == nil { + t.Errorf("%s: unresolved selector %s", fset.Position(s.Sel.Pos()), s.Sel.Name) + return false + } + return false + } + return false + } + return true + }) + } + + for id, obj := range uses { + if obj == nil { + t.Errorf("%s: Uses[%s] == nil", fset.Position(id.Pos()), id.Name) + } + } + + // check that each identifier in the source is found in uses or defs or both + var both []string + for _, f := range files { + ast.Inspect(f, func(n ast.Node) bool { + if x, ok := n.(*ast.Ident); ok { + var objects int + if _, found := uses[x]; found { + objects |= 1 + delete(uses, x) + } + if _, found := defs[x]; found { + objects |= 2 + delete(defs, x) + } + if objects == 0 { + t.Errorf("%s: unresolved identifier %s", fset.Position(x.Pos()), x.Name) + } else if objects == 3 { + both = append(both, x.Name) + } + return false + } + return true + }) + } + + // check the expected set of idents that are simultaneously uses and defs + sort.Strings(both) + if got, want := fmt.Sprint(both), "[Mutex Stringer error]"; got != want { + t.Errorf("simultaneous uses/defs = %s, want %s", got, want) + } + + // any left-over identifiers didn't exist in the source + for x := range uses { + t.Errorf("%s: identifier %s not present in source", fset.Position(x.Pos()), x.Name) + } + for x := range defs { + t.Errorf("%s: identifier %s not present in source", fset.Position(x.Pos()), x.Name) + } + + // TODO(gri) add tests to check ImplicitObj callbacks +} diff --git a/src/go/types/return.go b/src/go/types/return.go new file mode 100644 index 0000000..ee8c41a --- /dev/null +++ b/src/go/types/return.go @@ -0,0 +1,184 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements isTerminating. + +package types + +import ( + "go/ast" + "go/token" +) + +// isTerminating reports if s is a terminating statement. +// If s is labeled, label is the label name; otherwise s +// is "". +func (check *Checker) isTerminating(s ast.Stmt, label string) bool { + switch s := s.(type) { + default: + unreachable() + + case *ast.BadStmt, *ast.DeclStmt, *ast.EmptyStmt, *ast.SendStmt, + *ast.IncDecStmt, *ast.AssignStmt, *ast.GoStmt, *ast.DeferStmt, + *ast.RangeStmt: + // no chance + + case *ast.LabeledStmt: + return check.isTerminating(s.Stmt, s.Label.Name) + + case *ast.ExprStmt: + // calling the predeclared (possibly parenthesized) panic() function is terminating + if call, ok := unparen(s.X).(*ast.CallExpr); ok && check.isPanic[call] { + return true + } + + case *ast.ReturnStmt: + return true + + case *ast.BranchStmt: + if s.Tok == token.GOTO || s.Tok == token.FALLTHROUGH { + return true + } + + case *ast.BlockStmt: + return check.isTerminatingList(s.List, "") + + case *ast.IfStmt: + if s.Else != nil && + check.isTerminating(s.Body, "") && + check.isTerminating(s.Else, "") { + return true + } + + case *ast.SwitchStmt: + return check.isTerminatingSwitch(s.Body, label) + + case *ast.TypeSwitchStmt: + return check.isTerminatingSwitch(s.Body, label) + + case *ast.SelectStmt: + for _, s := range s.Body.List { + cc := s.(*ast.CommClause) + if !check.isTerminatingList(cc.Body, "") || hasBreakList(cc.Body, label, true) { + return false + } + + } + return true + + case *ast.ForStmt: + if s.Cond == nil && !hasBreak(s.Body, label, true) { + return true + } + } + + return false +} + +func (check *Checker) isTerminatingList(list []ast.Stmt, label string) bool { + // trailing empty statements are permitted - skip them + for i := len(list) - 1; i >= 0; i-- { + if _, ok := list[i].(*ast.EmptyStmt); !ok { + return check.isTerminating(list[i], label) + } + } + return false // all statements are empty +} + +func (check *Checker) isTerminatingSwitch(body *ast.BlockStmt, label string) bool { + hasDefault := false + for _, s := range body.List { + cc := s.(*ast.CaseClause) + if cc.List == nil { + hasDefault = true + } + if !check.isTerminatingList(cc.Body, "") || hasBreakList(cc.Body, label, true) { + return false + } + } + return hasDefault +} + +// TODO(gri) For nested breakable statements, the current implementation of hasBreak +// will traverse the same subtree repeatedly, once for each label. Replace +// with a single-pass label/break matching phase. + +// hasBreak reports if s is or contains a break statement +// referring to the label-ed statement or implicit-ly the +// closest outer breakable statement. +func hasBreak(s ast.Stmt, label string, implicit bool) bool { + switch s := s.(type) { + default: + unreachable() + + case *ast.BadStmt, *ast.DeclStmt, *ast.EmptyStmt, *ast.ExprStmt, + *ast.SendStmt, *ast.IncDecStmt, *ast.AssignStmt, *ast.GoStmt, + *ast.DeferStmt, *ast.ReturnStmt: + // no chance + + case *ast.LabeledStmt: + return hasBreak(s.Stmt, label, implicit) + + case *ast.BranchStmt: + if s.Tok == token.BREAK { + if s.Label == nil { + return implicit + } + if s.Label.Name == label { + return true + } + } + + case *ast.BlockStmt: + return hasBreakList(s.List, label, implicit) + + case *ast.IfStmt: + if hasBreak(s.Body, label, implicit) || + s.Else != nil && hasBreak(s.Else, label, implicit) { + return true + } + + case *ast.CaseClause: + return hasBreakList(s.Body, label, implicit) + + case *ast.SwitchStmt: + if label != "" && hasBreak(s.Body, label, false) { + return true + } + + case *ast.TypeSwitchStmt: + if label != "" && hasBreak(s.Body, label, false) { + return true + } + + case *ast.CommClause: + return hasBreakList(s.Body, label, implicit) + + case *ast.SelectStmt: + if label != "" && hasBreak(s.Body, label, false) { + return true + } + + case *ast.ForStmt: + if label != "" && hasBreak(s.Body, label, false) { + return true + } + + case *ast.RangeStmt: + if label != "" && hasBreak(s.Body, label, false) { + return true + } + } + + return false +} + +func hasBreakList(list []ast.Stmt, label string, implicit bool) bool { + for _, s := range list { + if hasBreak(s, label, implicit) { + return true + } + } + return false +} diff --git a/src/go/types/scope.go b/src/go/types/scope.go new file mode 100644 index 0000000..fc42ce6 --- /dev/null +++ b/src/go/types/scope.go @@ -0,0 +1,292 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements Scopes. + +package types + +import ( + "fmt" + "go/token" + "io" + "sort" + "strings" + "sync" +) + +// A Scope maintains a set of objects and links to its containing +// (parent) and contained (children) scopes. Objects may be inserted +// and looked up by name. The zero value for Scope is a ready-to-use +// empty scope. +type Scope struct { + parent *Scope + children []*Scope + number int // parent.children[number-1] is this scope; 0 if there is no parent + elems map[string]Object // lazily allocated + pos, end token.Pos // scope extent; may be invalid + comment string // for debugging only + isFunc bool // set if this is a function scope (internal use only) +} + +// NewScope returns a new, empty scope contained in the given parent +// scope, if any. The comment is for debugging only. +func NewScope(parent *Scope, pos, end token.Pos, comment string) *Scope { + s := &Scope{parent, nil, 0, nil, pos, end, comment, false} + // don't add children to Universe scope! + if parent != nil && parent != Universe { + parent.children = append(parent.children, s) + s.number = len(parent.children) + } + return s +} + +// Parent returns the scope's containing (parent) scope. +func (s *Scope) Parent() *Scope { return s.parent } + +// Len returns the number of scope elements. +func (s *Scope) Len() int { return len(s.elems) } + +// Names returns the scope's element names in sorted order. +func (s *Scope) Names() []string { + names := make([]string, len(s.elems)) + i := 0 + for name := range s.elems { + names[i] = name + i++ + } + sort.Strings(names) + return names +} + +// NumChildren returns the number of scopes nested in s. +func (s *Scope) NumChildren() int { return len(s.children) } + +// Child returns the i'th child scope for 0 <= i < NumChildren(). +func (s *Scope) Child(i int) *Scope { return s.children[i] } + +// Lookup returns the object in scope s with the given name if such an +// object exists; otherwise the result is nil. +func (s *Scope) Lookup(name string) Object { + return resolve(name, s.elems[name]) +} + +// LookupParent follows the parent chain of scopes starting with s until +// it finds a scope where Lookup(name) returns a non-nil object, and then +// returns that scope and object. If a valid position pos is provided, +// only objects that were declared at or before pos are considered. +// If no such scope and object exists, the result is (nil, nil). +// +// Note that obj.Parent() may be different from the returned scope if the +// object was inserted into the scope and already had a parent at that +// time (see Insert). This can only happen for dot-imported objects +// whose scope is the scope of the package that exported them. +func (s *Scope) LookupParent(name string, pos token.Pos) (*Scope, Object) { + for ; s != nil; s = s.parent { + if obj := s.Lookup(name); obj != nil && (!pos.IsValid() || obj.scopePos() <= pos) { + return s, obj + } + } + return nil, nil +} + +// Insert attempts to insert an object obj into scope s. +// If s already contains an alternative object alt with +// the same name, Insert leaves s unchanged and returns alt. +// Otherwise it inserts obj, sets the object's parent scope +// if not already set, and returns nil. +func (s *Scope) Insert(obj Object) Object { + name := obj.Name() + if alt := s.Lookup(name); alt != nil { + return alt + } + s.insert(name, obj) + if obj.Parent() == nil { + obj.setParent(s) + } + return nil +} + +// _InsertLazy is like Insert, but allows deferring construction of the +// inserted object until it's accessed with Lookup. The Object +// returned by resolve must have the same name as given to _InsertLazy. +// If s already contains an alternative object with the same name, +// _InsertLazy leaves s unchanged and returns false. Otherwise it +// records the binding and returns true. The object's parent scope +// will be set to s after resolve is called. +func (s *Scope) _InsertLazy(name string, resolve func() Object) bool { + if s.elems[name] != nil { + return false + } + s.insert(name, &lazyObject{parent: s, resolve: resolve}) + return true +} + +func (s *Scope) insert(name string, obj Object) { + if s.elems == nil { + s.elems = make(map[string]Object) + } + s.elems[name] = obj +} + +// squash merges s with its parent scope p by adding all +// objects of s to p, adding all children of s to the +// children of p, and removing s from p's children. +// The function f is called for each object obj in s which +// has an object alt in p. s should be discarded after +// having been squashed. +func (s *Scope) squash(err func(obj, alt Object)) { + p := s.parent + assert(p != nil) + for name, obj := range s.elems { + obj = resolve(name, obj) + obj.setParent(nil) + if alt := p.Insert(obj); alt != nil { + err(obj, alt) + } + } + + j := -1 // index of s in p.children + for i, ch := range p.children { + if ch == s { + j = i + break + } + } + assert(j >= 0) + k := len(p.children) - 1 + p.children[j] = p.children[k] + p.children = p.children[:k] + + p.children = append(p.children, s.children...) + + s.children = nil + s.elems = nil +} + +// Pos and End describe the scope's source code extent [pos, end). +// The results are guaranteed to be valid only if the type-checked +// AST has complete position information. The extent is undefined +// for Universe and package scopes. +func (s *Scope) Pos() token.Pos { return s.pos } +func (s *Scope) End() token.Pos { return s.end } + +// Contains reports whether pos is within the scope's extent. +// The result is guaranteed to be valid only if the type-checked +// AST has complete position information. +func (s *Scope) Contains(pos token.Pos) bool { + return s.pos <= pos && pos < s.end +} + +// Innermost returns the innermost (child) scope containing +// pos. If pos is not within any scope, the result is nil. +// The result is also nil for the Universe scope. +// The result is guaranteed to be valid only if the type-checked +// AST has complete position information. +func (s *Scope) Innermost(pos token.Pos) *Scope { + // Package scopes do not have extents since they may be + // discontiguous, so iterate over the package's files. + if s.parent == Universe { + for _, s := range s.children { + if inner := s.Innermost(pos); inner != nil { + return inner + } + } + } + + if s.Contains(pos) { + for _, s := range s.children { + if s.Contains(pos) { + return s.Innermost(pos) + } + } + return s + } + return nil +} + +// WriteTo writes a string representation of the scope to w, +// with the scope elements sorted by name. +// The level of indentation is controlled by n >= 0, with +// n == 0 for no indentation. +// If recurse is set, it also writes nested (children) scopes. +func (s *Scope) WriteTo(w io.Writer, n int, recurse bool) { + const ind = ". " + indn := strings.Repeat(ind, n) + + fmt.Fprintf(w, "%s%s scope %p {\n", indn, s.comment, s) + + indn1 := indn + ind + for _, name := range s.Names() { + fmt.Fprintf(w, "%s%s\n", indn1, s.Lookup(name)) + } + + if recurse { + for _, s := range s.children { + s.WriteTo(w, n+1, recurse) + } + } + + fmt.Fprintf(w, "%s}\n", indn) +} + +// String returns a string representation of the scope, for debugging. +func (s *Scope) String() string { + var buf strings.Builder + s.WriteTo(&buf, 0, false) + return buf.String() +} + +// A lazyObject represents an imported Object that has not been fully +// resolved yet by its importer. +type lazyObject struct { + parent *Scope + resolve func() Object + obj Object + once sync.Once +} + +// resolve returns the Object represented by obj, resolving lazy +// objects as appropriate. +func resolve(name string, obj Object) Object { + if lazy, ok := obj.(*lazyObject); ok { + lazy.once.Do(func() { + obj := lazy.resolve() + + if _, ok := obj.(*lazyObject); ok { + panic("recursive lazy object") + } + if obj.Name() != name { + panic("lazy object has unexpected name") + } + + if obj.Parent() == nil { + obj.setParent(lazy.parent) + } + lazy.obj = obj + }) + + obj = lazy.obj + } + return obj +} + +// stub implementations so *lazyObject implements Object and we can +// store them directly into Scope.elems. +func (*lazyObject) Parent() *Scope { panic("unreachable") } +func (*lazyObject) Pos() token.Pos { panic("unreachable") } +func (*lazyObject) Pkg() *Package { panic("unreachable") } +func (*lazyObject) Name() string { panic("unreachable") } +func (*lazyObject) Type() Type { panic("unreachable") } +func (*lazyObject) Exported() bool { panic("unreachable") } +func (*lazyObject) Id() string { panic("unreachable") } +func (*lazyObject) String() string { panic("unreachable") } +func (*lazyObject) order() uint32 { panic("unreachable") } +func (*lazyObject) color() color { panic("unreachable") } +func (*lazyObject) setType(Type) { panic("unreachable") } +func (*lazyObject) setOrder(uint32) { panic("unreachable") } +func (*lazyObject) setColor(color color) { panic("unreachable") } +func (*lazyObject) setParent(*Scope) { panic("unreachable") } +func (*lazyObject) sameId(pkg *Package, name string) bool { panic("unreachable") } +func (*lazyObject) scopePos() token.Pos { panic("unreachable") } +func (*lazyObject) setScopePos(pos token.Pos) { panic("unreachable") } diff --git a/src/go/types/selection.go b/src/go/types/selection.go new file mode 100644 index 0000000..09c304d --- /dev/null +++ b/src/go/types/selection.go @@ -0,0 +1,142 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements Selections. + +package types + +import ( + "bytes" + "fmt" +) + +// SelectionKind describes the kind of a selector expression x.f +// (excluding qualified identifiers). +type SelectionKind int + +const ( + FieldVal SelectionKind = iota // x.f is a struct field selector + MethodVal // x.f is a method selector + MethodExpr // x.f is a method expression +) + +// A Selection describes a selector expression x.f. +// For the declarations: +// +// type T struct{ x int; E } +// type E struct{} +// func (e E) m() {} +// var p *T +// +// the following relations exist: +// +// Selector Kind Recv Obj Type Index Indirect +// +// p.x FieldVal T x int {0} true +// p.m MethodVal *T m func() {1, 0} true +// T.m MethodExpr T m func(T) {1, 0} false +type Selection struct { + kind SelectionKind + recv Type // type of x + obj Object // object denoted by x.f + index []int // path from x to x.f + indirect bool // set if there was any pointer indirection on the path +} + +// Kind returns the selection kind. +func (s *Selection) Kind() SelectionKind { return s.kind } + +// Recv returns the type of x in x.f. +func (s *Selection) Recv() Type { return s.recv } + +// Obj returns the object denoted by x.f; a *Var for +// a field selection, and a *Func in all other cases. +func (s *Selection) Obj() Object { return s.obj } + +// Type returns the type of x.f, which may be different from the type of f. +// See Selection for more information. +func (s *Selection) Type() Type { + switch s.kind { + case MethodVal: + // The type of x.f is a method with its receiver type set + // to the type of x. + sig := *s.obj.(*Func).typ.(*Signature) + recv := *sig.recv + recv.typ = s.recv + sig.recv = &recv + return &sig + + case MethodExpr: + // The type of x.f is a function (without receiver) + // and an additional first argument with the same type as x. + // TODO(gri) Similar code is already in call.go - factor! + // TODO(gri) Compute this eagerly to avoid allocations. + sig := *s.obj.(*Func).typ.(*Signature) + arg0 := *sig.recv + sig.recv = nil + arg0.typ = s.recv + var params []*Var + if sig.params != nil { + params = sig.params.vars + } + sig.params = NewTuple(append([]*Var{&arg0}, params...)...) + return &sig + } + + // In all other cases, the type of x.f is the type of x. + return s.obj.Type() +} + +// Index describes the path from x to f in x.f. +// The last index entry is the field or method index of the type declaring f; +// either: +// +// 1. the list of declared methods of a named type; or +// 2. the list of methods of an interface type; or +// 3. the list of fields of a struct type. +// +// The earlier index entries are the indices of the embedded fields implicitly +// traversed to get from (the type of) x to f, starting at embedding depth 0. +func (s *Selection) Index() []int { return s.index } + +// Indirect reports whether any pointer indirection was required to get from +// x to f in x.f. +func (s *Selection) Indirect() bool { return s.indirect } + +func (s *Selection) String() string { return SelectionString(s, nil) } + +// SelectionString returns the string form of s. +// The Qualifier controls the printing of +// package-level objects, and may be nil. +// +// Examples: +// +// "field (T) f int" +// "method (T) f(X) Y" +// "method expr (T) f(X) Y" +func SelectionString(s *Selection, qf Qualifier) string { + var k string + switch s.kind { + case FieldVal: + k = "field " + case MethodVal: + k = "method " + case MethodExpr: + k = "method expr " + default: + unreachable() + } + var buf bytes.Buffer + buf.WriteString(k) + buf.WriteByte('(') + WriteType(&buf, s.Recv(), qf) + fmt.Fprintf(&buf, ") %s", s.obj.Name()) + if T := s.Type(); s.kind == FieldVal { + buf.WriteByte(' ') + WriteType(&buf, T, qf) + } else { + WriteSignature(&buf, T.(*Signature), qf) + } + return buf.String() +} diff --git a/src/go/types/self_test.go b/src/go/types/self_test.go new file mode 100644 index 0000000..a63f2b7 --- /dev/null +++ b/src/go/types/self_test.go @@ -0,0 +1,122 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "go/ast" + "go/importer" + "go/parser" + "go/token" + "internal/testenv" + "path" + "path/filepath" + "testing" + "time" + + . "go/types" +) + +func TestSelf(t *testing.T) { + testenv.MustHaveGoBuild(t) // The Go command is needed for the importer to determine the locations of stdlib .a files. + + fset := token.NewFileSet() + files, err := pkgFiles(fset, ".") + if err != nil { + t.Fatal(err) + } + + conf := Config{Importer: importer.Default()} + _, err = conf.Check("go/types", fset, files, nil) + if err != nil { + t.Fatal(err) + } +} + +func BenchmarkCheck(b *testing.B) { + testenv.MustHaveGoBuild(b) // The Go command is needed for the importer to determine the locations of stdlib .a files. + + for _, p := range []string{ + "net/http", + "go/parser", + "go/constant", + "runtime", + filepath.Join("go", "internal", "gcimporter"), + } { + b.Run(path.Base(p), func(b *testing.B) { + path := filepath.Join("..", "..", p) + for _, ignoreFuncBodies := range []bool{false, true} { + name := "funcbodies" + if ignoreFuncBodies { + name = "nofuncbodies" + } + b.Run(name, func(b *testing.B) { + b.Run("info", func(b *testing.B) { + runbench(b, path, ignoreFuncBodies, true) + }) + b.Run("noinfo", func(b *testing.B) { + runbench(b, path, ignoreFuncBodies, false) + }) + }) + } + }) + } +} + +func runbench(b *testing.B, path string, ignoreFuncBodies, writeInfo bool) { + fset := token.NewFileSet() + files, err := pkgFiles(fset, path) + if err != nil { + b.Fatal(err) + } + // determine line count + lines := 0 + fset.Iterate(func(f *token.File) bool { + lines += f.LineCount() + return true + }) + + b.ResetTimer() + start := time.Now() + for i := 0; i < b.N; i++ { + conf := Config{ + IgnoreFuncBodies: ignoreFuncBodies, + Importer: importer.Default(), + } + var info *Info + if writeInfo { + info = &Info{ + Types: make(map[ast.Expr]TypeAndValue), + Defs: make(map[*ast.Ident]Object), + Uses: make(map[*ast.Ident]Object), + Implicits: make(map[ast.Node]Object), + Selections: make(map[*ast.SelectorExpr]*Selection), + Scopes: make(map[ast.Node]*Scope), + } + } + if _, err := conf.Check(path, fset, files, info); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + b.ReportMetric(float64(lines)*float64(b.N)/time.Since(start).Seconds(), "lines/s") +} + +func pkgFiles(fset *token.FileSet, path string) ([]*ast.File, error) { + filenames, err := pkgFilenames(path) // from stdlib_test.go + if err != nil { + return nil, err + } + + var files []*ast.File + for _, filename := range filenames { + file, err := parser.ParseFile(fset, filename, nil, 0) + if err != nil { + return nil, err + } + files = append(files, file) + } + + return files, nil +} diff --git a/src/go/types/signature.go b/src/go/types/signature.go new file mode 100644 index 0000000..83460ea --- /dev/null +++ b/src/go/types/signature.go @@ -0,0 +1,321 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "fmt" + "go/ast" + "go/token" + . "internal/types/errors" +) + +// ---------------------------------------------------------------------------- +// API + +// A Signature represents a (non-builtin) function or method type. +// The receiver is ignored when comparing signatures for identity. +type Signature struct { + // We need to keep the scope in Signature (rather than passing it around + // and store it in the Func Object) because when type-checking a function + // literal we call the general type checker which returns a general Type. + // We then unpack the *Signature and use the scope for the literal body. + rparams *TypeParamList // receiver type parameters from left to right, or nil + tparams *TypeParamList // type parameters from left to right, or nil + scope *Scope // function scope for package-local and non-instantiated signatures; nil otherwise + recv *Var // nil if not a method + params *Tuple // (incoming) parameters from left to right; or nil + results *Tuple // (outgoing) results from left to right; or nil + variadic bool // true if the last parameter's type is of the form ...T (or string, for append built-in only) +} + +// NewSignature returns a new function type for the given receiver, parameters, +// and results, either of which may be nil. If variadic is set, the function +// is variadic, it must have at least one parameter, and the last parameter +// must be of unnamed slice type. +// +// Deprecated: Use NewSignatureType instead which allows for type parameters. +func NewSignature(recv *Var, params, results *Tuple, variadic bool) *Signature { + return NewSignatureType(recv, nil, nil, params, results, variadic) +} + +// NewSignatureType creates a new function type for the given receiver, +// receiver type parameters, type parameters, parameters, and results. If +// variadic is set, params must hold at least one parameter and the last +// parameter's core type must be of unnamed slice or bytestring type. +// If recv is non-nil, typeParams must be empty. If recvTypeParams is +// non-empty, recv must be non-nil. +func NewSignatureType(recv *Var, recvTypeParams, typeParams []*TypeParam, params, results *Tuple, variadic bool) *Signature { + if variadic { + n := params.Len() + if n == 0 { + panic("variadic function must have at least one parameter") + } + core := coreString(params.At(n - 1).typ) + if _, ok := core.(*Slice); !ok && !isString(core) { + panic(fmt.Sprintf("got %s, want variadic parameter with unnamed slice type or string as core type", core.String())) + } + } + sig := &Signature{recv: recv, params: params, results: results, variadic: variadic} + if len(recvTypeParams) != 0 { + if recv == nil { + panic("function with receiver type parameters must have a receiver") + } + sig.rparams = bindTParams(recvTypeParams) + } + if len(typeParams) != 0 { + if recv != nil { + panic("function with type parameters cannot have a receiver") + } + sig.tparams = bindTParams(typeParams) + } + return sig +} + +// Recv returns the receiver of signature s (if a method), or nil if a +// function. It is ignored when comparing signatures for identity. +// +// For an abstract method, Recv returns the enclosing interface either +// as a *Named or an *Interface. Due to embedding, an interface may +// contain methods whose receiver type is a different interface. +func (s *Signature) Recv() *Var { return s.recv } + +// TypeParams returns the type parameters of signature s, or nil. +func (s *Signature) TypeParams() *TypeParamList { return s.tparams } + +// RecvTypeParams returns the receiver type parameters of signature s, or nil. +func (s *Signature) RecvTypeParams() *TypeParamList { return s.rparams } + +// Params returns the parameters of signature s, or nil. +func (s *Signature) Params() *Tuple { return s.params } + +// Results returns the results of signature s, or nil. +func (s *Signature) Results() *Tuple { return s.results } + +// Variadic reports whether the signature s is variadic. +func (s *Signature) Variadic() bool { return s.variadic } + +func (t *Signature) Underlying() Type { return t } +func (t *Signature) String() string { return TypeString(t, nil) } + +// ---------------------------------------------------------------------------- +// Implementation + +// funcType type-checks a function or method type. +func (check *Checker) funcType(sig *Signature, recvPar *ast.FieldList, ftyp *ast.FuncType) { + check.openScope(ftyp, "function") + check.scope.isFunc = true + check.recordScope(ftyp, check.scope) + sig.scope = check.scope + defer check.closeScope() + + if recvPar != nil && len(recvPar.List) > 0 { + // collect generic receiver type parameters, if any + // - a receiver type parameter is like any other type parameter, except that it is declared implicitly + // - the receiver specification acts as local declaration for its type parameters, which may be blank + _, rname, rparams := check.unpackRecv(recvPar.List[0].Type, true) + if len(rparams) > 0 { + tparams := check.declareTypeParams(nil, rparams) + sig.rparams = bindTParams(tparams) + // Blank identifiers don't get declared, so naive type-checking of the + // receiver type expression would fail in Checker.collectParams below, + // when Checker.ident cannot resolve the _ to a type. + // + // Checker.recvTParamMap maps these blank identifiers to their type parameter + // types, so that they may be resolved in Checker.ident when they fail + // lookup in the scope. + for i, p := range rparams { + if p.Name == "_" { + if check.recvTParamMap == nil { + check.recvTParamMap = make(map[*ast.Ident]*TypeParam) + } + check.recvTParamMap[p] = tparams[i] + } + } + // determine receiver type to get its type parameters + // and the respective type parameter bounds + var recvTParams []*TypeParam + if rname != nil { + // recv should be a Named type (otherwise an error is reported elsewhere) + // Also: Don't report an error via genericType since it will be reported + // again when we type-check the signature. + // TODO(gri) maybe the receiver should be marked as invalid instead? + if recv, _ := check.genericType(rname, nil).(*Named); recv != nil { + recvTParams = recv.TypeParams().list() + } + } + // provide type parameter bounds + if len(tparams) == len(recvTParams) { + smap := makeRenameMap(recvTParams, tparams) + for i, tpar := range tparams { + recvTPar := recvTParams[i] + check.mono.recordCanon(tpar, recvTPar) + // recvTPar.bound is (possibly) parameterized in the context of the + // receiver type declaration. Substitute parameters for the current + // context. + tpar.bound = check.subst(tpar.obj.pos, recvTPar.bound, smap, nil, check.context()) + } + } else if len(tparams) < len(recvTParams) { + // Reporting an error here is a stop-gap measure to avoid crashes in the + // compiler when a type parameter/argument cannot be inferred later. It + // may lead to follow-on errors (see issues #51339, #51343). + // TODO(gri) find a better solution + got := measure(len(tparams), "type parameter") + check.errorf(recvPar, BadRecv, "got %s, but receiver base type declares %d", got, len(recvTParams)) + } + } + } + + if ftyp.TypeParams != nil { + check.collectTypeParams(&sig.tparams, ftyp.TypeParams) + // Always type-check method type parameters but complain that they are not allowed. + // (A separate check is needed when type-checking interface method signatures because + // they don't have a receiver specification.) + if recvPar != nil { + check.error(ftyp.TypeParams, InvalidMethodTypeParams, "methods cannot have type parameters") + } + } + + // Value (non-type) parameters' scope starts in the function body. Use a temporary scope for their + // declarations and then squash that scope into the parent scope (and report any redeclarations at + // that time). + scope := NewScope(check.scope, token.NoPos, token.NoPos, "function body (temp. scope)") + recvList, _ := check.collectParams(scope, recvPar, false) + params, variadic := check.collectParams(scope, ftyp.Params, true) + results, _ := check.collectParams(scope, ftyp.Results, false) + scope.squash(func(obj, alt Object) { + check.errorf(obj, DuplicateDecl, "%s redeclared in this block", obj.Name()) + check.reportAltDecl(alt) + }) + + if recvPar != nil { + // recv parameter list present (may be empty) + // spec: "The receiver is specified via an extra parameter section preceding the + // method name. That parameter section must declare a single parameter, the receiver." + var recv *Var + switch len(recvList) { + case 0: + // error reported by resolver + recv = NewParam(token.NoPos, nil, "", Typ[Invalid]) // ignore recv below + default: + // more than one receiver + check.error(recvList[len(recvList)-1], InvalidRecv, "method has multiple receivers") + fallthrough // continue with first receiver + case 1: + recv = recvList[0] + } + sig.recv = recv + + // Delay validation of receiver type as it may cause premature expansion + // of types the receiver type is dependent on (see issues #51232, #51233). + check.later(func() { + // spec: "The receiver type must be of the form T or *T where T is a type name." + rtyp, _ := deref(recv.typ) + if rtyp == Typ[Invalid] { + return // error was reported before + } + // spec: "The type denoted by T is called the receiver base type; it must not + // be a pointer or interface type and it must be declared in the same package + // as the method." + switch T := rtyp.(type) { + case *Named: + // The receiver type may be an instantiated type referred to + // by an alias (which cannot have receiver parameters for now). + if T.TypeArgs() != nil && sig.RecvTypeParams() == nil { + check.errorf(recv, InvalidRecv, "cannot define new methods on instantiated type %s", rtyp) + break + } + if T.obj.pkg != check.pkg { + check.errorf(recv, InvalidRecv, "cannot define new methods on non-local type %s", rtyp) + break + } + var cause string + switch u := T.under().(type) { + case *Basic: + // unsafe.Pointer is treated like a regular pointer + if u.kind == UnsafePointer { + cause = "unsafe.Pointer" + } + case *Pointer, *Interface: + cause = "pointer or interface type" + case *TypeParam: + // The underlying type of a receiver base type cannot be a + // type parameter: "type T[P any] P" is not a valid declaration. + unreachable() + } + if cause != "" { + check.errorf(recv, InvalidRecv, "invalid receiver type %s (%s)", rtyp, cause) + } + case *Basic: + check.errorf(recv, InvalidRecv, "cannot define new methods on non-local type %s", rtyp) + default: + check.errorf(recv, InvalidRecv, "invalid receiver type %s", recv.typ) + } + }).describef(recv, "validate receiver %s", recv) + } + + sig.params = NewTuple(params...) + sig.results = NewTuple(results...) + sig.variadic = variadic +} + +// collectParams declares the parameters of list in scope and returns the corresponding +// variable list. +func (check *Checker) collectParams(scope *Scope, list *ast.FieldList, variadicOk bool) (params []*Var, variadic bool) { + if list == nil { + return + } + + var named, anonymous bool + for i, field := range list.List { + ftype := field.Type + if t, _ := ftype.(*ast.Ellipsis); t != nil { + ftype = t.Elt + if variadicOk && i == len(list.List)-1 && len(field.Names) <= 1 { + variadic = true + } else { + check.softErrorf(t, MisplacedDotDotDot, "can only use ... with final parameter in list") + // ignore ... and continue + } + } + typ := check.varType(ftype) + // The parser ensures that f.Tag is nil and we don't + // care if a constructed AST contains a non-nil tag. + if len(field.Names) > 0 { + // named parameter + for _, name := range field.Names { + if name.Name == "" { + check.error(name, InvalidSyntaxTree, "anonymous parameter") + // ok to continue + } + par := NewParam(name.Pos(), check.pkg, name.Name, typ) + check.declare(scope, name, par, scope.pos) + params = append(params, par) + } + named = true + } else { + // anonymous parameter + par := NewParam(ftype.Pos(), check.pkg, "", typ) + check.recordImplicit(field, par) + params = append(params, par) + anonymous = true + } + } + + if named && anonymous { + check.error(list, InvalidSyntaxTree, "list contains both named and anonymous parameters") + // ok to continue + } + + // For a variadic function, change the last parameter's type from T to []T. + // Since we type-checked T rather than ...T, we also need to retro-actively + // record the type for ...T. + if variadic { + last := params[len(params)-1] + last.typ = &Slice{elem: last.typ} + check.recordTypeAndValue(list.List[len(list.List)-1].Type, typexpr, last.typ, nil) + } + + return +} diff --git a/src/go/types/sizeof_test.go b/src/go/types/sizeof_test.go new file mode 100644 index 0000000..d4ce0a7 --- /dev/null +++ b/src/go/types/sizeof_test.go @@ -0,0 +1,62 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "reflect" + "testing" +) + +// Signal size changes of important structures. +func TestSizeof(t *testing.T) { + const _64bit = ^uint(0)>>32 != 0 + + var tests = []struct { + val any // type as a value + _32bit uintptr // size on 32bit platforms + _64bit uintptr // size on 64bit platforms + }{ + // Types + {Basic{}, 16, 32}, + {Array{}, 16, 24}, + {Slice{}, 8, 16}, + {Struct{}, 24, 48}, + {Pointer{}, 8, 16}, + {Tuple{}, 12, 24}, + {Signature{}, 28, 56}, + {Union{}, 12, 24}, + {Interface{}, 40, 80}, + {Map{}, 16, 32}, + {Chan{}, 12, 24}, + {Named{}, 60, 112}, + {TypeParam{}, 28, 48}, + {term{}, 12, 24}, + + // Objects + {PkgName{}, 48, 88}, + {Const{}, 48, 88}, + {TypeName{}, 40, 72}, + {Var{}, 48, 88}, + {Func{}, 48, 88}, + {Label{}, 44, 80}, + {Builtin{}, 44, 80}, + {Nil{}, 40, 72}, + + // Misc + {Scope{}, 44, 88}, + {Package{}, 40, 80}, + {_TypeSet{}, 28, 56}, + } + for _, test := range tests { + got := reflect.TypeOf(test.val).Size() + want := test._32bit + if _64bit { + want = test._64bit + } + if got != want { + t.Errorf("unsafe.Sizeof(%T) = %d, want %d", test.val, got, want) + } + } +} diff --git a/src/go/types/sizes.go b/src/go/types/sizes.go new file mode 100644 index 0000000..cb5253b --- /dev/null +++ b/src/go/types/sizes.go @@ -0,0 +1,296 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements Sizes. + +package types + +// Sizes defines the sizing functions for package unsafe. +type Sizes interface { + // Alignof returns the alignment of a variable of type T. + // Alignof must implement the alignment guarantees required by the spec. + Alignof(T Type) int64 + + // Offsetsof returns the offsets of the given struct fields, in bytes. + // Offsetsof must implement the offset guarantees required by the spec. + Offsetsof(fields []*Var) []int64 + + // Sizeof returns the size of a variable of type T. + // Sizeof must implement the size guarantees required by the spec. + Sizeof(T Type) int64 +} + +// StdSizes is a convenience type for creating commonly used Sizes. +// It makes the following simplifying assumptions: +// +// - The size of explicitly sized basic types (int16, etc.) is the +// specified size. +// - The size of strings and interfaces is 2*WordSize. +// - The size of slices is 3*WordSize. +// - The size of an array of n elements corresponds to the size of +// a struct of n consecutive fields of the array's element type. +// - The size of a struct is the offset of the last field plus that +// field's size. As with all element types, if the struct is used +// in an array its size must first be aligned to a multiple of the +// struct's alignment. +// - All other types have size WordSize. +// - Arrays and structs are aligned per spec definition; all other +// types are naturally aligned with a maximum alignment MaxAlign. +// +// *StdSizes implements Sizes. +type StdSizes struct { + WordSize int64 // word size in bytes - must be >= 4 (32bits) + MaxAlign int64 // maximum alignment in bytes - must be >= 1 +} + +func (s *StdSizes) Alignof(T Type) int64 { + // For arrays and structs, alignment is defined in terms + // of alignment of the elements and fields, respectively. + switch t := under(T).(type) { + case *Array: + // spec: "For a variable x of array type: unsafe.Alignof(x) + // is the same as unsafe.Alignof(x[0]), but at least 1." + return s.Alignof(t.elem) + case *Struct: + if len(t.fields) == 0 && isSyncAtomicAlign64(T) { + // Special case: sync/atomic.align64 is an + // empty struct we recognize as a signal that + // the struct it contains must be + // 64-bit-aligned. + // + // This logic is equivalent to the logic in + // cmd/compile/internal/types/size.go:calcStructOffset + return 8 + } + + // spec: "For a variable x of struct type: unsafe.Alignof(x) + // is the largest of the values unsafe.Alignof(x.f) for each + // field f of x, but at least 1." + max := int64(1) + for _, f := range t.fields { + if a := s.Alignof(f.typ); a > max { + max = a + } + } + return max + case *Slice, *Interface: + // Multiword data structures are effectively structs + // in which each element has size WordSize. + // Type parameters lead to variable sizes/alignments; + // StdSizes.Alignof won't be called for them. + assert(!isTypeParam(T)) + return s.WordSize + case *Basic: + // Strings are like slices and interfaces. + if t.Info()&IsString != 0 { + return s.WordSize + } + case *TypeParam, *Union: + unreachable() + } + a := s.Sizeof(T) // may be 0 + // spec: "For a variable x of any type: unsafe.Alignof(x) is at least 1." + if a < 1 { + return 1 + } + // complex{64,128} are aligned like [2]float{32,64}. + if isComplex(T) { + a /= 2 + } + if a > s.MaxAlign { + return s.MaxAlign + } + return a +} + +func isSyncAtomicAlign64(T Type) bool { + named, ok := T.(*Named) + if !ok { + return false + } + obj := named.Obj() + return obj.Name() == "align64" && + obj.Pkg() != nil && + (obj.Pkg().Path() == "sync/atomic" || + obj.Pkg().Path() == "runtime/internal/atomic") +} + +func (s *StdSizes) Offsetsof(fields []*Var) []int64 { + offsets := make([]int64, len(fields)) + var o int64 + for i, f := range fields { + a := s.Alignof(f.typ) + o = align(o, a) + offsets[i] = o + o += s.Sizeof(f.typ) + } + return offsets +} + +var basicSizes = [...]byte{ + Bool: 1, + Int8: 1, + Int16: 2, + Int32: 4, + Int64: 8, + Uint8: 1, + Uint16: 2, + Uint32: 4, + Uint64: 8, + Float32: 4, + Float64: 8, + Complex64: 8, + Complex128: 16, +} + +func (s *StdSizes) Sizeof(T Type) int64 { + switch t := under(T).(type) { + case *Basic: + assert(isTyped(T)) + k := t.kind + if int(k) < len(basicSizes) { + if s := basicSizes[k]; s > 0 { + return int64(s) + } + } + if k == String { + return s.WordSize * 2 + } + case *Array: + n := t.len + if n <= 0 { + return 0 + } + // n > 0 + a := s.Alignof(t.elem) + z := s.Sizeof(t.elem) + return align(z, a)*(n-1) + z + case *Slice: + return s.WordSize * 3 + case *Struct: + n := t.NumFields() + if n == 0 { + return 0 + } + offsets := s.Offsetsof(t.fields) + return offsets[n-1] + s.Sizeof(t.fields[n-1].typ) + case *Interface: + // Type parameters lead to variable sizes/alignments; + // StdSizes.Sizeof won't be called for them. + assert(!isTypeParam(T)) + return s.WordSize * 2 + case *TypeParam, *Union: + unreachable() + } + return s.WordSize // catch-all +} + +// common architecture word sizes and alignments +var gcArchSizes = map[string]*StdSizes{ + "386": {4, 4}, + "amd64": {8, 8}, + "amd64p32": {4, 8}, + "arm": {4, 4}, + "arm64": {8, 8}, + "loong64": {8, 8}, + "mips": {4, 4}, + "mipsle": {4, 4}, + "mips64": {8, 8}, + "mips64le": {8, 8}, + "ppc64": {8, 8}, + "ppc64le": {8, 8}, + "riscv64": {8, 8}, + "s390x": {8, 8}, + "sparc64": {8, 8}, + "wasm": {8, 8}, + // When adding more architectures here, + // update the doc string of SizesFor below. +} + +// SizesFor returns the Sizes used by a compiler for an architecture. +// The result is nil if a compiler/architecture pair is not known. +// +// Supported architectures for compiler "gc": +// "386", "amd64", "amd64p32", "arm", "arm64", "loong64", "mips", "mipsle", +// "mips64", "mips64le", "ppc64", "ppc64le", "riscv64", "s390x", "sparc64", "wasm". +func SizesFor(compiler, arch string) Sizes { + var m map[string]*StdSizes + switch compiler { + case "gc": + m = gcArchSizes + case "gccgo": + m = gccgoArchSizes + default: + return nil + } + s, ok := m[arch] + if !ok { + return nil + } + return s +} + +// stdSizes is used if Config.Sizes == nil. +var stdSizes = SizesFor("gc", "amd64") + +func (conf *Config) alignof(T Type) int64 { + if s := conf.Sizes; s != nil { + if a := s.Alignof(T); a >= 1 { + return a + } + panic("Config.Sizes.Alignof returned an alignment < 1") + } + return stdSizes.Alignof(T) +} + +func (conf *Config) offsetsof(T *Struct) []int64 { + var offsets []int64 + if T.NumFields() > 0 { + // compute offsets on demand + if s := conf.Sizes; s != nil { + offsets = s.Offsetsof(T.fields) + // sanity checks + if len(offsets) != T.NumFields() { + panic("Config.Sizes.Offsetsof returned the wrong number of offsets") + } + for _, o := range offsets { + if o < 0 { + panic("Config.Sizes.Offsetsof returned an offset < 0") + } + } + } else { + offsets = stdSizes.Offsetsof(T.fields) + } + } + return offsets +} + +// offsetof returns the offset of the field specified via +// the index sequence relative to typ. All embedded fields +// must be structs (rather than pointer to structs). +func (conf *Config) offsetof(typ Type, index []int) int64 { + var o int64 + for _, i := range index { + s := under(typ).(*Struct) + o += conf.offsetsof(s)[i] + typ = s.fields[i].typ + } + return o +} + +func (conf *Config) sizeof(T Type) int64 { + if s := conf.Sizes; s != nil { + if z := s.Sizeof(T); z >= 0 { + return z + } + panic("Config.Sizes.Sizeof returned a size < 0") + } + return stdSizes.Sizeof(T) +} + +// align returns the smallest y >= x such that y % a == 0. +func align(x, a int64) int64 { + y := x + a - 1 + return y - y%a +} diff --git a/src/go/types/sizes_test.go b/src/go/types/sizes_test.go new file mode 100644 index 0000000..09ac9e2 --- /dev/null +++ b/src/go/types/sizes_test.go @@ -0,0 +1,142 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains tests for sizes. + +package types_test + +import ( + "go/ast" + "go/importer" + "go/token" + "go/types" + "internal/testenv" + "testing" +) + +// findStructType typechecks src and returns the first struct type encountered. +func findStructType(t *testing.T, src string) *types.Struct { + return findStructTypeConfig(t, src, &types.Config{}) +} + +func findStructTypeConfig(t *testing.T, src string, conf *types.Config) *types.Struct { + types_ := make(map[ast.Expr]types.TypeAndValue) + mustTypecheck("x", src, &types.Info{Types: types_}) + for _, tv := range types_ { + if ts, ok := tv.Type.(*types.Struct); ok { + return ts + } + } + t.Fatalf("failed to find a struct type in src:\n%s\n", src) + return nil +} + +// Issue 16316 +func TestMultipleSizeUse(t *testing.T) { + const src = ` +package main + +type S struct { + i int + b bool + s string + n int +} +` + ts := findStructType(t, src) + sizes := types.StdSizes{WordSize: 4, MaxAlign: 4} + if got := sizes.Sizeof(ts); got != 20 { + t.Errorf("Sizeof(%v) with WordSize 4 = %d want 20", ts, got) + } + sizes = types.StdSizes{WordSize: 8, MaxAlign: 8} + if got := sizes.Sizeof(ts); got != 40 { + t.Errorf("Sizeof(%v) with WordSize 8 = %d want 40", ts, got) + } +} + +// Issue 16464 +func TestAlignofNaclSlice(t *testing.T) { + const src = ` +package main + +var s struct { + x *int + y []byte +} +` + ts := findStructType(t, src) + sizes := &types.StdSizes{WordSize: 4, MaxAlign: 8} + var fields []*types.Var + // Make a copy manually :( + for i := 0; i < ts.NumFields(); i++ { + fields = append(fields, ts.Field(i)) + } + offsets := sizes.Offsetsof(fields) + if offsets[0] != 0 || offsets[1] != 4 { + t.Errorf("OffsetsOf(%v) = %v want %v", ts, offsets, []int{0, 4}) + } +} + +func TestIssue16902(t *testing.T) { + const src = ` +package a + +import "unsafe" + +const _ = unsafe.Offsetof(struct{ x int64 }{}.x) +` + fset := token.NewFileSet() + f := mustParse(fset, "x.go", src) + info := types.Info{Types: make(map[ast.Expr]types.TypeAndValue)} + conf := types.Config{ + Importer: importer.Default(), + Sizes: &types.StdSizes{WordSize: 8, MaxAlign: 8}, + } + _, err := conf.Check("x", fset, []*ast.File{f}, &info) + if err != nil { + t.Fatal(err) + } + for _, tv := range info.Types { + _ = conf.Sizes.Sizeof(tv.Type) + _ = conf.Sizes.Alignof(tv.Type) + } +} + +// Issue #53884. +func TestAtomicAlign(t *testing.T) { + testenv.MustHaveGoBuild(t) // The Go command is needed for the importer to determine the locations of stdlib .a files. + + const src = ` +package main + +import "sync/atomic" + +var s struct { + x int32 + y atomic.Int64 + z int64 +} +` + + want := []int64{0, 8, 16} + for _, arch := range []string{"386", "amd64"} { + t.Run(arch, func(t *testing.T) { + conf := types.Config{ + Importer: importer.Default(), + Sizes: types.SizesFor("gc", arch), + } + ts := findStructTypeConfig(t, src, &conf) + var fields []*types.Var + // Make a copy manually :( + for i := 0; i < ts.NumFields(); i++ { + fields = append(fields, ts.Field(i)) + } + + offsets := conf.Sizes.Offsetsof(fields) + if offsets[0] != want[0] || offsets[1] != want[1] || offsets[2] != want[2] { + t.Errorf("OffsetsOf(%v) = %v want %v", ts, offsets, want) + } + }) + } +} diff --git a/src/go/types/slice.go b/src/go/types/slice.go new file mode 100644 index 0000000..debdd81 --- /dev/null +++ b/src/go/types/slice.go @@ -0,0 +1,19 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +// A Slice represents a slice type. +type Slice struct { + elem Type +} + +// NewSlice returns a new slice type for the given element type. +func NewSlice(elem Type) *Slice { return &Slice{elem: elem} } + +// Elem returns the element type of slice s. +func (s *Slice) Elem() Type { return s.elem } + +func (t *Slice) Underlying() Type { return t } +func (t *Slice) String() string { return TypeString(t, nil) } diff --git a/src/go/types/stdlib_test.go b/src/go/types/stdlib_test.go new file mode 100644 index 0000000..c0c9fcf --- /dev/null +++ b/src/go/types/stdlib_test.go @@ -0,0 +1,355 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file tests types.Check by using it to +// typecheck the standard library and tests. + +package types_test + +import ( + "fmt" + "go/ast" + "go/build" + "go/importer" + "go/parser" + "go/scanner" + "go/token" + "internal/testenv" + "os" + "path/filepath" + "strings" + "testing" + "time" + + . "go/types" +) + +// The cmd/*/internal packages may have been deleted as part of a binary +// release. Import from source instead. +// +// (See https://golang.org/issue/43232 and +// https://github.com/golang/build/blob/df58bbac082bc87c4a3cdfe336d1ffe60bbaa916/cmd/release/release.go#L533-L545.) +// +// Use the same importer for all std lib tests to +// avoid repeated importing of the same packages. +var stdLibImporter = importer.ForCompiler(token.NewFileSet(), "source", nil) + +func TestStdlib(t *testing.T) { + testenv.MustHaveGoBuild(t) + + pkgCount := 0 + duration := walkPkgDirs(filepath.Join(testenv.GOROOT(t), "src"), func(dir string, filenames []string) { + typecheckFiles(t, dir, filenames) + pkgCount++ + }, t.Error) + + if testing.Verbose() { + fmt.Println(pkgCount, "packages typechecked in", duration) + } +} + +// firstComment returns the contents of the first non-empty comment in +// the given file, "skip", or the empty string. No matter the present +// comments, if any of them contains a build tag, the result is always +// "skip". Only comments before the "package" token and within the first +// 4K of the file are considered. +func firstComment(filename string) string { + f, err := os.Open(filename) + if err != nil { + return "" + } + defer f.Close() + + var src [4 << 10]byte // read at most 4KB + n, _ := f.Read(src[:]) + + var first string + var s scanner.Scanner + s.Init(fset.AddFile("", fset.Base(), n), src[:n], nil /* ignore errors */, scanner.ScanComments) + for { + _, tok, lit := s.Scan() + switch tok { + case token.COMMENT: + // remove trailing */ of multi-line comment + if lit[1] == '*' { + lit = lit[:len(lit)-2] + } + contents := strings.TrimSpace(lit[2:]) + if strings.HasPrefix(contents, "+build ") { + return "skip" + } + if first == "" { + first = contents // contents may be "" but that's ok + } + // continue as we may still see build tags + + case token.PACKAGE, token.EOF: + return first + } + } +} + +func testTestDir(t *testing.T, path string, ignore ...string) { + files, err := os.ReadDir(path) + if err != nil { + t.Fatal(err) + } + + excluded := make(map[string]bool) + for _, filename := range ignore { + excluded[filename] = true + } + + fset := token.NewFileSet() + for _, f := range files { + // filter directory contents + if f.IsDir() || !strings.HasSuffix(f.Name(), ".go") || excluded[f.Name()] { + continue + } + + // get per-file instructions + expectErrors := false + filename := filepath.Join(path, f.Name()) + goVersion := "" + if comment := firstComment(filename); comment != "" { + fields := strings.Fields(comment) + switch fields[0] { + case "skip", "compiledir": + continue // ignore this file + case "errorcheck": + expectErrors = true + for _, arg := range fields[1:] { + if arg == "-0" || arg == "-+" || arg == "-std" { + // Marked explicitly as not expecting errors (-0), + // or marked as compiling runtime/stdlib, which is only done + // to trigger runtime/stdlib-only error output. + // In both cases, the code should typecheck. + expectErrors = false + break + } + const prefix = "-lang=" + if strings.HasPrefix(arg, prefix) { + goVersion = arg[len(prefix):] + } + } + } + } + + // parse and type-check file + file, err := parser.ParseFile(fset, filename, nil, 0) + if err == nil { + conf := Config{GoVersion: goVersion, Importer: stdLibImporter} + _, err = conf.Check(filename, fset, []*ast.File{file}, nil) + } + + if expectErrors { + if err == nil { + t.Errorf("expected errors but found none in %s", filename) + } + } else { + if err != nil { + t.Error(err) + } + } + } +} + +func TestStdTest(t *testing.T) { + testenv.MustHaveGoBuild(t) + + if testing.Short() && testenv.Builder() == "" { + t.Skip("skipping in short mode") + } + + testTestDir(t, filepath.Join(testenv.GOROOT(t), "test"), + "cmplxdivide.go", // also needs file cmplxdivide1.go - ignore + "directive.go", // tests compiler rejection of bad directive placement - ignore + "directive2.go", // tests compiler rejection of bad directive placement - ignore + "embedfunc.go", // tests //go:embed + "embedvers.go", // tests //go:embed + "linkname2.go", // go/types doesn't check validity of //go:xxx directives + "linkname3.go", // go/types doesn't check validity of //go:xxx directives + ) +} + +func TestStdFixed(t *testing.T) { + testenv.MustHaveGoBuild(t) + + if testing.Short() && testenv.Builder() == "" { + t.Skip("skipping in short mode") + } + + testTestDir(t, filepath.Join(testenv.GOROOT(t), "test", "fixedbugs"), + "bug248.go", "bug302.go", "bug369.go", // complex test instructions - ignore + "issue6889.go", // gc-specific test + "issue11362.go", // canonical import path check + "issue16369.go", // go/types handles this correctly - not an issue + "issue18459.go", // go/types doesn't check validity of //go:xxx directives + "issue18882.go", // go/types doesn't check validity of //go:xxx directives + "issue20529.go", // go/types does not have constraints on stack size + "issue22200.go", // go/types does not have constraints on stack size + "issue22200b.go", // go/types does not have constraints on stack size + "issue25507.go", // go/types does not have constraints on stack size + "issue20780.go", // go/types does not have constraints on stack size + "bug251.go", // issue #34333 which was exposed with fix for #34151 + "issue42058a.go", // go/types does not have constraints on channel element size + "issue42058b.go", // go/types does not have constraints on channel element size + "issue48097.go", // go/types doesn't check validity of //go:xxx directives, and non-init bodyless function + "issue48230.go", // go/types doesn't check validity of //go:xxx directives + "issue49767.go", // go/types does not have constraints on channel element size + "issue49814.go", // go/types does not have constraints on array size + "issue56103.go", // anonymous interface cycles; will be a type checker error in 1.22 + + // These tests requires runtime/cgo.Incomplete, which is only available on some platforms. + // However, go/types does not know about build constraints. + "bug514.go", + "issue40954.go", + "issue42032.go", + "issue42076.go", + "issue46903.go", + "issue51733.go", + "notinheap2.go", + "notinheap3.go", + ) +} + +func TestStdKen(t *testing.T) { + testenv.MustHaveGoBuild(t) + + testTestDir(t, filepath.Join(testenv.GOROOT(t), "test", "ken")) +} + +// Package paths of excluded packages. +var excluded = map[string]bool{ + "builtin": true, + + // See #46027: some imports are missing for this submodule. + "crypto/internal/edwards25519/field/_asm": true, + "crypto/internal/bigmod/_asm": true, +} + +// typecheckFiles typechecks the given package files. +func typecheckFiles(t *testing.T, path string, filenames []string) { + fset := token.NewFileSet() + + // parse package files + var files []*ast.File + for _, filename := range filenames { + file, err := parser.ParseFile(fset, filename, nil, parser.AllErrors) + if err != nil { + // the parser error may be a list of individual errors; report them all + if list, ok := err.(scanner.ErrorList); ok { + for _, err := range list { + t.Error(err) + } + return + } + t.Error(err) + return + } + + if testing.Verbose() { + if len(files) == 0 { + fmt.Println("package", file.Name.Name) + } + fmt.Println("\t", filename) + } + + files = append(files, file) + } + + // typecheck package files + conf := Config{ + Error: func(err error) { + t.Helper() + t.Error(err) + }, + Importer: stdLibImporter, + } + info := Info{Uses: make(map[*ast.Ident]Object)} + conf.Check(path, fset, files, &info) + + // Perform checks of API invariants. + + // All Objects have a package, except predeclared ones. + errorError := Universe.Lookup("error").Type().Underlying().(*Interface).ExplicitMethod(0) // (error).Error + for id, obj := range info.Uses { + predeclared := obj == Universe.Lookup(obj.Name()) || obj == errorError + if predeclared == (obj.Pkg() != nil) { + posn := fset.Position(id.Pos()) + if predeclared { + t.Errorf("%s: predeclared object with package: %s", posn, obj) + } else { + t.Errorf("%s: user-defined object without package: %s", posn, obj) + } + } + } +} + +// pkgFilenames returns the list of package filenames for the given directory. +func pkgFilenames(dir string) ([]string, error) { + ctxt := build.Default + ctxt.CgoEnabled = false + pkg, err := ctxt.ImportDir(dir, 0) + if err != nil { + if _, nogo := err.(*build.NoGoError); nogo { + return nil, nil // no *.go files, not an error + } + return nil, err + } + if excluded[pkg.ImportPath] { + return nil, nil + } + var filenames []string + for _, name := range pkg.GoFiles { + filenames = append(filenames, filepath.Join(pkg.Dir, name)) + } + for _, name := range pkg.TestGoFiles { + filenames = append(filenames, filepath.Join(pkg.Dir, name)) + } + return filenames, nil +} + +func walkPkgDirs(dir string, pkgh func(dir string, filenames []string), errh func(args ...any)) time.Duration { + w := walker{time.Now(), 10 * time.Millisecond, pkgh, errh} + w.walk(dir) + return time.Since(w.start) +} + +type walker struct { + start time.Time + dmax time.Duration + pkgh func(dir string, filenames []string) + errh func(args ...any) +} + +func (w *walker) walk(dir string) { + // limit run time for short tests + if testing.Short() && time.Since(w.start) >= w.dmax { + return + } + + files, err := os.ReadDir(dir) + if err != nil { + w.errh(err) + return + } + + // apply pkgh to the files in directory dir + pkgFiles, err := pkgFilenames(dir) + if err != nil { + w.errh(err) + return + } + if pkgFiles != nil { + w.pkgh(dir, pkgFiles) + } + + // traverse subdirectories, but don't walk into testdata + for _, f := range files { + if f.IsDir() && f.Name() != "testdata" { + w.walk(filepath.Join(dir, f.Name())) + } + } +} diff --git a/src/go/types/stmt.go b/src/go/types/stmt.go new file mode 100644 index 0000000..ac6255d --- /dev/null +++ b/src/go/types/stmt.go @@ -0,0 +1,962 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements typechecking of statements. + +package types + +import ( + "go/ast" + "go/constant" + "go/token" + . "internal/types/errors" + "sort" +) + +func (check *Checker) funcBody(decl *declInfo, name string, sig *Signature, body *ast.BlockStmt, iota constant.Value) { + if check.conf.IgnoreFuncBodies { + panic("function body not ignored") + } + + if trace { + check.trace(body.Pos(), "-- %s: %s", name, sig) + } + + // set function scope extent + sig.scope.pos = body.Pos() + sig.scope.end = body.End() + + // save/restore current environment and set up function environment + // (and use 0 indentation at function start) + defer func(env environment, indent int) { + check.environment = env + check.indent = indent + }(check.environment, check.indent) + check.environment = environment{ + decl: decl, + scope: sig.scope, + iota: iota, + sig: sig, + } + check.indent = 0 + + check.stmtList(0, body.List) + + if check.hasLabel { + check.labels(body) + } + + if sig.results.Len() > 0 && !check.isTerminating(body, "") { + check.error(atPos(body.Rbrace), MissingReturn, "missing return") + } + + // spec: "Implementation restriction: A compiler may make it illegal to + // declare a variable inside a function body if the variable is never used." + check.usage(sig.scope) +} + +func (check *Checker) usage(scope *Scope) { + var unused []*Var + for name, elem := range scope.elems { + elem = resolve(name, elem) + if v, _ := elem.(*Var); v != nil && !v.used { + unused = append(unused, v) + } + } + sort.Slice(unused, func(i, j int) bool { + return unused[i].pos < unused[j].pos + }) + for _, v := range unused { + check.softErrorf(v, UnusedVar, "%s declared and not used", v.name) + } + + for _, scope := range scope.children { + // Don't go inside function literal scopes a second time; + // they are handled explicitly by funcBody. + if !scope.isFunc { + check.usage(scope) + } + } +} + +// stmtContext is a bitset describing which +// control-flow statements are permissible, +// and provides additional context information +// for better error messages. +type stmtContext uint + +const ( + // permissible control-flow statements + breakOk stmtContext = 1 << iota + continueOk + fallthroughOk + + // additional context information + finalSwitchCase + inTypeSwitch +) + +func (check *Checker) simpleStmt(s ast.Stmt) { + if s != nil { + check.stmt(0, s) + } +} + +func trimTrailingEmptyStmts(list []ast.Stmt) []ast.Stmt { + for i := len(list); i > 0; i-- { + if _, ok := list[i-1].(*ast.EmptyStmt); !ok { + return list[:i] + } + } + return nil +} + +func (check *Checker) stmtList(ctxt stmtContext, list []ast.Stmt) { + ok := ctxt&fallthroughOk != 0 + inner := ctxt &^ fallthroughOk + list = trimTrailingEmptyStmts(list) // trailing empty statements are "invisible" to fallthrough analysis + for i, s := range list { + inner := inner + if ok && i+1 == len(list) { + inner |= fallthroughOk + } + check.stmt(inner, s) + } +} + +func (check *Checker) multipleDefaults(list []ast.Stmt) { + var first ast.Stmt + for _, s := range list { + var d ast.Stmt + switch c := s.(type) { + case *ast.CaseClause: + if len(c.List) == 0 { + d = s + } + case *ast.CommClause: + if c.Comm == nil { + d = s + } + default: + check.error(s, InvalidSyntaxTree, "case/communication clause expected") + } + if d != nil { + if first != nil { + check.errorf(d, DuplicateDefault, "multiple defaults (first at %s)", check.fset.Position(first.Pos())) + } else { + first = d + } + } + } +} + +func (check *Checker) openScope(node ast.Node, comment string) { + scope := NewScope(check.scope, node.Pos(), node.End(), comment) + check.recordScope(node, scope) + check.scope = scope +} + +func (check *Checker) closeScope() { + check.scope = check.scope.Parent() +} + +func assignOp(op token.Token) token.Token { + // token_test.go verifies the token ordering this function relies on + if token.ADD_ASSIGN <= op && op <= token.AND_NOT_ASSIGN { + return op + (token.ADD - token.ADD_ASSIGN) + } + return token.ILLEGAL +} + +func (check *Checker) suspendedCall(keyword string, call *ast.CallExpr) { + var x operand + var msg string + var code Code + switch check.rawExpr(&x, call, nil, false) { + case conversion: + msg = "requires function call, not conversion" + code = InvalidDefer + if keyword == "go" { + code = InvalidGo + } + case expression: + msg = "discards result of" + code = UnusedResults + case statement: + return + default: + unreachable() + } + check.errorf(&x, code, "%s %s %s", keyword, msg, &x) +} + +// goVal returns the Go value for val, or nil. +func goVal(val constant.Value) any { + // val should exist, but be conservative and check + if val == nil { + return nil + } + // Match implementation restriction of other compilers. + // gc only checks duplicates for integer, floating-point + // and string values, so only create Go values for these + // types. + switch val.Kind() { + case constant.Int: + if x, ok := constant.Int64Val(val); ok { + return x + } + if x, ok := constant.Uint64Val(val); ok { + return x + } + case constant.Float: + if x, ok := constant.Float64Val(val); ok { + return x + } + case constant.String: + return constant.StringVal(val) + } + return nil +} + +// A valueMap maps a case value (of a basic Go type) to a list of positions +// where the same case value appeared, together with the corresponding case +// types. +// Since two case values may have the same "underlying" value but different +// types we need to also check the value's types (e.g., byte(1) vs myByte(1)) +// when the switch expression is of interface type. +type ( + valueMap map[any][]valueType // underlying Go value -> valueType + valueType struct { + pos token.Pos + typ Type + } +) + +func (check *Checker) caseValues(x *operand, values []ast.Expr, seen valueMap) { +L: + for _, e := range values { + var v operand + check.expr(&v, e) + if x.mode == invalid || v.mode == invalid { + continue L + } + check.convertUntyped(&v, x.typ) + if v.mode == invalid { + continue L + } + // Order matters: By comparing v against x, error positions are at the case values. + res := v // keep original v unchanged + check.comparison(&res, x, token.EQL, true) + if res.mode == invalid { + continue L + } + if v.mode != constant_ { + continue L // we're done + } + // look for duplicate values + if val := goVal(v.val); val != nil { + // look for duplicate types for a given value + // (quadratic algorithm, but these lists tend to be very short) + for _, vt := range seen[val] { + if Identical(v.typ, vt.typ) { + check.errorf(&v, DuplicateCase, "duplicate case %s in expression switch", &v) + check.error(atPos(vt.pos), DuplicateCase, "\tprevious case") // secondary error, \t indented + continue L + } + } + seen[val] = append(seen[val], valueType{v.Pos(), v.typ}) + } + } +} + +// isNil reports whether the expression e denotes the predeclared value nil. +func (check *Checker) isNil(e ast.Expr) bool { + // The only way to express the nil value is by literally writing nil (possibly in parentheses). + if name, _ := unparen(e).(*ast.Ident); name != nil { + _, ok := check.lookup(name.Name).(*Nil) + return ok + } + return false +} + +// If the type switch expression is invalid, x is nil. +func (check *Checker) caseTypes(x *operand, types []ast.Expr, seen map[Type]ast.Expr) (T Type) { + var dummy operand +L: + for _, e := range types { + // The spec allows the value nil instead of a type. + if check.isNil(e) { + T = nil + check.expr(&dummy, e) // run e through expr so we get the usual Info recordings + } else { + T = check.varType(e) + if T == Typ[Invalid] { + continue L + } + } + // look for duplicate types + // (quadratic algorithm, but type switches tend to be reasonably small) + for t, other := range seen { + if T == nil && t == nil || T != nil && t != nil && Identical(T, t) { + // talk about "case" rather than "type" because of nil case + Ts := "nil" + if T != nil { + Ts = TypeString(T, check.qualifier) + } + check.errorf(e, DuplicateCase, "duplicate case %s in type switch", Ts) + check.error(other, DuplicateCase, "\tprevious case") // secondary error, \t indented + continue L + } + } + seen[T] = e + if x != nil && T != nil { + check.typeAssertion(e, x, T, true) + } + } + return +} + +// TODO(gri) Once we are certain that typeHash is correct in all situations, use this version of caseTypes instead. +// (Currently it may be possible that different types have identical names and import paths due to ImporterFrom.) +// +// func (check *Checker) caseTypes(x *operand, xtyp *Interface, types []ast.Expr, seen map[string]ast.Expr) (T Type) { +// var dummy operand +// L: +// for _, e := range types { +// // The spec allows the value nil instead of a type. +// var hash string +// if check.isNil(e) { +// check.expr(&dummy, e) // run e through expr so we get the usual Info recordings +// T = nil +// hash = "" // avoid collision with a type named nil +// } else { +// T = check.varType(e) +// if T == Typ[Invalid] { +// continue L +// } +// hash = typeHash(T, nil) +// } +// // look for duplicate types +// if other := seen[hash]; other != nil { +// // talk about "case" rather than "type" because of nil case +// Ts := "nil" +// if T != nil { +// Ts = TypeString(T, check.qualifier) +// } +// var err error_ +// err.code = DuplicateCase +// err.errorf(e, "duplicate case %s in type switch", Ts) +// err.errorf(other, "previous case") +// check.report(&err) +// continue L +// } +// seen[hash] = e +// if T != nil { +// check.typeAssertion(e.Pos(), x, xtyp, T) +// } +// } +// return +// } + +// stmt typechecks statement s. +func (check *Checker) stmt(ctxt stmtContext, s ast.Stmt) { + // statements must end with the same top scope as they started with + if debug { + defer func(scope *Scope) { + // don't check if code is panicking + if p := recover(); p != nil { + panic(p) + } + assert(scope == check.scope) + }(check.scope) + } + + // process collected function literals before scope changes + defer check.processDelayed(len(check.delayed)) + + // reset context for statements of inner blocks + inner := ctxt &^ (fallthroughOk | finalSwitchCase | inTypeSwitch) + + switch s := s.(type) { + case *ast.BadStmt, *ast.EmptyStmt: + // ignore + + case *ast.DeclStmt: + check.declStmt(s.Decl) + + case *ast.LabeledStmt: + check.hasLabel = true + check.stmt(ctxt, s.Stmt) + + case *ast.ExprStmt: + // spec: "With the exception of specific built-in functions, + // function and method calls and receive operations can appear + // in statement context. Such statements may be parenthesized." + var x operand + kind := check.rawExpr(&x, s.X, nil, false) + var msg string + var code Code + switch x.mode { + default: + if kind == statement { + return + } + msg = "is not used" + code = UnusedExpr + case builtin: + msg = "must be called" + code = UncalledBuiltin + case typexpr: + msg = "is not an expression" + code = NotAnExpr + } + check.errorf(&x, code, "%s %s", &x, msg) + + case *ast.SendStmt: + var ch, val operand + check.expr(&ch, s.Chan) + check.expr(&val, s.Value) + if ch.mode == invalid || val.mode == invalid { + return + } + u := coreType(ch.typ) + if u == nil { + check.errorf(inNode(s, s.Arrow), InvalidSend, invalidOp+"cannot send to %s: no core type", &ch) + return + } + uch, _ := u.(*Chan) + if uch == nil { + check.errorf(inNode(s, s.Arrow), InvalidSend, invalidOp+"cannot send to non-channel %s", &ch) + return + } + if uch.dir == RecvOnly { + check.errorf(inNode(s, s.Arrow), InvalidSend, invalidOp+"cannot send to receive-only channel %s", &ch) + return + } + check.assignment(&val, uch.elem, "send") + + case *ast.IncDecStmt: + var op token.Token + switch s.Tok { + case token.INC: + op = token.ADD + case token.DEC: + op = token.SUB + default: + check.errorf(inNode(s, s.TokPos), InvalidSyntaxTree, "unknown inc/dec operation %s", s.Tok) + return + } + + var x operand + check.expr(&x, s.X) + if x.mode == invalid { + return + } + if !allNumeric(x.typ) { + check.errorf(s.X, NonNumericIncDec, invalidOp+"%s%s (non-numeric type %s)", s.X, s.Tok, x.typ) + return + } + + Y := &ast.BasicLit{ValuePos: s.X.Pos(), Kind: token.INT, Value: "1"} // use x's position + check.binary(&x, nil, s.X, Y, op, s.TokPos) + if x.mode == invalid { + return + } + check.assignVar(s.X, &x) + + case *ast.AssignStmt: + switch s.Tok { + case token.ASSIGN, token.DEFINE: + if len(s.Lhs) == 0 { + check.error(s, InvalidSyntaxTree, "missing lhs in assignment") + return + } + if s.Tok == token.DEFINE { + check.shortVarDecl(inNode(s, s.TokPos), s.Lhs, s.Rhs) + } else { + // regular assignment + check.assignVars(s.Lhs, s.Rhs) + } + + default: + // assignment operations + if len(s.Lhs) != 1 || len(s.Rhs) != 1 { + check.errorf(inNode(s, s.TokPos), MultiValAssignOp, "assignment operation %s requires single-valued expressions", s.Tok) + return + } + op := assignOp(s.Tok) + if op == token.ILLEGAL { + check.errorf(atPos(s.TokPos), InvalidSyntaxTree, "unknown assignment operation %s", s.Tok) + return + } + var x operand + check.binary(&x, nil, s.Lhs[0], s.Rhs[0], op, s.TokPos) + if x.mode == invalid { + return + } + check.assignVar(s.Lhs[0], &x) + } + + case *ast.GoStmt: + check.suspendedCall("go", s.Call) + + case *ast.DeferStmt: + check.suspendedCall("defer", s.Call) + + case *ast.ReturnStmt: + res := check.sig.results + // Return with implicit results allowed for function with named results. + // (If one is named, all are named.) + if len(s.Results) == 0 && res.Len() > 0 && res.vars[0].name != "" { + // spec: "Implementation restriction: A compiler may disallow an empty expression + // list in a "return" statement if a different entity (constant, type, or variable) + // with the same name as a result parameter is in scope at the place of the return." + for _, obj := range res.vars { + if alt := check.lookup(obj.name); alt != nil && alt != obj { + check.errorf(s, OutOfScopeResult, "result parameter %s not in scope at return", obj.name) + check.errorf(alt, OutOfScopeResult, "\tinner declaration of %s", obj) + // ok to continue + } + } + } else { + var lhs []*Var + if res.Len() > 0 { + lhs = res.vars + } + check.initVars(lhs, s.Results, s) + } + + case *ast.BranchStmt: + if s.Label != nil { + check.hasLabel = true + return // checked in 2nd pass (check.labels) + } + switch s.Tok { + case token.BREAK: + if ctxt&breakOk == 0 { + check.error(s, MisplacedBreak, "break not in for, switch, or select statement") + } + case token.CONTINUE: + if ctxt&continueOk == 0 { + check.error(s, MisplacedContinue, "continue not in for statement") + } + case token.FALLTHROUGH: + if ctxt&fallthroughOk == 0 { + var msg string + switch { + case ctxt&finalSwitchCase != 0: + msg = "cannot fallthrough final case in switch" + case ctxt&inTypeSwitch != 0: + msg = "cannot fallthrough in type switch" + default: + msg = "fallthrough statement out of place" + } + check.error(s, MisplacedFallthrough, msg) + } + default: + check.errorf(s, InvalidSyntaxTree, "branch statement: %s", s.Tok) + } + + case *ast.BlockStmt: + check.openScope(s, "block") + defer check.closeScope() + + check.stmtList(inner, s.List) + + case *ast.IfStmt: + check.openScope(s, "if") + defer check.closeScope() + + check.simpleStmt(s.Init) + var x operand + check.expr(&x, s.Cond) + if x.mode != invalid && !allBoolean(x.typ) { + check.error(s.Cond, InvalidCond, "non-boolean condition in if statement") + } + check.stmt(inner, s.Body) + // The parser produces a correct AST but if it was modified + // elsewhere the else branch may be invalid. Check again. + switch s.Else.(type) { + case nil, *ast.BadStmt: + // valid or error already reported + case *ast.IfStmt, *ast.BlockStmt: + check.stmt(inner, s.Else) + default: + check.error(s.Else, InvalidSyntaxTree, "invalid else branch in if statement") + } + + case *ast.SwitchStmt: + inner |= breakOk + check.openScope(s, "switch") + defer check.closeScope() + + check.simpleStmt(s.Init) + var x operand + if s.Tag != nil { + check.expr(&x, s.Tag) + // By checking assignment of x to an invisible temporary + // (as a compiler would), we get all the relevant checks. + check.assignment(&x, nil, "switch expression") + if x.mode != invalid && !Comparable(x.typ) && !hasNil(x.typ) { + check.errorf(&x, InvalidExprSwitch, "cannot switch on %s (%s is not comparable)", &x, x.typ) + x.mode = invalid + } + } else { + // spec: "A missing switch expression is + // equivalent to the boolean value true." + x.mode = constant_ + x.typ = Typ[Bool] + x.val = constant.MakeBool(true) + x.expr = &ast.Ident{NamePos: s.Body.Lbrace, Name: "true"} + } + + check.multipleDefaults(s.Body.List) + + seen := make(valueMap) // map of seen case values to positions and types + for i, c := range s.Body.List { + clause, _ := c.(*ast.CaseClause) + if clause == nil { + check.error(c, InvalidSyntaxTree, "incorrect expression switch case") + continue + } + check.caseValues(&x, clause.List, seen) + check.openScope(clause, "case") + inner := inner + if i+1 < len(s.Body.List) { + inner |= fallthroughOk + } else { + inner |= finalSwitchCase + } + check.stmtList(inner, clause.Body) + check.closeScope() + } + + case *ast.TypeSwitchStmt: + inner |= breakOk | inTypeSwitch + check.openScope(s, "type switch") + defer check.closeScope() + + check.simpleStmt(s.Init) + + // A type switch guard must be of the form: + // + // TypeSwitchGuard = [ identifier ":=" ] PrimaryExpr "." "(" "type" ")" . + // + // The parser is checking syntactic correctness; + // remaining syntactic errors are considered AST errors here. + // TODO(gri) better factoring of error handling (invalid ASTs) + // + var lhs *ast.Ident // lhs identifier or nil + var rhs ast.Expr + switch guard := s.Assign.(type) { + case *ast.ExprStmt: + rhs = guard.X + case *ast.AssignStmt: + if len(guard.Lhs) != 1 || guard.Tok != token.DEFINE || len(guard.Rhs) != 1 { + check.error(s, InvalidSyntaxTree, "incorrect form of type switch guard") + return + } + + lhs, _ = guard.Lhs[0].(*ast.Ident) + if lhs == nil { + check.error(s, InvalidSyntaxTree, "incorrect form of type switch guard") + return + } + + if lhs.Name == "_" { + // _ := x.(type) is an invalid short variable declaration + check.softErrorf(lhs, NoNewVar, "no new variable on left side of :=") + lhs = nil // avoid declared and not used error below + } else { + check.recordDef(lhs, nil) // lhs variable is implicitly declared in each cause clause + } + + rhs = guard.Rhs[0] + + default: + check.error(s, InvalidSyntaxTree, "incorrect form of type switch guard") + return + } + + // rhs must be of the form: expr.(type) and expr must be an ordinary interface + expr, _ := rhs.(*ast.TypeAssertExpr) + if expr == nil || expr.Type != nil { + check.error(s, InvalidSyntaxTree, "incorrect form of type switch guard") + return + } + var x operand + check.expr(&x, expr.X) + if x.mode == invalid { + return + } + // TODO(gri) we may want to permit type switches on type parameter values at some point + var sx *operand // switch expression against which cases are compared against; nil if invalid + if isTypeParam(x.typ) { + check.errorf(&x, InvalidTypeSwitch, "cannot use type switch on type parameter value %s", &x) + } else { + if _, ok := under(x.typ).(*Interface); ok { + sx = &x + } else { + check.errorf(&x, InvalidTypeSwitch, "%s is not an interface", &x) + } + } + + check.multipleDefaults(s.Body.List) + + var lhsVars []*Var // list of implicitly declared lhs variables + seen := make(map[Type]ast.Expr) // map of seen types to positions + for _, s := range s.Body.List { + clause, _ := s.(*ast.CaseClause) + if clause == nil { + check.error(s, InvalidSyntaxTree, "incorrect type switch case") + continue + } + // Check each type in this type switch case. + T := check.caseTypes(sx, clause.List, seen) + check.openScope(clause, "case") + // If lhs exists, declare a corresponding variable in the case-local scope. + if lhs != nil { + // spec: "The TypeSwitchGuard may include a short variable declaration. + // When that form is used, the variable is declared at the beginning of + // the implicit block in each clause. In clauses with a case listing + // exactly one type, the variable has that type; otherwise, the variable + // has the type of the expression in the TypeSwitchGuard." + if len(clause.List) != 1 || T == nil { + T = x.typ + } + obj := NewVar(lhs.Pos(), check.pkg, lhs.Name, T) + scopePos := clause.Pos() + token.Pos(len("default")) // for default clause (len(List) == 0) + if n := len(clause.List); n > 0 { + scopePos = clause.List[n-1].End() + } + check.declare(check.scope, nil, obj, scopePos) + check.recordImplicit(clause, obj) + // For the "declared and not used" error, all lhs variables act as + // one; i.e., if any one of them is 'used', all of them are 'used'. + // Collect them for later analysis. + lhsVars = append(lhsVars, obj) + } + check.stmtList(inner, clause.Body) + check.closeScope() + } + + // If lhs exists, we must have at least one lhs variable that was used. + if lhs != nil { + var used bool + for _, v := range lhsVars { + if v.used { + used = true + } + v.used = true // avoid usage error when checking entire function + } + if !used { + check.softErrorf(lhs, UnusedVar, "%s declared and not used", lhs.Name) + } + } + + case *ast.SelectStmt: + inner |= breakOk + + check.multipleDefaults(s.Body.List) + + for _, s := range s.Body.List { + clause, _ := s.(*ast.CommClause) + if clause == nil { + continue // error reported before + } + + // clause.Comm must be a SendStmt, RecvStmt, or default case + valid := false + var rhs ast.Expr // rhs of RecvStmt, or nil + switch s := clause.Comm.(type) { + case nil, *ast.SendStmt: + valid = true + case *ast.AssignStmt: + if len(s.Rhs) == 1 { + rhs = s.Rhs[0] + } + case *ast.ExprStmt: + rhs = s.X + } + + // if present, rhs must be a receive operation + if rhs != nil { + if x, _ := unparen(rhs).(*ast.UnaryExpr); x != nil && x.Op == token.ARROW { + valid = true + } + } + + if !valid { + check.error(clause.Comm, InvalidSelectCase, "select case must be send or receive (possibly with assignment)") + continue + } + + check.openScope(s, "case") + if clause.Comm != nil { + check.stmt(inner, clause.Comm) + } + check.stmtList(inner, clause.Body) + check.closeScope() + } + + case *ast.ForStmt: + inner |= breakOk | continueOk + check.openScope(s, "for") + defer check.closeScope() + + check.simpleStmt(s.Init) + if s.Cond != nil { + var x operand + check.expr(&x, s.Cond) + if x.mode != invalid && !allBoolean(x.typ) { + check.error(s.Cond, InvalidCond, "non-boolean condition in for statement") + } + } + check.simpleStmt(s.Post) + // spec: "The init statement may be a short variable + // declaration, but the post statement must not." + if s, _ := s.Post.(*ast.AssignStmt); s != nil && s.Tok == token.DEFINE { + check.softErrorf(s, InvalidPostDecl, "cannot declare in post statement") + // Don't call useLHS here because we want to use the lhs in + // this erroneous statement so that we don't get errors about + // these lhs variables being declared and not used. + check.use(s.Lhs...) // avoid follow-up errors + } + check.stmt(inner, s.Body) + + case *ast.RangeStmt: + inner |= breakOk | continueOk + + // check expression to iterate over + var x operand + check.expr(&x, s.X) + + // determine key/value types + var key, val Type + if x.mode != invalid { + // Ranging over a type parameter is permitted if it has a core type. + var cause string + u := coreType(x.typ) + switch t := u.(type) { + case nil: + cause = check.sprintf("%s has no core type", x.typ) + case *Chan: + if s.Value != nil { + check.softErrorf(s.Value, InvalidIterVar, "range over %s permits only one iteration variable", &x) + // ok to continue + } + if t.dir == SendOnly { + cause = "receive from send-only channel" + } + } + key, val = rangeKeyVal(u) + if key == nil || cause != "" { + if cause == "" { + check.softErrorf(&x, InvalidRangeExpr, "cannot range over %s", &x) + } else { + check.softErrorf(&x, InvalidRangeExpr, "cannot range over %s (%s)", &x, cause) + } + // ok to continue + } + } + + // Open the for-statement block scope now, after the range clause. + // Iteration variables declared with := need to go in this scope (was issue #51437). + check.openScope(s, "range") + defer check.closeScope() + + // check assignment to/declaration of iteration variables + // (irregular assignment, cannot easily map to existing assignment checks) + + // lhs expressions and initialization value (rhs) types + lhs := [2]ast.Expr{s.Key, s.Value} + rhs := [2]Type{key, val} // key, val may be nil + + if s.Tok == token.DEFINE { + // short variable declaration + var vars []*Var + for i, lhs := range lhs { + if lhs == nil { + continue + } + + // determine lhs variable + var obj *Var + if ident, _ := lhs.(*ast.Ident); ident != nil { + // declare new variable + name := ident.Name + obj = NewVar(ident.Pos(), check.pkg, name, nil) + check.recordDef(ident, obj) + // _ variables don't count as new variables + if name != "_" { + vars = append(vars, obj) + } + } else { + check.errorf(lhs, InvalidSyntaxTree, "cannot declare %s", lhs) + obj = NewVar(lhs.Pos(), check.pkg, "_", nil) // dummy variable + } + + // initialize lhs variable + if typ := rhs[i]; typ != nil { + x.mode = value + x.expr = lhs // we don't have a better rhs expression to use here + x.typ = typ + check.initVar(obj, &x, "range clause") + } else { + obj.typ = Typ[Invalid] + obj.used = true // don't complain about unused variable + } + } + + // declare variables + if len(vars) > 0 { + scopePos := s.Body.Pos() + for _, obj := range vars { + check.declare(check.scope, nil /* recordDef already called */, obj, scopePos) + } + } else { + check.error(inNode(s, s.TokPos), NoNewVar, "no new variables on left side of :=") + } + } else { + // ordinary assignment + for i, lhs := range lhs { + if lhs == nil { + continue + } + if typ := rhs[i]; typ != nil { + x.mode = value + x.expr = lhs // we don't have a better rhs expression to use here + x.typ = typ + check.assignVar(lhs, &x) + } + } + } + + check.stmt(inner, s.Body) + + default: + check.error(s, InvalidSyntaxTree, "invalid statement") + } +} + +// rangeKeyVal returns the key and value type produced by a range clause +// over an expression of type typ. If the range clause is not permitted +// the results are nil. +func rangeKeyVal(typ Type) (key, val Type) { + switch typ := arrayPtrDeref(typ).(type) { + case *Basic: + if isString(typ) { + return Typ[Int], universeRune // use 'rune' name + } + case *Array: + return Typ[Int], typ.elem + case *Slice: + return Typ[Int], typ.elem + case *Map: + return typ.key, typ.elem + case *Chan: + return typ.elem, Typ[Invalid] + } + return +} diff --git a/src/go/types/struct.go b/src/go/types/struct.go new file mode 100644 index 0000000..2ed0e6d --- /dev/null +++ b/src/go/types/struct.go @@ -0,0 +1,218 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "go/ast" + "go/token" + . "internal/types/errors" + "strconv" +) + +// ---------------------------------------------------------------------------- +// API + +// A Struct represents a struct type. +type Struct struct { + fields []*Var // fields != nil indicates the struct is set up (possibly with len(fields) == 0) + tags []string // field tags; nil if there are no tags +} + +// NewStruct returns a new struct with the given fields and corresponding field tags. +// If a field with index i has a tag, tags[i] must be that tag, but len(tags) may be +// only as long as required to hold the tag with the largest index i. Consequently, +// if no field has a tag, tags may be nil. +func NewStruct(fields []*Var, tags []string) *Struct { + var fset objset + for _, f := range fields { + if f.name != "_" && fset.insert(f) != nil { + panic("multiple fields with the same name") + } + } + if len(tags) > len(fields) { + panic("more tags than fields") + } + s := &Struct{fields: fields, tags: tags} + s.markComplete() + return s +} + +// NumFields returns the number of fields in the struct (including blank and embedded fields). +func (s *Struct) NumFields() int { return len(s.fields) } + +// Field returns the i'th field for 0 <= i < NumFields(). +func (s *Struct) Field(i int) *Var { return s.fields[i] } + +// Tag returns the i'th field tag for 0 <= i < NumFields(). +func (s *Struct) Tag(i int) string { + if i < len(s.tags) { + return s.tags[i] + } + return "" +} + +func (t *Struct) Underlying() Type { return t } +func (t *Struct) String() string { return TypeString(t, nil) } + +// ---------------------------------------------------------------------------- +// Implementation + +func (s *Struct) markComplete() { + if s.fields == nil { + s.fields = make([]*Var, 0) + } +} + +func (check *Checker) structType(styp *Struct, e *ast.StructType) { + list := e.Fields + if list == nil { + styp.markComplete() + return + } + + // struct fields and tags + var fields []*Var + var tags []string + + // for double-declaration checks + var fset objset + + // current field typ and tag + var typ Type + var tag string + add := func(ident *ast.Ident, embedded bool, pos token.Pos) { + if tag != "" && tags == nil { + tags = make([]string, len(fields)) + } + if tags != nil { + tags = append(tags, tag) + } + + name := ident.Name + fld := NewField(pos, check.pkg, name, typ, embedded) + // spec: "Within a struct, non-blank field names must be unique." + if name == "_" || check.declareInSet(&fset, pos, fld) { + fields = append(fields, fld) + check.recordDef(ident, fld) + } + } + + // addInvalid adds an embedded field of invalid type to the struct for + // fields with errors; this keeps the number of struct fields in sync + // with the source as long as the fields are _ or have different names + // (issue #25627). + addInvalid := func(ident *ast.Ident, pos token.Pos) { + typ = Typ[Invalid] + tag = "" + add(ident, true, pos) + } + + for _, f := range list.List { + typ = check.varType(f.Type) + tag = check.tag(f.Tag) + if len(f.Names) > 0 { + // named fields + for _, name := range f.Names { + add(name, false, name.Pos()) + } + } else { + // embedded field + // spec: "An embedded type must be specified as a type name T or as a + // pointer to a non-interface type name *T, and T itself may not be a + // pointer type." + pos := f.Type.Pos() + name := embeddedFieldIdent(f.Type) + if name == nil { + check.errorf(f.Type, InvalidSyntaxTree, "embedded field type %s has no name", f.Type) + name = ast.NewIdent("_") + name.NamePos = pos + addInvalid(name, pos) + continue + } + add(name, true, pos) + + // Because we have a name, typ must be of the form T or *T, where T is the name + // of a (named or alias) type, and t (= deref(typ)) must be the type of T. + // We must delay this check to the end because we don't want to instantiate + // (via under(t)) a possibly incomplete type. + + // for use in the closure below + embeddedTyp := typ + embeddedPos := f.Type + + check.later(func() { + t, isPtr := deref(embeddedTyp) + switch u := under(t).(type) { + case *Basic: + if t == Typ[Invalid] { + // error was reported before + return + } + // unsafe.Pointer is treated like a regular pointer + if u.kind == UnsafePointer { + check.error(embeddedPos, InvalidPtrEmbed, "embedded field type cannot be unsafe.Pointer") + } + case *Pointer: + check.error(embeddedPos, InvalidPtrEmbed, "embedded field type cannot be a pointer") + case *Interface: + if isTypeParam(t) { + // The error code here is inconsistent with other error codes for + // invalid embedding, because this restriction may be relaxed in the + // future, and so it did not warrant a new error code. + check.error(embeddedPos, MisplacedTypeParam, "embedded field type cannot be a (pointer to a) type parameter") + break + } + if isPtr { + check.error(embeddedPos, InvalidPtrEmbed, "embedded field type cannot be a pointer to an interface") + } + } + }).describef(embeddedPos, "check embedded type %s", embeddedTyp) + } + } + + styp.fields = fields + styp.tags = tags + styp.markComplete() +} + +func embeddedFieldIdent(e ast.Expr) *ast.Ident { + switch e := e.(type) { + case *ast.Ident: + return e + case *ast.StarExpr: + // *T is valid, but **T is not + if _, ok := e.X.(*ast.StarExpr); !ok { + return embeddedFieldIdent(e.X) + } + case *ast.SelectorExpr: + return e.Sel + case *ast.IndexExpr: + return embeddedFieldIdent(e.X) + case *ast.IndexListExpr: + return embeddedFieldIdent(e.X) + } + return nil // invalid embedded field +} + +func (check *Checker) declareInSet(oset *objset, pos token.Pos, obj Object) bool { + if alt := oset.insert(obj); alt != nil { + check.errorf(atPos(pos), DuplicateDecl, "%s redeclared", obj.Name()) + check.reportAltDecl(alt) + return false + } + return true +} + +func (check *Checker) tag(t *ast.BasicLit) string { + if t != nil { + if t.Kind == token.STRING { + if val, err := strconv.Unquote(t.Value); err == nil { + return val + } + } + check.errorf(t, InvalidSyntaxTree, "incorrect tag syntax: %q", t.Value) + } + return "" +} diff --git a/src/go/types/subst.go b/src/go/types/subst.go new file mode 100644 index 0000000..5a49c04 --- /dev/null +++ b/src/go/types/subst.go @@ -0,0 +1,421 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements type parameter substitution. + +package types + +import ( + "go/token" +) + +type substMap map[*TypeParam]Type + +// makeSubstMap creates a new substitution map mapping tpars[i] to targs[i]. +// If targs[i] is nil, tpars[i] is not substituted. +func makeSubstMap(tpars []*TypeParam, targs []Type) substMap { + assert(len(tpars) == len(targs)) + proj := make(substMap, len(tpars)) + for i, tpar := range tpars { + proj[tpar] = targs[i] + } + return proj +} + +// makeRenameMap is like makeSubstMap, but creates a map used to rename type +// parameters in from with the type parameters in to. +func makeRenameMap(from, to []*TypeParam) substMap { + assert(len(from) == len(to)) + proj := make(substMap, len(from)) + for i, tpar := range from { + proj[tpar] = to[i] + } + return proj +} + +func (m substMap) empty() bool { + return len(m) == 0 +} + +func (m substMap) lookup(tpar *TypeParam) Type { + if t := m[tpar]; t != nil { + return t + } + return tpar +} + +// subst returns the type typ with its type parameters tpars replaced by the +// corresponding type arguments targs, recursively. subst is pure in the sense +// that it doesn't modify the incoming type. If a substitution took place, the +// result type is different from the incoming type. +// +// If expanding is non-nil, it is the instance type currently being expanded. +// One of expanding or ctxt must be non-nil. +func (check *Checker) subst(pos token.Pos, typ Type, smap substMap, expanding *Named, ctxt *Context) Type { + assert(expanding != nil || ctxt != nil) + + if smap.empty() { + return typ + } + + // common cases + switch t := typ.(type) { + case *Basic: + return typ // nothing to do + case *TypeParam: + return smap.lookup(t) + } + + // general case + subst := subster{ + pos: pos, + smap: smap, + check: check, + expanding: expanding, + ctxt: ctxt, + } + return subst.typ(typ) +} + +type subster struct { + pos token.Pos + smap substMap + check *Checker // nil if called via Instantiate + expanding *Named // if non-nil, the instance that is being expanded + ctxt *Context +} + +func (subst *subster) typ(typ Type) Type { + switch t := typ.(type) { + case nil: + // Call typOrNil if it's possible that typ is nil. + panic("nil typ") + + case *Basic: + // nothing to do + + case *Array: + elem := subst.typOrNil(t.elem) + if elem != t.elem { + return &Array{len: t.len, elem: elem} + } + + case *Slice: + elem := subst.typOrNil(t.elem) + if elem != t.elem { + return &Slice{elem: elem} + } + + case *Struct: + if fields, copied := subst.varList(t.fields); copied { + s := &Struct{fields: fields, tags: t.tags} + s.markComplete() + return s + } + + case *Pointer: + base := subst.typ(t.base) + if base != t.base { + return &Pointer{base: base} + } + + case *Tuple: + return subst.tuple(t) + + case *Signature: + // Preserve the receiver: it is handled during *Interface and *Named type + // substitution. + // + // Naively doing the substitution here can lead to an infinite recursion in + // the case where the receiver is an interface. For example, consider the + // following declaration: + // + // type T[A any] struct { f interface{ m() } } + // + // In this case, the type of f is an interface that is itself the receiver + // type of all of its methods. Because we have no type name to break + // cycles, substituting in the recv results in an infinite loop of + // recv->interface->recv->interface->... + recv := t.recv + + params := subst.tuple(t.params) + results := subst.tuple(t.results) + if params != t.params || results != t.results { + return &Signature{ + rparams: t.rparams, + // TODO(rFindley) why can't we nil out tparams here, rather than in instantiate? + tparams: t.tparams, + // instantiated signatures have a nil scope + recv: recv, + params: params, + results: results, + variadic: t.variadic, + } + } + + case *Union: + terms, copied := subst.termlist(t.terms) + if copied { + // term list substitution may introduce duplicate terms (unlikely but possible). + // This is ok; lazy type set computation will determine the actual type set + // in normal form. + return &Union{terms} + } + + case *Interface: + methods, mcopied := subst.funcList(t.methods) + embeddeds, ecopied := subst.typeList(t.embeddeds) + if mcopied || ecopied { + iface := subst.check.newInterface() + iface.embeddeds = embeddeds + iface.implicit = t.implicit + iface.complete = t.complete + // If we've changed the interface type, we may need to replace its + // receiver if the receiver type is the original interface. Receivers of + // *Named type are replaced during named type expansion. + // + // Notably, it's possible to reach here and not create a new *Interface, + // even though the receiver type may be parameterized. For example: + // + // type T[P any] interface{ m() } + // + // In this case the interface will not be substituted here, because its + // method signatures do not depend on the type parameter P, but we still + // need to create new interface methods to hold the instantiated + // receiver. This is handled by Named.expandUnderlying. + iface.methods, _ = replaceRecvType(methods, t, iface) + return iface + } + + case *Map: + key := subst.typ(t.key) + elem := subst.typ(t.elem) + if key != t.key || elem != t.elem { + return &Map{key: key, elem: elem} + } + + case *Chan: + elem := subst.typ(t.elem) + if elem != t.elem { + return &Chan{dir: t.dir, elem: elem} + } + + case *Named: + // dump is for debugging + dump := func(string, ...any) {} + if subst.check != nil && trace { + subst.check.indent++ + defer func() { + subst.check.indent-- + }() + dump = func(format string, args ...any) { + subst.check.trace(subst.pos, format, args...) + } + } + + // subst is called during expansion, so in this function we need to be + // careful not to call any methods that would cause t to be expanded: doing + // so would result in deadlock. + // + // So we call t.Origin().TypeParams() rather than t.TypeParams(). + orig := t.Origin() + n := orig.TypeParams().Len() + if n == 0 { + dump(">>> %s is not parameterized", t) + return t // type is not parameterized + } + + var newTArgs []Type + if t.TypeArgs().Len() != n { + return Typ[Invalid] // error reported elsewhere + } + + // already instantiated + dump(">>> %s already instantiated", t) + // For each (existing) type argument targ, determine if it needs + // to be substituted; i.e., if it is or contains a type parameter + // that has a type argument for it. + for i, targ := range t.TypeArgs().list() { + dump(">>> %d targ = %s", i, targ) + new_targ := subst.typ(targ) + if new_targ != targ { + dump(">>> substituted %d targ %s => %s", i, targ, new_targ) + if newTArgs == nil { + newTArgs = make([]Type, n) + copy(newTArgs, t.TypeArgs().list()) + } + newTArgs[i] = new_targ + } + } + + if newTArgs == nil { + dump(">>> nothing to substitute in %s", t) + return t // nothing to substitute + } + + // Create a new instance and populate the context to avoid endless + // recursion. The position used here is irrelevant because validation only + // occurs on t (we don't call validType on named), but we use subst.pos to + // help with debugging. + return subst.check.instance(subst.pos, orig, newTArgs, subst.expanding, subst.ctxt) + + case *TypeParam: + return subst.smap.lookup(t) + + default: + unreachable() + } + + return typ +} + +// typOrNil is like typ but if the argument is nil it is replaced with Typ[Invalid]. +// A nil type may appear in pathological cases such as type T[P any] []func(_ T([]_)) +// where an array/slice element is accessed before it is set up. +func (subst *subster) typOrNil(typ Type) Type { + if typ == nil { + return Typ[Invalid] + } + return subst.typ(typ) +} + +func (subst *subster) var_(v *Var) *Var { + if v != nil { + if typ := subst.typ(v.typ); typ != v.typ { + return substVar(v, typ) + } + } + return v +} + +func substVar(v *Var, typ Type) *Var { + copy := *v + copy.typ = typ + copy.origin = v.Origin() + return © +} + +func (subst *subster) tuple(t *Tuple) *Tuple { + if t != nil { + if vars, copied := subst.varList(t.vars); copied { + return &Tuple{vars: vars} + } + } + return t +} + +func (subst *subster) varList(in []*Var) (out []*Var, copied bool) { + out = in + for i, v := range in { + if w := subst.var_(v); w != v { + if !copied { + // first variable that got substituted => allocate new out slice + // and copy all variables + new := make([]*Var, len(in)) + copy(new, out) + out = new + copied = true + } + out[i] = w + } + } + return +} + +func (subst *subster) func_(f *Func) *Func { + if f != nil { + if typ := subst.typ(f.typ); typ != f.typ { + return substFunc(f, typ) + } + } + return f +} + +func substFunc(f *Func, typ Type) *Func { + copy := *f + copy.typ = typ + copy.origin = f.Origin() + return © +} + +func (subst *subster) funcList(in []*Func) (out []*Func, copied bool) { + out = in + for i, f := range in { + if g := subst.func_(f); g != f { + if !copied { + // first function that got substituted => allocate new out slice + // and copy all functions + new := make([]*Func, len(in)) + copy(new, out) + out = new + copied = true + } + out[i] = g + } + } + return +} + +func (subst *subster) typeList(in []Type) (out []Type, copied bool) { + out = in + for i, t := range in { + if u := subst.typ(t); u != t { + if !copied { + // first function that got substituted => allocate new out slice + // and copy all functions + new := make([]Type, len(in)) + copy(new, out) + out = new + copied = true + } + out[i] = u + } + } + return +} + +func (subst *subster) termlist(in []*Term) (out []*Term, copied bool) { + out = in + for i, t := range in { + if u := subst.typ(t.typ); u != t.typ { + if !copied { + // first function that got substituted => allocate new out slice + // and copy all functions + new := make([]*Term, len(in)) + copy(new, out) + out = new + copied = true + } + out[i] = NewTerm(t.tilde, u) + } + } + return +} + +// replaceRecvType updates any function receivers that have type old to have +// type new. It does not modify the input slice; if modifications are required, +// the input slice and any affected signatures will be copied before mutating. +// +// The resulting out slice contains the updated functions, and copied reports +// if anything was modified. +func replaceRecvType(in []*Func, old, new Type) (out []*Func, copied bool) { + out = in + for i, method := range in { + sig := method.Type().(*Signature) + if sig.recv != nil && sig.recv.Type() == old { + if !copied { + // Allocate a new methods slice before mutating for the first time. + // This is defensive, as we may share methods across instantiations of + // a given interface type if they do not get substituted. + out = make([]*Func, len(in)) + copy(out, in) + copied = true + } + newsig := *sig + newsig.recv = substVar(sig.recv, new) + out[i] = substFunc(method, &newsig) + } + } + return +} diff --git a/src/go/types/termlist.go b/src/go/types/termlist.go new file mode 100644 index 0000000..83a02ee --- /dev/null +++ b/src/go/types/termlist.go @@ -0,0 +1,161 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import "strings" + +// A termlist represents the type set represented by the union +// t1 āˆŖ y2 āˆŖ ... tn of the type sets of the terms t1 to tn. +// A termlist is in normal form if all terms are disjoint. +// termlist operations don't require the operands to be in +// normal form. +type termlist []*term + +// allTermlist represents the set of all types. +// It is in normal form. +var allTermlist = termlist{new(term)} + +// termSep is the separator used between individual terms. +const termSep = " | " + +// String prints the termlist exactly (without normalization). +func (xl termlist) String() string { + if len(xl) == 0 { + return "āˆ…" + } + var buf strings.Builder + for i, x := range xl { + if i > 0 { + buf.WriteString(termSep) + } + buf.WriteString(x.String()) + } + return buf.String() +} + +// isEmpty reports whether the termlist xl represents the empty set of types. +func (xl termlist) isEmpty() bool { + // If there's a non-nil term, the entire list is not empty. + // If the termlist is in normal form, this requires at most + // one iteration. + for _, x := range xl { + if x != nil { + return false + } + } + return true +} + +// isAll reports whether the termlist xl represents the set of all types. +func (xl termlist) isAll() bool { + // If there's a š“¤ term, the entire list is š“¤. + // If the termlist is in normal form, this requires at most + // one iteration. + for _, x := range xl { + if x != nil && x.typ == nil { + return true + } + } + return false +} + +// norm returns the normal form of xl. +func (xl termlist) norm() termlist { + // Quadratic algorithm, but good enough for now. + // TODO(gri) fix asymptotic performance + used := make([]bool, len(xl)) + var rl termlist + for i, xi := range xl { + if xi == nil || used[i] { + continue + } + for j := i + 1; j < len(xl); j++ { + xj := xl[j] + if xj == nil || used[j] { + continue + } + if u1, u2 := xi.union(xj); u2 == nil { + // If we encounter a š“¤ term, the entire list is š“¤. + // Exit early. + // (Note that this is not just an optimization; + // if we continue, we may end up with a š“¤ term + // and other terms and the result would not be + // in normal form.) + if u1.typ == nil { + return allTermlist + } + xi = u1 + used[j] = true // xj is now unioned into xi - ignore it in future iterations + } + } + rl = append(rl, xi) + } + return rl +} + +// union returns the union xl āˆŖ yl. +func (xl termlist) union(yl termlist) termlist { + return append(xl, yl...).norm() +} + +// intersect returns the intersection xl āˆ© yl. +func (xl termlist) intersect(yl termlist) termlist { + if xl.isEmpty() || yl.isEmpty() { + return nil + } + + // Quadratic algorithm, but good enough for now. + // TODO(gri) fix asymptotic performance + var rl termlist + for _, x := range xl { + for _, y := range yl { + if r := x.intersect(y); r != nil { + rl = append(rl, r) + } + } + } + return rl.norm() +} + +// equal reports whether xl and yl represent the same type set. +func (xl termlist) equal(yl termlist) bool { + // TODO(gri) this should be more efficient + return xl.subsetOf(yl) && yl.subsetOf(xl) +} + +// includes reports whether t āˆˆ xl. +func (xl termlist) includes(t Type) bool { + for _, x := range xl { + if x.includes(t) { + return true + } + } + return false +} + +// supersetOf reports whether y āŠ† xl. +func (xl termlist) supersetOf(y *term) bool { + for _, x := range xl { + if y.subsetOf(x) { + return true + } + } + return false +} + +// subsetOf reports whether xl āŠ† yl. +func (xl termlist) subsetOf(yl termlist) bool { + if yl.isEmpty() { + return xl.isEmpty() + } + + // each term x of xl must be a subset of yl + for _, x := range xl { + if !yl.supersetOf(x) { + return false // x is not a subset yl + } + } + return true +} diff --git a/src/go/types/termlist_test.go b/src/go/types/termlist_test.go new file mode 100644 index 0000000..0ff687e --- /dev/null +++ b/src/go/types/termlist_test.go @@ -0,0 +1,284 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "strings" + "testing" +) + +// maketl makes a term list from a string of the term list. +func maketl(s string) termlist { + s = strings.ReplaceAll(s, " ", "") + names := strings.Split(s, "|") + r := make(termlist, len(names)) + for i, n := range names { + r[i] = testTerm(n) + } + return r +} + +func TestTermlistAll(t *testing.T) { + if !allTermlist.isAll() { + t.Errorf("allTermlist is not the set of all types") + } +} + +func TestTermlistString(t *testing.T) { + for _, want := range []string{ + "āˆ…", + "š“¤", + "int", + "~int", + "myInt", + "āˆ… | āˆ…", + "š“¤ | š“¤", + "āˆ… | š“¤ | int", + "āˆ… | š“¤ | int | myInt", + } { + if got := maketl(want).String(); got != want { + t.Errorf("(%v).String() == %v", want, got) + } + } +} + +func TestTermlistIsEmpty(t *testing.T) { + for test, want := range map[string]bool{ + "āˆ…": true, + "āˆ… | āˆ…": true, + "āˆ… | āˆ… | š“¤": false, + "āˆ… | āˆ… | myInt": false, + "š“¤": false, + "š“¤ | int": false, + "š“¤ | myInt | āˆ…": false, + } { + xl := maketl(test) + got := xl.isEmpty() + if got != want { + t.Errorf("(%v).isEmpty() == %v; want %v", test, got, want) + } + } +} + +func TestTermlistIsAll(t *testing.T) { + for test, want := range map[string]bool{ + "āˆ…": false, + "āˆ… | āˆ…": false, + "int | ~string": false, + "~int | myInt": false, + "āˆ… | āˆ… | š“¤": true, + "š“¤": true, + "š“¤ | int": true, + "myInt | š“¤": true, + } { + xl := maketl(test) + got := xl.isAll() + if got != want { + t.Errorf("(%v).isAll() == %v; want %v", test, got, want) + } + } +} + +func TestTermlistNorm(t *testing.T) { + for _, test := range []struct { + xl, want string + }{ + {"āˆ…", "āˆ…"}, + {"āˆ… | āˆ…", "āˆ…"}, + {"āˆ… | int", "int"}, + {"āˆ… | myInt", "myInt"}, + {"š“¤ | int", "š“¤"}, + {"š“¤ | myInt", "š“¤"}, + {"int | myInt", "int | myInt"}, + {"~int | int", "~int"}, + {"~int | myInt", "~int"}, + {"int | ~string | int", "int | ~string"}, + {"~int | string | š“¤ | ~string | int", "š“¤"}, + {"~int | string | myInt | ~string | int", "~int | ~string"}, + } { + xl := maketl(test.xl) + got := maketl(test.xl).norm() + if got.String() != test.want { + t.Errorf("(%v).norm() = %v; want %v", xl, got, test.want) + } + } +} + +func TestTermlistUnion(t *testing.T) { + for _, test := range []struct { + xl, yl, want string + }{ + + {"āˆ…", "āˆ…", "āˆ…"}, + {"āˆ…", "š“¤", "š“¤"}, + {"āˆ…", "int", "int"}, + {"š“¤", "~int", "š“¤"}, + {"int", "~int", "~int"}, + {"int", "string", "int | string"}, + {"int", "myInt", "int | myInt"}, + {"~int", "myInt", "~int"}, + {"int | string", "~string", "int | ~string"}, + {"~int | string", "~string | int", "~int | ~string"}, + {"~int | string | āˆ…", "~string | int", "~int | ~string"}, + {"~int | myInt | āˆ…", "~string | int", "~int | ~string"}, + {"~int | string | š“¤", "~string | int", "š“¤"}, + {"~int | string | myInt", "~string | int", "~int | ~string"}, + } { + xl := maketl(test.xl) + yl := maketl(test.yl) + got := xl.union(yl).String() + if got != test.want { + t.Errorf("(%v).union(%v) = %v; want %v", test.xl, test.yl, got, test.want) + } + } +} + +func TestTermlistIntersect(t *testing.T) { + for _, test := range []struct { + xl, yl, want string + }{ + + {"āˆ…", "āˆ…", "āˆ…"}, + {"āˆ…", "š“¤", "āˆ…"}, + {"āˆ…", "int", "āˆ…"}, + {"āˆ…", "myInt", "āˆ…"}, + {"š“¤", "~int", "~int"}, + {"š“¤", "myInt", "myInt"}, + {"int", "~int", "int"}, + {"int", "string", "āˆ…"}, + {"int", "myInt", "āˆ…"}, + {"~int", "myInt", "myInt"}, + {"int | string", "~string", "string"}, + {"~int | string", "~string | int", "int | string"}, + {"~int | string | āˆ…", "~string | int", "int | string"}, + {"~int | myInt | āˆ…", "~string | int", "int"}, + {"~int | string | š“¤", "~string | int", "int | ~string"}, + {"~int | string | myInt", "~string | int", "int | string"}, + } { + xl := maketl(test.xl) + yl := maketl(test.yl) + got := xl.intersect(yl).String() + if got != test.want { + t.Errorf("(%v).intersect(%v) = %v; want %v", test.xl, test.yl, got, test.want) + } + } +} + +func TestTermlistEqual(t *testing.T) { + for _, test := range []struct { + xl, yl string + want bool + }{ + {"āˆ…", "āˆ…", true}, + {"āˆ…", "š“¤", false}, + {"š“¤", "š“¤", true}, + {"š“¤ | int", "š“¤", true}, + {"š“¤ | int", "string | š“¤", true}, + {"š“¤ | myInt", "string | š“¤", true}, + {"int | ~string", "string | int", false}, + {"~int | string", "string | myInt", false}, + {"int | ~string | āˆ…", "string | int | ~string", true}, + } { + xl := maketl(test.xl) + yl := maketl(test.yl) + got := xl.equal(yl) + if got != test.want { + t.Errorf("(%v).equal(%v) = %v; want %v", test.xl, test.yl, got, test.want) + } + } +} + +func TestTermlistIncludes(t *testing.T) { + for _, test := range []struct { + xl, typ string + want bool + }{ + {"āˆ…", "int", false}, + {"š“¤", "int", true}, + {"~int", "int", true}, + {"int", "string", false}, + {"~int", "string", false}, + {"~int", "myInt", true}, + {"int | string", "string", true}, + {"~int | string", "int", true}, + {"~int | string", "myInt", true}, + {"~int | myInt | āˆ…", "myInt", true}, + {"myInt | āˆ… | š“¤", "int", true}, + } { + xl := maketl(test.xl) + yl := testTerm(test.typ).typ + got := xl.includes(yl) + if got != test.want { + t.Errorf("(%v).includes(%v) = %v; want %v", test.xl, yl, got, test.want) + } + } +} + +func TestTermlistSupersetOf(t *testing.T) { + for _, test := range []struct { + xl, typ string + want bool + }{ + {"āˆ…", "āˆ…", true}, + {"āˆ…", "š“¤", false}, + {"āˆ…", "int", false}, + {"š“¤", "āˆ…", true}, + {"š“¤", "š“¤", true}, + {"š“¤", "int", true}, + {"š“¤", "~int", true}, + {"š“¤", "myInt", true}, + {"~int", "int", true}, + {"~int", "~int", true}, + {"~int", "myInt", true}, + {"int", "~int", false}, + {"myInt", "~int", false}, + {"int", "string", false}, + {"~int", "string", false}, + {"int | string", "string", true}, + {"int | string", "~string", false}, + {"~int | string", "int", true}, + {"~int | string", "myInt", true}, + {"~int | string | āˆ…", "string", true}, + {"~string | āˆ… | š“¤", "myInt", true}, + } { + xl := maketl(test.xl) + y := testTerm(test.typ) + got := xl.supersetOf(y) + if got != test.want { + t.Errorf("(%v).supersetOf(%v) = %v; want %v", test.xl, y, got, test.want) + } + } +} + +func TestTermlistSubsetOf(t *testing.T) { + for _, test := range []struct { + xl, yl string + want bool + }{ + {"āˆ…", "āˆ…", true}, + {"āˆ…", "š“¤", true}, + {"š“¤", "āˆ…", false}, + {"š“¤", "š“¤", true}, + {"int", "int | string", true}, + {"~int", "int | string", false}, + {"~int", "myInt | string", false}, + {"myInt", "~int | string", true}, + {"~int", "string | string | int | ~int", true}, + {"myInt", "string | string | ~int", true}, + {"int | string", "string", false}, + {"int | string", "string | int", true}, + {"int | ~string", "string | int", false}, + {"myInt | ~string", "string | int | š“¤", true}, + {"int | ~string", "string | int | āˆ… | string", false}, + {"int | myInt", "string | ~int | āˆ… | string", true}, + } { + xl := maketl(test.xl) + yl := maketl(test.yl) + got := xl.subsetOf(yl) + if got != test.want { + t.Errorf("(%v).subsetOf(%v) = %v; want %v", test.xl, test.yl, got, test.want) + } + } +} diff --git a/src/go/types/testdata/local/issue47996.go b/src/go/types/testdata/local/issue47996.go new file mode 100644 index 0000000..4d28920 --- /dev/null +++ b/src/go/types/testdata/local/issue47996.go @@ -0,0 +1,9 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +// don't crash +// TODO(gri) make this test work for go/types +// func T[P] m() {} diff --git a/src/go/types/testdata/local/shifts.go b/src/go/types/testdata/local/shifts.go new file mode 100644 index 0000000..cf847d3 --- /dev/null +++ b/src/go/types/testdata/local/shifts.go @@ -0,0 +1,27 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The following shift tests are disabled in the shared +// testdata/check/shifts.go file because they don't work +// correctly with types2 at the moment. See issue #52080. +// Make sure we keep testing them with go/types. +// +// TODO(gri) Once #52080 is fixed, this file can be +// deleted in favor of the re-enabled tests +// in the shared file. + +package p + +func _() { + var s uint + + _ = int32(0x80000000 /* ERROR "overflows int32" */ << s) + // TODO(rfindley) Eliminate the redundant error here. + _ = int32(( /* ERROR "truncated to int32" */ 0x80000000 /* ERROR "truncated to int32" */ + 0i) << s) + + _ = int(1 + 0i<<0) + _ = int((1 + 0i) << s) + _ = int(1.0 << s) + _ = int(complex(1, 0) << s) +} diff --git a/src/go/types/testdata/manual.go b/src/go/types/testdata/manual.go new file mode 100644 index 0000000..a7caee9 --- /dev/null +++ b/src/go/types/testdata/manual.go @@ -0,0 +1,9 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is tested when running "go test -run Manual" +// without source arguments. Use for one-off debugging. + +package p + diff --git a/src/go/types/token_test.go b/src/go/types/token_test.go new file mode 100644 index 0000000..705bb29 --- /dev/null +++ b/src/go/types/token_test.go @@ -0,0 +1,47 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file checks invariants of token.Token ordering that we rely on +// since package go/token doesn't provide any guarantees at the moment. + +package types + +import ( + "go/token" + "testing" +) + +var assignOps = map[token.Token]token.Token{ + token.ADD_ASSIGN: token.ADD, + token.SUB_ASSIGN: token.SUB, + token.MUL_ASSIGN: token.MUL, + token.QUO_ASSIGN: token.QUO, + token.REM_ASSIGN: token.REM, + token.AND_ASSIGN: token.AND, + token.OR_ASSIGN: token.OR, + token.XOR_ASSIGN: token.XOR, + token.SHL_ASSIGN: token.SHL, + token.SHR_ASSIGN: token.SHR, + token.AND_NOT_ASSIGN: token.AND_NOT, +} + +func TestZeroTok(t *testing.T) { + // zero value for token.Token must be token.ILLEGAL + var zero token.Token + if token.ILLEGAL != zero { + t.Errorf("%s == %d; want 0", token.ILLEGAL, zero) + } +} + +func TestAssignOp(t *testing.T) { + // there are fewer than 256 tokens + for i := 0; i < 256; i++ { + tok := token.Token(i) + got := assignOp(tok) + want := assignOps[tok] + if got != want { + t.Errorf("for assignOp(%s): got %s; want %s", tok, got, want) + } + } +} diff --git a/src/go/types/tuple.go b/src/go/types/tuple.go new file mode 100644 index 0000000..e85c5aa --- /dev/null +++ b/src/go/types/tuple.go @@ -0,0 +1,34 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +// A Tuple represents an ordered list of variables; a nil *Tuple is a valid (empty) tuple. +// Tuples are used as components of signatures and to represent the type of multiple +// assignments; they are not first class types of Go. +type Tuple struct { + vars []*Var +} + +// NewTuple returns a new tuple for the given variables. +func NewTuple(x ...*Var) *Tuple { + if len(x) > 0 { + return &Tuple{vars: x} + } + return nil +} + +// Len returns the number variables of tuple t. +func (t *Tuple) Len() int { + if t != nil { + return len(t.vars) + } + return 0 +} + +// At returns the i'th variable of tuple t. +func (t *Tuple) At(i int) *Var { return t.vars[i] } + +func (t *Tuple) Underlying() Type { return t } +func (t *Tuple) String() string { return TypeString(t, nil) } diff --git a/src/go/types/type.go b/src/go/types/type.go new file mode 100644 index 0000000..1306375 --- /dev/null +++ b/src/go/types/type.go @@ -0,0 +1,124 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +// A Type represents a type of Go. +// All types implement the Type interface. +type Type interface { + // Underlying returns the underlying type of a type. + Underlying() Type + + // String returns a string representation of a type. + String() string +} + +// under returns the true expanded underlying type. +// If it doesn't exist, the result is Typ[Invalid]. +// under must only be called when a type is known +// to be fully set up. +func under(t Type) Type { + if t, _ := t.(*Named); t != nil { + return t.under() + } + return t.Underlying() +} + +// If t is not a type parameter, coreType returns the underlying type. +// If t is a type parameter, coreType returns the single underlying +// type of all types in its type set if it exists, or nil otherwise. If the +// type set contains only unrestricted and restricted channel types (with +// identical element types), the single underlying type is the restricted +// channel type if the restrictions are always the same, or nil otherwise. +func coreType(t Type) Type { + tpar, _ := t.(*TypeParam) + if tpar == nil { + return under(t) + } + + var su Type + if tpar.underIs(func(u Type) bool { + if u == nil { + return false + } + if su != nil { + u = match(su, u) + if u == nil { + return false + } + } + // su == nil || match(su, u) != nil + su = u + return true + }) { + return su + } + return nil +} + +// coreString is like coreType but also considers []byte +// and strings as identical. In this case, if successful and we saw +// a string, the result is of type (possibly untyped) string. +func coreString(t Type) Type { + tpar, _ := t.(*TypeParam) + if tpar == nil { + return under(t) // string or untyped string + } + + var su Type + hasString := false + if tpar.underIs(func(u Type) bool { + if u == nil { + return false + } + if isString(u) { + u = NewSlice(universeByte) + hasString = true + } + if su != nil { + u = match(su, u) + if u == nil { + return false + } + } + // su == nil || match(su, u) != nil + su = u + return true + }) { + if hasString { + return Typ[String] + } + return su + } + return nil +} + +// If x and y are identical, match returns x. +// If x and y are identical channels but for their direction +// and one of them is unrestricted, match returns the channel +// with the restricted direction. +// In all other cases, match returns nil. +func match(x, y Type) Type { + // Common case: we don't have channels. + if Identical(x, y) { + return x + } + + // We may have channels that differ in direction only. + if x, _ := x.(*Chan); x != nil { + if y, _ := y.(*Chan); y != nil && Identical(x.elem, y.elem) { + // We have channels that differ in direction only. + // If there's an unrestricted channel, select the restricted one. + switch { + case x.dir == SendRecv: + return y + case y.dir == SendRecv: + return x + } + } + } + + // types are different + return nil +} diff --git a/src/go/types/typelists.go b/src/go/types/typelists.go new file mode 100644 index 0000000..0f24135 --- /dev/null +++ b/src/go/types/typelists.go @@ -0,0 +1,69 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +// TypeParamList holds a list of type parameters. +type TypeParamList struct{ tparams []*TypeParam } + +// Len returns the number of type parameters in the list. +// It is safe to call on a nil receiver. +func (l *TypeParamList) Len() int { return len(l.list()) } + +// At returns the i'th type parameter in the list. +func (l *TypeParamList) At(i int) *TypeParam { return l.tparams[i] } + +// list is for internal use where we expect a []*TypeParam. +// TODO(rfindley): list should probably be eliminated: we can pass around a +// TypeParamList instead. +func (l *TypeParamList) list() []*TypeParam { + if l == nil { + return nil + } + return l.tparams +} + +// TypeList holds a list of types. +type TypeList struct{ types []Type } + +// newTypeList returns a new TypeList with the types in list. +func newTypeList(list []Type) *TypeList { + if len(list) == 0 { + return nil + } + return &TypeList{list} +} + +// Len returns the number of types in the list. +// It is safe to call on a nil receiver. +func (l *TypeList) Len() int { return len(l.list()) } + +// At returns the i'th type in the list. +func (l *TypeList) At(i int) Type { return l.types[i] } + +// list is for internal use where we expect a []Type. +// TODO(rfindley): list should probably be eliminated: we can pass around a +// TypeList instead. +func (l *TypeList) list() []Type { + if l == nil { + return nil + } + return l.types +} + +// ---------------------------------------------------------------------------- +// Implementation + +func bindTParams(list []*TypeParam) *TypeParamList { + if len(list) == 0 { + return nil + } + for i, typ := range list { + if typ.index >= 0 { + panic("type parameter bound more than once") + } + typ.index = i + } + return &TypeParamList{tparams: list} +} diff --git a/src/go/types/typeparam.go b/src/go/types/typeparam.go new file mode 100644 index 0000000..40d96ac --- /dev/null +++ b/src/go/types/typeparam.go @@ -0,0 +1,158 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "sync/atomic" +) + +// Note: This is a uint32 rather than a uint64 because the +// respective 64 bit atomic instructions are not available +// on all platforms. +var lastID uint32 + +// nextID returns a value increasing monotonically by 1 with +// each call, starting with 1. It may be called concurrently. +func nextID() uint64 { return uint64(atomic.AddUint32(&lastID, 1)) } + +// A TypeParam represents a type parameter type. +type TypeParam struct { + check *Checker // for lazy type bound completion + id uint64 // unique id, for debugging only + obj *TypeName // corresponding type name + index int // type parameter index in source order, starting at 0 + bound Type // any type, but underlying is eventually *Interface for correct programs (see TypeParam.iface) +} + +// NewTypeParam returns a new TypeParam. Type parameters may be set on a Named +// or Signature type by calling SetTypeParams. Setting a type parameter on more +// than one type will result in a panic. +// +// The constraint argument can be nil, and set later via SetConstraint. If the +// constraint is non-nil, it must be fully defined. +func NewTypeParam(obj *TypeName, constraint Type) *TypeParam { + return (*Checker)(nil).newTypeParam(obj, constraint) +} + +// check may be nil +func (check *Checker) newTypeParam(obj *TypeName, constraint Type) *TypeParam { + // Always increment lastID, even if it is not used. + id := nextID() + if check != nil { + check.nextID++ + id = check.nextID + } + typ := &TypeParam{check: check, id: id, obj: obj, index: -1, bound: constraint} + if obj.typ == nil { + obj.typ = typ + } + // iface may mutate typ.bound, so we must ensure that iface() is called + // at least once before the resulting TypeParam escapes. + if check != nil { + check.needsCleanup(typ) + } else if constraint != nil { + typ.iface() + } + return typ +} + +// Index returns the index of the type param within its param list, or -1 if +// the type parameter has not yet been bound to a type. +func (t *TypeParam) Index() int { + return t.index +} + +// Obj returns the type name for t. +func (t *TypeParam) Obj() *TypeName { return t.obj } + +// Constraint returns the type constraint specified for t. +func (t *TypeParam) Constraint() Type { + return t.bound +} + +// SetConstraint sets the type constraint for t. +// +// It must be called by users of NewTypeParam after the bound's underlying is +// fully defined, and before using the type parameter in any way other than to +// form other types. Once SetConstraint returns the receiver, t is safe for +// concurrent use. +func (t *TypeParam) SetConstraint(bound Type) { + if bound == nil { + panic("nil constraint") + } + t.bound = bound + // iface may mutate t.bound (if bound is not an interface), so ensure that + // this is done before returning. + t.iface() +} + +func (t *TypeParam) Underlying() Type { + return t.iface() +} + +func (t *TypeParam) String() string { return TypeString(t, nil) } + +// ---------------------------------------------------------------------------- +// Implementation + +func (t *TypeParam) cleanup() { + t.iface() + t.check = nil +} + +// iface returns the constraint interface of t. +func (t *TypeParam) iface() *Interface { + bound := t.bound + + // determine constraint interface + var ityp *Interface + switch u := under(bound).(type) { + case *Basic: + if u == Typ[Invalid] { + // error is reported elsewhere + return &emptyInterface + } + case *Interface: + if isTypeParam(bound) { + // error is reported in Checker.collectTypeParams + return &emptyInterface + } + ityp = u + } + + // If we don't have an interface, wrap constraint into an implicit interface. + if ityp == nil { + ityp = NewInterfaceType(nil, []Type{bound}) + ityp.implicit = true + t.bound = ityp // update t.bound for next time (optimization) + } + + // compute type set if necessary + if ityp.tset == nil { + // pos is used for tracing output; start with the type parameter position. + pos := t.obj.pos + // use the (original or possibly instantiated) type bound position if we have one + if n, _ := bound.(*Named); n != nil { + pos = n.obj.pos + } + computeInterfaceTypeSet(t.check, pos, ityp) + } + + return ityp +} + +// is calls f with the specific type terms of t's constraint and reports whether +// all calls to f returned true. If there are no specific terms, is +// returns the result of f(nil). +func (t *TypeParam) is(f func(*term) bool) bool { + return t.iface().typeSet().is(f) +} + +// underIs calls f with the underlying types of the specific type terms +// of t's constraint and reports whether all calls to f returned true. +// If there are no specific terms, underIs returns the result of f(nil). +func (t *TypeParam) underIs(f func(Type) bool) bool { + return t.iface().typeSet().underIs(f) +} diff --git a/src/go/types/typeset.go b/src/go/types/typeset.go new file mode 100644 index 0000000..4facce0 --- /dev/null +++ b/src/go/types/typeset.go @@ -0,0 +1,417 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "go/token" + . "internal/types/errors" + "sort" + "strings" +) + +// ---------------------------------------------------------------------------- +// API + +// A _TypeSet represents the type set of an interface. +// Because of existing language restrictions, methods can be "factored out" +// from the terms. The actual type set is the intersection of the type set +// implied by the methods and the type set described by the terms and the +// comparable bit. To test whether a type is included in a type set +// ("implements" relation), the type must implement all methods _and_ be +// an element of the type set described by the terms and the comparable bit. +// If the term list describes the set of all types and comparable is true, +// only comparable types are meant; in all other cases comparable is false. +type _TypeSet struct { + methods []*Func // all methods of the interface; sorted by unique ID + terms termlist // type terms of the type set + comparable bool // invariant: !comparable || terms.isAll() +} + +// IsEmpty reports whether type set s is the empty set. +func (s *_TypeSet) IsEmpty() bool { return s.terms.isEmpty() } + +// IsAll reports whether type set s is the set of all types (corresponding to the empty interface). +func (s *_TypeSet) IsAll() bool { return s.IsMethodSet() && len(s.methods) == 0 } + +// IsMethodSet reports whether the interface t is fully described by its method set. +func (s *_TypeSet) IsMethodSet() bool { return !s.comparable && s.terms.isAll() } + +// IsComparable reports whether each type in the set is comparable. +func (s *_TypeSet) IsComparable(seen map[Type]bool) bool { + if s.terms.isAll() { + return s.comparable + } + return s.is(func(t *term) bool { + return t != nil && comparable(t.typ, false, seen, nil) + }) +} + +// NumMethods returns the number of methods available. +func (s *_TypeSet) NumMethods() int { return len(s.methods) } + +// Method returns the i'th method of type set s for 0 <= i < s.NumMethods(). +// The methods are ordered by their unique ID. +func (s *_TypeSet) Method(i int) *Func { return s.methods[i] } + +// LookupMethod returns the index of and method with matching package and name, or (-1, nil). +func (s *_TypeSet) LookupMethod(pkg *Package, name string, foldCase bool) (int, *Func) { + return lookupMethod(s.methods, pkg, name, foldCase) +} + +func (s *_TypeSet) String() string { + switch { + case s.IsEmpty(): + return "āˆ…" + case s.IsAll(): + return "š“¤" + } + + hasMethods := len(s.methods) > 0 + hasTerms := s.hasTerms() + + var buf strings.Builder + buf.WriteByte('{') + if s.comparable { + buf.WriteString("comparable") + if hasMethods || hasTerms { + buf.WriteString("; ") + } + } + for i, m := range s.methods { + if i > 0 { + buf.WriteString("; ") + } + buf.WriteString(m.String()) + } + if hasMethods && hasTerms { + buf.WriteString("; ") + } + if hasTerms { + buf.WriteString(s.terms.String()) + } + buf.WriteString("}") + return buf.String() +} + +// ---------------------------------------------------------------------------- +// Implementation + +// hasTerms reports whether the type set has specific type terms. +func (s *_TypeSet) hasTerms() bool { return !s.terms.isEmpty() && !s.terms.isAll() } + +// subsetOf reports whether s1 āŠ† s2. +func (s1 *_TypeSet) subsetOf(s2 *_TypeSet) bool { return s1.terms.subsetOf(s2.terms) } + +// TODO(gri) TypeSet.is and TypeSet.underIs should probably also go into termlist.go + +// is calls f with the specific type terms of s and reports whether +// all calls to f returned true. If there are no specific terms, is +// returns the result of f(nil). +func (s *_TypeSet) is(f func(*term) bool) bool { + if !s.hasTerms() { + return f(nil) + } + for _, t := range s.terms { + assert(t.typ != nil) + if !f(t) { + return false + } + } + return true +} + +// underIs calls f with the underlying types of the specific type terms +// of s and reports whether all calls to f returned true. If there are +// no specific terms, underIs returns the result of f(nil). +func (s *_TypeSet) underIs(f func(Type) bool) bool { + if !s.hasTerms() { + return f(nil) + } + for _, t := range s.terms { + assert(t.typ != nil) + // x == under(x) for ~x terms + u := t.typ + if !t.tilde { + u = under(u) + } + if debug { + assert(Identical(u, under(u))) + } + if !f(u) { + return false + } + } + return true +} + +// topTypeSet may be used as type set for the empty interface. +var topTypeSet = _TypeSet{terms: allTermlist} + +// computeInterfaceTypeSet may be called with check == nil. +func computeInterfaceTypeSet(check *Checker, pos token.Pos, ityp *Interface) *_TypeSet { + if ityp.tset != nil { + return ityp.tset + } + + // If the interface is not fully set up yet, the type set will + // not be complete, which may lead to errors when using the + // type set (e.g. missing method). Don't compute a partial type + // set (and don't store it!), so that we still compute the full + // type set eventually. Instead, return the top type set and + // let any follow-on errors play out. + // + // TODO(gri) Consider recording when this happens and reporting + // it as an error (but only if there were no other errors so to + // to not have unnecessary follow-on errors). + if !ityp.complete { + return &topTypeSet + } + + if check != nil && trace { + // Types don't generally have position information. + // If we don't have a valid pos provided, try to use + // one close enough. + if !pos.IsValid() && len(ityp.methods) > 0 { + pos = ityp.methods[0].pos + } + + check.trace(pos, "-- type set for %s", ityp) + check.indent++ + defer func() { + check.indent-- + check.trace(pos, "=> %s ", ityp.typeSet()) + }() + } + + // An infinitely expanding interface (due to a cycle) is detected + // elsewhere (Checker.validType), so here we simply assume we only + // have valid interfaces. Mark the interface as complete to avoid + // infinite recursion if the validType check occurs later for some + // reason. + ityp.tset = &_TypeSet{terms: allTermlist} // TODO(gri) is this sufficient? + + var unionSets map[*Union]*_TypeSet + if check != nil { + if check.unionTypeSets == nil { + check.unionTypeSets = make(map[*Union]*_TypeSet) + } + unionSets = check.unionTypeSets + } else { + unionSets = make(map[*Union]*_TypeSet) + } + + // Methods of embedded interfaces are collected unchanged; i.e., the identity + // of a method I.m's Func Object of an interface I is the same as that of + // the method m in an interface that embeds interface I. On the other hand, + // if a method is embedded via multiple overlapping embedded interfaces, we + // don't provide a guarantee which "original m" got chosen for the embedding + // interface. See also issue #34421. + // + // If we don't care to provide this identity guarantee anymore, instead of + // reusing the original method in embeddings, we can clone the method's Func + // Object and give it the position of a corresponding embedded interface. Then + // we can get rid of the mpos map below and simply use the cloned method's + // position. + + var seen objset + var allMethods []*Func + mpos := make(map[*Func]token.Pos) // method specification or method embedding position, for good error messages + addMethod := func(pos token.Pos, m *Func, explicit bool) { + switch other := seen.insert(m); { + case other == nil: + allMethods = append(allMethods, m) + mpos[m] = pos + case explicit: + if check != nil { + check.errorf(atPos(pos), DuplicateDecl, "duplicate method %s", m.name) + check.errorf(atPos(mpos[other.(*Func)]), DuplicateDecl, "\tother declaration of %s", m.name) // secondary error, \t indented + } + default: + // We have a duplicate method name in an embedded (not explicitly declared) method. + // Check method signatures after all types are computed (issue #33656). + // If we're pre-go1.14 (overlapping embeddings are not permitted), report that + // error here as well (even though we could do it eagerly) because it's the same + // error message. + if check != nil { + check.later(func() { + if !check.allowVersion(m.pkg, 1, 14) || !Identical(m.typ, other.Type()) { + check.errorf(atPos(pos), DuplicateDecl, "duplicate method %s", m.name) + check.errorf(atPos(mpos[other.(*Func)]), DuplicateDecl, "\tother declaration of %s", m.name) // secondary error, \t indented + } + }).describef(atPos(pos), "duplicate method check for %s", m.name) + } + } + } + + for _, m := range ityp.methods { + addMethod(m.pos, m, true) + } + + // collect embedded elements + allTerms := allTermlist + allComparable := false + for i, typ := range ityp.embeddeds { + // The embedding position is nil for imported interfaces + // and also for interface copies after substitution (but + // in that case we don't need to report errors again). + var pos token.Pos // embedding position + if ityp.embedPos != nil { + pos = (*ityp.embedPos)[i] + } + var comparable bool + var terms termlist + switch u := under(typ).(type) { + case *Interface: + // For now we don't permit type parameters as constraints. + assert(!isTypeParam(typ)) + tset := computeInterfaceTypeSet(check, pos, u) + // If typ is local, an error was already reported where typ is specified/defined. + if check != nil && check.isImportedConstraint(typ) && !check.allowVersion(check.pkg, 1, 18) { + check.errorf(atPos(pos), UnsupportedFeature, "embedding constraint interface %s requires go1.18 or later", typ) + continue + } + comparable = tset.comparable + for _, m := range tset.methods { + addMethod(pos, m, false) // use embedding position pos rather than m.pos + } + terms = tset.terms + case *Union: + if check != nil && !check.allowVersion(check.pkg, 1, 18) { + check.errorf(atPos(pos), UnsupportedFeature, "embedding interface element %s requires go1.18 or later", u) + continue + } + tset := computeUnionTypeSet(check, unionSets, pos, u) + if tset == &invalidTypeSet { + continue // ignore invalid unions + } + assert(!tset.comparable) + assert(len(tset.methods) == 0) + terms = tset.terms + default: + if u == Typ[Invalid] { + continue + } + if check != nil && !check.allowVersion(check.pkg, 1, 18) { + check.errorf(atPos(pos), UnsupportedFeature, "embedding non-interface type %s requires go1.18 or later", typ) + continue + } + terms = termlist{{false, typ}} + } + + // The type set of an interface is the intersection of the type sets of all its elements. + // Due to language restrictions, only embedded interfaces can add methods, they are handled + // separately. Here we only need to intersect the term lists and comparable bits. + allTerms, allComparable = intersectTermLists(allTerms, allComparable, terms, comparable) + } + ityp.embedPos = nil // not needed anymore (errors have been reported) + + ityp.tset.comparable = allComparable + if len(allMethods) != 0 { + sortMethods(allMethods) + ityp.tset.methods = allMethods + } + ityp.tset.terms = allTerms + + return ityp.tset +} + +// TODO(gri) The intersectTermLists function belongs to the termlist implementation. +// The comparable type set may also be best represented as a term (using +// a special type). + +// intersectTermLists computes the intersection of two term lists and respective comparable bits. +// xcomp, ycomp are valid only if xterms.isAll() and yterms.isAll() respectively. +func intersectTermLists(xterms termlist, xcomp bool, yterms termlist, ycomp bool) (termlist, bool) { + terms := xterms.intersect(yterms) + // If one of xterms or yterms is marked as comparable, + // the result must only include comparable types. + comp := xcomp || ycomp + if comp && !terms.isAll() { + // only keep comparable terms + i := 0 + for _, t := range terms { + assert(t.typ != nil) + if comparable(t.typ, false /* strictly comparable */, nil, nil) { + terms[i] = t + i++ + } + } + terms = terms[:i] + if !terms.isAll() { + comp = false + } + } + assert(!comp || terms.isAll()) // comparable invariant + return terms, comp +} + +func sortMethods(list []*Func) { + sort.Sort(byUniqueMethodName(list)) +} + +func assertSortedMethods(list []*Func) { + if !debug { + panic("assertSortedMethods called outside debug mode") + } + if !sort.IsSorted(byUniqueMethodName(list)) { + panic("methods not sorted") + } +} + +// byUniqueMethodName method lists can be sorted by their unique method names. +type byUniqueMethodName []*Func + +func (a byUniqueMethodName) Len() int { return len(a) } +func (a byUniqueMethodName) Less(i, j int) bool { return a[i].Id() < a[j].Id() } +func (a byUniqueMethodName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// invalidTypeSet is a singleton type set to signal an invalid type set +// due to an error. It's also a valid empty type set, so consumers of +// type sets may choose to ignore it. +var invalidTypeSet _TypeSet + +// computeUnionTypeSet may be called with check == nil. +// The result is &invalidTypeSet if the union overflows. +func computeUnionTypeSet(check *Checker, unionSets map[*Union]*_TypeSet, pos token.Pos, utyp *Union) *_TypeSet { + if tset, _ := unionSets[utyp]; tset != nil { + return tset + } + + // avoid infinite recursion (see also computeInterfaceTypeSet) + unionSets[utyp] = new(_TypeSet) + + var allTerms termlist + for _, t := range utyp.terms { + var terms termlist + u := under(t.typ) + if ui, _ := u.(*Interface); ui != nil { + // For now we don't permit type parameters as constraints. + assert(!isTypeParam(t.typ)) + terms = computeInterfaceTypeSet(check, pos, ui).terms + } else if u == Typ[Invalid] { + continue + } else { + if t.tilde && !Identical(t.typ, u) { + // There is no underlying type which is t.typ. + // The corresponding type set is empty. + t = nil // āˆ… term + } + terms = termlist{(*term)(t)} + } + // The type set of a union expression is the union + // of the type sets of each term. + allTerms = allTerms.union(terms) + if len(allTerms) > maxTermCount { + if check != nil { + check.errorf(atPos(pos), InvalidUnion, "cannot handle more than %d union terms (implementation limitation)", maxTermCount) + } + unionSets[utyp] = &invalidTypeSet + return unionSets[utyp] + } + } + unionSets[utyp].terms = allTerms + + return unionSets[utyp] +} diff --git a/src/go/types/typeset_test.go b/src/go/types/typeset_test.go new file mode 100644 index 0000000..5156092 --- /dev/null +++ b/src/go/types/typeset_test.go @@ -0,0 +1,81 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "go/ast" + "go/parser" + "go/token" + "testing" +) + +func TestInvalidTypeSet(t *testing.T) { + if !invalidTypeSet.IsEmpty() { + t.Error("invalidTypeSet is not empty") + } +} + +func TestTypeSetString(t *testing.T) { + for body, want := range map[string]string{ + "{}": "š“¤", + "{int}": "{int}", + "{~int}": "{~int}", + "{int|string}": "{int | string}", + "{int; string}": "āˆ…", + + "{comparable}": "{comparable}", + "{comparable; int}": "{int}", + "{~int; comparable}": "{~int}", + "{int|string; comparable}": "{int | string}", + "{comparable; int; string}": "āˆ…", + + "{m()}": "{func (p.T).m()}", + "{m1(); m2() int }": "{func (p.T).m1(); func (p.T).m2() int}", + "{error}": "{func (error).Error() string}", + "{m(); comparable}": "{comparable; func (p.T).m()}", + "{m1(); comparable; m2() int }": "{comparable; func (p.T).m1(); func (p.T).m2() int}", + "{comparable; error}": "{comparable; func (error).Error() string}", + + "{m(); comparable; int|float32|string}": "{func (p.T).m(); int | float32 | string}", + "{m1(); int; m2(); comparable }": "{func (p.T).m1(); func (p.T).m2(); int}", + + "{E}; type E interface{}": "š“¤", + "{E}; type E interface{int;string}": "āˆ…", + "{E}; type E interface{comparable}": "{comparable}", + } { + // parse + src := "package p; type T interface" + body + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, "p.go", src, parser.AllErrors) + if file == nil { + t.Fatalf("%s: %v (invalid test case)", body, err) + } + + // type check + var conf Config + pkg, err := conf.Check(file.Name.Name, fset, []*ast.File{file}, nil) + if err != nil { + t.Fatalf("%s: %v (invalid test case)", body, err) + } + + // lookup T + obj := pkg.scope.Lookup("T") + if obj == nil { + t.Fatalf("%s: T not found (invalid test case)", body) + } + T, ok := under(obj.Type()).(*Interface) + if !ok { + t.Fatalf("%s: %v is not an interface (invalid test case)", body, obj) + } + + // verify test case + got := T.typeSet().String() + if got != want { + t.Errorf("%s: got %s; want %s", body, got, want) + } + } +} + +// TODO(gri) add more tests diff --git a/src/go/types/typestring.go b/src/go/types/typestring.go new file mode 100644 index 0000000..cfeb7eb --- /dev/null +++ b/src/go/types/typestring.go @@ -0,0 +1,491 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements printing of types. + +package types + +import ( + "bytes" + "fmt" + "go/token" + "sort" + "strconv" + "strings" + "unicode/utf8" +) + +// A Qualifier controls how named package-level objects are printed in +// calls to TypeString, ObjectString, and SelectionString. +// +// These three formatting routines call the Qualifier for each +// package-level object O, and if the Qualifier returns a non-empty +// string p, the object is printed in the form p.O. +// If it returns an empty string, only the object name O is printed. +// +// Using a nil Qualifier is equivalent to using (*Package).Path: the +// object is qualified by the import path, e.g., "encoding/json.Marshal". +type Qualifier func(*Package) string + +// RelativeTo returns a Qualifier that fully qualifies members of +// all packages other than pkg. +func RelativeTo(pkg *Package) Qualifier { + if pkg == nil { + return nil + } + return func(other *Package) string { + if pkg == other { + return "" // same package; unqualified + } + return other.Path() + } +} + +// TypeString returns the string representation of typ. +// The Qualifier controls the printing of +// package-level objects, and may be nil. +func TypeString(typ Type, qf Qualifier) string { + var buf bytes.Buffer + WriteType(&buf, typ, qf) + return buf.String() +} + +// WriteType writes the string representation of typ to buf. +// The Qualifier controls the printing of +// package-level objects, and may be nil. +func WriteType(buf *bytes.Buffer, typ Type, qf Qualifier) { + newTypeWriter(buf, qf).typ(typ) +} + +// WriteSignature writes the representation of the signature sig to buf, +// without a leading "func" keyword. The Qualifier controls the printing +// of package-level objects, and may be nil. +func WriteSignature(buf *bytes.Buffer, sig *Signature, qf Qualifier) { + newTypeWriter(buf, qf).signature(sig) +} + +type typeWriter struct { + buf *bytes.Buffer + seen map[Type]bool + qf Qualifier + ctxt *Context // if non-nil, we are type hashing + tparams *TypeParamList // local type parameters + paramNames bool // if set, write function parameter names, otherwise, write types only + tpSubscripts bool // if set, write type parameter indices as subscripts + pkgInfo bool // package-annotate first unexported-type field to avoid confusing type description +} + +func newTypeWriter(buf *bytes.Buffer, qf Qualifier) *typeWriter { + return &typeWriter{buf, make(map[Type]bool), qf, nil, nil, true, false, false} +} + +func newTypeHasher(buf *bytes.Buffer, ctxt *Context) *typeWriter { + assert(ctxt != nil) + return &typeWriter{buf, make(map[Type]bool), nil, ctxt, nil, false, false, false} +} + +func (w *typeWriter) byte(b byte) { + if w.ctxt != nil { + if b == ' ' { + b = '#' + } + w.buf.WriteByte(b) + return + } + w.buf.WriteByte(b) + if b == ',' || b == ';' { + w.buf.WriteByte(' ') + } +} + +func (w *typeWriter) string(s string) { + w.buf.WriteString(s) +} + +func (w *typeWriter) error(msg string) { + if w.ctxt != nil { + panic(msg) + } + w.buf.WriteString("<" + msg + ">") +} + +func (w *typeWriter) typ(typ Type) { + if w.seen[typ] { + w.error("cycle to " + goTypeName(typ)) + return + } + w.seen[typ] = true + defer delete(w.seen, typ) + + switch t := typ.(type) { + case nil: + w.error("nil") + + case *Basic: + // exported basic types go into package unsafe + // (currently this is just unsafe.Pointer) + if token.IsExported(t.name) { + if obj, _ := Unsafe.scope.Lookup(t.name).(*TypeName); obj != nil { + w.typeName(obj) + break + } + } + w.string(t.name) + + case *Array: + w.byte('[') + w.string(strconv.FormatInt(t.len, 10)) + w.byte(']') + w.typ(t.elem) + + case *Slice: + w.string("[]") + w.typ(t.elem) + + case *Struct: + w.string("struct{") + for i, f := range t.fields { + if i > 0 { + w.byte(';') + } + + // If disambiguating one struct for another, look for the first unexported field. + // Do this first in case of nested structs; tag the first-outermost field. + pkgAnnotate := false + if w.qf == nil && w.pkgInfo && !token.IsExported(f.name) { + // note for embedded types, type name is field name, and "string" etc are lower case hence unexported. + pkgAnnotate = true + w.pkgInfo = false // only tag once + } + + // This doesn't do the right thing for embedded type + // aliases where we should print the alias name, not + // the aliased type (see issue #44410). + if !f.embedded { + w.string(f.name) + w.byte(' ') + } + w.typ(f.typ) + if pkgAnnotate { + w.string(" /* package ") + w.string(f.pkg.Path()) + w.string(" */ ") + } + if tag := t.Tag(i); tag != "" { + w.byte(' ') + // TODO(rfindley) If tag contains blanks, replacing them with '#' + // in Context.TypeHash may produce another tag + // accidentally. + w.string(strconv.Quote(tag)) + } + } + w.byte('}') + + case *Pointer: + w.byte('*') + w.typ(t.base) + + case *Tuple: + w.tuple(t, false) + + case *Signature: + w.string("func") + w.signature(t) + + case *Union: + // Unions only appear as (syntactic) embedded elements + // in interfaces and syntactically cannot be empty. + if t.Len() == 0 { + w.error("empty union") + break + } + for i, t := range t.terms { + if i > 0 { + w.string(termSep) + } + if t.tilde { + w.byte('~') + } + w.typ(t.typ) + } + + case *Interface: + if w.ctxt == nil { + if t == universeAny.Type() { + // When not hashing, we can try to improve type strings by writing "any" + // for a type that is pointer-identical to universeAny. This logic should + // be deprecated by more robust handling for aliases. + w.string("any") + break + } + if t == universeComparable.Type().(*Named).underlying { + w.string("interface{comparable}") + break + } + } + if t.implicit { + if len(t.methods) == 0 && len(t.embeddeds) == 1 { + w.typ(t.embeddeds[0]) + break + } + // Something's wrong with the implicit interface. + // Print it as such and continue. + w.string("/* implicit */ ") + } + w.string("interface{") + first := true + if w.ctxt != nil { + w.typeSet(t.typeSet()) + } else { + for _, m := range t.methods { + if !first { + w.byte(';') + } + first = false + w.string(m.name) + w.signature(m.typ.(*Signature)) + } + for _, typ := range t.embeddeds { + if !first { + w.byte(';') + } + first = false + w.typ(typ) + } + } + w.byte('}') + + case *Map: + w.string("map[") + w.typ(t.key) + w.byte(']') + w.typ(t.elem) + + case *Chan: + var s string + var parens bool + switch t.dir { + case SendRecv: + s = "chan " + // chan (<-chan T) requires parentheses + if c, _ := t.elem.(*Chan); c != nil && c.dir == RecvOnly { + parens = true + } + case SendOnly: + s = "chan<- " + case RecvOnly: + s = "<-chan " + default: + w.error("unknown channel direction") + } + w.string(s) + if parens { + w.byte('(') + } + w.typ(t.elem) + if parens { + w.byte(')') + } + + case *Named: + // If hashing, write a unique prefix for t to represent its identity, since + // named type identity is pointer identity. + if w.ctxt != nil { + w.string(strconv.Itoa(w.ctxt.getID(t))) + } + w.typeName(t.obj) // when hashing written for readability of the hash only + if t.inst != nil { + // instantiated type + w.typeList(t.inst.targs.list()) + } else if w.ctxt == nil && t.TypeParams().Len() != 0 { // For type hashing, don't need to format the TypeParams + // parameterized type + w.tParamList(t.TypeParams().list()) + } + + case *TypeParam: + if t.obj == nil { + w.error("unnamed type parameter") + break + } + if i := tparamIndex(w.tparams.list(), t); i >= 0 { + // The names of type parameters that are declared by the type being + // hashed are not part of the type identity. Replace them with a + // placeholder indicating their index. + w.string(fmt.Sprintf("$%d", i)) + } else { + w.string(t.obj.name) + if w.tpSubscripts || w.ctxt != nil { + w.string(subscript(t.id)) + } + } + + default: + // For externally defined implementations of Type. + // Note: In this case cycles won't be caught. + w.string(t.String()) + } +} + +// typeSet writes a canonical hash for an interface type set. +func (w *typeWriter) typeSet(s *_TypeSet) { + assert(w.ctxt != nil) + first := true + for _, m := range s.methods { + if !first { + w.byte(';') + } + first = false + w.string(m.name) + w.signature(m.typ.(*Signature)) + } + switch { + case s.terms.isAll(): + // nothing to do + case s.terms.isEmpty(): + w.string(s.terms.String()) + default: + var termHashes []string + for _, term := range s.terms { + // terms are not canonically sorted, so we sort their hashes instead. + var buf bytes.Buffer + if term.tilde { + buf.WriteByte('~') + } + newTypeHasher(&buf, w.ctxt).typ(term.typ) + termHashes = append(termHashes, buf.String()) + } + sort.Strings(termHashes) + if !first { + w.byte(';') + } + w.string(strings.Join(termHashes, "|")) + } +} + +func (w *typeWriter) typeList(list []Type) { + w.byte('[') + for i, typ := range list { + if i > 0 { + w.byte(',') + } + w.typ(typ) + } + w.byte(']') +} + +func (w *typeWriter) tParamList(list []*TypeParam) { + w.byte('[') + var prev Type + for i, tpar := range list { + // Determine the type parameter and its constraint. + // list is expected to hold type parameter names, + // but don't crash if that's not the case. + if tpar == nil { + w.error("nil type parameter") + continue + } + if i > 0 { + if tpar.bound != prev { + // bound changed - write previous one before advancing + w.byte(' ') + w.typ(prev) + } + w.byte(',') + } + prev = tpar.bound + w.typ(tpar) + } + if prev != nil { + w.byte(' ') + w.typ(prev) + } + w.byte(']') +} + +func (w *typeWriter) typeName(obj *TypeName) { + w.string(packagePrefix(obj.pkg, w.qf)) + w.string(obj.name) +} + +func (w *typeWriter) tuple(tup *Tuple, variadic bool) { + w.byte('(') + if tup != nil { + for i, v := range tup.vars { + if i > 0 { + w.byte(',') + } + // parameter names are ignored for type identity and thus type hashes + if w.ctxt == nil && v.name != "" && w.paramNames { + w.string(v.name) + w.byte(' ') + } + typ := v.typ + if variadic && i == len(tup.vars)-1 { + if s, ok := typ.(*Slice); ok { + w.string("...") + typ = s.elem + } else { + // special case: + // append(s, "foo"...) leads to signature func([]byte, string...) + if t, _ := under(typ).(*Basic); t == nil || t.kind != String { + w.error("expected string type") + continue + } + w.typ(typ) + w.string("...") + continue + } + } + w.typ(typ) + } + } + w.byte(')') +} + +func (w *typeWriter) signature(sig *Signature) { + if sig.TypeParams().Len() != 0 { + if w.ctxt != nil { + assert(w.tparams == nil) + w.tparams = sig.TypeParams() + defer func() { + w.tparams = nil + }() + } + w.tParamList(sig.TypeParams().list()) + } + + w.tuple(sig.params, sig.variadic) + + n := sig.results.Len() + if n == 0 { + // no result + return + } + + w.byte(' ') + if n == 1 && (w.ctxt != nil || sig.results.vars[0].name == "") { + // single unnamed result (if type hashing, name must be ignored) + w.typ(sig.results.vars[0].typ) + return + } + + // multiple or named result(s) + w.tuple(sig.results, false) +} + +// subscript returns the decimal (utf8) representation of x using subscript digits. +func subscript(x uint64) string { + const w = len("ā‚€") // all digits 0...9 have the same utf8 width + var buf [32 * w]byte + i := len(buf) + for { + i -= w + utf8.EncodeRune(buf[i:], 'ā‚€'+rune(x%10)) // 'ā‚€' == U+2080 + x /= 10 + if x == 0 { + break + } + } + return string(buf[i:]) +} diff --git a/src/go/types/typestring_test.go b/src/go/types/typestring_test.go new file mode 100644 index 0000000..e73f241 --- /dev/null +++ b/src/go/types/typestring_test.go @@ -0,0 +1,167 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "internal/testenv" + "testing" + + . "go/types" +) + +const filename = "" + +type testEntry struct { + src, str string +} + +// dup returns a testEntry where both src and str are the same. +func dup(s string) testEntry { + return testEntry{s, s} +} + +// types that don't depend on any other type declarations +var independentTestTypes = []testEntry{ + // basic types + dup("int"), + dup("float32"), + dup("string"), + + // arrays + dup("[10]int"), + + // slices + dup("[]int"), + dup("[][]int"), + + // structs + dup("struct{}"), + dup("struct{x int}"), + {`struct { + x, y int + z float32 "foo" + }`, `struct{x int; y int; z float32 "foo"}`}, + {`struct { + string + elems []complex128 + }`, `struct{string; elems []complex128}`}, + + // pointers + dup("*int"), + dup("***struct{}"), + dup("*struct{a int; b float32}"), + + // functions + dup("func()"), + dup("func(x int)"), + {"func(x, y int)", "func(x int, y int)"}, + {"func(x, y int, z string)", "func(x int, y int, z string)"}, + dup("func(int)"), + {"func(int, string, byte)", "func(int, string, byte)"}, + + dup("func() int"), + {"func() (string)", "func() string"}, + dup("func() (u int)"), + {"func() (u, v int, w string)", "func() (u int, v int, w string)"}, + + dup("func(int) string"), + dup("func(x int) string"), + dup("func(x int) (u string)"), + {"func(x, y int) (u string)", "func(x int, y int) (u string)"}, + + dup("func(...int) string"), + dup("func(x ...int) string"), + dup("func(x ...int) (u string)"), + {"func(x int, y ...int) (u string)", "func(x int, y ...int) (u string)"}, + + // interfaces + dup("interface{}"), + dup("interface{m()}"), + dup(`interface{String() string; m(int) float32}`), + dup("interface{int | float32 | complex128}"), + dup("interface{int | ~float32 | ~complex128}"), + dup("any"), + dup("interface{comparable}"), + // TODO(gri) adjust test for EvalCompositeTest + // {"comparable", "interface{comparable}"}, + // {"error", "interface{Error() string}"}, + + // maps + dup("map[string]int"), + {"map[struct{x, y int}][]byte", "map[struct{x int; y int}][]byte"}, + + // channels + dup("chan<- chan int"), + dup("chan<- <-chan int"), + dup("<-chan <-chan int"), + dup("chan (<-chan int)"), + dup("chan<- func()"), + dup("<-chan []func() int"), +} + +// types that depend on other type declarations (src in TestTypes) +var dependentTestTypes = []testEntry{ + // interfaces + dup(`interface{io.Reader; io.Writer}`), + dup(`interface{m() int; io.Writer}`), + {`interface{m() interface{T}}`, `interface{m() interface{p.T}}`}, +} + +func TestTypeString(t *testing.T) { + // The Go command is needed for the importer to determine the locations of stdlib .a files. + testenv.MustHaveGoBuild(t) + + var tests []testEntry + tests = append(tests, independentTestTypes...) + tests = append(tests, dependentTestTypes...) + + for _, test := range tests { + src := `package p; import "io"; type _ io.Writer; type T ` + test.src + pkg, err := typecheck(filename, src, nil) + if err != nil { + t.Errorf("%s: %s", src, err) + continue + } + obj := pkg.Scope().Lookup("T") + if obj == nil { + t.Errorf("%s: T not found", test.src) + continue + } + typ := obj.Type().Underlying() + if got := typ.String(); got != test.str { + t.Errorf("%s: got %s, want %s", test.src, got, test.str) + } + } +} + +func TestQualifiedTypeString(t *testing.T) { + p := mustTypecheck("p.go", "package p; type T int", nil) + q := mustTypecheck("q.go", "package q", nil) + + pT := p.Scope().Lookup("T").Type() + for _, test := range []struct { + typ Type + this *Package + want string + }{ + {nil, nil, ""}, + {pT, nil, "p.T"}, + {pT, p, "T"}, + {pT, q, "p.T"}, + {NewPointer(pT), p, "*T"}, + {NewPointer(pT), q, "*p.T"}, + } { + qualifier := func(pkg *Package) string { + if pkg != test.this { + return pkg.Name() + } + return "" + } + if got := TypeString(test.typ, qualifier); got != test.want { + t.Errorf("TypeString(%s, %s) = %s, want %s", + test.this, test.typ, got, test.want) + } + } +} diff --git a/src/go/types/typeterm.go b/src/go/types/typeterm.go new file mode 100644 index 0000000..a7b8969 --- /dev/null +++ b/src/go/types/typeterm.go @@ -0,0 +1,165 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +// A term describes elementary type sets: +// +// āˆ…: (*term)(nil) == āˆ… // set of no types (empty set) +// š“¤: &term{} == š“¤ // set of all types (š“¤niverse) +// T: &term{false, T} == {T} // set of type T +// ~t: &term{true, t} == {t' | under(t') == t} // set of types with underlying type t +type term struct { + tilde bool // valid if typ != nil + typ Type +} + +func (x *term) String() string { + switch { + case x == nil: + return "āˆ…" + case x.typ == nil: + return "š“¤" + case x.tilde: + return "~" + x.typ.String() + default: + return x.typ.String() + } +} + +// equal reports whether x and y represent the same type set. +func (x *term) equal(y *term) bool { + // easy cases + switch { + case x == nil || y == nil: + return x == y + case x.typ == nil || y.typ == nil: + return x.typ == y.typ + } + // āˆ… āŠ‚ x, y āŠ‚ š“¤ + + return x.tilde == y.tilde && Identical(x.typ, y.typ) +} + +// union returns the union x āˆŖ y: zero, one, or two non-nil terms. +func (x *term) union(y *term) (_, _ *term) { + // easy cases + switch { + case x == nil && y == nil: + return nil, nil // āˆ… āˆŖ āˆ… == āˆ… + case x == nil: + return y, nil // āˆ… āˆŖ y == y + case y == nil: + return x, nil // x āˆŖ āˆ… == x + case x.typ == nil: + return x, nil // š“¤ āˆŖ y == š“¤ + case y.typ == nil: + return y, nil // x āˆŖ š“¤ == š“¤ + } + // āˆ… āŠ‚ x, y āŠ‚ š“¤ + + if x.disjoint(y) { + return x, y // x āˆŖ y == (x, y) if x āˆ© y == āˆ… + } + // x.typ == y.typ + + // ~t āˆŖ ~t == ~t + // ~t āˆŖ T == ~t + // T āˆŖ ~t == ~t + // T āˆŖ T == T + if x.tilde || !y.tilde { + return x, nil + } + return y, nil +} + +// intersect returns the intersection x āˆ© y. +func (x *term) intersect(y *term) *term { + // easy cases + switch { + case x == nil || y == nil: + return nil // āˆ… āˆ© y == āˆ… and āˆ© āˆ… == āˆ… + case x.typ == nil: + return y // š“¤ āˆ© y == y + case y.typ == nil: + return x // x āˆ© š“¤ == x + } + // āˆ… āŠ‚ x, y āŠ‚ š“¤ + + if x.disjoint(y) { + return nil // x āˆ© y == āˆ… if x āˆ© y == āˆ… + } + // x.typ == y.typ + + // ~t āˆ© ~t == ~t + // ~t āˆ© T == T + // T āˆ© ~t == T + // T āˆ© T == T + if !x.tilde || y.tilde { + return x + } + return y +} + +// includes reports whether t āˆˆ x. +func (x *term) includes(t Type) bool { + // easy cases + switch { + case x == nil: + return false // t āˆˆ āˆ… == false + case x.typ == nil: + return true // t āˆˆ š“¤ == true + } + // āˆ… āŠ‚ x āŠ‚ š“¤ + + u := t + if x.tilde { + u = under(u) + } + return Identical(x.typ, u) +} + +// subsetOf reports whether x āŠ† y. +func (x *term) subsetOf(y *term) bool { + // easy cases + switch { + case x == nil: + return true // āˆ… āŠ† y == true + case y == nil: + return false // x āŠ† āˆ… == false since x != āˆ… + case y.typ == nil: + return true // x āŠ† š“¤ == true + case x.typ == nil: + return false // š“¤ āŠ† y == false since y != š“¤ + } + // āˆ… āŠ‚ x, y āŠ‚ š“¤ + + if x.disjoint(y) { + return false // x āŠ† y == false if x āˆ© y == āˆ… + } + // x.typ == y.typ + + // ~t āŠ† ~t == true + // ~t āŠ† T == false + // T āŠ† ~t == true + // T āŠ† T == true + return !x.tilde || y.tilde +} + +// disjoint reports whether x āˆ© y == āˆ…. +// x.typ and y.typ must not be nil. +func (x *term) disjoint(y *term) bool { + if debug && (x.typ == nil || y.typ == nil) { + panic("invalid argument(s)") + } + ux := x.typ + if y.tilde { + ux = under(ux) + } + uy := y.typ + if x.tilde { + uy = under(uy) + } + return !Identical(ux, uy) +} diff --git a/src/go/types/typeterm_test.go b/src/go/types/typeterm_test.go new file mode 100644 index 0000000..24a1410 --- /dev/null +++ b/src/go/types/typeterm_test.go @@ -0,0 +1,240 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "go/token" + "strings" + "testing" +) + +var myInt = func() Type { + tname := NewTypeName(token.NoPos, nil, "myInt", nil) + return NewNamed(tname, Typ[Int], nil) +}() + +var testTerms = map[string]*term{ + "āˆ…": nil, + "š“¤": {}, + "int": {false, Typ[Int]}, + "~int": {true, Typ[Int]}, + "string": {false, Typ[String]}, + "~string": {true, Typ[String]}, + "myInt": {false, myInt}, +} + +func TestTermString(t *testing.T) { + for want, x := range testTerms { + if got := x.String(); got != want { + t.Errorf("%v.String() == %v; want %v", x, got, want) + } + } +} + +func split(s string, n int) []string { + r := strings.Split(s, " ") + if len(r) != n { + panic("invalid test case: " + s) + } + return r +} + +func testTerm(name string) *term { + r, ok := testTerms[name] + if !ok { + panic("invalid test argument: " + name) + } + return r +} + +func TestTermEqual(t *testing.T) { + for _, test := range []string{ + "āˆ… āˆ… T", + "š“¤ š“¤ T", + "int int T", + "~int ~int T", + "myInt myInt T", + "āˆ… š“¤ F", + "āˆ… int F", + "āˆ… ~int F", + "š“¤ int F", + "š“¤ ~int F", + "š“¤ myInt F", + "int ~int F", + "int myInt F", + "~int myInt F", + } { + args := split(test, 3) + x := testTerm(args[0]) + y := testTerm(args[1]) + want := args[2] == "T" + if got := x.equal(y); got != want { + t.Errorf("%v.equal(%v) = %v; want %v", x, y, got, want) + } + // equal is symmetric + x, y = y, x + if got := x.equal(y); got != want { + t.Errorf("%v.equal(%v) = %v; want %v", x, y, got, want) + } + } +} + +func TestTermUnion(t *testing.T) { + for _, test := range []string{ + "āˆ… āˆ… āˆ… āˆ…", + "āˆ… š“¤ š“¤ āˆ…", + "āˆ… int int āˆ…", + "āˆ… ~int ~int āˆ…", + "āˆ… myInt myInt āˆ…", + "š“¤ š“¤ š“¤ āˆ…", + "š“¤ int š“¤ āˆ…", + "š“¤ ~int š“¤ āˆ…", + "š“¤ myInt š“¤ āˆ…", + "int int int āˆ…", + "int ~int ~int āˆ…", + "int string int string", + "int ~string int ~string", + "int myInt int myInt", + "~int ~string ~int ~string", + "~int myInt ~int āˆ…", + + // union is symmetric, but the result order isn't - repeat symmetric cases explicitly + "š“¤ āˆ… š“¤ āˆ…", + "int āˆ… int āˆ…", + "~int āˆ… ~int āˆ…", + "myInt āˆ… myInt āˆ…", + "int š“¤ š“¤ āˆ…", + "~int š“¤ š“¤ āˆ…", + "myInt š“¤ š“¤ āˆ…", + "~int int ~int āˆ…", + "string int string int", + "~string int ~string int", + "myInt int myInt int", + "~string ~int ~string ~int", + "myInt ~int ~int āˆ…", + } { + args := split(test, 4) + x := testTerm(args[0]) + y := testTerm(args[1]) + want1 := testTerm(args[2]) + want2 := testTerm(args[3]) + if got1, got2 := x.union(y); !got1.equal(want1) || !got2.equal(want2) { + t.Errorf("%v.union(%v) = %v, %v; want %v, %v", x, y, got1, got2, want1, want2) + } + } +} + +func TestTermIntersection(t *testing.T) { + for _, test := range []string{ + "āˆ… āˆ… āˆ…", + "āˆ… š“¤ āˆ…", + "āˆ… int āˆ…", + "āˆ… ~int āˆ…", + "āˆ… myInt āˆ…", + "š“¤ š“¤ š“¤", + "š“¤ int int", + "š“¤ ~int ~int", + "š“¤ myInt myInt", + "int int int", + "int ~int int", + "int string āˆ…", + "int ~string āˆ…", + "int string āˆ…", + "~int ~string āˆ…", + "~int myInt myInt", + } { + args := split(test, 3) + x := testTerm(args[0]) + y := testTerm(args[1]) + want := testTerm(args[2]) + if got := x.intersect(y); !got.equal(want) { + t.Errorf("%v.intersect(%v) = %v; want %v", x, y, got, want) + } + // intersect is symmetric + x, y = y, x + if got := x.intersect(y); !got.equal(want) { + t.Errorf("%v.intersect(%v) = %v; want %v", x, y, got, want) + } + } +} + +func TestTermIncludes(t *testing.T) { + for _, test := range []string{ + "āˆ… int F", + "š“¤ int T", + "int int T", + "~int int T", + "~int myInt T", + "string int F", + "~string int F", + "myInt int F", + } { + args := split(test, 3) + x := testTerm(args[0]) + y := testTerm(args[1]).typ + want := args[2] == "T" + if got := x.includes(y); got != want { + t.Errorf("%v.includes(%v) = %v; want %v", x, y, got, want) + } + } +} + +func TestTermSubsetOf(t *testing.T) { + for _, test := range []string{ + "āˆ… āˆ… T", + "š“¤ š“¤ T", + "int int T", + "~int ~int T", + "myInt myInt T", + "āˆ… š“¤ T", + "āˆ… int T", + "āˆ… ~int T", + "āˆ… myInt T", + "š“¤ int F", + "š“¤ ~int F", + "š“¤ myInt F", + "int ~int T", + "int myInt F", + "~int myInt F", + "myInt int F", + "myInt ~int T", + } { + args := split(test, 3) + x := testTerm(args[0]) + y := testTerm(args[1]) + want := args[2] == "T" + if got := x.subsetOf(y); got != want { + t.Errorf("%v.subsetOf(%v) = %v; want %v", x, y, got, want) + } + } +} + +func TestTermDisjoint(t *testing.T) { + for _, test := range []string{ + "int int F", + "~int ~int F", + "int ~int F", + "int string T", + "int ~string T", + "int myInt T", + "~int ~string T", + "~int myInt F", + "string myInt T", + "~string myInt T", + } { + args := split(test, 3) + x := testTerm(args[0]) + y := testTerm(args[1]) + want := args[2] == "T" + if got := x.disjoint(y); got != want { + t.Errorf("%v.disjoint(%v) = %v; want %v", x, y, got, want) + } + // disjoint is symmetric + x, y = y, x + if got := x.disjoint(y); got != want { + t.Errorf("%v.disjoint(%v) = %v; want %v", x, y, got, want) + } + } +} diff --git a/src/go/types/typexpr.go b/src/go/types/typexpr.go new file mode 100644 index 0000000..03817dd --- /dev/null +++ b/src/go/types/typexpr.go @@ -0,0 +1,521 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements type-checking of identifiers and type expressions. + +package types + +import ( + "fmt" + "go/ast" + "go/constant" + "go/internal/typeparams" + . "internal/types/errors" + "strings" +) + +// ident type-checks identifier e and initializes x with the value or type of e. +// If an error occurred, x.mode is set to invalid. +// For the meaning of def, see Checker.definedType, below. +// If wantType is set, the identifier e is expected to denote a type. +func (check *Checker) ident(x *operand, e *ast.Ident, def *Named, wantType bool) { + x.mode = invalid + x.expr = e + + // Note that we cannot use check.lookup here because the returned scope + // may be different from obj.Parent(). See also Scope.LookupParent doc. + scope, obj := check.scope.LookupParent(e.Name, check.pos) + switch obj { + case nil: + if e.Name == "_" { + // Blank identifiers are never declared, but the current identifier may + // be a placeholder for a receiver type parameter. In this case we can + // resolve its type and object from Checker.recvTParamMap. + if tpar := check.recvTParamMap[e]; tpar != nil { + x.mode = typexpr + x.typ = tpar + } else { + check.error(e, InvalidBlank, "cannot use _ as value or type") + } + } else { + check.errorf(e, UndeclaredName, "undefined: %s", e.Name) + } + return + case universeAny, universeComparable: + if !check.allowVersion(check.pkg, 1, 18) { + check.versionErrorf(e, "go1.18", "predeclared %s", e.Name) + return // avoid follow-on errors + } + } + check.recordUse(e, obj) + + // Type-check the object. + // Only call Checker.objDecl if the object doesn't have a type yet + // (in which case we must actually determine it) or the object is a + // TypeName and we also want a type (in which case we might detect + // a cycle which needs to be reported). Otherwise we can skip the + // call and avoid a possible cycle error in favor of the more + // informative "not a type/value" error that this function's caller + // will issue (see issue #25790). + typ := obj.Type() + if _, gotType := obj.(*TypeName); typ == nil || gotType && wantType { + check.objDecl(obj, def) + typ = obj.Type() // type must have been assigned by Checker.objDecl + } + assert(typ != nil) + + // The object may have been dot-imported. + // If so, mark the respective package as used. + // (This code is only needed for dot-imports. Without them, + // we only have to mark variables, see *Var case below). + if pkgName := check.dotImportMap[dotImportKey{scope, obj.Name()}]; pkgName != nil { + pkgName.used = true + } + + switch obj := obj.(type) { + case *PkgName: + check.errorf(e, InvalidPkgUse, "use of package %s not in selector", obj.name) + return + + case *Const: + check.addDeclDep(obj) + if typ == Typ[Invalid] { + return + } + if obj == universeIota { + if check.iota == nil { + check.error(e, InvalidIota, "cannot use iota outside constant declaration") + return + } + x.val = check.iota + } else { + x.val = obj.val + } + assert(x.val != nil) + x.mode = constant_ + + case *TypeName: + if check.isBrokenAlias(obj) { + check.errorf(e, InvalidDeclCycle, "invalid use of type alias %s in recursive type (see issue #50729)", obj.name) + return + } + x.mode = typexpr + + case *Var: + // It's ok to mark non-local variables, but ignore variables + // from other packages to avoid potential race conditions with + // dot-imported variables. + if obj.pkg == check.pkg { + obj.used = true + } + check.addDeclDep(obj) + if typ == Typ[Invalid] { + return + } + x.mode = variable + + case *Func: + check.addDeclDep(obj) + x.mode = value + + case *Builtin: + x.id = obj.id + x.mode = builtin + + case *Nil: + x.mode = value + + default: + unreachable() + } + + x.typ = typ +} + +// typ type-checks the type expression e and returns its type, or Typ[Invalid]. +// The type must not be an (uninstantiated) generic type. +func (check *Checker) typ(e ast.Expr) Type { + return check.definedType(e, nil) +} + +// varType type-checks the type expression e and returns its type, or Typ[Invalid]. +// The type must not be an (uninstantiated) generic type and it must not be a +// constraint interface. +func (check *Checker) varType(e ast.Expr) Type { + typ := check.definedType(e, nil) + check.validVarType(e, typ) + return typ +} + +// validVarType reports an error if typ is a constraint interface. +// The expression e is used for error reporting, if any. +func (check *Checker) validVarType(e ast.Expr, typ Type) { + // If we have a type parameter there's nothing to do. + if isTypeParam(typ) { + return + } + + // We don't want to call under() or complete interfaces while we are in + // the middle of type-checking parameter declarations that might belong + // to interface methods. Delay this check to the end of type-checking. + check.later(func() { + if t, _ := under(typ).(*Interface); t != nil { + tset := computeInterfaceTypeSet(check, e.Pos(), t) // TODO(gri) is this the correct position? + if !tset.IsMethodSet() { + if tset.comparable { + check.softErrorf(e, MisplacedConstraintIface, "cannot use type %s outside a type constraint: interface is (or embeds) comparable", typ) + } else { + check.softErrorf(e, MisplacedConstraintIface, "cannot use type %s outside a type constraint: interface contains type constraints", typ) + } + } + } + }).describef(e, "check var type %s", typ) +} + +// definedType is like typ but also accepts a type name def. +// If def != nil, e is the type specification for the defined type def, declared +// in a type declaration, and def.underlying will be set to the type of e before +// any components of e are type-checked. +func (check *Checker) definedType(e ast.Expr, def *Named) Type { + typ := check.typInternal(e, def) + assert(isTyped(typ)) + if isGeneric(typ) { + check.errorf(e, WrongTypeArgCount, "cannot use generic type %s without instantiation", typ) + typ = Typ[Invalid] + } + check.recordTypeAndValue(e, typexpr, typ, nil) + return typ +} + +// genericType is like typ but the type must be an (uninstantiated) generic +// type. If cause is non-nil and the type expression was a valid type but not +// generic, cause will be populated with a message describing the error. +func (check *Checker) genericType(e ast.Expr, cause *string) Type { + typ := check.typInternal(e, nil) + assert(isTyped(typ)) + if typ != Typ[Invalid] && !isGeneric(typ) { + if cause != nil { + *cause = check.sprintf("%s is not a generic type", typ) + } + typ = Typ[Invalid] + } + // TODO(gri) what is the correct call below? + check.recordTypeAndValue(e, typexpr, typ, nil) + return typ +} + +// goTypeName returns the Go type name for typ and +// removes any occurrences of "types." from that name. +func goTypeName(typ Type) string { + return strings.ReplaceAll(fmt.Sprintf("%T", typ), "types.", "") +} + +// typInternal drives type checking of types. +// Must only be called by definedType or genericType. +func (check *Checker) typInternal(e0 ast.Expr, def *Named) (T Type) { + if trace { + check.trace(e0.Pos(), "-- type %s", e0) + check.indent++ + defer func() { + check.indent-- + var under Type + if T != nil { + // Calling under() here may lead to endless instantiations. + // Test case: type T[P any] *T[P] + under = safeUnderlying(T) + } + if T == under { + check.trace(e0.Pos(), "=> %s // %s", T, goTypeName(T)) + } else { + check.trace(e0.Pos(), "=> %s (under = %s) // %s", T, under, goTypeName(T)) + } + }() + } + + switch e := e0.(type) { + case *ast.BadExpr: + // ignore - error reported before + + case *ast.Ident: + var x operand + check.ident(&x, e, def, true) + + switch x.mode { + case typexpr: + typ := x.typ + def.setUnderlying(typ) + return typ + case invalid: + // ignore - error reported before + case novalue: + check.errorf(&x, NotAType, "%s used as type", &x) + default: + check.errorf(&x, NotAType, "%s is not a type", &x) + } + + case *ast.SelectorExpr: + var x operand + check.selector(&x, e, def, true) + + switch x.mode { + case typexpr: + typ := x.typ + def.setUnderlying(typ) + return typ + case invalid: + // ignore - error reported before + case novalue: + check.errorf(&x, NotAType, "%s used as type", &x) + default: + check.errorf(&x, NotAType, "%s is not a type", &x) + } + + case *ast.IndexExpr, *ast.IndexListExpr: + ix := typeparams.UnpackIndexExpr(e) + if !check.allowVersion(check.pkg, 1, 18) { + check.softErrorf(inNode(e, ix.Lbrack), UnsupportedFeature, "type instantiation requires go1.18 or later") + } + return check.instantiatedType(ix, def) + + case *ast.ParenExpr: + // Generic types must be instantiated before they can be used in any form. + // Consequently, generic types cannot be parenthesized. + return check.definedType(e.X, def) + + case *ast.ArrayType: + if e.Len == nil { + typ := new(Slice) + def.setUnderlying(typ) + typ.elem = check.varType(e.Elt) + return typ + } + + typ := new(Array) + def.setUnderlying(typ) + // Provide a more specific error when encountering a [...] array + // rather than leaving it to the handling of the ... expression. + if _, ok := e.Len.(*ast.Ellipsis); ok { + check.error(e.Len, BadDotDotDotSyntax, "invalid use of [...] array (outside a composite literal)") + typ.len = -1 + } else { + typ.len = check.arrayLength(e.Len) + } + typ.elem = check.varType(e.Elt) + if typ.len >= 0 { + return typ + } + // report error if we encountered [...] + + case *ast.Ellipsis: + // dots are handled explicitly where they are legal + // (array composite literals and parameter lists) + check.error(e, InvalidDotDotDot, "invalid use of '...'") + check.use(e.Elt) + + case *ast.StructType: + typ := new(Struct) + def.setUnderlying(typ) + check.structType(typ, e) + return typ + + case *ast.StarExpr: + typ := new(Pointer) + typ.base = Typ[Invalid] // avoid nil base in invalid recursive type declaration + def.setUnderlying(typ) + typ.base = check.varType(e.X) + return typ + + case *ast.FuncType: + typ := new(Signature) + def.setUnderlying(typ) + check.funcType(typ, nil, e) + return typ + + case *ast.InterfaceType: + typ := check.newInterface() + def.setUnderlying(typ) + check.interfaceType(typ, e, def) + return typ + + case *ast.MapType: + typ := new(Map) + def.setUnderlying(typ) + + typ.key = check.varType(e.Key) + typ.elem = check.varType(e.Value) + + // spec: "The comparison operators == and != must be fully defined + // for operands of the key type; thus the key type must not be a + // function, map, or slice." + // + // Delay this check because it requires fully setup types; + // it is safe to continue in any case (was issue 6667). + check.later(func() { + if !Comparable(typ.key) { + var why string + if isTypeParam(typ.key) { + why = " (missing comparable constraint)" + } + check.errorf(e.Key, IncomparableMapKey, "invalid map key type %s%s", typ.key, why) + } + }).describef(e.Key, "check map key %s", typ.key) + + return typ + + case *ast.ChanType: + typ := new(Chan) + def.setUnderlying(typ) + + dir := SendRecv + switch e.Dir { + case ast.SEND | ast.RECV: + // nothing to do + case ast.SEND: + dir = SendOnly + case ast.RECV: + dir = RecvOnly + default: + check.errorf(e, InvalidSyntaxTree, "unknown channel direction %d", e.Dir) + // ok to continue + } + + typ.dir = dir + typ.elem = check.varType(e.Value) + return typ + + default: + check.errorf(e0, NotAType, "%s is not a type", e0) + check.use(e0) + } + + typ := Typ[Invalid] + def.setUnderlying(typ) + return typ +} + +func (check *Checker) instantiatedType(ix *typeparams.IndexExpr, def *Named) (res Type) { + if trace { + check.trace(ix.Pos(), "-- instantiating type %s with %s", ix.X, ix.Indices) + check.indent++ + defer func() { + check.indent-- + // Don't format the underlying here. It will always be nil. + check.trace(ix.Pos(), "=> %s", res) + }() + } + + var cause string + gtyp := check.genericType(ix.X, &cause) + if cause != "" { + check.errorf(ix.Orig, NotAGenericType, invalidOp+"%s (%s)", ix.Orig, cause) + } + if gtyp == Typ[Invalid] { + return gtyp // error already reported + } + + orig, _ := gtyp.(*Named) + if orig == nil { + panic(fmt.Sprintf("%v: cannot instantiate %v", ix.Pos(), gtyp)) + } + + // evaluate arguments + targs := check.typeList(ix.Indices) + if targs == nil { + def.setUnderlying(Typ[Invalid]) // avoid errors later due to lazy instantiation + return Typ[Invalid] + } + + // create the instance + inst := check.instance(ix.Pos(), orig, targs, nil, check.context()).(*Named) + def.setUnderlying(inst) + + // orig.tparams may not be set up, so we need to do expansion later. + check.later(func() { + // This is an instance from the source, not from recursive substitution, + // and so it must be resolved during type-checking so that we can report + // errors. + check.recordInstance(ix.Orig, inst.TypeArgs().list(), inst) + + if check.validateTArgLen(ix.Pos(), inst.TypeParams().Len(), inst.TypeArgs().Len()) { + if i, err := check.verify(ix.Pos(), inst.TypeParams().list(), inst.TypeArgs().list(), check.context()); err != nil { + // best position for error reporting + pos := ix.Pos() + if i < len(ix.Indices) { + pos = ix.Indices[i].Pos() + } + check.softErrorf(atPos(pos), InvalidTypeArg, err.Error()) + } else { + check.mono.recordInstance(check.pkg, ix.Pos(), inst.TypeParams().list(), inst.TypeArgs().list(), ix.Indices) + } + } + + // TODO(rfindley): remove this call: we don't need to call validType here, + // as cycles can only occur for types used inside a Named type declaration, + // and so it suffices to call validType from declared types. + check.validType(inst) + }).describef(ix, "resolve instance %s", inst) + + return inst +} + +// arrayLength type-checks the array length expression e +// and returns the constant length >= 0, or a value < 0 +// to indicate an error (and thus an unknown length). +func (check *Checker) arrayLength(e ast.Expr) int64 { + // If e is an identifier, the array declaration might be an + // attempt at a parameterized type declaration with missing + // constraint. Provide an error message that mentions array + // length. + if name, _ := e.(*ast.Ident); name != nil { + obj := check.lookup(name.Name) + if obj == nil { + check.errorf(name, InvalidArrayLen, "undefined array length %s or missing type constraint", name.Name) + return -1 + } + if _, ok := obj.(*Const); !ok { + check.errorf(name, InvalidArrayLen, "invalid array length %s", name.Name) + return -1 + } + } + + var x operand + check.expr(&x, e) + if x.mode != constant_ { + if x.mode != invalid { + check.errorf(&x, InvalidArrayLen, "array length %s must be constant", &x) + } + return -1 + } + + if isUntyped(x.typ) || isInteger(x.typ) { + if val := constant.ToInt(x.val); val.Kind() == constant.Int { + if representableConst(val, check, Typ[Int], nil) { + if n, ok := constant.Int64Val(val); ok && n >= 0 { + return n + } + check.errorf(&x, InvalidArrayLen, "invalid array length %s", &x) + return -1 + } + } + } + + check.errorf(&x, InvalidArrayLen, "array length %s must be integer", &x) + return -1 +} + +// typeList provides the list of types corresponding to the incoming expression list. +// If an error occurred, the result is nil, but all list elements were type-checked. +func (check *Checker) typeList(list []ast.Expr) []Type { + res := make([]Type, len(list)) // res != nil even if len(list) == 0 + for i, x := range list { + t := check.varType(x) + if t == Typ[Invalid] { + res = nil + } + if res != nil { + res[i] = t + } + } + return res +} diff --git a/src/go/types/unify.go b/src/go/types/unify.go new file mode 100644 index 0000000..602e304 --- /dev/null +++ b/src/go/types/unify.go @@ -0,0 +1,583 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements type unification. + +package types + +import ( + "bytes" + "fmt" + "strings" +) + +// The unifier maintains two separate sets of type parameters x and y +// which are used to resolve type parameters in the x and y arguments +// provided to the unify call. For unidirectional unification, only +// one of these sets (say x) is provided, and then type parameters are +// only resolved for the x argument passed to unify, not the y argument +// (even if that also contains possibly the same type parameters). This +// is crucial to infer the type parameters of self-recursive calls: +// +// func f[P any](a P) { f(a) } +// +// For the call f(a) we want to infer that the type argument for P is P. +// During unification, the parameter type P must be resolved to the type +// parameter P ("x" side), but the argument type P must be left alone so +// that unification resolves the type parameter P to P. +// +// For bidirectional unification, both sets are provided. This enables +// unification to go from argument to parameter type and vice versa. +// For constraint type inference, we use bidirectional unification +// where both the x and y type parameters are identical. This is done +// by setting up one of them (using init) and then assigning its value +// to the other. + +const ( + // Upper limit for recursion depth. Used to catch infinite recursions + // due to implementation issues (e.g., see issues #48619, #48656). + unificationDepthLimit = 50 + + // Whether to panic when unificationDepthLimit is reached. + // If disabled, a recursion depth overflow results in a (quiet) + // unification failure. + panicAtUnificationDepthLimit = true + + // If enableCoreTypeUnification is set, unification will consider + // the core types, if any, of non-local (unbound) type parameters. + enableCoreTypeUnification = true + + // If traceInference is set, unification will print a trace of its operation. + // Interpretation of trace: + // x ā‰” y attempt to unify types x and y + // p āžž y type parameter p is set to type y (p is inferred to be y) + // p ā‡„ q type parameters p and q match (p is inferred to be q and vice versa) + // x ā‰¢ y types x and y cannot be unified + // [p, q, ...] āžž [x, y, ...] mapping from type parameters to types + traceInference = false +) + +// A unifier maintains the current type parameters for x and y +// and the respective types inferred for each type parameter. +// A unifier is created by calling newUnifier. +type unifier struct { + exact bool + x, y tparamsList // x and y must initialized via tparamsList.init + types []Type // inferred types, shared by x and y + depth int // recursion depth during unification +} + +// newUnifier returns a new unifier. +// If exact is set, unification requires unified types to match +// exactly. If exact is not set, a named type's underlying type +// is considered if unification would fail otherwise, and the +// direction of channels is ignored. +// TODO(gri) exact is not set anymore by a caller. Consider removing it. +func newUnifier(exact bool) *unifier { + u := &unifier{exact: exact} + u.x.unifier = u + u.y.unifier = u + return u +} + +// unify attempts to unify x and y and reports whether it succeeded. +func (u *unifier) unify(x, y Type) bool { + return u.nify(x, y, nil) +} + +func (u *unifier) tracef(format string, args ...interface{}) { + fmt.Println(strings.Repeat(". ", u.depth) + sprintf(nil, nil, true, format, args...)) +} + +// A tparamsList describes a list of type parameters and the types inferred for them. +type tparamsList struct { + unifier *unifier + tparams []*TypeParam + // For each tparams element, there is a corresponding type slot index in indices. + // index < 0: unifier.types[-index-1] == nil + // index == 0: no type slot allocated yet + // index > 0: unifier.types[index-1] == typ + // Joined tparams elements share the same type slot and thus have the same index. + // By using a negative index for nil types we don't need to check unifier.types + // to see if we have a type or not. + indices []int // len(d.indices) == len(d.tparams) +} + +// String returns a string representation for a tparamsList. For debugging. +func (d *tparamsList) String() string { + var buf bytes.Buffer + w := newTypeWriter(&buf, nil) + w.byte('[') + for i, tpar := range d.tparams { + if i > 0 { + w.string(", ") + } + w.typ(tpar) + w.string(": ") + w.typ(d.at(i)) + } + w.byte(']') + return buf.String() +} + +// init initializes d with the given type parameters. +// The type parameters must be in the order in which they appear in their declaration +// (this ensures that the tparams indices match the respective type parameter index). +func (d *tparamsList) init(tparams []*TypeParam) { + if len(tparams) == 0 { + return + } + if debug { + for i, tpar := range tparams { + assert(i == tpar.index) + } + } + d.tparams = tparams + d.indices = make([]int, len(tparams)) +} + +// join unifies the i'th type parameter of x with the j'th type parameter of y. +// If both type parameters already have a type associated with them and they are +// not joined, join fails and returns false. +func (u *unifier) join(i, j int) bool { + if traceInference { + u.tracef("%s ā‡„ %s", u.x.tparams[i], u.y.tparams[j]) + } + ti := u.x.indices[i] + tj := u.y.indices[j] + switch { + case ti == 0 && tj == 0: + // Neither type parameter has a type slot associated with them. + // Allocate a new joined nil type slot (negative index). + u.types = append(u.types, nil) + u.x.indices[i] = -len(u.types) + u.y.indices[j] = -len(u.types) + case ti == 0: + // The type parameter for x has no type slot yet. Use slot of y. + u.x.indices[i] = tj + case tj == 0: + // The type parameter for y has no type slot yet. Use slot of x. + u.y.indices[j] = ti + + // Both type parameters have a slot: ti != 0 && tj != 0. + case ti == tj: + // Both type parameters already share the same slot. Nothing to do. + break + case ti > 0 && tj > 0: + // Both type parameters have (possibly different) inferred types. Cannot join. + // TODO(gri) Should we check if types are identical? Investigate. + return false + case ti > 0: + // Only the type parameter for x has an inferred type. Use x slot for y. + u.y.setIndex(j, ti) + // This case is handled like the default case. + // case tj > 0: + // // Only the type parameter for y has an inferred type. Use y slot for x. + // u.x.setIndex(i, tj) + default: + // Neither type parameter has an inferred type. Use y slot for x + // (or x slot for y, it doesn't matter). + u.x.setIndex(i, tj) + } + return true +} + +// If typ is a type parameter of d, index returns the type parameter index. +// Otherwise, the result is < 0. +func (d *tparamsList) index(typ Type) int { + if tpar, ok := typ.(*TypeParam); ok { + return tparamIndex(d.tparams, tpar) + } + return -1 +} + +// If tpar is a type parameter in list, tparamIndex returns the type parameter index. +// Otherwise, the result is < 0. tpar must not be nil. +func tparamIndex(list []*TypeParam, tpar *TypeParam) int { + // Once a type parameter is bound its index is >= 0. However, there are some + // code paths (namely tracing and type hashing) by which it is possible to + // arrive here with a type parameter that has not been bound, hence the check + // for 0 <= i below. + // TODO(rfindley): investigate a better approach for guarding against using + // unbound type parameters. + if i := tpar.index; 0 <= i && i < len(list) && list[i] == tpar { + return i + } + return -1 +} + +// setIndex sets the type slot index for the i'th type parameter +// (and all its joined parameters) to tj. The type parameter +// must have a (possibly nil) type slot associated with it. +func (d *tparamsList) setIndex(i, tj int) { + ti := d.indices[i] + assert(ti != 0 && tj != 0) + for k, tk := range d.indices { + if tk == ti { + d.indices[k] = tj + } + } +} + +// at returns the type set for the i'th type parameter; or nil. +func (d *tparamsList) at(i int) Type { + if ti := d.indices[i]; ti > 0 { + return d.unifier.types[ti-1] + } + return nil +} + +// set sets the type typ for the i'th type parameter; +// typ must not be nil and it must not have been set before. +func (d *tparamsList) set(i int, typ Type) { + assert(typ != nil) + u := d.unifier + if traceInference { + u.tracef("%s āžž %s", d.tparams[i], typ) + } + switch ti := d.indices[i]; { + case ti < 0: + u.types[-ti-1] = typ + d.setIndex(i, -ti) + case ti == 0: + u.types = append(u.types, typ) + d.indices[i] = len(u.types) + default: + panic("type already set") + } +} + +// unknowns returns the number of type parameters for which no type has been set yet. +func (d *tparamsList) unknowns() int { + n := 0 + for _, ti := range d.indices { + if ti <= 0 { + n++ + } + } + return n +} + +// types returns the list of inferred types (via unification) for the type parameters +// described by d, and an index. If all types were inferred, the returned index is < 0. +// Otherwise, it is the index of the first type parameter which couldn't be inferred; +// i.e., for which list[index] is nil. +func (d *tparamsList) types() (list []Type, index int) { + list = make([]Type, len(d.tparams)) + index = -1 + for i := range d.tparams { + t := d.at(i) + list[i] = t + if index < 0 && t == nil { + index = i + } + } + return +} + +func (u *unifier) nifyEq(x, y Type, p *ifacePair) bool { + return x == y || u.nify(x, y, p) +} + +// nify implements the core unification algorithm which is an +// adapted version of Checker.identical. For changes to that +// code the corresponding changes should be made here. +// Must not be called directly from outside the unifier. +func (u *unifier) nify(x, y Type, p *ifacePair) (result bool) { + if traceInference { + u.tracef("%s ā‰” %s", x, y) + } + + // Stop gap for cases where unification fails. + if u.depth >= unificationDepthLimit { + if traceInference { + u.tracef("depth %d >= %d", u.depth, unificationDepthLimit) + } + if panicAtUnificationDepthLimit { + panic("unification reached recursion depth limit") + } + return false + } + u.depth++ + defer func() { + u.depth-- + if traceInference && !result { + u.tracef("%s ā‰¢ %s", x, y) + } + }() + + if !u.exact { + // If exact unification is known to fail because we attempt to + // match a type name against an unnamed type literal, consider + // the underlying type of the named type. + // (We use !hasName to exclude any type with a name, including + // basic types and type parameters; the rest are unamed types.) + if nx, _ := x.(*Named); nx != nil && !hasName(y) { + if traceInference { + u.tracef("under %s ā‰” %s", nx, y) + } + return u.nify(nx.under(), y, p) + } else if ny, _ := y.(*Named); ny != nil && !hasName(x) { + if traceInference { + u.tracef("%s ā‰” under %s", x, ny) + } + return u.nify(x, ny.under(), p) + } + } + + // Cases where at least one of x or y is a type parameter. + switch i, j := u.x.index(x), u.y.index(y); { + case i >= 0 && j >= 0: + // both x and y are type parameters + if u.join(i, j) { + return true + } + // both x and y have an inferred type - they must match + return u.nifyEq(u.x.at(i), u.y.at(j), p) + + case i >= 0: + // x is a type parameter, y is not + if tx := u.x.at(i); tx != nil { + return u.nifyEq(tx, y, p) + } + // otherwise, infer type from y + u.x.set(i, y) + return true + + case j >= 0: + // y is a type parameter, x is not + if ty := u.y.at(j); ty != nil { + return u.nifyEq(x, ty, p) + } + // otherwise, infer type from x + u.y.set(j, x) + return true + } + + // If we get here and x or y is a type parameter, they are type parameters + // from outside our declaration list. Try to unify their core types, if any + // (see issue #50755 for a test case). + if enableCoreTypeUnification && !u.exact { + if isTypeParam(x) && !hasName(y) { + // When considering the type parameter for unification + // we look at the adjusted core term (adjusted core type + // with tilde information). + // If the adjusted core type is a named type N; the + // corresponding core type is under(N). Since !u.exact + // and y doesn't have a name, unification will end up + // comparing under(N) to y, so we can just use the core + // type instead. And we can ignore the tilde because we + // already look at the underlying types on both sides + // and we have known types on both sides. + // Optimization. + if cx := coreType(x); cx != nil { + if traceInference { + u.tracef("core %s ā‰” %s", x, y) + } + return u.nify(cx, y, p) + } + } else if isTypeParam(y) && !hasName(x) { + // see comment above + if cy := coreType(y); cy != nil { + if traceInference { + u.tracef("%s ā‰” core %s", x, y) + } + return u.nify(x, cy, p) + } + } + } + + // For type unification, do not shortcut (x == y) for identical + // types. Instead keep comparing them element-wise to unify the + // matching (and equal type parameter types). A simple test case + // where this matters is: func f[P any](a P) { f(a) } . + + switch x := x.(type) { + case *Basic: + // Basic types are singletons except for the rune and byte + // aliases, thus we cannot solely rely on the x == y check + // above. See also comment in TypeName.IsAlias. + if y, ok := y.(*Basic); ok { + return x.kind == y.kind + } + + case *Array: + // Two array types are identical if they have identical element types + // and the same array length. + if y, ok := y.(*Array); ok { + // If one or both array lengths are unknown (< 0) due to some error, + // assume they are the same to avoid spurious follow-on errors. + return (x.len < 0 || y.len < 0 || x.len == y.len) && u.nify(x.elem, y.elem, p) + } + + case *Slice: + // Two slice types are identical if they have identical element types. + if y, ok := y.(*Slice); ok { + return u.nify(x.elem, y.elem, p) + } + + case *Struct: + // Two struct types are identical if they have the same sequence of fields, + // and if corresponding fields have the same names, and identical types, + // and identical tags. Two embedded fields are considered to have the same + // name. Lower-case field names from different packages are always different. + if y, ok := y.(*Struct); ok { + if x.NumFields() == y.NumFields() { + for i, f := range x.fields { + g := y.fields[i] + if f.embedded != g.embedded || + x.Tag(i) != y.Tag(i) || + !f.sameId(g.pkg, g.name) || + !u.nify(f.typ, g.typ, p) { + return false + } + } + return true + } + } + + case *Pointer: + // Two pointer types are identical if they have identical base types. + if y, ok := y.(*Pointer); ok { + return u.nify(x.base, y.base, p) + } + + case *Tuple: + // Two tuples types are identical if they have the same number of elements + // and corresponding elements have identical types. + if y, ok := y.(*Tuple); ok { + if x.Len() == y.Len() { + if x != nil { + for i, v := range x.vars { + w := y.vars[i] + if !u.nify(v.typ, w.typ, p) { + return false + } + } + } + return true + } + } + + case *Signature: + // Two function types are identical if they have the same number of parameters + // and result values, corresponding parameter and result types are identical, + // and either both functions are variadic or neither is. Parameter and result + // names are not required to match. + // TODO(gri) handle type parameters or document why we can ignore them. + if y, ok := y.(*Signature); ok { + return x.variadic == y.variadic && + u.nify(x.params, y.params, p) && + u.nify(x.results, y.results, p) + } + + case *Interface: + // Two interface types are identical if they have the same set of methods with + // the same names and identical function types. Lower-case method names from + // different packages are always different. The order of the methods is irrelevant. + if y, ok := y.(*Interface); ok { + xset := x.typeSet() + yset := y.typeSet() + if xset.comparable != yset.comparable { + return false + } + if !xset.terms.equal(yset.terms) { + return false + } + a := xset.methods + b := yset.methods + if len(a) == len(b) { + // Interface types are the only types where cycles can occur + // that are not "terminated" via named types; and such cycles + // can only be created via method parameter types that are + // anonymous interfaces (directly or indirectly) embedding + // the current interface. Example: + // + // type T interface { + // m() interface{T} + // } + // + // If two such (differently named) interfaces are compared, + // endless recursion occurs if the cycle is not detected. + // + // If x and y were compared before, they must be equal + // (if they were not, the recursion would have stopped); + // search the ifacePair stack for the same pair. + // + // This is a quadratic algorithm, but in practice these stacks + // are extremely short (bounded by the nesting depth of interface + // type declarations that recur via parameter types, an extremely + // rare occurrence). An alternative implementation might use a + // "visited" map, but that is probably less efficient overall. + q := &ifacePair{x, y, p} + for p != nil { + if p.identical(q) { + return true // same pair was compared before + } + p = p.prev + } + if debug { + assertSortedMethods(a) + assertSortedMethods(b) + } + for i, f := range a { + g := b[i] + if f.Id() != g.Id() || !u.nify(f.typ, g.typ, q) { + return false + } + } + return true + } + } + + case *Map: + // Two map types are identical if they have identical key and value types. + if y, ok := y.(*Map); ok { + return u.nify(x.key, y.key, p) && u.nify(x.elem, y.elem, p) + } + + case *Chan: + // Two channel types are identical if they have identical value types. + if y, ok := y.(*Chan); ok { + return (!u.exact || x.dir == y.dir) && u.nify(x.elem, y.elem, p) + } + + case *Named: + // TODO(gri) This code differs now from the parallel code in Checker.identical. Investigate. + if y, ok := y.(*Named); ok { + xargs := x.TypeArgs().list() + yargs := y.TypeArgs().list() + + if len(xargs) != len(yargs) { + return false + } + + // TODO(gri) This is not always correct: two types may have the same names + // in the same package if one of them is nested in a function. + // Extremely unlikely but we need an always correct solution. + if x.obj.pkg == y.obj.pkg && x.obj.name == y.obj.name { + for i, x := range xargs { + if !u.nify(x, yargs[i], p) { + return false + } + } + return true + } + } + + case *TypeParam: + // Two type parameters (which are not part of the type parameters of the + // enclosing type as those are handled in the beginning of this function) + // are identical if they originate in the same declaration. + return x == y + + case nil: + // avoid a crash in case of nil type + + default: + panic(sprintf(nil, nil, true, "u.nify(%s, %s), u.x.tparams = %s", x, y, u.x.tparams)) + } + + return false +} diff --git a/src/go/types/union.go b/src/go/types/union.go new file mode 100644 index 0000000..9509afe --- /dev/null +++ b/src/go/types/union.go @@ -0,0 +1,200 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "go/ast" + "go/token" + . "internal/types/errors" +) + +// ---------------------------------------------------------------------------- +// API + +// A Union represents a union of terms embedded in an interface. +type Union struct { + terms []*Term // list of syntactical terms (not a canonicalized termlist) +} + +// NewUnion returns a new Union type with the given terms. +// It is an error to create an empty union; they are syntactically not possible. +func NewUnion(terms []*Term) *Union { + if len(terms) == 0 { + panic("empty union") + } + return &Union{terms} +} + +func (u *Union) Len() int { return len(u.terms) } +func (u *Union) Term(i int) *Term { return u.terms[i] } + +func (u *Union) Underlying() Type { return u } +func (u *Union) String() string { return TypeString(u, nil) } + +// A Term represents a term in a Union. +type Term term + +// NewTerm returns a new union term. +func NewTerm(tilde bool, typ Type) *Term { return &Term{tilde, typ} } + +func (t *Term) Tilde() bool { return t.tilde } +func (t *Term) Type() Type { return t.typ } +func (t *Term) String() string { return (*term)(t).String() } + +// ---------------------------------------------------------------------------- +// Implementation + +// Avoid excessive type-checking times due to quadratic termlist operations. +const maxTermCount = 100 + +// parseUnion parses uexpr as a union of expressions. +// The result is a Union type, or Typ[Invalid] for some errors. +func parseUnion(check *Checker, uexpr ast.Expr) Type { + blist, tlist := flattenUnion(nil, uexpr) + assert(len(blist) == len(tlist)-1) + + var terms []*Term + + var u Type + for i, x := range tlist { + term := parseTilde(check, x) + if len(tlist) == 1 && !term.tilde { + // Single type. Ok to return early because all relevant + // checks have been performed in parseTilde (no need to + // run through term validity check below). + return term.typ // typ already recorded through check.typ in parseTilde + } + if len(terms) >= maxTermCount { + if u != Typ[Invalid] { + check.errorf(x, InvalidUnion, "cannot handle more than %d union terms (implementation limitation)", maxTermCount) + u = Typ[Invalid] + } + } else { + terms = append(terms, term) + u = &Union{terms} + } + + if i > 0 { + check.recordTypeAndValue(blist[i-1], typexpr, u, nil) + } + } + + if u == Typ[Invalid] { + return u + } + + // Check validity of terms. + // Do this check later because it requires types to be set up. + // Note: This is a quadratic algorithm, but unions tend to be short. + check.later(func() { + for i, t := range terms { + if t.typ == Typ[Invalid] { + continue + } + + u := under(t.typ) + f, _ := u.(*Interface) + if t.tilde { + if f != nil { + check.errorf(tlist[i], InvalidUnion, "invalid use of ~ (%s is an interface)", t.typ) + continue // don't report another error for t + } + + if !Identical(u, t.typ) { + check.errorf(tlist[i], InvalidUnion, "invalid use of ~ (underlying type of %s is %s)", t.typ, u) + continue + } + } + + // Stand-alone embedded interfaces are ok and are handled by the single-type case + // in the beginning. Embedded interfaces with tilde are excluded above. If we reach + // here, we must have at least two terms in the syntactic term list (but not necessarily + // in the term list of the union's type set). + if f != nil { + tset := f.typeSet() + switch { + case tset.NumMethods() != 0: + check.errorf(tlist[i], InvalidUnion, "cannot use %s in union (%s contains methods)", t, t) + case t.typ == universeComparable.Type(): + check.error(tlist[i], InvalidUnion, "cannot use comparable in union") + case tset.comparable: + check.errorf(tlist[i], InvalidUnion, "cannot use %s in union (%s embeds comparable)", t, t) + } + continue // terms with interface types are not subject to the no-overlap rule + } + + // Report overlapping (non-disjoint) terms such as + // a|a, a|~a, ~a|~a, and ~a|A (where under(A) == a). + if j := overlappingTerm(terms[:i], t); j >= 0 { + check.softErrorf(tlist[i], InvalidUnion, "overlapping terms %s and %s", t, terms[j]) + } + } + }).describef(uexpr, "check term validity %s", uexpr) + + return u +} + +func parseTilde(check *Checker, tx ast.Expr) *Term { + x := tx + var tilde bool + if op, _ := x.(*ast.UnaryExpr); op != nil && op.Op == token.TILDE { + x = op.X + tilde = true + } + typ := check.typ(x) + // Embedding stand-alone type parameters is not permitted (issue #47127). + // We don't need this restriction anymore if we make the underlying type of a type + // parameter its constraint interface: if we embed a lone type parameter, we will + // simply use its underlying type (like we do for other named, embedded interfaces), + // and since the underlying type is an interface the embedding is well defined. + if isTypeParam(typ) { + if tilde { + check.errorf(x, MisplacedTypeParam, "type in term %s cannot be a type parameter", tx) + } else { + check.error(x, MisplacedTypeParam, "term cannot be a type parameter") + } + typ = Typ[Invalid] + } + term := NewTerm(tilde, typ) + if tilde { + check.recordTypeAndValue(tx, typexpr, &Union{[]*Term{term}}, nil) + } + return term +} + +// overlappingTerm reports the index of the term x in terms which is +// overlapping (not disjoint) from y. The result is < 0 if there is no +// such term. The type of term y must not be an interface, and terms +// with an interface type are ignored in the terms list. +func overlappingTerm(terms []*Term, y *Term) int { + assert(!IsInterface(y.typ)) + for i, x := range terms { + if IsInterface(x.typ) { + continue + } + // disjoint requires non-nil, non-top arguments, + // and non-interface types as term types. + if debug { + if x == nil || x.typ == nil || y == nil || y.typ == nil { + panic("empty or top union term") + } + } + if !(*term)(x).disjoint((*term)(y)) { + return i + } + } + return -1 +} + +// flattenUnion walks a union type expression of the form A | B | C | ..., +// extracting both the binary exprs (blist) and leaf types (tlist). +func flattenUnion(list []ast.Expr, x ast.Expr) (blist, tlist []ast.Expr) { + if o, _ := x.(*ast.BinaryExpr); o != nil && o.Op == token.OR { + blist, tlist = flattenUnion(list, o.X) + blist = append(blist, o) + x = o.Y + } + return blist, append(tlist, x) +} diff --git a/src/go/types/universe.go b/src/go/types/universe.go new file mode 100644 index 0000000..9103fca --- /dev/null +++ b/src/go/types/universe.go @@ -0,0 +1,284 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file sets up the universe scope and the unsafe package. + +package types + +import ( + "go/constant" + "go/token" + "strings" +) + +// The Universe scope contains all predeclared objects of Go. +// It is the outermost scope of any chain of nested scopes. +var Universe *Scope + +// The Unsafe package is the package returned by an importer +// for the import path "unsafe". +var Unsafe *Package + +var ( + universeIota Object + universeByte Type // uint8 alias, but has name "byte" + universeRune Type // int32 alias, but has name "rune" + universeAny Object + universeError Type + universeComparable Object +) + +// Typ contains the predeclared *Basic types indexed by their +// corresponding BasicKind. +// +// The *Basic type for Typ[Byte] will have the name "uint8". +// Use Universe.Lookup("byte").Type() to obtain the specific +// alias basic type named "byte" (and analogous for "rune"). +var Typ = []*Basic{ + Invalid: {Invalid, 0, "invalid type"}, + + Bool: {Bool, IsBoolean, "bool"}, + Int: {Int, IsInteger, "int"}, + Int8: {Int8, IsInteger, "int8"}, + Int16: {Int16, IsInteger, "int16"}, + Int32: {Int32, IsInteger, "int32"}, + Int64: {Int64, IsInteger, "int64"}, + Uint: {Uint, IsInteger | IsUnsigned, "uint"}, + Uint8: {Uint8, IsInteger | IsUnsigned, "uint8"}, + Uint16: {Uint16, IsInteger | IsUnsigned, "uint16"}, + Uint32: {Uint32, IsInteger | IsUnsigned, "uint32"}, + Uint64: {Uint64, IsInteger | IsUnsigned, "uint64"}, + Uintptr: {Uintptr, IsInteger | IsUnsigned, "uintptr"}, + Float32: {Float32, IsFloat, "float32"}, + Float64: {Float64, IsFloat, "float64"}, + Complex64: {Complex64, IsComplex, "complex64"}, + Complex128: {Complex128, IsComplex, "complex128"}, + String: {String, IsString, "string"}, + UnsafePointer: {UnsafePointer, 0, "Pointer"}, + + UntypedBool: {UntypedBool, IsBoolean | IsUntyped, "untyped bool"}, + UntypedInt: {UntypedInt, IsInteger | IsUntyped, "untyped int"}, + UntypedRune: {UntypedRune, IsInteger | IsUntyped, "untyped rune"}, + UntypedFloat: {UntypedFloat, IsFloat | IsUntyped, "untyped float"}, + UntypedComplex: {UntypedComplex, IsComplex | IsUntyped, "untyped complex"}, + UntypedString: {UntypedString, IsString | IsUntyped, "untyped string"}, + UntypedNil: {UntypedNil, IsUntyped, "untyped nil"}, +} + +var aliases = [...]*Basic{ + {Byte, IsInteger | IsUnsigned, "byte"}, + {Rune, IsInteger, "rune"}, +} + +func defPredeclaredTypes() { + for _, t := range Typ { + def(NewTypeName(token.NoPos, nil, t.name, t)) + } + for _, t := range aliases { + def(NewTypeName(token.NoPos, nil, t.name, t)) + } + + // type any = interface{} + // Note: don't use &emptyInterface for the type of any. Using a unique + // pointer allows us to detect any and format it as "any" rather than + // interface{}, which clarifies user-facing error messages significantly. + def(NewTypeName(token.NoPos, nil, "any", &Interface{complete: true, tset: &topTypeSet})) + + // type error interface{ Error() string } + { + obj := NewTypeName(token.NoPos, nil, "error", nil) + obj.setColor(black) + typ := NewNamed(obj, nil, nil) + + // error.Error() string + recv := NewVar(token.NoPos, nil, "", typ) + res := NewVar(token.NoPos, nil, "", Typ[String]) + sig := NewSignatureType(recv, nil, nil, nil, NewTuple(res), false) + err := NewFunc(token.NoPos, nil, "Error", sig) + + // interface{ Error() string } + ityp := &Interface{methods: []*Func{err}, complete: true} + computeInterfaceTypeSet(nil, token.NoPos, ityp) // prevent races due to lazy computation of tset + + typ.SetUnderlying(ityp) + def(obj) + } + + // type comparable interface{} // marked as comparable + { + obj := NewTypeName(token.NoPos, nil, "comparable", nil) + obj.setColor(black) + typ := NewNamed(obj, nil, nil) + + // interface{} // marked as comparable + ityp := &Interface{complete: true, tset: &_TypeSet{nil, allTermlist, true}} + + typ.SetUnderlying(ityp) + def(obj) + } +} + +var predeclaredConsts = [...]struct { + name string + kind BasicKind + val constant.Value +}{ + {"true", UntypedBool, constant.MakeBool(true)}, + {"false", UntypedBool, constant.MakeBool(false)}, + {"iota", UntypedInt, constant.MakeInt64(0)}, +} + +func defPredeclaredConsts() { + for _, c := range predeclaredConsts { + def(NewConst(token.NoPos, nil, c.name, Typ[c.kind], c.val)) + } +} + +func defPredeclaredNil() { + def(&Nil{object{name: "nil", typ: Typ[UntypedNil], color_: black}}) +} + +// A builtinId is the id of a builtin function. +type builtinId int + +const ( + // universe scope + _Append builtinId = iota + _Cap + _Clear + _Close + _Complex + _Copy + _Delete + _Imag + _Len + _Make + _New + _Panic + _Print + _Println + _Real + _Recover + + // package unsafe + _Add + _Alignof + _Offsetof + _Sizeof + _Slice + _SliceData + _String + _StringData + + // testing support + _Assert + _Trace +) + +var predeclaredFuncs = [...]struct { + name string + nargs int + variadic bool + kind exprKind +}{ + _Append: {"append", 1, true, expression}, + _Cap: {"cap", 1, false, expression}, + _Clear: {"clear", 1, false, statement}, + _Close: {"close", 1, false, statement}, + _Complex: {"complex", 2, false, expression}, + _Copy: {"copy", 2, false, statement}, + _Delete: {"delete", 2, false, statement}, + _Imag: {"imag", 1, false, expression}, + _Len: {"len", 1, false, expression}, + _Make: {"make", 1, true, expression}, + _New: {"new", 1, false, expression}, + _Panic: {"panic", 1, false, statement}, + _Print: {"print", 0, true, statement}, + _Println: {"println", 0, true, statement}, + _Real: {"real", 1, false, expression}, + _Recover: {"recover", 0, false, statement}, + + _Add: {"Add", 2, false, expression}, + _Alignof: {"Alignof", 1, false, expression}, + _Offsetof: {"Offsetof", 1, false, expression}, + _Sizeof: {"Sizeof", 1, false, expression}, + _Slice: {"Slice", 2, false, expression}, + _SliceData: {"SliceData", 1, false, expression}, + _String: {"String", 2, false, expression}, + _StringData: {"StringData", 1, false, expression}, + + _Assert: {"assert", 1, false, statement}, + _Trace: {"trace", 0, true, statement}, +} + +func defPredeclaredFuncs() { + for i := range predeclaredFuncs { + id := builtinId(i) + if id == _Assert || id == _Trace { + continue // only define these in testing environment + } + def(newBuiltin(id)) + } +} + +// DefPredeclaredTestFuncs defines the assert and trace built-ins. +// These built-ins are intended for debugging and testing of this +// package only. +func DefPredeclaredTestFuncs() { + if Universe.Lookup("assert") != nil { + return // already defined + } + def(newBuiltin(_Assert)) + def(newBuiltin(_Trace)) +} + +func init() { + Universe = NewScope(nil, token.NoPos, token.NoPos, "universe") + Unsafe = NewPackage("unsafe", "unsafe") + Unsafe.complete = true + + defPredeclaredTypes() + defPredeclaredConsts() + defPredeclaredNil() + defPredeclaredFuncs() + + universeIota = Universe.Lookup("iota") + universeByte = Universe.Lookup("byte").Type() + universeRune = Universe.Lookup("rune").Type() + universeAny = Universe.Lookup("any") + universeError = Universe.Lookup("error").Type() + universeComparable = Universe.Lookup("comparable") +} + +// Objects with names containing blanks are internal and not entered into +// a scope. Objects with exported names are inserted in the unsafe package +// scope; other objects are inserted in the universe scope. +func def(obj Object) { + assert(obj.color() == black) + name := obj.Name() + if strings.Contains(name, " ") { + return // nothing to do + } + // fix Obj link for named types + if typ, _ := obj.Type().(*Named); typ != nil { + typ.obj = obj.(*TypeName) + } + // exported identifiers go into package unsafe + scope := Universe + if obj.Exported() { + scope = Unsafe.scope + // set Pkg field + switch obj := obj.(type) { + case *TypeName: + obj.pkg = Unsafe + case *Builtin: + obj.pkg = Unsafe + default: + unreachable() + } + } + if scope.Insert(obj) != nil { + panic("double declaration of predeclared identifier") + } +} diff --git a/src/go/types/validtype.go b/src/go/types/validtype.go new file mode 100644 index 0000000..d62c398 --- /dev/null +++ b/src/go/types/validtype.go @@ -0,0 +1,256 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +// validType verifies that the given type does not "expand" indefinitely +// producing a cycle in the type graph. +// (Cycles involving alias types, as in "type A = [10]A" are detected +// earlier, via the objDecl cycle detection mechanism.) +func (check *Checker) validType(typ *Named) { + check.validType0(typ, nil, nil) +} + +// validType0 checks if the given type is valid. If typ is a type parameter +// its value is looked up in the type argument list of the instantiated +// (enclosing) type, if it exists. Otherwise the type parameter must be from +// an enclosing function and can be ignored. +// The nest list describes the stack (the "nest in memory") of types which +// contain (or embed in the case of interfaces) other types. For instance, a +// struct named S which contains a field of named type F contains (the memory +// of) F in S, leading to the nest S->F. If a type appears in its own nest +// (say S->F->S) we have an invalid recursive type. The path list is the full +// path of named types in a cycle, it is only needed for error reporting. +func (check *Checker) validType0(typ Type, nest, path []*Named) bool { + switch t := typ.(type) { + case nil: + // We should never see a nil type but be conservative and panic + // only in debug mode. + if debug { + panic("validType0(nil)") + } + + case *Array: + return check.validType0(t.elem, nest, path) + + case *Struct: + for _, f := range t.fields { + if !check.validType0(f.typ, nest, path) { + return false + } + } + + case *Union: + for _, t := range t.terms { + if !check.validType0(t.typ, nest, path) { + return false + } + } + + case *Interface: + for _, etyp := range t.embeddeds { + if !check.validType0(etyp, nest, path) { + return false + } + } + + case *Named: + // Exit early if we already know t is valid. + // This is purely an optimization but it prevents excessive computation + // times in pathological cases such as testdata/fixedbugs/issue6977.go. + // (Note: The valids map could also be allocated locally, once for each + // validType call.) + if check.valids.lookup(t) != nil { + break + } + + // Don't report a 2nd error if we already know the type is invalid + // (e.g., if a cycle was detected earlier, via under). + // Note: ensure that t.orig is fully resolved by calling Underlying(). + if t.Underlying() == Typ[Invalid] { + return false + } + + // If the current type t is also found in nest, (the memory of) t is + // embedded in itself, indicating an invalid recursive type. + for _, e := range nest { + if Identical(e, t) { + // We have a cycle. If t != t.Origin() then t is an instance of + // the generic type t.Origin(). Because t is in the nest, t must + // occur within the definition (RHS) of the generic type t.Origin(), + // directly or indirectly, after expansion of the RHS. + // Therefore t.Origin() must be invalid, no matter how it is + // instantiated since the instantiation t of t.Origin() happens + // inside t.Origin()'s RHS and thus is always the same and always + // present. + // Therefore we can mark the underlying of both t and t.Origin() + // as invalid. If t is not an instance of a generic type, t and + // t.Origin() are the same. + // Furthermore, because we check all types in a package for validity + // before type checking is complete, any exported type that is invalid + // will have an invalid underlying type and we can't reach here with + // such a type (invalid types are excluded above). + // Thus, if we reach here with a type t, both t and t.Origin() (if + // different in the first place) must be from the current package; + // they cannot have been imported. + // Therefore it is safe to change their underlying types; there is + // no chance for a race condition (the types of the current package + // are not yet available to other goroutines). + assert(t.obj.pkg == check.pkg) + assert(t.Origin().obj.pkg == check.pkg) + t.underlying = Typ[Invalid] + t.Origin().underlying = Typ[Invalid] + + // Find the starting point of the cycle and report it. + // Because each type in nest must also appear in path (see invariant below), + // type t must be in path since it was found in nest. But not every type in path + // is in nest. Specifically t may appear in path with an earlier index than the + // index of t in nest. Search again. + for start, p := range path { + if Identical(p, t) { + check.cycleError(makeObjList(path[start:])) + return false + } + } + panic("cycle start not found") + } + } + + // No cycle was found. Check the RHS of t. + // Every type added to nest is also added to path; thus every type that is in nest + // must also be in path (invariant). But not every type in path is in nest, since + // nest may be pruned (see below, *TypeParam case). + if !check.validType0(t.Origin().fromRHS, append(nest, t), append(path, t)) { + return false + } + + check.valids.add(t) // t is valid + + case *TypeParam: + // A type parameter stands for the type (argument) it was instantiated with. + // Check the corresponding type argument for validity if we are in an + // instantiated type. + if len(nest) > 0 { + inst := nest[len(nest)-1] // the type instance + // Find the corresponding type argument for the type parameter + // and proceed with checking that type argument. + for i, tparam := range inst.TypeParams().list() { + // The type parameter and type argument lists should + // match in length but be careful in case of errors. + if t == tparam && i < inst.TypeArgs().Len() { + targ := inst.TypeArgs().At(i) + // The type argument must be valid in the enclosing + // type (where inst was instantiated), hence we must + // check targ's validity in the type nest excluding + // the current (instantiated) type (see the example + // at the end of this file). + // For error reporting we keep the full path. + return check.validType0(targ, nest[:len(nest)-1], path) + } + } + } + } + + return true +} + +// makeObjList returns the list of type name objects for the given +// list of named types. +func makeObjList(tlist []*Named) []Object { + olist := make([]Object, len(tlist)) + for i, t := range tlist { + olist[i] = t.obj + } + return olist +} + +// Here is an example illustrating why we need to exclude the +// instantiated type from nest when evaluating the validity of +// a type parameter. Given the declarations +// +// var _ A[A[string]] +// +// type A[P any] struct { _ B[P] } +// type B[P any] struct { _ P } +// +// we want to determine if the type A[A[string]] is valid. +// We start evaluating A[A[string]] outside any type nest: +// +// A[A[string]] +// nest = +// path = +// +// The RHS of A is now evaluated in the A[A[string]] nest: +// +// struct{_ B[Pā‚]} +// nest = A[A[string]] +// path = A[A[string]] +// +// The struct has a single field of type B[Pā‚] with which +// we continue: +// +// B[Pā‚] +// nest = A[A[string]] +// path = A[A[string]] +// +// struct{_ Pā‚‚} +// nest = A[A[string]]->B[P] +// path = A[A[string]]->B[P] +// +// Eventually we reach the type parameter P of type B (Pā‚‚): +// +// Pā‚‚ +// nest = A[A[string]]->B[P] +// path = A[A[string]]->B[P] +// +// The type argument for P of B is the type parameter P of A (Pā‚). +// It must be evaluated in the type nest that existed when B was +// instantiated: +// +// Pā‚ +// nest = A[A[string]] <== type nest at B's instantiation time +// path = A[A[string]]->B[P] +// +// If we'd use the current nest it would correspond to the path +// which will be wrong as we will see shortly. P's type argument +// is A[string], which again must be evaluated in the type nest +// that existed when A was instantiated with A[string]. That type +// nest is empty: +// +// A[string] +// nest = <== type nest at A's instantiation time +// path = A[A[string]]->B[P] +// +// Evaluation then proceeds as before for A[string]: +// +// struct{_ B[Pā‚]} +// nest = A[string] +// path = A[A[string]]->B[P]->A[string] +// +// Now we reach B[P] again. If we had not adjusted nest, it would +// correspond to path, and we would find B[P] in nest, indicating +// a cycle, which would clearly be wrong since there's no cycle in +// A[string]: +// +// B[Pā‚] +// nest = A[string] +// path = A[A[string]]->B[P]->A[string] <== path contains B[P]! +// +// But because we use the correct type nest, evaluation proceeds without +// errors and we get the evaluation sequence: +// +// struct{_ Pā‚‚} +// nest = A[string]->B[P] +// path = A[A[string]]->B[P]->A[string]->B[P] +// Pā‚‚ +// nest = A[string]->B[P] +// path = A[A[string]]->B[P]->A[string]->B[P] +// Pā‚ +// nest = A[string] +// path = A[A[string]]->B[P]->A[string]->B[P] +// string +// nest = +// path = A[A[string]]->B[P]->A[string]->B[P] +// +// At this point we're done and A[A[string]] and is valid. diff --git a/src/go/types/version.go b/src/go/types/version.go new file mode 100644 index 0000000..3958ec9 --- /dev/null +++ b/src/go/types/version.go @@ -0,0 +1,83 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types + +import ( + "fmt" + "go/ast" + "go/token" + . "internal/types/errors" + "regexp" + "strconv" + "strings" +) + +// langCompat reports an error if the representation of a numeric +// literal is not compatible with the current language version. +func (check *Checker) langCompat(lit *ast.BasicLit) { + s := lit.Value + if len(s) <= 2 || check.allowVersion(check.pkg, 1, 13) { + return + } + // len(s) > 2 + if strings.Contains(s, "_") { + check.error(lit, UnsupportedFeature, "underscores in numeric literals requires go1.13 or later") + return + } + if s[0] != '0' { + return + } + radix := s[1] + if radix == 'b' || radix == 'B' { + check.error(lit, UnsupportedFeature, "binary literals requires go1.13 or later") + return + } + if radix == 'o' || radix == 'O' { + check.error(lit, UnsupportedFeature, "0o/0O-style octal literals requires go1.13 or later") + return + } + if lit.Kind != token.INT && (radix == 'x' || radix == 'X') { + check.error(lit, UnsupportedFeature, "hexadecimal floating-point literals requires go1.13 or later") + } +} + +// allowVersion reports whether the given package +// is allowed to use version major.minor. +func (check *Checker) allowVersion(pkg *Package, major, minor int) bool { + // We assume that imported packages have all been checked, + // so we only have to check for the local package. + if pkg != check.pkg { + return true + } + ma, mi := check.version.major, check.version.minor + return ma == 0 && mi == 0 || ma > major || ma == major && mi >= minor +} + +type version struct { + major, minor int +} + +// parseGoVersion parses a Go version string (such as "go1.12") +// and returns the version, or an error. If s is the empty +// string, the version is 0.0. +func parseGoVersion(s string) (v version, err error) { + if s == "" { + return + } + matches := goVersionRx.FindStringSubmatch(s) + if matches == nil { + err = fmt.Errorf(`should be something like "go1.12"`) + return + } + v.major, err = strconv.Atoi(matches[1]) + if err != nil { + return + } + v.minor, err = strconv.Atoi(matches[2]) + return +} + +// goVersionRx matches a Go version string, e.g. "go1.12". +var goVersionRx = regexp.MustCompile(`^go([1-9]\d*)\.(0|[1-9]\d*)$`) -- cgit v1.2.3