diff --git a/.travis.yml b/.travis.yml index 964b903136..f7f8311130 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,15 +15,6 @@ install: - cp -L -R $GOPATH/src/k8s.io/kubernetes/vendor/ $GOPATH/src/ - rm -r $GOPATH/src/k8s.io/kubernetes/vendor/ -# (2) Fetch md-check along with all its dependencies. -- git clone --depth=50 --branch=master https://github.com/kubernetes/md-check $HOME/gopath/src/k8s.io/md-check -- go get -t -v k8s.io/md-check - -# (3) Build mungedocs -- go install ./_tools/mungedocs - script: - go test -v k8s.io/kubernetes.github.io/test -- $GOPATH/bin/md-check --root-dir=$HOME/gopath/src/k8s.io/kubernetes.github.io - ./verify-docs-format.sh -- $GOPATH/bin/mungedocs --verbose --verify --upstream=origin --root-dir=$HOME/gopath/src/k8s.io/kubernetes.github.io/docs/ --repo-root=$HOME/gopath/src/k8s.io/kubernetes.github.io --skip-munges=remove-whitespace,blank-lines-surround-preformatted,header-lines,sync-examples,analytics,analytics,kubectl-dash-f,table-of-contents,md-links,kubectl-dash-f diff --git a/_tools/mungedocs/README.md b/_tools/mungedocs/README.md deleted file mode 100644 index 5fe3ed106c..0000000000 --- a/_tools/mungedocs/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Documentation Mungers - -Basically this is like lint/gofmt for md docs. - -It basically does the following: -- iterate over all files in the given doc root. -- for each file split it into a slice (mungeLines) of lines (mungeLine) -- a mungeline has metadata about each line typically determined by a 'fast' regex. - - metadata contains things like 'is inside a preformatted block' - - contains a markdown header - - has a link to another file - - etc.. - - if you have a really slow regex with a lot of backtracking you might want to write a fast one to limit how often you run the slow one. -- each munger is then called in turn - - they are given the mungeLines - - they create an entirely new set of mungeLines with their modifications - - the new set is returned -- the new set is then fed into the next munger. -- in the end we might commit the end mungeLines to the file or not (--verify) - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cmd/mungedocs/README.md?pixel)]() diff --git a/_tools/mungedocs/analytics.go b/_tools/mungedocs/analytics.go deleted file mode 100644 index a7eaefa080..0000000000 --- a/_tools/mungedocs/analytics.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "strings" -) - -const analyticsMungeTag = "GENERATED_ANALYTICS" -const analyticsLinePrefix = "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/" - -func updateAnalytics(fileName string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - fileName, err := makeRepoRelative(fileName, fileName) - if err != nil { - return mlines, err - } - - link := fmt.Sprintf(analyticsLinePrefix+"%s?pixel)]()", fileName) - insertLines := getMungeLines(link) - mlines, err = removeMacroBlock(analyticsMungeTag, mlines) - if err != nil { - return mlines, err - } - - // Remove floating analytics links not surrounded by the munge tags. - for _, mline := range mlines { - if mline.preformatted || mline.header || mline.beginTag || mline.endTag { - out = append(out, mline) - continue - } - if strings.HasPrefix(mline.data, analyticsLinePrefix) { - continue - } - out = append(out, mline) - } - out = appendMacroBlock(out, analyticsMungeTag) - out, err = updateMacroBlock(out, analyticsMungeTag, insertLines) - if err != nil { - return mlines, err - } - return out, nil -} diff --git a/_tools/mungedocs/analytics_test.go b/_tools/mungedocs/analytics_test.go deleted file mode 100644 index 99306f2fe0..0000000000 --- a/_tools/mungedocs/analytics_test.go +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" -) - -func TestAnalytics(t *testing.T) { - b := beginMungeTag("GENERATED_ANALYTICS") - e := endMungeTag("GENERATED_ANALYTICS") - var cases = []struct { - in string - expected string - }{ - { - "aoeu", - "aoeu" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + "\n" + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()", - "aoeu" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n", - "aoeu" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n", - "aoeu" + "\n" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "prefix" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + - "\n" + "suffix", - "prefix" + "\n" + "suffix" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n", - "aoeu" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - out, err := updateAnalytics("path/to/file-name.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - - if !expected.Equal(out) { - t.Errorf("Case %d Expected \n\n%v\n\n but got \n\n%v\n\n", i, expected.String(), out.String()) - } - } -} diff --git a/_tools/mungedocs/example_syncer.go b/_tools/mungedocs/example_syncer.go deleted file mode 100644 index c15255be43..0000000000 --- a/_tools/mungedocs/example_syncer.go +++ /dev/null @@ -1,121 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "io/ioutil" - "regexp" - "strings" -) - -const exampleToken = "EXAMPLE" - -const exampleLineStart = " -// -// ```yaml -// foo: -// bar: -// ``` -// -// [Download example](../../examples/guestbook/frontend-service.yaml?raw=true) -// -func syncExamples(filePath string, mlines mungeLines) (mungeLines, error) { - var err error - type exampleTag struct { - token string - linkText string - fileType string - } - exampleTags := []exampleTag{} - - // collect all example Tags - for _, mline := range mlines { - if mline.preformatted || !mline.beginTag { - continue - } - line := mline.data - if !strings.HasPrefix(line, exampleLineStart) { - continue - } - match := exampleMungeTagRE.FindStringSubmatch(line) - if len(match) < 4 { - err = fmt.Errorf("Found unparsable EXAMPLE munge line %v", line) - return mlines, err - } - tag := exampleTag{ - token: exampleToken + " " + match[1], - linkText: match[1], - fileType: match[3], - } - exampleTags = append(exampleTags, tag) - } - // update all example Tags - for _, tag := range exampleTags { - ft := "" - if tag.fileType == "json" { - ft = "json" - } - if tag.fileType == "yaml" { - ft = "yaml" - } - example, err := exampleContent(filePath, tag.linkText, ft) - if err != nil { - return mlines, err - } - mlines, err = updateMacroBlock(mlines, tag.token, example) - if err != nil { - return mlines, err - } - } - return mlines, nil -} - -// exampleContent retrieves the content of the file at linkPath -func exampleContent(filePath, linkPath, fileType string) (mungeLines, error) { - repoRel, err := makeRepoRelative(linkPath, filePath) - if err != nil { - return nil, err - } - - fileRel, err := makeFileRelative(linkPath, filePath) - if err != nil { - return nil, err - } - - dat, err := ioutil.ReadFile(repoRel) - if err != nil { - return nil, err - } - - // remove leading and trailing spaces and newlines - trimmedFileContent := strings.TrimSpace(string(dat)) - content := fmt.Sprintf("\n```%s\n%s\n```\n\n[Download example](%s?raw=true)", fileType, trimmedFileContent, fileRel) - out := getMungeLines(content) - return out, nil -} diff --git a/_tools/mungedocs/example_syncer_test.go b/_tools/mungedocs/example_syncer_test.go deleted file mode 100644 index 3d51b28c9d..0000000000 --- a/_tools/mungedocs/example_syncer_test.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" -) - -func Test_syncExamples(t *testing.T) { - var podExample = `apiVersion: v1 -kind: Pod -metadata: - name: nginx -spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 -` - var textExample = `some text -` - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "\n\n", - "\n\n```yaml\n" + podExample + "```\n\n[Download example](testdata/pod.yaml?raw=true)\n\n", - }, - { - "\n\n", - "\n\n```yaml\n" + podExample + "```\n\n[Download example](../mungedocs/testdata/pod.yaml?raw=true)\n\n", - }, - { - "\n\n", - "\n\n```\n" + textExample + "```\n\n[Download example](testdata/example.txt?raw=true)\n\n", - }, - } - repoRoot = "" - for _, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := syncExamples("filename.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - - if !expected.Equal(actual) { - t.Errorf("Expected example \n'%q' but got \n'%q'", expected.String(), actual.String()) - } - } -} diff --git a/_tools/mungedocs/headers.go b/_tools/mungedocs/headers.go deleted file mode 100644 index e23ae7536e..0000000000 --- a/_tools/mungedocs/headers.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "regexp" -) - -var headerRegex = regexp.MustCompile(`^(#+)\s*(.*)$`) - -func fixHeaderLine(mlines mungeLines, newlines mungeLines, linenum int) mungeLines { - var out mungeLines - - mline := mlines[linenum] - line := mlines[linenum].data - - matches := headerRegex.FindStringSubmatch(line) - if matches == nil { - out = append(out, mline) - return out - } - - // There must be a blank line before the # (unless first line in file) - if linenum != 0 { - newlen := len(newlines) - if newlines[newlen-1].data != "" { - out = append(out, blankMungeLine) - } - } - - // There must be a space AFTER the ##'s - newline := fmt.Sprintf("%s %s", matches[1], matches[2]) - newmline := newMungeLine(newline) - out = append(out, newmline) - - // The next line needs to be a blank line (unless last line in file) - if len(mlines) > linenum+1 && mlines[linenum+1].data != "" { - out = append(out, blankMungeLine) - } - return out -} - -// Header lines need whitespace around them and after the #s. -func updateHeaderLines(filePath string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - for i, mline := range mlines { - if mline.preformatted { - out = append(out, mline) - continue - } - if !mline.header { - out = append(out, mline) - continue - } - newLines := fixHeaderLine(mlines, out, i) - out = append(out, newLines...) - } - return out, nil -} diff --git a/_tools/mungedocs/headers_test.go b/_tools/mungedocs/headers_test.go deleted file mode 100644 index d308b01034..0000000000 --- a/_tools/mungedocs/headers_test.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" -) - -func TestHeaderLines(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "# ok", - "# ok", - }, - { - "## ok", - "## ok", - }, - { - "##### ok", - "##### ok", - }, - { - "##fix", - "## fix", - }, - { - "foo\n\n##fix\n\nbar", - "foo\n\n## fix\n\nbar", - }, - { - "foo\n##fix\nbar", - "foo\n\n## fix\n\nbar", - }, - { - "foo\n```\n##fix\n```\nbar", - "foo\n```\n##fix\n```\nbar", - }, - { - "foo\n#fix1\n##fix2\nbar", - "foo\n\n# fix1\n\n## fix2\n\nbar", - }, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updateHeaderLines("filename.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - - if !actual.Equal(expected) { - t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) - } - } -} diff --git a/_tools/mungedocs/kubectl_dash_f.go b/_tools/mungedocs/kubectl_dash_f.go deleted file mode 100644 index 2dae48cc85..0000000000 --- a/_tools/mungedocs/kubectl_dash_f.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "os" - "path" - "strings" -) - -// Looks for lines that have kubectl commands with -f flags and files that -// don't exist. -func updateKubectlFileTargets(file string, mlines mungeLines) (mungeLines, error) { - var errors []string - for i, mline := range mlines { - if !mline.preformatted { - continue - } - if err := lookForKubectl(mline.data, i); err != nil { - errors = append(errors, err.Error()) - } - } - err := error(nil) - if len(errors) != 0 { - err = fmt.Errorf("%s", strings.Join(errors, "\n")) - } - return mlines, err -} - -func lookForKubectl(line string, lineNum int) error { - fields := strings.Fields(line) - for i := range fields { - if fields[i] == "kubectl" { - return gotKubectl(lineNum, fields, i) - } - } - return nil -} - -func gotKubectl(lineNum int, fields []string, fieldNum int) error { - for i := fieldNum + 1; i < len(fields); i++ { - switch fields[i] { - case "create", "update", "replace", "delete": - return gotCommand(lineNum, fields, i) - } - } - return nil -} - -func gotCommand(lineNum int, fields []string, fieldNum int) error { - for i := fieldNum + 1; i < len(fields); i++ { - if strings.HasPrefix(fields[i], "-f") { - return gotDashF(lineNum, fields, i) - } - } - return nil -} - -func gotDashF(lineNum int, fields []string, fieldNum int) error { - target := "" - if fields[fieldNum] == "-f" { - if fieldNum+1 == len(fields) { - return fmt.Errorf("ran out of fields after '-f'") - } - target = fields[fieldNum+1] - } else { - target = fields[fieldNum][2:] - } - // Turn dirs into file-like names. - target = strings.TrimRight(target, "/") - - // Now exclude special-cases - - if target == "-" || target == "FILENAME" { - // stdin and "FILENAME" are OK - return nil - } - if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") { - // URLs are ok - return nil - } - if strings.HasPrefix(target, "./") { - // Same-dir files are usually created in the same example - return nil - } - if strings.HasPrefix(target, "~/") { - // Home directory may also be created by the same example - return nil - } - if strings.HasPrefix(target, "/") { - // Absolute paths tend to be /tmp/* and created in the same example. - return nil - } - if strings.HasPrefix(target, "$") { - // Allow the start of the target to be an environment - // variable that points to the root of the kubernetes - // repo. - split := strings.SplitN(target, "/", 2) - if len(split) == 2 { - target = split[1] - } - } - - // If we got here we expect the file to exist. - _, err := os.Stat(path.Join(repoRoot, target)) - if os.IsNotExist(err) { - return fmt.Errorf("%d: target file %q does not exist", lineNum, target) - } - return err -} diff --git a/_tools/mungedocs/kubectl_dash_f_test.go b/_tools/mungedocs/kubectl_dash_f_test.go deleted file mode 100644 index 6f18fd547a..0000000000 --- a/_tools/mungedocs/kubectl_dash_f_test.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import "testing" - -func TestKubectlDashF(t *testing.T) { - var cases = []struct { - in string - ok bool - }{ - // No match - {"", true}, - { - "Foo\nBar\n", - true, - }, - { - "Foo\nkubectl blah blech\nBar", - true, - }, - { - "Foo\n```shell\nkubectl blah blech\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create blech\n```\nBar", - true, - }, - // Special cases - { - "Foo\n```\nkubectl -blah create -f -\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f-\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f FILENAME\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -fFILENAME\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f http://google.com/foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -fhttp://google.com/foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f ./foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f./foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f /foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f/foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f~/foobar\n```\nBar", - true, - }, - // Real checks - { - "Foo\n```\nkubectl -blah create -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -fmungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah update -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah update -fmungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah replace -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah replace -fmungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah delete -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah delete -fmungedocs.go\n```\nBar", - true, - }, - // Failures - { - "Foo\n```\nkubectl -blah delete -f does_not_exist\n```\nBar", - false, - }, - { - "Foo\n```\nkubectl -blah delete -fdoes_not_exist\n```\nBar", - false, - }, - } - for i, c := range cases { - repoRoot = "" - in := getMungeLines(c.in) - _, err := updateKubectlFileTargets("filename.md", in) - if err != nil && c.ok { - t.Errorf("case[%d]: expected success, got %v", i, err) - } - if err == nil && !c.ok { - t.Errorf("case[%d]: unexpected success", i) - } - } -} diff --git a/_tools/mungedocs/links.go b/_tools/mungedocs/links.go deleted file mode 100644 index c05cdfa187..0000000000 --- a/_tools/mungedocs/links.go +++ /dev/null @@ -1,238 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "errors" - "fmt" - "net/url" - "os" - "path" - "regexp" - "strings" -) - -var ( - // Finds markdown links of the form [foo](bar "alt-text"). - linkRE = regexp.MustCompile(`\[([^]]*)\]\(([^)]*)\)`) - // Finds markdown link typos of the form (foo)[bar] - badLinkRE = regexp.MustCompile(`\([^]()]*\)\[[^]()]*\]`) - // Splits the link target into link target and alt-text. - altTextRE = regexp.MustCompile(`([^)]*)( ".*")`) -) - -func processLink(in string, filePath string) (string, error) { - var errs []string - out := linkRE.ReplaceAllStringFunc(in, func(in string) string { - var err error - match := linkRE.FindStringSubmatch(in) - if match == nil { - errs = append(errs, fmt.Sprintf("Detected this line had a link, but unable to parse, %v", in)) - return "" - } - // match[0] is the entire expression; - visibleText := match[1] - linkText := match[2] - altText := "" - if parts := altTextRE.FindStringSubmatch(linkText); parts != nil { - linkText = parts[1] - altText = parts[2] - } - - // clean up some random garbage I found in our docs. - linkText = strings.Trim(linkText, " ") - linkText = strings.Trim(linkText, "\n") - linkText = strings.Trim(linkText, " ") - - u, terr := url.Parse(linkText) - if terr != nil { - errs = append(errs, fmt.Sprintf("link %q is unparsable: %v", linkText, terr)) - return in - } - - if u.Host != "" && u.Host != "github.com" { - // We only care about relative links and links within github. - return in - } - - suggestedVisibleText := visibleText - if u.Path != "" && !strings.HasPrefix(linkText, "TODO:") { - newPath, targetExists := checkPath(filePath, path.Clean(u.Path)) - if !targetExists { - errs = append(errs, fmt.Sprintf("%q: target not found", linkText)) - return in - } - u.Path = newPath - if strings.HasPrefix(u.Path, "/") { - u.Host = "github.com" - u.Scheme = "https" - } else { - // Remove host and scheme from relative paths - u.Host = "" - u.Scheme = "" - } - // Make the visible text show the absolute path if it's - // not nested in or beneath the current directory. - if strings.HasPrefix(u.Path, "..") { - dir := path.Dir(filePath) - suggestedVisibleText, err = makeRepoRelative(path.Join(dir, u.Path), filePath) - if err != nil { - errs = append(errs, fmt.Sprintf("%q: unable to make path relative", filePath)) - return in - } - } else { - suggestedVisibleText = u.Path - } - var unescaped string - if unescaped, err = url.QueryUnescape(u.String()); err != nil { - // Remove %28 type stuff, be nice to humans. - // And don't fight with the toc generator. - linkText = unescaped - } else { - linkText = u.String() - } - } - // If the current visible text is trying to be a file name, use - // the correct file name. - if strings.HasSuffix(visibleText, ".md") && !strings.ContainsAny(visibleText, ` '"`+"`") { - visibleText = suggestedVisibleText - } - - return fmt.Sprintf("[%s](%s)", visibleText, linkText+altText) - }) - if len(errs) != 0 { - return "", errors.New(strings.Join(errs, ",")) - } - return out, nil -} - -// updateLinks assumes lines has links in markdown syntax, and verifies that -// any relative links actually point to files that exist. -func updateLinks(filePath string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - allErrs := []string{} - - for lineNum, mline := range mlines { - if mline.preformatted { - out = append(out, mline) - continue - } - if badMatch := badLinkRE.FindString(mline.data); badMatch != "" { - allErrs = append(allErrs, - fmt.Sprintf("On line %d: found backwards markdown link %q", lineNum, badMatch)) - } - if !mline.link { - out = append(out, mline) - continue - } - line, err := processLink(mline.data, filePath) - if err != nil { - var s = fmt.Sprintf("On line %d: %s", lineNum, err.Error()) - err := errors.New(s) - allErrs = append(allErrs, err.Error()) - } - ml := newMungeLine(line) - out = append(out, ml) - } - err := error(nil) - if len(allErrs) != 0 { - err = fmt.Errorf("%s", strings.Join(allErrs, "\n")) - } - return out, err -} - -// We have to append together before path.Clean will be able to tell that stuff -// like ../docs isn't needed. -func cleanPath(dirPath, linkPath string) string { - clean := path.Clean(path.Join(dirPath, linkPath)) - if strings.HasPrefix(clean, dirPath+"/") { - out := strings.TrimPrefix(clean, dirPath+"/") - if out != linkPath { - fmt.Printf("%s -> %s\n", linkPath, out) - } - return out - } - return linkPath -} - -func checkPath(filePath, linkPath string) (newPath string, ok bool) { - dir := path.Dir(filePath) - absFilePrefixes := []string{ - "/kubernetes/kubernetes/blob/master/", - "/kubernetes/kubernetes/tree/master/", - } - for _, prefix := range absFilePrefixes { - if strings.HasPrefix(linkPath, prefix) { - linkPath = strings.TrimPrefix(linkPath, prefix) - // Now linkPath is relative to the root of the repo. The below - // loop that adds ../ at the beginning of the path should find - // the right path. - break - } - } - if strings.HasPrefix(linkPath, "/") { - // These links might go to e.g. the github issues page, or a - // file at a particular revision, or another github project - // entirely. - return linkPath, true - } - linkPath = cleanPath(dir, linkPath) - - // Fast exit if the link is already correct. - if info, err := os.Stat(path.Join(dir, linkPath)); err == nil { - if info.IsDir() { - return linkPath + "/", true - } - return linkPath, true - } - - for strings.HasPrefix(linkPath, "../") { - linkPath = strings.TrimPrefix(linkPath, "../") - } - - // Fix - vs _ automatically - nameMungers := []func(string) string{ - func(s string) string { return s }, - func(s string) string { return strings.Replace(s, "-", "_", -1) }, - func(s string) string { return strings.Replace(s, "_", "-", -1) }, - } - // Fix being moved into/out of admin (replace "admin" with directory - // you're doing mass movements to/from). - pathMungers := []func(string) string{ - func(s string) string { return s }, - func(s string) string { return path.Join("admin", s) }, - func(s string) string { return strings.TrimPrefix(s, "admin/") }, - } - - for _, namer := range nameMungers { - for _, pather := range pathMungers { - newPath = pather(namer(linkPath)) - for i := 0; i < 7; i++ { - // The file must exist. - target := path.Join(dir, newPath) - if info, err := os.Stat(target); err == nil { - if info.IsDir() { - return newPath + "/", true - } - return cleanPath(dir, newPath), true - } - newPath = path.Join("..", newPath) - } - } - } - return linkPath, false -} diff --git a/_tools/mungedocs/links_test.go b/_tools/mungedocs/links_test.go deleted file mode 100644 index 37f8e53171..0000000000 --- a/_tools/mungedocs/links_test.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "testing" -) - -var _ = fmt.Printf - -func TestBadLinks(t *testing.T) { - var cases = []struct { - in string - }{ - {"[NOTREADME](https://github.com/kubernetes/kubernetes/tree/master/NOTREADME.md)"}, - {"[NOTREADME](https://github.com/kubernetes/kubernetes/tree/master/docs/NOTREADME.md)"}, - {"[NOTREADME](../NOTREADME.md)"}, - } - for _, c := range cases { - in := getMungeLines(c.in) - _, err := updateLinks("filename.md", in) - if err == nil { - t.Errorf("Expected error") - } - } -} -func TestGoodLinks(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - {"[README](https://lwn.net)", - "[README](https://lwn.net)"}, - // _ to - - {"[README](https://github.com/kubernetes/kubernetes/tree/master/cmd/mungedocs/testdata/test_dashes.md)", - "[README](../../cmd/mungedocs/testdata/test-dashes.md)"}, - // - to _ - {"[README](../../cmd/mungedocs/testdata/test-underscores.md)", - "[README](../../cmd/mungedocs/testdata/test_underscores.md)"}, - - // Does this even make sense? i dunno - {"[README](/docs/README.md)", - "[README](https://github.com/docs/README.md)"}, - {"[README](/kubernetes/kubernetes/tree/master/cmd/mungedocs/testdata/README.md)", - "[README](../../cmd/mungedocs/testdata/README.md)"}, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updateLinks("filename.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - - if !actual.Equal(expected) { - t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) - } - } -} diff --git a/_tools/mungedocs/mungedocs.go b/_tools/mungedocs/mungedocs.go deleted file mode 100644 index ef3eb2da1a..0000000000 --- a/_tools/mungedocs/mungedocs.go +++ /dev/null @@ -1,235 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "errors" - "flag" - "fmt" - "io/ioutil" - "os" - "path" - "path/filepath" - "strings" -) - -var ( - verbose = flag.Bool("verbose", false, "On verification failure, emit pre-munge and post-munge versions.") - verify = flag.Bool("verify", false, "Exit with status 1 if files would have needed changes but do not change.") - norecurse = flag.Bool("norecurse", false, "Only process the files of --root-dir.") - upstream = flag.String("upstream", "upstream", "The name of the upstream Git remote to pull from") - rootDir = flag.String("root-dir", "", "Root directory containing documents to be processed.") - // "repo-root" seems like a dumb name, this is the relative path (from rootDir) to get to the repoRoot - relRoot = flag.String("repo-root", "..", `Appended to --root-dir to get the repository root. -It's done this way so that generally you just have to set --root-dir. -Examples: - * --root-dir=docs/ --repo-root=.. means the repository root is ./ - * --root-dir=/usr/local/long/path/repo/docs/ --repo-root=.. means the repository root is /usr/local/long/path/repo/ - * --root-dir=/usr/local/long/path/repo/docs/admin --repo-root=../.. means the repository root is /usr/local/long/path/repo/`) - skipMunges = flag.String("skip-munges", "", "Comma-separated list of munges to *not* run. Available munges are: "+availableMungeList) - repoRoot string - - ErrChangesNeeded = errors.New("mungedocs: changes required") - - // All of the munge operations to perform. - // TODO: allow selection from command line. (e.g., just check links in the examples directory.) - allMunges = []munge{ - // Simple "check something" functions must run first. - {"preformat-balance", checkPreformatBalance}, - // Functions which modify state. - {"remove-whitespace", updateWhitespace}, - {"table-of-contents", updateTOC}, - {"md-links", updateLinks}, - {"blank-lines-surround-preformatted", updatePreformatted}, - {"header-lines", updateHeaderLines}, - {"analytics", updateAnalytics}, - {"kubectl-dash-f", updateKubectlFileTargets}, - {"sync-examples", syncExamples}, - } - availableMungeList = func() string { - names := []string{} - for _, m := range allMunges { - names = append(names, m.name) - } - return strings.Join(names, ",") - }() -) - -// a munge processes a document, returning an updated document xor an error. -// The fn is NOT allowed to mutate 'before', if changes are needed it must copy -// data into a new byte array and return that. -type munge struct { - name string - fn func(filePath string, mlines mungeLines) (after mungeLines, err error) -} - -type fileProcessor struct { - // Which munge functions should we call? - munges []munge - - // Are we allowed to make changes? - verifyOnly bool -} - -// Either change a file or verify that it needs no changes (according to modify argument) -func (f fileProcessor) visit(path string) error { - if !strings.HasSuffix(path, ".md") { - return nil - } - - fileBytes, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - mungeLines := getMungeLines(string(fileBytes)) - - modificationsMade := false - errFound := false - filePrinted := false - for _, munge := range f.munges { - after, err := munge.fn(path, mungeLines) - if err != nil || !after.Equal(mungeLines) { - if !filePrinted { - fmt.Printf("%s\n----\n", path) - filePrinted = true - } - fmt.Printf("%s:\n", munge.name) - if *verbose { - if len(mungeLines) <= 20 { - fmt.Printf("INPUT: <<<%v>>>\n", mungeLines) - fmt.Printf("MUNGED: <<<%v>>>\n", after) - } else { - fmt.Printf("not printing failed chunk: too many lines\n") - } - } - if err != nil { - fmt.Println(err) - errFound = true - } else { - fmt.Println("contents were modified") - modificationsMade = true - } - fmt.Println("") - } - mungeLines = after - } - - // Write out new file with any changes. - if modificationsMade { - if f.verifyOnly { - // We're not allowed to make changes. - return ErrChangesNeeded - } - ioutil.WriteFile(path, mungeLines.Bytes(), 0644) - } - if errFound { - return ErrChangesNeeded - } - - return nil -} - -func newWalkFunc(fp *fileProcessor, changesNeeded *bool) filepath.WalkFunc { - return func(path string, info os.FileInfo, err error) error { - stat, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - if path != *rootDir && stat.IsDir() && *norecurse { - return filepath.SkipDir - } - if err := fp.visit(path); err != nil { - *changesNeeded = true - if err != ErrChangesNeeded { - return err - } - } - return nil - } -} - -func wantedMunges() (filtered []munge) { - skipList := strings.Split(*skipMunges, ",") - skipped := map[string]bool{} - for _, m := range skipList { - if len(m) > 0 { - skipped[m] = true - } - } - for _, m := range allMunges { - if !skipped[m.name] { - filtered = append(filtered, m) - } else { - // Remove from the map so we can verify that everything - // requested was in fact valid. - delete(skipped, m.name) - } - } - if len(skipped) != 0 { - fmt.Fprintf(os.Stderr, "ERROR: requested to skip %v, but these are not valid munges. (valid: %v)\n", skipped, availableMungeList) - os.Exit(1) - } - return filtered -} - -func main() { - var err error - flag.Parse() - - if *rootDir == "" { - fmt.Fprintf(os.Stderr, "usage: %s [--help] [--verify] [--norecurse] --root-dir [--skip-munges=] [--upstream=] \n", flag.Arg(0)) - os.Exit(1) - } - - repoRoot = path.Join(*rootDir, *relRoot) - repoRoot, err = filepath.Abs(repoRoot) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) - os.Exit(2) - } - - fp := fileProcessor{ - munges: wantedMunges(), - verifyOnly: *verify, - } - - // For each markdown file under source docs root, process the doc. - // - If any error occurs: exit with failure (exit >1). - // - If verify is true: exit 0 if no changes needed, exit 1 if changes - // needed. - // - If verify is false: exit 0 if changes successfully made or no - // changes needed, exit 1 if manual changes are needed. - var changesNeeded bool - - err = filepath.Walk(*rootDir, newWalkFunc(&fp, &changesNeeded)) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) - os.Exit(2) - } - if changesNeeded { - if *verify { - fmt.Fprintf(os.Stderr, "FAIL: changes needed but not made due to --verify\n") - } else { - fmt.Fprintf(os.Stderr, "FAIL: some manual changes are still required.\n") - } - os.Exit(1) - } -} diff --git a/_tools/mungedocs/preformatted.go b/_tools/mungedocs/preformatted.go deleted file mode 100644 index 582ba981a1..0000000000 --- a/_tools/mungedocs/preformatted.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import "fmt" - -// Blocks of ``` need to have blank lines on both sides or they don't look -// right in HTML. -func updatePreformatted(filePath string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - inpreformat := false - for i, mline := range mlines { - if !inpreformat && mline.preformatted { - if i == 0 || out[len(out)-1].data != "" { - out = append(out, blankMungeLine) - } - // start of a preformat block - inpreformat = true - } - out = append(out, mline) - if inpreformat && !mline.preformatted { - if i >= len(mlines)-2 || mlines[i+1].data != "" { - out = append(out, blankMungeLine) - } - inpreformat = false - } - } - return out, nil -} - -// If the file ends on a preformatted line, there must have been an imbalance. -func checkPreformatBalance(filePath string, mlines mungeLines) (mungeLines, error) { - if len(mlines) > 0 && mlines[len(mlines)-1].preformatted { - return mlines, fmt.Errorf("unbalanced triple backtick delimiters") - } - return mlines, nil -} diff --git a/_tools/mungedocs/preformatted_test.go b/_tools/mungedocs/preformatted_test.go deleted file mode 100644 index ec93854ba0..0000000000 --- a/_tools/mungedocs/preformatted_test.go +++ /dev/null @@ -1,98 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" -) - -func TestPreformatted(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "```\nbob\n```", - "\n```\nbob\n```\n\n", - }, - { - "```\nbob\n```\n```\nnotbob\n```\n", - "\n```\nbob\n```\n\n```\nnotbob\n```\n\n", - }, - { - "```bob```\n", - "```bob```\n", - }, - { - " ```\n bob\n ```", - "\n ```\n bob\n ```\n\n", - }, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updatePreformatted("filename.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - - if !actual.Equal(expected) { - t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) - } - } -} - -func TestPreformattedImbalance(t *testing.T) { - var cases = []struct { - in string - ok bool - }{ - {"", true}, - {"```\nin\n```", true}, - {"```\nin\n```\nout", true}, - {"```", false}, - {"```\nin\n```\nout\n```", false}, - } - for i, c := range cases { - in := getMungeLines(c.in) - out, err := checkPreformatBalance("filename.md", in) - if err != nil && c.ok { - t.Errorf("case[%d]: expected success", i) - } - if err == nil && !c.ok { - t.Errorf("case[%d]: expected failure", i) - } - // Even in case of misformat, return all the text, - // so that the user's work is not lost. - if !equalMungeLines(out, in) { - t.Errorf("case[%d]: expected munged text to be identical to input text", i) - } - } -} - -func equalMungeLines(a, b mungeLines) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} diff --git a/_tools/mungedocs/testdata/README.md b/_tools/mungedocs/testdata/README.md deleted file mode 100644 index 7b57bd29ea..0000000000 --- a/_tools/mungedocs/testdata/README.md +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/_tools/mungedocs/testdata/example.txt b/_tools/mungedocs/testdata/example.txt deleted file mode 100644 index 7b57bd29ea..0000000000 --- a/_tools/mungedocs/testdata/example.txt +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/_tools/mungedocs/testdata/pod.yaml b/_tools/mungedocs/testdata/pod.yaml deleted file mode 100644 index 89920b83a9..0000000000 --- a/_tools/mungedocs/testdata/pod.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: nginx -spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 \ No newline at end of file diff --git a/_tools/mungedocs/testdata/test-dashes.md b/_tools/mungedocs/testdata/test-dashes.md deleted file mode 100644 index 7b57bd29ea..0000000000 --- a/_tools/mungedocs/testdata/test-dashes.md +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/_tools/mungedocs/testdata/test_underscores.md b/_tools/mungedocs/testdata/test_underscores.md deleted file mode 100644 index 7b57bd29ea..0000000000 --- a/_tools/mungedocs/testdata/test_underscores.md +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/_tools/mungedocs/toc.go b/_tools/mungedocs/toc.go deleted file mode 100644 index 649a954bf5..0000000000 --- a/_tools/mungedocs/toc.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "regexp" - "strings" -) - -const tocMungeTag = "GENERATED_TOC" - -var r = regexp.MustCompile("[^A-Za-z0-9-]") - -// inserts/updates a table of contents in markdown file. -// -// First, builds a ToC. -// Then, finds the magic macro block tags and replaces anything between those with -// the ToC, thereby updating any previously inserted ToC. -// -// TODO(erictune): put this in own package with tests -func updateTOC(filePath string, mlines mungeLines) (mungeLines, error) { - toc := buildTOC(mlines) - updatedMarkdown, err := updateMacroBlock(mlines, tocMungeTag, toc) - if err != nil { - return mlines, err - } - return updatedMarkdown, nil -} - -// builds table of contents for markdown file -// -// First scans for all section headers (lines that begin with "#" but not within code quotes) -// and builds a table of contents from those. Assumes bookmarks for those will be -// like #each-word-in-heading-in-lowercases-with-dashes-instead-of-spaces. -// builds the ToC. - -func buildTOC(mlines mungeLines) mungeLines { - var out mungeLines - bookmarks := map[string]int{} - - for _, mline := range mlines { - if mline.preformatted || !mline.header { - continue - } - // Add a blank line after the munge start tag - if len(out) == 0 { - out = append(out, blankMungeLine) - } - line := mline.data - noSharps := strings.TrimLeft(line, "#") - numSharps := len(line) - len(noSharps) - heading := strings.Trim(noSharps, " \n") - if numSharps > 0 { - indent := strings.Repeat(" ", numSharps-1) - bookmark := strings.Replace(strings.ToLower(heading), " ", "-", -1) - // remove symbols (except for -) in bookmarks - bookmark = r.ReplaceAllString(bookmark, "") - // Incremental counter for duplicate bookmarks - next := bookmarks[bookmark] - bookmarks[bookmark] = next + 1 - if next > 0 { - bookmark = fmt.Sprintf("%s-%d", bookmark, next) - } - tocLine := fmt.Sprintf("%s- [%s](#%s)", indent, heading, bookmark) - out = append(out, newMungeLine(tocLine)) - } - - } - // Add a blank line before the munge end tag - if len(out) != 0 { - out = append(out, blankMungeLine) - } - return out -} diff --git a/_tools/mungedocs/toc_test.go b/_tools/mungedocs/toc_test.go deleted file mode 100644 index 95ff354ed5..0000000000 --- a/_tools/mungedocs/toc_test.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" -) - -func Test_buildTOC(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - {"Lorem ipsum\ndolor sit amet\n", ""}, - { - "# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n", - "\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n", - }, - { - "# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n```bash\n#!/bin/sh\n```", - "\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n", - }, - { - "# Title\nLorem ipsum \n## Section Heading\n### Ok, why doesn't this work? ...add 4 *more* `symbols`!\ndolor sit amet\n", - "\n- [Title](#title)\n - [Section Heading](#section-heading)\n - [Ok, why doesn't this work? ...add 4 *more* `symbols`!](#ok-why-doesnt-this-work-add-4-more-symbols)\n\n", - }, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual := buildTOC(in) - if !expected.Equal(actual) { - t.Errorf("Case[%d] Expected TOC '%v' but got '%v'", i, expected.String(), actual.String()) - } - } -} - -func Test_updateTOC(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "Lorem ipsum\ndolor sit amet\n", - "Lorem ipsum\ndolor sit amet\n", - }, - { - "# Title\nLorem ipsum \n**table of contents**\n\nold cruft\n\n## Section Heading\ndolor sit amet\n", - "# Title\nLorem ipsum \n**table of contents**\n\n\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n\n## Section Heading\ndolor sit amet\n", - }, - } - for _, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updateTOC("filename.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - if !expected.Equal(actual) { - t.Errorf("Expected TOC '%v' but got '%v'", expected.String(), actual.String()) - } - } -} diff --git a/_tools/mungedocs/util.go b/_tools/mungedocs/util.go deleted file mode 100644 index c25e1d1976..0000000000 --- a/_tools/mungedocs/util.go +++ /dev/null @@ -1,291 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "path" - "path/filepath" - "regexp" - "strings" - "unicode" -) - -// Replaces the text between matching "beginMark" and "endMark" within the -// document represented by "lines" with "insertThis". -// -// Delimiters should occupy own line. -// Returns copy of document with modifications. -func updateMacroBlock(mlines mungeLines, token string, insertThis mungeLines) (mungeLines, error) { - beginMark := beginMungeTag(token) - endMark := endMungeTag(token) - var out mungeLines - betweenBeginAndEnd := false - for _, mline := range mlines { - if mline.preformatted && !betweenBeginAndEnd { - out = append(out, mline) - continue - } - line := mline.data - if mline.beginTag && line == beginMark { - if betweenBeginAndEnd { - return nil, fmt.Errorf("found second begin mark while updating macro blocks") - } - betweenBeginAndEnd = true - out = append(out, mline) - } else if mline.endTag && line == endMark { - if !betweenBeginAndEnd { - return nil, fmt.Errorf("found end mark without begin mark while updating macro blocks") - } - betweenBeginAndEnd = false - out = append(out, insertThis...) - out = append(out, mline) - } else { - if !betweenBeginAndEnd { - out = append(out, mline) - } - } - } - if betweenBeginAndEnd { - return nil, fmt.Errorf("never found closing end mark while updating macro blocks") - } - return out, nil -} - -// Tests that a document, represented as a slice of lines, has a line. Ignores -// leading and trailing space. -func hasLine(lines mungeLines, needle string) bool { - for _, mline := range lines { - haystack := strings.TrimSpace(mline.data) - if haystack == needle { - return true - } - } - return false -} - -func removeMacroBlock(token string, mlines mungeLines) (mungeLines, error) { - beginMark := beginMungeTag(token) - endMark := endMungeTag(token) - var out mungeLines - betweenBeginAndEnd := false - for _, mline := range mlines { - if mline.preformatted { - out = append(out, mline) - continue - } - line := mline.data - if mline.beginTag && line == beginMark { - if betweenBeginAndEnd { - return nil, fmt.Errorf("found second begin mark while updating macro blocks") - } - betweenBeginAndEnd = true - } else if mline.endTag && line == endMark { - if !betweenBeginAndEnd { - return nil, fmt.Errorf("found end mark without begin mark while updating macro blocks") - } - betweenBeginAndEnd = false - } else { - if !betweenBeginAndEnd { - out = append(out, mline) - } - } - } - if betweenBeginAndEnd { - return nil, fmt.Errorf("never found closing end mark while updating macro blocks") - } - return out, nil -} - -// Add a macro block to the beginning of a set of lines -func prependMacroBlock(token string, mlines mungeLines) mungeLines { - beginLine := newMungeLine(beginMungeTag(token)) - endLine := newMungeLine(endMungeTag(token)) - out := mungeLines{beginLine, endLine} - if len(mlines) > 0 && mlines[0].data != "" { - out = append(out, blankMungeLine) - } - return append(out, mlines...) -} - -// Add a macro block to the end of a set of lines -func appendMacroBlock(mlines mungeLines, token string) mungeLines { - beginLine := newMungeLine(beginMungeTag(token)) - endLine := newMungeLine(endMungeTag(token)) - out := mlines - if len(mlines) > 0 && mlines[len(mlines)-1].data != "" { - out = append(out, blankMungeLine) - } - return append(out, beginLine, endLine) -} - -// Tests that a document, represented as a slice of lines, has a macro block. -func hasMacroBlock(lines mungeLines, token string) bool { - beginMark := beginMungeTag(token) - endMark := endMungeTag(token) - - foundBegin := false - for _, mline := range lines { - if mline.preformatted { - continue - } - if !mline.beginTag && !mline.endTag { - continue - } - line := mline.data - switch { - case !foundBegin && line == beginMark: - foundBegin = true - case foundBegin && line == endMark: - return true - } - } - return false -} - -// Returns the canonical begin-tag for a given description. This does not -// include the trailing newline. -func beginMungeTag(desc string) string { - return fmt.Sprintf("", desc) -} - -// Returns the canonical end-tag for a given description. This does not -// include the trailing newline. -func endMungeTag(desc string) string { - return fmt.Sprintf("", desc) -} - -type mungeLine struct { - data string - preformatted bool - header bool - link bool - beginTag bool - endTag bool -} - -type mungeLines []mungeLine - -func (m1 mungeLines) Equal(m2 mungeLines) bool { - if len(m1) != len(m2) { - return false - } - for i := range m1 { - if m1[i].data != m2[i].data { - return false - } - } - return true -} - -func (mlines mungeLines) String() string { - slice := []string{} - for _, mline := range mlines { - slice = append(slice, mline.data) - } - s := strings.Join(slice, "\n") - // We need to tack on an extra newline at the end of the file - return s + "\n" -} - -func (mlines mungeLines) Bytes() []byte { - return []byte(mlines.String()) -} - -var ( - // Finds all preformatted block start/stops. - preformatRE = regexp.MustCompile("^\\s*```") - notPreformatRE = regexp.MustCompile("^\\s*```.*```") - // Is this line a header? - mlHeaderRE = regexp.MustCompile(`^#`) - // Is there a link on this line? - mlLinkRE = regexp.MustCompile(`\[[^]]*\]\([^)]*\)`) - beginTagRE = regexp.MustCompile(`