Import kubernetes/community docs (#6863)

* Import docs from other repos * imported test docs * fix README copy * Add ToC YAML and index page for imported docs. * modify update-imported-docs.sh * Add `imported` to global ToCs * Add nav to imported section * remove unnecessary scope for docs/imported * update script w/ comments and directory root fixes * use genCmd variable * display output of generation command * Update instructions * Add Community to /docs/imported/index.md * fix travis errors * update README.md and fix community docs issues * simplify code, add comment * make sure the binary is used instead of `go run` * add automated handling for links from imported files
2018-01-10 20:03:16 -08:00
parent 6dfb0356b8
commit a95ca06339
14 changed files with 1067 additions and 37 deletions
@@ -1,5 +0,0 @@
-update-imported-docs/update-imported-docs.go will update the target files generated from other repos.
-You should modify update-imported-docs/config.yaml to reflect the desired src and dst path.
-```
-go run update-imported-docs/update-imported-docs.go
-```
@@ -0,0 +1,75 @@
+# Update imported docs
+
+This script updates the target files generated from other repos listed in the <config.yml> file, which is specified as the command line argument.
+
+## Requirements
+
+Imported docs must follow these guidelines:
+
+1. Be listed somewhere in the `/_data/imported.yml` table of contents file.
+1. Have `title` defined in the front matter. For example:
+
+    ```
+    ---
+    title: Title Displayed in Table of Contents
+    ---
+
+    Rest of the .md file...
+    ```
+
+1. Adhere to the [Documentation Style Guide](/docs/home/contribute/style-guide/).
+
+## Usage
+
+From within this directory, run the following command:
+
+```
+./update-imported-docs <config.yaml>
+```
+
+The output should look similar to the following:
+
+```
+Website root directory: /Users/someuser/git/kubernetes-website
+
+            *   *   *
+
+Cloning repo "community"...
+
+            *   *   *
+
+Docs imported! Run 'git add .' 'git commit -m <comment>' and 'git push' to upload them.
+```
+
+## Config file format
+
+Each config file may contain multiple repos, which will be imported together. You should modify the corresponding `update-imported-docs/<config.yml>` file to reflect the desired `src` and `dst` paths.
+
+You may also create new config files for different groups of documents to import. The following is an example of the YAML file format:
+
+```
+repos:
+- name: kubernetes                          #tmp directory name
+  remote: https://github.com/kubernetes/kubernetes.git
+  branch: release-1.9
+  generate-command: hack/generate-docs.sh   #optional command to run
+  files:
+  - src: docs/admin/cloud-controller-manager.md
+    dst: docs/reference/generated/cloud-controller-manager.md
+  - src: docs/admin/kube-apiserver.md
+    dst: docs/reference/generated/kube-apiserver.md
+- name: community                           #tmp directory name
+  remote: https://github.com/kubernetes/community.git
+  branch: master
+  files:
+  - src: contributors/devel/README.md
+    dst: docs/imported/community/devel.md
+  - src: contributors/guide/README.md
+    dst: docs/imported/community/guide.md
+```
+
+Note: `generate-command` is an optional entry, which can be used to run a given command to auto-generate the docs from within that repo.
+
+## Fixing Links
+
+To fix relative links within your imported files, set the repo config's `gen-absolute-links` value to `true`. You can see an example of this in [`community.yml`](community.yml).
@@ -0,0 +1,14 @@
+repos:
+- name: community
+  remote: https://github.com/kubernetes/community.git
+  branch: master
+  gen-absolute-links: true
+  files:
+  - src: contributors/devel/README.md
+    dst: docs/imported/community/devel.md
+  - src: contributors/guide/README.md
+    dst: docs/imported/community/guide.md
+  - src: mentoring/README.md
+    dst: docs/imported/community/mentoring.md
+  - src: keps/1-kubernetes-enhancement-proposal-process.md
+    dst: docs/imported/community/keps.md
@@ -2,6 +2,7 @@ repos:
 - name: kubernetes
  remote: https://github.com/kubernetes/kubernetes.git
  branch: release-1.9
+  generate-command: hack/generate-docs.sh
  files:
  - src: docs/admin/cloud-controller-manager.md
    dst: docs/reference/generated/cloud-controller-manager.md
@@ -21,6 +22,7 @@ repos:
  remote: https://github.com/kubernetes/federation.git
 #  # Change this to a release branch when federation has release branches.
  branch: master
+  generate-command: hack/generate-docs.sh
  files:
  - src: docs/admin/federation-apiserver.md
    dst: docs/reference/generated/federation-apiserver.md
@@ -1,35 +1,66 @@
 package main

 import (
+  "bufio"
  "fmt"
-  "io"
  "io/ioutil"
  "os"
  "os/exec"
  "path"
  "path/filepath"
  "regexp"
-
+  "strings"
  "github.com/ghodss/yaml"
 )

 func main() {
-  websiteRepo, err := os.Getwd()
-  checkError(err)

-  content, err := ioutil.ReadFile("update-imported-docs/config.yaml")
-  if err != nil {
-    fmt.Fprintf(os.Stderr, "error when reading file: %v\n", err)
+  //get command line arguments without executable
+  clArgs := os.Args[1:]
+
+  //check that an argument has been passed in
+  if len(clArgs) == 0 {
+    fmt.Fprintf(os.Stderr, "Please specify a config file as a command line argument.\n")
+    os.Exit(1)
+  }
+  configFile := clArgs[0]
+
+  //get directory of executable
+  ex, err := os.Executable()
+  checkError(err)
+  exPath := filepath.Dir(ex) //file path of updated-imported-docs executable
+  suffix := filepath.Base(exPath) //should be "updated-imported-docs"
+
+  //check if suffix is "updated-imported-docs"
+  if suffix != "update-imported-docs" {
+    fmt.Fprintf(os.Stderr, "Instead of `go run update-imported-docs.go <config.yml>`, use the compiled binary `./update-imported-docs <config.yml>`\n")
    os.Exit(1)
  }

+  //set root directory of website
+  websiteRepo := filepath.Clean(strings.TrimSuffix(exPath,suffix)) //path of parent directory
+  fmt.Fprintf(os.Stdout, "Website root directory: %s\n", websiteRepo)
+
+  //read config.yaml file specified by first command line argument
+  content, err := ioutil.ReadFile(configFile)
+  if err != nil {
+    fmt.Fprintf(os.Stderr, "Error when reading file: %v\n", err)
+    os.Exit(1)
+  }
+
+  //convert contents of config.yml file into a map
  var config map[string]interface{}
  err = yaml.Unmarshal(content, &config)
  if err != nil {
-    fmt.Fprintf(os.Stderr, "error when unmarshal the config file: %v\n", err)
+    fmt.Fprintf(os.Stderr, "Error when unmarshal the config file: %v\n", err)
    os.Exit(1)
  }

+  //change working directory to website root
+  err = os.Chdir(websiteRepo)
+  checkError(err)
+
+  //clean out temp directory
  tmpDir := "/tmp/update_docs"
  os.RemoveAll(tmpDir)
  os.Mkdir(tmpDir, 0750)
@@ -40,32 +71,74 @@ func main() {
  // title: ***
  // notile: ***
  // ---
-  titleRegex := regexp.MustCompile("^---\n(.*\n)*---\n")
+  titleRegex := regexp.MustCompile("^---\ntitle:(.*\n)*?---\n")

+  // To extract repo path prefix from `remote`
+  remoteGitRegex := regexp.MustCompile("(https://.*)\\.git$")
+
+  //execute for each repo
  repos := config["repos"].([]interface{})
  for _, repo := range repos {
    err = os.Chdir(tmpDir)
    checkError(err)

+    //get config info for repo, clone repo locally
    r := repo.(map[string]interface{})
    repoName := r["name"].(string)
+    remotePathMatch := remoteGitRegex.FindAllStringSubmatch(r["remote"].(string), -1)
+    if (len(remotePathMatch) == 0) {
+      fmt.Fprintf(os.Stderr, "\n\t\t\t!\t!\t!\n\nInvalid remote path %q. Schema should look like: https://<url>.git\n", r["remote"].(string))
+      os.Exit(1)
+    }
+    remotePrefix := fmt.Sprintf("%s/tree/master", remotePathMatch[0][1])
+
    cmd := "git"
    args := []string{"clone", "--depth=1", "-b", r["branch"].(string), r["remote"].(string), repoName}
-    fmt.Fprintf(os.Stdout, "Cloning repo %q\n", repoName)
+    fmt.Fprintf(os.Stdout, "\n\t\t\t*\t*\t*\n\nCloning repo %q...\n", repoName)
    if err := exec.Command(cmd, args...).Run(); err != nil {
-      fmt.Fprintf(os.Stderr, "error when cloning repo %q: %v\n", repoName, err)
+      fmt.Fprintf(os.Stderr, "\n\t\t\t!\t!\t!\n\nError when cloning repo %q: %v\n", repoName, err)
      os.Exit(1)
    }

    err = os.Chdir(repoName)
    checkError(err)

-    fmt.Fprintf(os.Stdout, "Generating docs for repo %q\n", repoName)
-    if err := exec.Command("hack/generate-docs.sh").Run(); err != nil {
-      fmt.Fprintf(os.Stderr, "error when generating docs for repo %q: %v\n", repoName, err)
-      os.Exit(1)
+    //if generate-command is specified in the repo config,
+    //run the command for that repo, e.g. "hack/generate-docs.sh"
+    if r["generate-command"] != nil {
+      genCmd := r["generate-command"].(string)
+
+      fmt.Fprintf(os.Stdout, "Generating docs for repo %q with %q...\n\n", repoName, genCmd)
+      cmd := exec.Command(genCmd)
+      cmdReader, err := cmd.StdoutPipe()
+      if err != nil {
+        fmt.Fprintf(os.Stderr, "\n\t\t\t!\t!\t!\n\nError when generating docs for repo %q: %v\n", repoName, err)
+        os.Exit(1)
+      }
+
+      //display running output of generate command
+      scanner := bufio.NewScanner(cmdReader)
+      go func() {
+        for scanner.Scan() {
+          fmt.Printf("generator output | %s\n", scanner.Text())
+        }
+      }()
+
+      err = cmd.Start()
+      if err != nil {
+        fmt.Fprintln(os.Stderr, "Error starting %q command\n", genCmd, err)
+        os.Exit(1)
+      }
+
+      err = cmd.Wait()
+      if err != nil {
+        fmt.Fprintln(os.Stderr, "Error waiting for %q command\n", genCmd, err)
+        os.Exit(1)
+      }
+
    }

+    //copy and rename files from src -> dst specified in config
    err = os.Chdir(websiteRepo)
    checkError(err)
    files := r["files"].([]interface{})
@@ -73,6 +146,7 @@ func main() {
      f := file.(map[string]interface{})
      src := f["src"].(string)
      dst := f["dst"].(string)
+      srcDir := filepath.Dir(src)
      absSrc, err := filepath.Abs(path.Join(tmpDir, repoName, src))
      checkError(err)
      absDst, err := filepath.Abs(dst)
@@ -82,38 +156,53 @@ func main() {
      titleBlock := titleRegex.Find(content)
      content, err = ioutil.ReadFile(absSrc)
      checkError(err)
+
+      // Write to new output file
      dstFile, err := os.OpenFile(absDst, os.O_RDWR|os.O_CREATE, 0755)
      checkError(err)
      defer dstFile.Close()
      _, err = dstFile.Write(titleBlock)
      checkError(err)
+
+      // Process content if necessary
+      if r["gen-absolute-links"] != nil {
+        content = processLinks(content, remotePrefix, srcDir)
+      }
      _, err = dstFile.Write(content)
      checkError(err)
      dstFile.Sync()
    }
  }
-  fmt.Fprintf(os.Stdout, "Docs imported! Run 'git add .' 'git commit -m <comment>' and 'git push' to upload them\n")
+  fmt.Fprintf(os.Stdout, "\n\t\t\t*\t*\t*\n\nDocs imported! Run 'git add .' 'git commit -m <comment>' and 'git push' to upload them.\n")
 }

-func copyFile(src, dst string) error {
-  sf, err := os.Open(src)
-  if err != nil {
-    return err
-  }
-  defer sf.Close()
+//
+func processLinks(content []byte, remotePrefix string, subPath string) []byte {
+  // To catch anything of the form [text](url)
+  linkRegex := regexp.MustCompile("(\\[.+?\\])\\(([^\\s\\)]+)\\)")
+  // Regexes to skip
+  absUrlRegex := regexp.MustCompile("https*://")
+  mailRegex := regexp.MustCompile("mailto:")

-  df, err := os.Create(dst)
-  if err != nil {
-    return err
-  }
-  defer df.Close()
+  processedContent := linkRegex.ReplaceAllFunc(content, func(b []byte) []byte {
+    if (absUrlRegex.Match(b) || mailRegex.Match(b)) {
+      return b // no processing needed
+    }
+    match := linkRegex.FindAllStringSubmatch(string(b), -1)
+    url := match[0][2]
+    if url[0] == '#' { // link on current page
+      return b
+    } else if url[0] == '/' { // link at root of repo
+      return []byte(fmt.Sprintf("%s(%s/%s)", match[0][1], remotePrefix, url[1:]))
+    } else { // link relative to current page
+      return []byte(fmt.Sprintf("%s(%s/%s/%s)", match[0][1], remotePrefix, subPath, url))
+    }
+  })

-  _, err = io.Copy(df, sf)
-  if err != nil {
-    return err
-  }
+  h1Regex := regexp.MustCompile("^(# .*)?\n")
+  processedContent = h1Regex.ReplaceAll(processedContent, []byte(""))

-  return df.Sync()
+  return processedContent
 }

 func checkError(err error) {